[Trunk | Orcid Service]:

1. pom.xml: 
        a. Added timestamp in properties.
        b. In <build> added <resources> with filtering set to true.
2. GlobalVars.java: Added GlobalVars class for date of deploy & date of build.
3. UoaOrcidServiceApplication.java: 
        a. Added  @PropertySource("classpath:authorization.properties")
        b. In @EnableConfigurationProperties added GlobalVars.class.
4. OrcidServiceCheckDeployController.java: Added exposing date of deploy and date of build.
5. orcidservice.properties: Added property orcidservice.globalVars.buildDate=@timestamp@.
6. Metrics.java: Added class for saving/getting metrics for orcid KPIs.
7. MetricsDAO.java & MongoDBMetricsDAO.java: DAOs to handle new collections in DB (metrics).
8. MetricsService.java: Added methods to calculate and get metrics for orcid KPIs.
9. MetricsController.java: Added API call methods to calculate and return metrics for orcid KPIs (only /report/metrics is needed for Prometheus).
This commit is contained in:
Konstantina Galouni 2021-09-10 07:57:45 +00:00
parent 6a714c115c
commit 2273f3c295
11 changed files with 344 additions and 3 deletions

View File

@ -21,6 +21,8 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.version>1.8</java.version>
<timestamp>${maven.build.timestamp}</timestamp>
<maven.build.timestamp.format>E MMM dd HH:mm:ss z yyyy</maven.build.timestamp.format>
</properties>
<dependencies>
@ -107,6 +109,12 @@
<!--</plugin>-->
</plugins>
<finalName>uoa-orcid-service</finalName>
<resources>
<resource>
<directory>src/main/resources</directory>
<filtering>true</filtering>
</resource>
</resources>
</build>

View File

@ -1,6 +1,7 @@
package eu.dnetlib.uoaorcidservice;
import eu.dnetlib.uoaauthorizationlibrary.configuration.AuthorizationConfiguration;
import eu.dnetlib.uoaorcidservice.configuration.GlobalVars;
import eu.dnetlib.uoaorcidservice.configuration.properties.MongoConfig;
//import eu.dnetlib.uoaauthorizationlibrary.configuration.AuthorizationConfiguration;
import eu.dnetlib.uoaorcidservice.configuration.properties.OrcidConfig;
@ -14,10 +15,11 @@ import org.springframework.context.annotation.PropertySources;
@SpringBootApplication(scanBasePackages = {"eu.dnetlib.uoaorcidservice"})
@PropertySources({
@PropertySource("classpath:authorization.properties"),
@PropertySource("classpath:orcidservice.properties"),
@PropertySource(value = "classpath:dnet-override.properties", ignoreResourceNotFound = true)
})
@EnableConfigurationProperties({MongoConfig.class, AESUtils.class, OrcidConfig.class})
@EnableConfigurationProperties({MongoConfig.class, AESUtils.class, OrcidConfig.class, GlobalVars.class})
@Import(AuthorizationConfiguration.class)
public class UoaOrcidServiceApplication {

View File

@ -0,0 +1,22 @@
package eu.dnetlib.uoaorcidservice.configuration;
import org.springframework.boot.context.properties.ConfigurationProperties;
import java.util.Date;
@ConfigurationProperties("orcidservice.globalVars")
public class GlobalVars {
public static Date date = new Date();
private Date buildDate;
public String getBuildDate() {
if(buildDate == null) {
return null;
}
return buildDate.toString();
}
public void setBuildDate(Date buildDate) {
this.buildDate = buildDate;
}
}

View File

@ -75,4 +75,15 @@ public class MetricsController {
public List<Object> countTotalWorks() {
return metricsService.countTotalWorks();
}
@RequestMapping(value = "/report/metrics", method = RequestMethod.GET, produces={"text/plain"})
public String getMetrics() {
return metricsService.getMetrics();
}
@RequestMapping(value = "/report/calc", method = RequestMethod.GET)
public String calculateMetrics() {
metricsService.calculateMetrics();
return "ALL GOOD";
}
}

View File

@ -3,11 +3,15 @@ package eu.dnetlib.uoaorcidservice.controllers;
import com.mongodb.BasicDBObject;
import com.mongodb.CommandResult;
import com.mongodb.DBObject;
import com.mongodb.MongoTimeoutException;
import eu.dnetlib.uoaorcidservice.configuration.GlobalVars;
import eu.dnetlib.uoaorcidservice.configuration.mongo.MongoConnection;
import eu.dnetlib.uoaorcidservice.configuration.properties.MongoConfig;
import eu.dnetlib.uoaorcidservice.configuration.properties.OrcidConfig;
import eu.dnetlib.uoaorcidservice.handlers.utils.AESUtils;
import org.apache.log4j.Logger;
import org.bson.*;
import org.bson.conversions.Bson;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.security.access.prepost.PreAuthorize;
@ -16,6 +20,7 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
@ -33,6 +38,9 @@ public class OrcidServiceCheckDeployController {
@Autowired
private OrcidConfig orcidConfig;
@Autowired
private GlobalVars globalVars;
@Autowired
private AESUtils aesUtils;
@ -56,6 +64,25 @@ public class OrcidServiceCheckDeployController {
response.put("Mongo catch: error", e.getMessage());
}
// Document ping = new Document("ping", "1");
// try {
// Document answer = mt.getDb().runCommand(ping);//.command(ping);
//
// for(String key : answer.keySet()) {
// log.debug("answer pair: "+key + ": "+answer.get(key));
// }
//
// if (answer != null && answer.get("ok") != null && (Double)answer.get("ok") == 1.0d) {
// response.put("Mongo try: everything is ok", answer.toString());//.getErrorMessage());
// } else {
// response.put("Mongo try: error", answer.toString());//.getErrorMessage());
// }
// } catch (MongoTimeoutException e) {
// response.put("Mongo catch timeout: error", e.getMessage());
// } catch (Exception e) {
// response.put("Mongo catch: error", e.getMessage());
// }
response.put("orcidservice.mongodb.database", mongoConfig.getDatabase());
response.put("orcidservice.mongodb.host", mongoConfig.getHost());
response.put("orcidservice.mongodb.port", mongoConfig.getPort()+"");
@ -69,6 +96,12 @@ public class OrcidServiceCheckDeployController {
response.put("orcidservice.encryption.password", aesUtils.getPassword() == null ? null : "[unexposed value]");
if(globalVars.date != null) {
response.put("Date of deploy", globalVars.date.toString());
}
if(globalVars.getBuildDate() != null) {
response.put("Date of build", globalVars.getBuildDate());
}
return response;
}
}

View File

@ -4,6 +4,8 @@ import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.web.ErrorAttributes;
import org.springframework.boot.autoconfigure.web.ErrorController;
//import org.springframework.boot.web.servlet.error.ErrorAttributes;
//import org.springframework.boot.web.servlet.error.ErrorController;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.util.Assert;
@ -12,6 +14,7 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.context.request.RequestAttributes;
import org.springframework.web.context.request.ServletRequestAttributes;
import org.springframework.web.context.request.ServletWebRequest;
import javax.servlet.RequestDispatcher;
import javax.servlet.http.HttpServletRequest;
@ -93,6 +96,7 @@ public class SimpleErrorController implements ErrorController {
private Map<String, Object> getErrorAttributes(HttpServletRequest aRequest, boolean includeStackTrace) {
RequestAttributes requestAttributes = new ServletRequestAttributes(aRequest);
// ServletWebRequest requestAttributes = new ServletWebRequest(aRequest);
return errorAttributes.getErrorAttributes(requestAttributes, includeStackTrace);
}
}

View File

@ -0,0 +1,18 @@
package eu.dnetlib.uoaorcidservice.dao;
import eu.dnetlib.uoaorcidservice.entities.Metrics;
import java.util.List;
import java.util.Optional;
public interface MetricsDAO {
List<Metrics> findAll();
Optional<Metrics> findById(String Id);
Metrics save(Metrics metrics);
void deleteAll();
void deleteById(String Id);
}

View File

@ -0,0 +1,20 @@
package eu.dnetlib.uoaorcidservice.dao;
import eu.dnetlib.uoaorcidservice.entities.Metrics;
import org.springframework.data.mongodb.repository.MongoRepository;
import java.util.List;
import java.util.Optional;
public interface MongoDBMetricsDAO extends MetricsDAO, MongoRepository<Metrics, String> {
List<Metrics> findAll();
Optional<Metrics> findById(String Id);
Metrics save(Metrics metrics);
void deleteAll();
void deleteById(String Id);
}

View File

@ -0,0 +1,80 @@
package eu.dnetlib.uoaorcidservice.entities;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.springframework.data.annotation.Id;
import java.util.Date;
import java.util.List;
enum Identifier {
// Do not rename or remove existring values. This may cause problems with already stored values in DB
current, previous
}
public class Metrics {
@Id
@JsonProperty("_id")
private Identifier id;
private int total_works;
private int total_users;
// private List<Object> works_per_month;
private List<Object> works_per_dashboard;
private Date date;
public String getId() {
if(id == null) {
return null;
}
return id.name();
}
public void setId(String id) {
if(id == null) {
this.id = null;
} else {
Identifier identifier = Identifier.valueOf(id);
this.id = identifier;
}
}
public int getTotal_works() {
return total_works;
}
public void setTotal_works(int total_works) {
this.total_works = total_works;
}
public int getTotal_users() {
return total_users;
}
public void setTotal_users(int total_users) {
this.total_users = total_users;
}
// public List<Object> getWorks_per_month() {
// return works_per_month;
// }
//
// public void setWorks_per_month(List<Object> works_per_month) {
// this.works_per_month = works_per_month;
// }
public List<Object> getWorks_per_dashboard() {
return works_per_dashboard;
}
public void setWorks_per_dashboard(List<Object> works_per_dashboard) {
this.works_per_dashboard = works_per_dashboard;
}
public Date getDate() {
return date;
}
public void setDate(Date date) {
this.date = date;
}
}

View File

@ -1,15 +1,21 @@
package eu.dnetlib.uoaorcidservice.services;
import eu.dnetlib.uoaorcidservice.dao.MongoDBUserTokensDAO;
import eu.dnetlib.uoaorcidservice.dao.MetricsDAO;
import eu.dnetlib.uoaorcidservice.dao.customDAOs.MongoDBUserTokensDAOCustom;
import eu.dnetlib.uoaorcidservice.dao.customDAOs.MongoDBWorkDAOCustom;
import eu.dnetlib.uoaorcidservice.entities.Metrics;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.util.List;
import java.time.Instant;
import java.util.*;
@EnableScheduling
@Service
public class MetricsService {
private final Logger log = Logger.getLogger(this.getClass());
@ -22,6 +28,9 @@ public class MetricsService {
@Qualifier("mongoDBUserTokensDAO")
private MongoDBUserTokensDAOCustom userTokensDAO;
@Autowired
private MetricsDAO metricsDAO;
public List<Object> countWorksPerDashboard() {
return workDAO.worksPerDashboard();
}
@ -65,4 +74,137 @@ public class MetricsService {
public List<Object> countTotalWorks() {
return workDAO.totalWorks();
}
// every day at midnight
// @Scheduled(cron = "0 0 0 * * ?")
// // every 5 mins for testing
@Scheduled(cron = "0 0/5 * * * *")
public List<Object> calculateMetrics() {
log.info("Calculate metrics and add them in DB");
Optional<Metrics> oldMetrics = metricsDAO.findById("current");
if(oldMetrics.isPresent()) {
oldMetrics.get().setId("previous");
metricsDAO.save(oldMetrics.get());
}
Metrics metrics = new Metrics();
metrics.setId("current");
List<Object> totalWorks = countTotalWorks();
if(totalWorks != null && totalWorks.get(0) != null) {
Map<String, Integer> works = (HashMap<String, Integer>) totalWorks.get(0);
metrics.setTotal_works(works.get("works"));
} else {
metrics.setTotal_works(0);
}
List<Object> totalUsers = countTotalUniqueUsers();
if(totalUsers != null && totalUsers.get(0) != null) {
Map<String, Integer> users = (HashMap<String, Integer>) totalUsers.get(0);
metrics.setTotal_users(users.get("users"));
} else {
metrics.setTotal_users(0);
}
// metrics.setWorks_per_month(countWorksPerYearAndMonth());
List<Object> works_per_dashboard = countWorksPerDashboard();
for(Object dashboardWorksObj : works_per_dashboard) {
Map<String, Object> dashboardWorks = (HashMap<String, Object>) dashboardWorksObj;
String[] dashboard_elements = ((String) dashboardWorks.get("dashboard")).split("_", 2);
dashboardWorks.put("environment", dashboard_elements[0]);
dashboardWorks.put("dashboard", dashboard_elements[1]);
// if(dashboard_elements.length > 2) {
// dashboardWorks.put("dashboardName", dashboard_elements[2]);
// }
}
metrics.setWorks_per_dashboard(works_per_dashboard);
metrics.setDate(new Date());
metricsDAO.save(metrics);
// Total works linked: http://duffy.di.uoa.gr:8080/uoa-orcid-service/report/totalWorks
// Total users: (unique/ per orcid): http://duffy.di.uoa.gr:8080/uoa-orcid-service/report/totalUniqueUsers
// Works linked per month: http://duffy.di.uoa.gr:8080/uoa-orcid-service/report/worksPerYearAndMonth (not needed?)
// Works linked per portal (not only explore): http://duffy.di.uoa.gr:8080/uoa-orcid-service/report/worksPerDashboard
return totalWorks;
}
public String getMetrics() {
// # TYPE aai_registered_users_total gauge
// aai_registered_users_total 50
//# TYPE aai_logins_total counter
// aai_logins_total 1742
//# TYPE aai_api_requests_total counter
// aai_api_requests_total 0
// aai_last_metrics_updater_run_timestamp_seconds 1619157852
Optional<Metrics> metrics_optional = metricsDAO.findById("current");
if(!metrics_optional.isPresent()) {
return null;
}
Metrics metrics = metrics_optional.get();
String response = "";
// response += "# TYPE explore_orcid_works_total gauge\n";
// response += "explore_orcid_works_total "+metrics.getTotal_works() + "\n";
response += "# TYPE orcid_users gauge\n";
response += "orcid_users "+metrics.getTotal_users() + "\n";
// if(metrics.getWorks_per_month() != null) {
// for(Object monthlyWorksObj : metrics.getWorks_per_month()) {
// Map<String, Integer> monthlyWorks = (HashMap<String, Integer>) monthlyWorksObj;
// response += "# TYPE orcid_total_works_"+monthlyWorks.get("month")+"_"+monthlyWorks.get("year")+" counter\n";
// response += "orcid_total_works_"+monthlyWorks.get("month")+"_"+monthlyWorks.get("year")+" "+monthlyWorks.get("works") + "\n";
//
// }
// }
if(metrics.getWorks_per_dashboard() != null) {
// 2nd approach
// Map<String, Integer> worksPerEnvironment = new HashMap<>();
// 3rd approach
Map<String, Integer> worksPerDashboard = new HashMap<>();
// 1st approach
// response += "# TYPE orcid_works gauge\n";
for(Object dashboardWorksObj : metrics.getWorks_per_dashboard()) {
Map<String, Object> dashboardWorks = (HashMap<String, Object>) dashboardWorksObj;
// 1st approach - e.g. orcid_works{envoronment="production" portal="explore"} 10
// response += "orcid_works{environment=\""+dashboardWorks.get("environment")+"\" portal=\""+dashboardWorks.get("dashboard")+"\"}"+" "+dashboardWorks.get("works") + "\n";
// 2nd approach - e.g. explore_orcid_works_total{envoronment="production"} 10
// if(worksPerEnvironment.containsKey(dashboardWorks.get("environment"))) {
// int worksSoFar = worksPerEnvironment.get(dashboardWorks.get("environment"));
// worksPerEnvironment.put((String)dashboardWorks.get("environment"), (Integer) dashboardWorks.get("works") + worksSoFar);
// } else {
// worksPerEnvironment.put((String)dashboardWorks.get("environment"), (Integer) dashboardWorks.get("works"));
// }
// 3rd approach - e.g. orcid_works{portal="explore"} 10
if(worksPerDashboard.containsKey(dashboardWorks.get("dashboard"))) {
int worksSoFar = worksPerDashboard.get(dashboardWorks.get("dashboard"));
worksPerDashboard.put((String)dashboardWorks.get("dashboard"), (Integer) dashboardWorks.get("works") + worksSoFar);
} else {
worksPerDashboard.put((String)dashboardWorks.get("dashboard"), (Integer) dashboardWorks.get("works"));
}
}
// 2nd approach
// response += "# TYPE explore_orcid_works_total gauge\n";
// for(Map.Entry<String, Integer> envWorks : worksPerEnvironment.entrySet()) {
// response += "explore_orcid_works_total{environment=\""+envWorks.getKey()+"\"} "+envWorks.getValue() + "\n";
// }
// 3rd approach
response += "# TYPE orcid_works gauge\n";
for(Map.Entry<String, Integer> envWorks : worksPerDashboard.entrySet()) {
response += "orcid_works{portal=\""+envWorks.getKey()+"\"} "+envWorks.getValue() + "\n";
}
}
Instant instant = metrics.getDate().toInstant();
response += "orcid_last_metrics_updater_run_timestamp_seconds "+instant.getEpochSecond()+"\n";
return response;
}
}

View File

@ -7,6 +7,7 @@ orcidservice.orcid.apiURL=https://api.sandbox.orcid.org/v3.0/
orcidservice.orcid.tokenURL=https://sandbox.orcid.org/oauth/token
orcidservice.orcid.clientId=APP-A5M3KTX6NCN67L91
orcidservice.orcid.clientSecret=96b20d71-ae06-4286-bb00-9172536c1ad4
orcidservice.globalVars.buildDate=@timestamp@
#orcidservice.encryption.password=...