[Trunk | Orcid Service]:
1. pom.xml: a. Added timestamp in properties. b. In <build> added <resources> with filtering set to true. 2. GlobalVars.java: Added GlobalVars class for date of deploy & date of build. 3. UoaOrcidServiceApplication.java: a. Added @PropertySource("classpath:authorization.properties") b. In @EnableConfigurationProperties added GlobalVars.class. 4. OrcidServiceCheckDeployController.java: Added exposing date of deploy and date of build. 5. orcidservice.properties: Added property orcidservice.globalVars.buildDate=@timestamp@. 6. Metrics.java: Added class for saving/getting metrics for orcid KPIs. 7. MetricsDAO.java & MongoDBMetricsDAO.java: DAOs to handle new collections in DB (metrics). 8. MetricsService.java: Added methods to calculate and get metrics for orcid KPIs. 9. MetricsController.java: Added API call methods to calculate and return metrics for orcid KPIs (only /report/metrics is needed for Prometheus).
This commit is contained in:
parent
6a714c115c
commit
2273f3c295
8
pom.xml
8
pom.xml
|
@ -21,6 +21,8 @@
|
|||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
|
||||
<java.version>1.8</java.version>
|
||||
<timestamp>${maven.build.timestamp}</timestamp>
|
||||
<maven.build.timestamp.format>E MMM dd HH:mm:ss z yyyy</maven.build.timestamp.format>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
@ -107,6 +109,12 @@
|
|||
<!--</plugin>-->
|
||||
</plugins>
|
||||
<finalName>uoa-orcid-service</finalName>
|
||||
<resources>
|
||||
<resource>
|
||||
<directory>src/main/resources</directory>
|
||||
<filtering>true</filtering>
|
||||
</resource>
|
||||
</resources>
|
||||
</build>
|
||||
|
||||
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
package eu.dnetlib.uoaorcidservice;
|
||||
|
||||
import eu.dnetlib.uoaauthorizationlibrary.configuration.AuthorizationConfiguration;
|
||||
import eu.dnetlib.uoaorcidservice.configuration.GlobalVars;
|
||||
import eu.dnetlib.uoaorcidservice.configuration.properties.MongoConfig;
|
||||
//import eu.dnetlib.uoaauthorizationlibrary.configuration.AuthorizationConfiguration;
|
||||
import eu.dnetlib.uoaorcidservice.configuration.properties.OrcidConfig;
|
||||
|
@ -14,10 +15,11 @@ import org.springframework.context.annotation.PropertySources;
|
|||
|
||||
@SpringBootApplication(scanBasePackages = {"eu.dnetlib.uoaorcidservice"})
|
||||
@PropertySources({
|
||||
@PropertySource("classpath:authorization.properties"),
|
||||
@PropertySource("classpath:orcidservice.properties"),
|
||||
@PropertySource(value = "classpath:dnet-override.properties", ignoreResourceNotFound = true)
|
||||
})
|
||||
@EnableConfigurationProperties({MongoConfig.class, AESUtils.class, OrcidConfig.class})
|
||||
@EnableConfigurationProperties({MongoConfig.class, AESUtils.class, OrcidConfig.class, GlobalVars.class})
|
||||
@Import(AuthorizationConfiguration.class)
|
||||
public class UoaOrcidServiceApplication {
|
||||
|
||||
|
|
|
@ -0,0 +1,22 @@
|
|||
package eu.dnetlib.uoaorcidservice.configuration;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
@ConfigurationProperties("orcidservice.globalVars")
|
||||
public class GlobalVars {
|
||||
public static Date date = new Date();
|
||||
private Date buildDate;
|
||||
|
||||
public String getBuildDate() {
|
||||
if(buildDate == null) {
|
||||
return null;
|
||||
}
|
||||
return buildDate.toString();
|
||||
}
|
||||
|
||||
public void setBuildDate(Date buildDate) {
|
||||
this.buildDate = buildDate;
|
||||
}
|
||||
}
|
|
@ -75,4 +75,15 @@ public class MetricsController {
|
|||
public List<Object> countTotalWorks() {
|
||||
return metricsService.countTotalWorks();
|
||||
}
|
||||
|
||||
@RequestMapping(value = "/report/metrics", method = RequestMethod.GET, produces={"text/plain"})
|
||||
public String getMetrics() {
|
||||
return metricsService.getMetrics();
|
||||
}
|
||||
|
||||
@RequestMapping(value = "/report/calc", method = RequestMethod.GET)
|
||||
public String calculateMetrics() {
|
||||
metricsService.calculateMetrics();
|
||||
return "ALL GOOD";
|
||||
}
|
||||
}
|
||||
|
|
|
@ -3,11 +3,15 @@ package eu.dnetlib.uoaorcidservice.controllers;
|
|||
import com.mongodb.BasicDBObject;
|
||||
import com.mongodb.CommandResult;
|
||||
import com.mongodb.DBObject;
|
||||
import com.mongodb.MongoTimeoutException;
|
||||
import eu.dnetlib.uoaorcidservice.configuration.GlobalVars;
|
||||
import eu.dnetlib.uoaorcidservice.configuration.mongo.MongoConnection;
|
||||
import eu.dnetlib.uoaorcidservice.configuration.properties.MongoConfig;
|
||||
import eu.dnetlib.uoaorcidservice.configuration.properties.OrcidConfig;
|
||||
import eu.dnetlib.uoaorcidservice.handlers.utils.AESUtils;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.bson.*;
|
||||
import org.bson.conversions.Bson;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.data.mongodb.core.MongoTemplate;
|
||||
import org.springframework.security.access.prepost.PreAuthorize;
|
||||
|
@ -16,6 +20,7 @@ import org.springframework.web.bind.annotation.RequestMapping;
|
|||
import org.springframework.web.bind.annotation.RequestMethod;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
|
@ -33,6 +38,9 @@ public class OrcidServiceCheckDeployController {
|
|||
@Autowired
|
||||
private OrcidConfig orcidConfig;
|
||||
|
||||
@Autowired
|
||||
private GlobalVars globalVars;
|
||||
|
||||
@Autowired
|
||||
private AESUtils aesUtils;
|
||||
|
||||
|
@ -56,6 +64,25 @@ public class OrcidServiceCheckDeployController {
|
|||
response.put("Mongo catch: error", e.getMessage());
|
||||
}
|
||||
|
||||
// Document ping = new Document("ping", "1");
|
||||
// try {
|
||||
// Document answer = mt.getDb().runCommand(ping);//.command(ping);
|
||||
//
|
||||
// for(String key : answer.keySet()) {
|
||||
// log.debug("answer pair: "+key + ": "+answer.get(key));
|
||||
// }
|
||||
//
|
||||
// if (answer != null && answer.get("ok") != null && (Double)answer.get("ok") == 1.0d) {
|
||||
// response.put("Mongo try: everything is ok", answer.toString());//.getErrorMessage());
|
||||
// } else {
|
||||
// response.put("Mongo try: error", answer.toString());//.getErrorMessage());
|
||||
// }
|
||||
// } catch (MongoTimeoutException e) {
|
||||
// response.put("Mongo catch timeout: error", e.getMessage());
|
||||
// } catch (Exception e) {
|
||||
// response.put("Mongo catch: error", e.getMessage());
|
||||
// }
|
||||
|
||||
response.put("orcidservice.mongodb.database", mongoConfig.getDatabase());
|
||||
response.put("orcidservice.mongodb.host", mongoConfig.getHost());
|
||||
response.put("orcidservice.mongodb.port", mongoConfig.getPort()+"");
|
||||
|
@ -69,6 +96,12 @@ public class OrcidServiceCheckDeployController {
|
|||
|
||||
response.put("orcidservice.encryption.password", aesUtils.getPassword() == null ? null : "[unexposed value]");
|
||||
|
||||
if(globalVars.date != null) {
|
||||
response.put("Date of deploy", globalVars.date.toString());
|
||||
}
|
||||
if(globalVars.getBuildDate() != null) {
|
||||
response.put("Date of build", globalVars.getBuildDate());
|
||||
}
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -4,6 +4,8 @@ import org.apache.log4j.Logger;
|
|||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.autoconfigure.web.ErrorAttributes;
|
||||
import org.springframework.boot.autoconfigure.web.ErrorController;
|
||||
//import org.springframework.boot.web.servlet.error.ErrorAttributes;
|
||||
//import org.springframework.boot.web.servlet.error.ErrorController;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.util.Assert;
|
||||
|
@ -12,6 +14,7 @@ import org.springframework.web.bind.annotation.RequestMapping;
|
|||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.context.request.RequestAttributes;
|
||||
import org.springframework.web.context.request.ServletRequestAttributes;
|
||||
import org.springframework.web.context.request.ServletWebRequest;
|
||||
|
||||
import javax.servlet.RequestDispatcher;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
|
@ -93,6 +96,7 @@ public class SimpleErrorController implements ErrorController {
|
|||
|
||||
private Map<String, Object> getErrorAttributes(HttpServletRequest aRequest, boolean includeStackTrace) {
|
||||
RequestAttributes requestAttributes = new ServletRequestAttributes(aRequest);
|
||||
// ServletWebRequest requestAttributes = new ServletWebRequest(aRequest);
|
||||
return errorAttributes.getErrorAttributes(requestAttributes, includeStackTrace);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,18 @@
|
|||
package eu.dnetlib.uoaorcidservice.dao;
|
||||
|
||||
import eu.dnetlib.uoaorcidservice.entities.Metrics;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
public interface MetricsDAO {
|
||||
List<Metrics> findAll();
|
||||
|
||||
Optional<Metrics> findById(String Id);
|
||||
|
||||
Metrics save(Metrics metrics);
|
||||
|
||||
void deleteAll();
|
||||
|
||||
void deleteById(String Id);
|
||||
}
|
|
@ -0,0 +1,20 @@
|
|||
package eu.dnetlib.uoaorcidservice.dao;
|
||||
|
||||
import eu.dnetlib.uoaorcidservice.entities.Metrics;
|
||||
import org.springframework.data.mongodb.repository.MongoRepository;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
public interface MongoDBMetricsDAO extends MetricsDAO, MongoRepository<Metrics, String> {
|
||||
|
||||
List<Metrics> findAll();
|
||||
|
||||
Optional<Metrics> findById(String Id);
|
||||
|
||||
Metrics save(Metrics metrics);
|
||||
|
||||
void deleteAll();
|
||||
|
||||
void deleteById(String Id);
|
||||
}
|
|
@ -0,0 +1,80 @@
|
|||
package eu.dnetlib.uoaorcidservice.entities;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import org.springframework.data.annotation.Id;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
enum Identifier {
|
||||
// Do not rename or remove existring values. This may cause problems with already stored values in DB
|
||||
current, previous
|
||||
}
|
||||
|
||||
public class Metrics {
|
||||
@Id
|
||||
@JsonProperty("_id")
|
||||
private Identifier id;
|
||||
|
||||
private int total_works;
|
||||
private int total_users;
|
||||
// private List<Object> works_per_month;
|
||||
private List<Object> works_per_dashboard;
|
||||
private Date date;
|
||||
|
||||
public String getId() {
|
||||
if(id == null) {
|
||||
return null;
|
||||
}
|
||||
return id.name();
|
||||
}
|
||||
|
||||
public void setId(String id) {
|
||||
if(id == null) {
|
||||
this.id = null;
|
||||
} else {
|
||||
Identifier identifier = Identifier.valueOf(id);
|
||||
this.id = identifier;
|
||||
}
|
||||
}
|
||||
|
||||
public int getTotal_works() {
|
||||
return total_works;
|
||||
}
|
||||
|
||||
public void setTotal_works(int total_works) {
|
||||
this.total_works = total_works;
|
||||
}
|
||||
|
||||
public int getTotal_users() {
|
||||
return total_users;
|
||||
}
|
||||
|
||||
public void setTotal_users(int total_users) {
|
||||
this.total_users = total_users;
|
||||
}
|
||||
|
||||
// public List<Object> getWorks_per_month() {
|
||||
// return works_per_month;
|
||||
// }
|
||||
//
|
||||
// public void setWorks_per_month(List<Object> works_per_month) {
|
||||
// this.works_per_month = works_per_month;
|
||||
// }
|
||||
|
||||
public List<Object> getWorks_per_dashboard() {
|
||||
return works_per_dashboard;
|
||||
}
|
||||
|
||||
public void setWorks_per_dashboard(List<Object> works_per_dashboard) {
|
||||
this.works_per_dashboard = works_per_dashboard;
|
||||
}
|
||||
|
||||
public Date getDate() {
|
||||
return date;
|
||||
}
|
||||
|
||||
public void setDate(Date date) {
|
||||
this.date = date;
|
||||
}
|
||||
}
|
|
@ -1,15 +1,21 @@
|
|||
package eu.dnetlib.uoaorcidservice.services;
|
||||
|
||||
import eu.dnetlib.uoaorcidservice.dao.MongoDBUserTokensDAO;
|
||||
import eu.dnetlib.uoaorcidservice.dao.MetricsDAO;
|
||||
import eu.dnetlib.uoaorcidservice.dao.customDAOs.MongoDBUserTokensDAOCustom;
|
||||
import eu.dnetlib.uoaorcidservice.dao.customDAOs.MongoDBWorkDAOCustom;
|
||||
import eu.dnetlib.uoaorcidservice.entities.Metrics;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
import java.time.Instant;
|
||||
import java.util.*;
|
||||
|
||||
|
||||
@EnableScheduling
|
||||
@Service
|
||||
public class MetricsService {
|
||||
private final Logger log = Logger.getLogger(this.getClass());
|
||||
|
@ -22,6 +28,9 @@ public class MetricsService {
|
|||
@Qualifier("mongoDBUserTokensDAO")
|
||||
private MongoDBUserTokensDAOCustom userTokensDAO;
|
||||
|
||||
@Autowired
|
||||
private MetricsDAO metricsDAO;
|
||||
|
||||
public List<Object> countWorksPerDashboard() {
|
||||
return workDAO.worksPerDashboard();
|
||||
}
|
||||
|
@ -65,4 +74,137 @@ public class MetricsService {
|
|||
public List<Object> countTotalWorks() {
|
||||
return workDAO.totalWorks();
|
||||
}
|
||||
|
||||
|
||||
// every day at midnight
|
||||
// @Scheduled(cron = "0 0 0 * * ?")
|
||||
// // every 5 mins for testing
|
||||
@Scheduled(cron = "0 0/5 * * * *")
|
||||
public List<Object> calculateMetrics() {
|
||||
log.info("Calculate metrics and add them in DB");
|
||||
|
||||
Optional<Metrics> oldMetrics = metricsDAO.findById("current");
|
||||
if(oldMetrics.isPresent()) {
|
||||
oldMetrics.get().setId("previous");
|
||||
metricsDAO.save(oldMetrics.get());
|
||||
}
|
||||
|
||||
Metrics metrics = new Metrics();
|
||||
metrics.setId("current");
|
||||
List<Object> totalWorks = countTotalWorks();
|
||||
if(totalWorks != null && totalWorks.get(0) != null) {
|
||||
Map<String, Integer> works = (HashMap<String, Integer>) totalWorks.get(0);
|
||||
metrics.setTotal_works(works.get("works"));
|
||||
} else {
|
||||
metrics.setTotal_works(0);
|
||||
}
|
||||
List<Object> totalUsers = countTotalUniqueUsers();
|
||||
if(totalUsers != null && totalUsers.get(0) != null) {
|
||||
Map<String, Integer> users = (HashMap<String, Integer>) totalUsers.get(0);
|
||||
metrics.setTotal_users(users.get("users"));
|
||||
} else {
|
||||
metrics.setTotal_users(0);
|
||||
}
|
||||
// metrics.setWorks_per_month(countWorksPerYearAndMonth());
|
||||
List<Object> works_per_dashboard = countWorksPerDashboard();
|
||||
for(Object dashboardWorksObj : works_per_dashboard) {
|
||||
Map<String, Object> dashboardWorks = (HashMap<String, Object>) dashboardWorksObj;
|
||||
String[] dashboard_elements = ((String) dashboardWorks.get("dashboard")).split("_", 2);
|
||||
dashboardWorks.put("environment", dashboard_elements[0]);
|
||||
dashboardWorks.put("dashboard", dashboard_elements[1]);
|
||||
// if(dashboard_elements.length > 2) {
|
||||
// dashboardWorks.put("dashboardName", dashboard_elements[2]);
|
||||
// }
|
||||
}
|
||||
metrics.setWorks_per_dashboard(works_per_dashboard);
|
||||
metrics.setDate(new Date());
|
||||
|
||||
metricsDAO.save(metrics);
|
||||
|
||||
// Total works linked: http://duffy.di.uoa.gr:8080/uoa-orcid-service/report/totalWorks
|
||||
// Total users: (unique/ per orcid): http://duffy.di.uoa.gr:8080/uoa-orcid-service/report/totalUniqueUsers
|
||||
// Works linked per month: http://duffy.di.uoa.gr:8080/uoa-orcid-service/report/worksPerYearAndMonth (not needed?)
|
||||
// Works linked per portal (not only explore): http://duffy.di.uoa.gr:8080/uoa-orcid-service/report/worksPerDashboard
|
||||
return totalWorks;
|
||||
}
|
||||
|
||||
public String getMetrics() {
|
||||
// # TYPE aai_registered_users_total gauge
|
||||
// aai_registered_users_total 50
|
||||
//# TYPE aai_logins_total counter
|
||||
// aai_logins_total 1742
|
||||
//# TYPE aai_api_requests_total counter
|
||||
// aai_api_requests_total 0
|
||||
// aai_last_metrics_updater_run_timestamp_seconds 1619157852
|
||||
|
||||
Optional<Metrics> metrics_optional = metricsDAO.findById("current");
|
||||
if(!metrics_optional.isPresent()) {
|
||||
return null;
|
||||
}
|
||||
Metrics metrics = metrics_optional.get();
|
||||
String response = "";
|
||||
|
||||
// response += "# TYPE explore_orcid_works_total gauge\n";
|
||||
// response += "explore_orcid_works_total "+metrics.getTotal_works() + "\n";
|
||||
|
||||
response += "# TYPE orcid_users gauge\n";
|
||||
response += "orcid_users "+metrics.getTotal_users() + "\n";
|
||||
|
||||
|
||||
// if(metrics.getWorks_per_month() != null) {
|
||||
// for(Object monthlyWorksObj : metrics.getWorks_per_month()) {
|
||||
// Map<String, Integer> monthlyWorks = (HashMap<String, Integer>) monthlyWorksObj;
|
||||
// response += "# TYPE orcid_total_works_"+monthlyWorks.get("month")+"_"+monthlyWorks.get("year")+" counter\n";
|
||||
// response += "orcid_total_works_"+monthlyWorks.get("month")+"_"+monthlyWorks.get("year")+" "+monthlyWorks.get("works") + "\n";
|
||||
//
|
||||
// }
|
||||
// }
|
||||
|
||||
if(metrics.getWorks_per_dashboard() != null) {
|
||||
// 2nd approach
|
||||
// Map<String, Integer> worksPerEnvironment = new HashMap<>();
|
||||
// 3rd approach
|
||||
Map<String, Integer> worksPerDashboard = new HashMap<>();
|
||||
// 1st approach
|
||||
// response += "# TYPE orcid_works gauge\n";
|
||||
|
||||
for(Object dashboardWorksObj : metrics.getWorks_per_dashboard()) {
|
||||
Map<String, Object> dashboardWorks = (HashMap<String, Object>) dashboardWorksObj;
|
||||
// 1st approach - e.g. orcid_works{envoronment="production" portal="explore"} 10
|
||||
// response += "orcid_works{environment=\""+dashboardWorks.get("environment")+"\" portal=\""+dashboardWorks.get("dashboard")+"\"}"+" "+dashboardWorks.get("works") + "\n";
|
||||
|
||||
// 2nd approach - e.g. explore_orcid_works_total{envoronment="production"} 10
|
||||
// if(worksPerEnvironment.containsKey(dashboardWorks.get("environment"))) {
|
||||
// int worksSoFar = worksPerEnvironment.get(dashboardWorks.get("environment"));
|
||||
// worksPerEnvironment.put((String)dashboardWorks.get("environment"), (Integer) dashboardWorks.get("works") + worksSoFar);
|
||||
// } else {
|
||||
// worksPerEnvironment.put((String)dashboardWorks.get("environment"), (Integer) dashboardWorks.get("works"));
|
||||
// }
|
||||
|
||||
// 3rd approach - e.g. orcid_works{portal="explore"} 10
|
||||
if(worksPerDashboard.containsKey(dashboardWorks.get("dashboard"))) {
|
||||
int worksSoFar = worksPerDashboard.get(dashboardWorks.get("dashboard"));
|
||||
worksPerDashboard.put((String)dashboardWorks.get("dashboard"), (Integer) dashboardWorks.get("works") + worksSoFar);
|
||||
} else {
|
||||
worksPerDashboard.put((String)dashboardWorks.get("dashboard"), (Integer) dashboardWorks.get("works"));
|
||||
}
|
||||
}
|
||||
// 2nd approach
|
||||
// response += "# TYPE explore_orcid_works_total gauge\n";
|
||||
// for(Map.Entry<String, Integer> envWorks : worksPerEnvironment.entrySet()) {
|
||||
// response += "explore_orcid_works_total{environment=\""+envWorks.getKey()+"\"} "+envWorks.getValue() + "\n";
|
||||
// }
|
||||
|
||||
// 3rd approach
|
||||
response += "# TYPE orcid_works gauge\n";
|
||||
for(Map.Entry<String, Integer> envWorks : worksPerDashboard.entrySet()) {
|
||||
response += "orcid_works{portal=\""+envWorks.getKey()+"\"} "+envWorks.getValue() + "\n";
|
||||
}
|
||||
}
|
||||
|
||||
Instant instant = metrics.getDate().toInstant();
|
||||
response += "orcid_last_metrics_updater_run_timestamp_seconds "+instant.getEpochSecond()+"\n";
|
||||
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -7,6 +7,7 @@ orcidservice.orcid.apiURL=https://api.sandbox.orcid.org/v3.0/
|
|||
orcidservice.orcid.tokenURL=https://sandbox.orcid.org/oauth/token
|
||||
orcidservice.orcid.clientId=APP-A5M3KTX6NCN67L91
|
||||
orcidservice.orcid.clientSecret=96b20d71-ae06-4286-bb00-9172536c1ad4
|
||||
orcidservice.globalVars.buildDate=@timestamp@
|
||||
|
||||
#orcidservice.encryption.password=...
|
||||
|
||||
|
|
Loading…
Reference in New Issue