diff --git a/pom.xml b/pom.xml index 6720c51..bbd2136 100644 --- a/pom.xml +++ b/pom.xml @@ -21,6 +21,8 @@ UTF-8 UTF-8 1.8 + ${maven.build.timestamp} + E MMM dd HH:mm:ss z yyyy @@ -107,6 +109,12 @@ uoa-orcid-service + + + src/main/resources + true + + diff --git a/src/main/java/eu/dnetlib/uoaorcidservice/UoaOrcidServiceApplication.java b/src/main/java/eu/dnetlib/uoaorcidservice/UoaOrcidServiceApplication.java index 51db317..cb96468 100644 --- a/src/main/java/eu/dnetlib/uoaorcidservice/UoaOrcidServiceApplication.java +++ b/src/main/java/eu/dnetlib/uoaorcidservice/UoaOrcidServiceApplication.java @@ -1,6 +1,7 @@ package eu.dnetlib.uoaorcidservice; import eu.dnetlib.uoaauthorizationlibrary.configuration.AuthorizationConfiguration; +import eu.dnetlib.uoaorcidservice.configuration.GlobalVars; import eu.dnetlib.uoaorcidservice.configuration.properties.MongoConfig; //import eu.dnetlib.uoaauthorizationlibrary.configuration.AuthorizationConfiguration; import eu.dnetlib.uoaorcidservice.configuration.properties.OrcidConfig; @@ -14,10 +15,11 @@ import org.springframework.context.annotation.PropertySources; @SpringBootApplication(scanBasePackages = {"eu.dnetlib.uoaorcidservice"}) @PropertySources({ + @PropertySource("classpath:authorization.properties"), @PropertySource("classpath:orcidservice.properties"), @PropertySource(value = "classpath:dnet-override.properties", ignoreResourceNotFound = true) }) -@EnableConfigurationProperties({MongoConfig.class, AESUtils.class, OrcidConfig.class}) +@EnableConfigurationProperties({MongoConfig.class, AESUtils.class, OrcidConfig.class, GlobalVars.class}) @Import(AuthorizationConfiguration.class) public class UoaOrcidServiceApplication { diff --git a/src/main/java/eu/dnetlib/uoaorcidservice/configuration/GlobalVars.java b/src/main/java/eu/dnetlib/uoaorcidservice/configuration/GlobalVars.java new file mode 100644 index 0000000..e80bb24 --- /dev/null +++ b/src/main/java/eu/dnetlib/uoaorcidservice/configuration/GlobalVars.java @@ -0,0 +1,22 @@ +package eu.dnetlib.uoaorcidservice.configuration; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +import java.util.Date; + +@ConfigurationProperties("orcidservice.globalVars") +public class GlobalVars { + public static Date date = new Date(); + private Date buildDate; + + public String getBuildDate() { + if(buildDate == null) { + return null; + } + return buildDate.toString(); + } + + public void setBuildDate(Date buildDate) { + this.buildDate = buildDate; + } +} diff --git a/src/main/java/eu/dnetlib/uoaorcidservice/controllers/MetricsController.java b/src/main/java/eu/dnetlib/uoaorcidservice/controllers/MetricsController.java index fe3621b..9b96f00 100644 --- a/src/main/java/eu/dnetlib/uoaorcidservice/controllers/MetricsController.java +++ b/src/main/java/eu/dnetlib/uoaorcidservice/controllers/MetricsController.java @@ -75,4 +75,15 @@ public class MetricsController { public List countTotalWorks() { return metricsService.countTotalWorks(); } + + @RequestMapping(value = "/report/metrics", method = RequestMethod.GET, produces={"text/plain"}) + public String getMetrics() { + return metricsService.getMetrics(); + } + + @RequestMapping(value = "/report/calc", method = RequestMethod.GET) + public String calculateMetrics() { + metricsService.calculateMetrics(); + return "ALL GOOD"; + } } diff --git a/src/main/java/eu/dnetlib/uoaorcidservice/controllers/OrcidServiceCheckDeployController.java b/src/main/java/eu/dnetlib/uoaorcidservice/controllers/OrcidServiceCheckDeployController.java index 25e7ef9..6716511 100644 --- a/src/main/java/eu/dnetlib/uoaorcidservice/controllers/OrcidServiceCheckDeployController.java +++ b/src/main/java/eu/dnetlib/uoaorcidservice/controllers/OrcidServiceCheckDeployController.java @@ -3,11 +3,15 @@ package eu.dnetlib.uoaorcidservice.controllers; import com.mongodb.BasicDBObject; import com.mongodb.CommandResult; import com.mongodb.DBObject; +import com.mongodb.MongoTimeoutException; +import eu.dnetlib.uoaorcidservice.configuration.GlobalVars; import eu.dnetlib.uoaorcidservice.configuration.mongo.MongoConnection; import eu.dnetlib.uoaorcidservice.configuration.properties.MongoConfig; import eu.dnetlib.uoaorcidservice.configuration.properties.OrcidConfig; import eu.dnetlib.uoaorcidservice.handlers.utils.AESUtils; import org.apache.log4j.Logger; +import org.bson.*; +import org.bson.conversions.Bson; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.security.access.prepost.PreAuthorize; @@ -16,6 +20,7 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; +import java.util.Date; import java.util.HashMap; import java.util.Map; @@ -33,6 +38,9 @@ public class OrcidServiceCheckDeployController { @Autowired private OrcidConfig orcidConfig; + @Autowired + private GlobalVars globalVars; + @Autowired private AESUtils aesUtils; @@ -56,6 +64,25 @@ public class OrcidServiceCheckDeployController { response.put("Mongo catch: error", e.getMessage()); } +// Document ping = new Document("ping", "1"); +// try { +// Document answer = mt.getDb().runCommand(ping);//.command(ping); +// +// for(String key : answer.keySet()) { +// log.debug("answer pair: "+key + ": "+answer.get(key)); +// } +// +// if (answer != null && answer.get("ok") != null && (Double)answer.get("ok") == 1.0d) { +// response.put("Mongo try: everything is ok", answer.toString());//.getErrorMessage()); +// } else { +// response.put("Mongo try: error", answer.toString());//.getErrorMessage()); +// } +// } catch (MongoTimeoutException e) { +// response.put("Mongo catch timeout: error", e.getMessage()); +// } catch (Exception e) { +// response.put("Mongo catch: error", e.getMessage()); +// } + response.put("orcidservice.mongodb.database", mongoConfig.getDatabase()); response.put("orcidservice.mongodb.host", mongoConfig.getHost()); response.put("orcidservice.mongodb.port", mongoConfig.getPort()+""); @@ -69,6 +96,12 @@ public class OrcidServiceCheckDeployController { response.put("orcidservice.encryption.password", aesUtils.getPassword() == null ? null : "[unexposed value]"); + if(globalVars.date != null) { + response.put("Date of deploy", globalVars.date.toString()); + } + if(globalVars.getBuildDate() != null) { + response.put("Date of build", globalVars.getBuildDate()); + } return response; } } diff --git a/src/main/java/eu/dnetlib/uoaorcidservice/controllers/SimpleErrorController.java b/src/main/java/eu/dnetlib/uoaorcidservice/controllers/SimpleErrorController.java index f2587b7..e62b1ec 100644 --- a/src/main/java/eu/dnetlib/uoaorcidservice/controllers/SimpleErrorController.java +++ b/src/main/java/eu/dnetlib/uoaorcidservice/controllers/SimpleErrorController.java @@ -4,6 +4,8 @@ import org.apache.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.web.ErrorAttributes; import org.springframework.boot.autoconfigure.web.ErrorController; +//import org.springframework.boot.web.servlet.error.ErrorAttributes; +//import org.springframework.boot.web.servlet.error.ErrorController; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.util.Assert; @@ -12,6 +14,7 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.RequestAttributes; import org.springframework.web.context.request.ServletRequestAttributes; +import org.springframework.web.context.request.ServletWebRequest; import javax.servlet.RequestDispatcher; import javax.servlet.http.HttpServletRequest; @@ -93,6 +96,7 @@ public class SimpleErrorController implements ErrorController { private Map getErrorAttributes(HttpServletRequest aRequest, boolean includeStackTrace) { RequestAttributes requestAttributes = new ServletRequestAttributes(aRequest); +// ServletWebRequest requestAttributes = new ServletWebRequest(aRequest); return errorAttributes.getErrorAttributes(requestAttributes, includeStackTrace); } } \ No newline at end of file diff --git a/src/main/java/eu/dnetlib/uoaorcidservice/dao/MetricsDAO.java b/src/main/java/eu/dnetlib/uoaorcidservice/dao/MetricsDAO.java new file mode 100644 index 0000000..b5eb880 --- /dev/null +++ b/src/main/java/eu/dnetlib/uoaorcidservice/dao/MetricsDAO.java @@ -0,0 +1,18 @@ +package eu.dnetlib.uoaorcidservice.dao; + +import eu.dnetlib.uoaorcidservice.entities.Metrics; + +import java.util.List; +import java.util.Optional; + +public interface MetricsDAO { + List findAll(); + + Optional findById(String Id); + + Metrics save(Metrics metrics); + + void deleteAll(); + + void deleteById(String Id); +} diff --git a/src/main/java/eu/dnetlib/uoaorcidservice/dao/MongoDBMetricsDAO.java b/src/main/java/eu/dnetlib/uoaorcidservice/dao/MongoDBMetricsDAO.java new file mode 100644 index 0000000..c474cc4 --- /dev/null +++ b/src/main/java/eu/dnetlib/uoaorcidservice/dao/MongoDBMetricsDAO.java @@ -0,0 +1,20 @@ +package eu.dnetlib.uoaorcidservice.dao; + +import eu.dnetlib.uoaorcidservice.entities.Metrics; +import org.springframework.data.mongodb.repository.MongoRepository; + +import java.util.List; +import java.util.Optional; + +public interface MongoDBMetricsDAO extends MetricsDAO, MongoRepository { + + List findAll(); + + Optional findById(String Id); + + Metrics save(Metrics metrics); + + void deleteAll(); + + void deleteById(String Id); +} diff --git a/src/main/java/eu/dnetlib/uoaorcidservice/entities/Metrics.java b/src/main/java/eu/dnetlib/uoaorcidservice/entities/Metrics.java new file mode 100644 index 0000000..e4b9685 --- /dev/null +++ b/src/main/java/eu/dnetlib/uoaorcidservice/entities/Metrics.java @@ -0,0 +1,80 @@ +package eu.dnetlib.uoaorcidservice.entities; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.springframework.data.annotation.Id; + +import java.util.Date; +import java.util.List; + +enum Identifier { + // Do not rename or remove existring values. This may cause problems with already stored values in DB + current, previous +} + +public class Metrics { + @Id + @JsonProperty("_id") + private Identifier id; + + private int total_works; + private int total_users; +// private List works_per_month; + private List works_per_dashboard; + private Date date; + + public String getId() { + if(id == null) { + return null; + } + return id.name(); + } + + public void setId(String id) { + if(id == null) { + this.id = null; + } else { + Identifier identifier = Identifier.valueOf(id); + this.id = identifier; + } + } + + public int getTotal_works() { + return total_works; + } + + public void setTotal_works(int total_works) { + this.total_works = total_works; + } + + public int getTotal_users() { + return total_users; + } + + public void setTotal_users(int total_users) { + this.total_users = total_users; + } + +// public List getWorks_per_month() { +// return works_per_month; +// } +// +// public void setWorks_per_month(List works_per_month) { +// this.works_per_month = works_per_month; +// } + + public List getWorks_per_dashboard() { + return works_per_dashboard; + } + + public void setWorks_per_dashboard(List works_per_dashboard) { + this.works_per_dashboard = works_per_dashboard; + } + + public Date getDate() { + return date; + } + + public void setDate(Date date) { + this.date = date; + } +} diff --git a/src/main/java/eu/dnetlib/uoaorcidservice/services/MetricsService.java b/src/main/java/eu/dnetlib/uoaorcidservice/services/MetricsService.java index 0ec2743..e4a4861 100644 --- a/src/main/java/eu/dnetlib/uoaorcidservice/services/MetricsService.java +++ b/src/main/java/eu/dnetlib/uoaorcidservice/services/MetricsService.java @@ -1,15 +1,21 @@ package eu.dnetlib.uoaorcidservice.services; -import eu.dnetlib.uoaorcidservice.dao.MongoDBUserTokensDAO; +import eu.dnetlib.uoaorcidservice.dao.MetricsDAO; import eu.dnetlib.uoaorcidservice.dao.customDAOs.MongoDBUserTokensDAOCustom; import eu.dnetlib.uoaorcidservice.dao.customDAOs.MongoDBWorkDAOCustom; +import eu.dnetlib.uoaorcidservice.entities.Metrics; import org.apache.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; -import java.util.List; +import java.time.Instant; +import java.util.*; + +@EnableScheduling @Service public class MetricsService { private final Logger log = Logger.getLogger(this.getClass()); @@ -22,6 +28,9 @@ public class MetricsService { @Qualifier("mongoDBUserTokensDAO") private MongoDBUserTokensDAOCustom userTokensDAO; + @Autowired + private MetricsDAO metricsDAO; + public List countWorksPerDashboard() { return workDAO.worksPerDashboard(); } @@ -65,4 +74,137 @@ public class MetricsService { public List countTotalWorks() { return workDAO.totalWorks(); } + + + // every day at midnight +// @Scheduled(cron = "0 0 0 * * ?") +// // every 5 mins for testing + @Scheduled(cron = "0 0/5 * * * *") + public List calculateMetrics() { + log.info("Calculate metrics and add them in DB"); + + Optional oldMetrics = metricsDAO.findById("current"); + if(oldMetrics.isPresent()) { + oldMetrics.get().setId("previous"); + metricsDAO.save(oldMetrics.get()); + } + + Metrics metrics = new Metrics(); + metrics.setId("current"); + List totalWorks = countTotalWorks(); + if(totalWorks != null && totalWorks.get(0) != null) { + Map works = (HashMap) totalWorks.get(0); + metrics.setTotal_works(works.get("works")); + } else { + metrics.setTotal_works(0); + } + List totalUsers = countTotalUniqueUsers(); + if(totalUsers != null && totalUsers.get(0) != null) { + Map users = (HashMap) totalUsers.get(0); + metrics.setTotal_users(users.get("users")); + } else { + metrics.setTotal_users(0); + } +// metrics.setWorks_per_month(countWorksPerYearAndMonth()); + List works_per_dashboard = countWorksPerDashboard(); + for(Object dashboardWorksObj : works_per_dashboard) { + Map dashboardWorks = (HashMap) dashboardWorksObj; + String[] dashboard_elements = ((String) dashboardWorks.get("dashboard")).split("_", 2); + dashboardWorks.put("environment", dashboard_elements[0]); + dashboardWorks.put("dashboard", dashboard_elements[1]); +// if(dashboard_elements.length > 2) { +// dashboardWorks.put("dashboardName", dashboard_elements[2]); +// } + } + metrics.setWorks_per_dashboard(works_per_dashboard); + metrics.setDate(new Date()); + + metricsDAO.save(metrics); + +// Total works linked: http://duffy.di.uoa.gr:8080/uoa-orcid-service/report/totalWorks +// Total users: (unique/ per orcid): http://duffy.di.uoa.gr:8080/uoa-orcid-service/report/totalUniqueUsers +// Works linked per month: http://duffy.di.uoa.gr:8080/uoa-orcid-service/report/worksPerYearAndMonth (not needed?) +// Works linked per portal (not only explore): http://duffy.di.uoa.gr:8080/uoa-orcid-service/report/worksPerDashboard + return totalWorks; + } + + public String getMetrics() { +// # TYPE aai_registered_users_total gauge +// aai_registered_users_total 50 +//# TYPE aai_logins_total counter +// aai_logins_total 1742 +//# TYPE aai_api_requests_total counter +// aai_api_requests_total 0 +// aai_last_metrics_updater_run_timestamp_seconds 1619157852 + + Optional metrics_optional = metricsDAO.findById("current"); + if(!metrics_optional.isPresent()) { + return null; + } + Metrics metrics = metrics_optional.get(); + String response = ""; + +// response += "# TYPE explore_orcid_works_total gauge\n"; +// response += "explore_orcid_works_total "+metrics.getTotal_works() + "\n"; + + response += "# TYPE orcid_users gauge\n"; + response += "orcid_users "+metrics.getTotal_users() + "\n"; + + +// if(metrics.getWorks_per_month() != null) { +// for(Object monthlyWorksObj : metrics.getWorks_per_month()) { +// Map monthlyWorks = (HashMap) monthlyWorksObj; +// response += "# TYPE orcid_total_works_"+monthlyWorks.get("month")+"_"+monthlyWorks.get("year")+" counter\n"; +// response += "orcid_total_works_"+monthlyWorks.get("month")+"_"+monthlyWorks.get("year")+" "+monthlyWorks.get("works") + "\n"; +// +// } +// } + + if(metrics.getWorks_per_dashboard() != null) { + // 2nd approach +// Map worksPerEnvironment = new HashMap<>(); + // 3rd approach + Map worksPerDashboard = new HashMap<>(); + // 1st approach +// response += "# TYPE orcid_works gauge\n"; + + for(Object dashboardWorksObj : metrics.getWorks_per_dashboard()) { + Map dashboardWorks = (HashMap) dashboardWorksObj; + // 1st approach - e.g. orcid_works{envoronment="production" portal="explore"} 10 +// response += "orcid_works{environment=\""+dashboardWorks.get("environment")+"\" portal=\""+dashboardWorks.get("dashboard")+"\"}"+" "+dashboardWorks.get("works") + "\n"; + + // 2nd approach - e.g. explore_orcid_works_total{envoronment="production"} 10 +// if(worksPerEnvironment.containsKey(dashboardWorks.get("environment"))) { +// int worksSoFar = worksPerEnvironment.get(dashboardWorks.get("environment")); +// worksPerEnvironment.put((String)dashboardWorks.get("environment"), (Integer) dashboardWorks.get("works") + worksSoFar); +// } else { +// worksPerEnvironment.put((String)dashboardWorks.get("environment"), (Integer) dashboardWorks.get("works")); +// } + + // 3rd approach - e.g. orcid_works{portal="explore"} 10 + if(worksPerDashboard.containsKey(dashboardWorks.get("dashboard"))) { + int worksSoFar = worksPerDashboard.get(dashboardWorks.get("dashboard")); + worksPerDashboard.put((String)dashboardWorks.get("dashboard"), (Integer) dashboardWorks.get("works") + worksSoFar); + } else { + worksPerDashboard.put((String)dashboardWorks.get("dashboard"), (Integer) dashboardWorks.get("works")); + } + } + // 2nd approach +// response += "# TYPE explore_orcid_works_total gauge\n"; +// for(Map.Entry envWorks : worksPerEnvironment.entrySet()) { +// response += "explore_orcid_works_total{environment=\""+envWorks.getKey()+"\"} "+envWorks.getValue() + "\n"; +// } + + // 3rd approach + response += "# TYPE orcid_works gauge\n"; + for(Map.Entry envWorks : worksPerDashboard.entrySet()) { + response += "orcid_works{portal=\""+envWorks.getKey()+"\"} "+envWorks.getValue() + "\n"; + } + } + + Instant instant = metrics.getDate().toInstant(); + response += "orcid_last_metrics_updater_run_timestamp_seconds "+instant.getEpochSecond()+"\n"; + + return response; + } } diff --git a/src/main/resources/orcidservice.properties b/src/main/resources/orcidservice.properties index af19dc1..a2c3323 100644 --- a/src/main/resources/orcidservice.properties +++ b/src/main/resources/orcidservice.properties @@ -7,6 +7,7 @@ orcidservice.orcid.apiURL=https://api.sandbox.orcid.org/v3.0/ orcidservice.orcid.tokenURL=https://sandbox.orcid.org/oauth/token orcidservice.orcid.clientId=APP-A5M3KTX6NCN67L91 orcidservice.orcid.clientSecret=96b20d71-ae06-4286-bb00-9172536c1ad4 +orcidservice.globalVars.buildDate=@timestamp@ #orcidservice.encryption.password=...