From aad37cd81e954337932c0888b458edfdfdffb69d Mon Sep 17 00:00:00 2001
From: LSmyrnaios <lsmyrnaios@gmail.com>
Date: Tue, 18 Oct 2022 15:00:26 +0300
Subject: [PATCH] Add the "StatsController", which brings the
 "getNumberOfPayloads" and "getNumberOfRecordsInspected" endpoints.

---
 .../controllers/StatsController.java          | 77 +++++++++++++++++++
 1 file changed, 77 insertions(+)
 create mode 100644 src/main/java/eu/openaire/urls_controller/controllers/StatsController.java

diff --git a/src/main/java/eu/openaire/urls_controller/controllers/StatsController.java b/src/main/java/eu/openaire/urls_controller/controllers/StatsController.java
new file mode 100644
index 0000000..8b2fa77
--- /dev/null
+++ b/src/main/java/eu/openaire/urls_controller/controllers/StatsController.java
@@ -0,0 +1,77 @@
+package eu.openaire.urls_controller.controllers;
+
+
+import eu.openaire.urls_controller.configuration.ImpalaConnector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+/**
+ *  This controller returns statistics for the database.
+ */
+@RestController
+@RequestMapping("/stats")
+public class StatsController {
+
+    private static final Logger logger = LoggerFactory.getLogger(StatsController.class);
+
+    @Autowired
+    private JdbcTemplate jdbcTemplate;
+
+    @GetMapping("getNumberOfPayloads")
+    public ResponseEntity<?> getNumberOfPayloads()
+    {
+        logger.info("Received a \"getNumberOfPayloads\" request.");
+
+        String getPayloadsNumberQuery = "select count(id) from " + ImpalaConnector.databaseName + ".payload";
+        try {
+            Object result = jdbcTemplate.queryForObject(getPayloadsNumberQuery, Integer.class);
+            if ( result != null ) {
+                int numOfPayloads = (int) result;
+                logger.info("Number of payloads in the database \"" + ImpalaConnector.databaseName + "\" is " + numOfPayloads);
+                return new ResponseEntity<>(numOfPayloads, HttpStatus.OK);
+            } else
+                return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("The payloads' number could not be retrieved from the database \"" + ImpalaConnector.databaseName + "\" using the getPayloadsNumberQuery: " + getPayloadsNumberQuery);
+        } catch (Exception e) {
+            String errorMsg = "Problem when executing \"getPayloadsNumberQuery\": " + getPayloadsNumberQuery;
+            logger.error(errorMsg, e);
+            return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorMsg);
+            // We may get a "Class Cast Exception", in case the Impala returns a non-integer value.
+        }
+    }
+
+
+    @GetMapping("getNumberOfRecordsInspected")
+    public ResponseEntity<?> getNumberOfRecordsInspected()
+    {
+        // Note that until all the records are inspected, the "attempt" table contains all the inspected records +very few duplicates (id-url) which come from the publications-database.
+        // After all the records are inspected, it contains duplicate records of more and more id-urls, as time goes one, since for every eligible record the Service re-attmepts to get the full-text.
+        // So in order to get the number of inspected records, we want the distinct number, which at some point it will remain stable, even though the Service will try aganin and again some of the records.
+        // Before all the records are inspected, this endpoint will report all the inspected records MINUS the duplicate records which come straight from the "publication" table.
+
+        logger.info("Received a \"getNumberOfRecordsInspected\" request.");
+
+        String getInspectedRecordsNumberQuery = "select count(dist.id) from (select distinct id, original_url from " + ImpalaConnector.databaseName + ".attempt) as dist";
+        try {
+            Object result = jdbcTemplate.queryForObject(getInspectedRecordsNumberQuery, Integer.class);
+            if ( result != null ) {
+                int numOfInspectedRecords = (int) result;
+                logger.info("Number of inspected records from the database \"" + ImpalaConnector.databaseName + "\" is " + numOfInspectedRecords);
+                return new ResponseEntity<>(numOfInspectedRecords, HttpStatus.OK);
+            } else
+                return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("The inspected records' number could not be retrieved from the database \"" + ImpalaConnector.databaseName + "\" using the getInspectedRecordsNumberQuery: " + getInspectedRecordsNumberQuery);
+        } catch (Exception e) {
+            String errorMsg = "Problem when executing \"getInspectedRecordsNumberQuery\": " + getInspectedRecordsNumberQuery;
+            logger.error(errorMsg, e);
+            return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorMsg);
+            // We may get a "Class Cast Exception", in case the Impala returns a non-integer value.
+        }
+    }
+
+}