michal.politowski
/
UrlsController
forked from lsmyrnaios/UrlsController
1
0
Fork 0
Code Pull Requests Activity

Refactor the "StatsController"-code, by offloading it to a dedicated "StatsService".

This commit is contained in:
Lampros Smyrnaios 2023-02-09 19:25:48 +02:00
parent 49fefefafd
commit 2253f05bf5
2 changed files with 76 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
src/main/java/eu/openaire/urls_controller/controllers/StatsController.java
View File

@ -2,13 +2,11 @@ package eu.openaire.urls_controller.controllers;
import eu.openaire.urls_controller.configuration.ImpalaConnector;
import eu.openaire.urls_controller.services.StatsService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.dao.EmptyResultDataAccessException;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@ -23,7 +21,8 @@ public class StatsController {
private static final Logger logger = LoggerFactory.getLogger(StatsController.class);
@Autowired
private JdbcTemplate jdbcTemplate;
private StatsService statsService;
/**
* This endpoint returns the total number of payloads existing in the database, independently of the way they were aggregated.
@ -33,7 +32,7 @@ public class StatsController {
public ResponseEntity<?> getNumberOfAllPayloads() {
logger.info("Received a \"getNumberOfAllPayloads\" request.");
final String getPayloadsNumberQuery = "select count(id) from " + ImpalaConnector.databaseName + ".payload";
return getNumberOfPayloads(getPayloadsNumberQuery, "");
return statsService.getNumberOfPayloads(getPayloadsNumberQuery, "");
}
@ -45,7 +44,7 @@ public class StatsController {
public ResponseEntity<?> getNumberOfPayloadsAggregatedByService() {
logger.info("Received a \"getNumberOfPayloadsAggregatedByService\" request.");
String getPayloadsAggregatedQuery = "select count(id) from " + ImpalaConnector.databaseName + ".payload where `date` >= cast(cast('2021-01-01' as timestamp) as bigint)";
return getNumberOfPayloads(getPayloadsAggregatedQuery, "retrieved by the PDF Aggregation Service");
return statsService.getNumberOfPayloads(getPayloadsAggregatedQuery, "retrieved by the PDF Aggregation Service");
}
@ -55,50 +54,7 @@ public class StatsController {
@GetMapping("getNumberOfRecordsInspected")
public ResponseEntity<?> getNumberOfRecordsInspected()
{
// Note that until all the records are inspected, the "attempt" table contains all the inspected records PLUS very few duplicates (id-url) which come from the publications-database.
// After all the records are inspected, it contains duplicate records of more and more id-urls, as time goes one, since for every eligible record the Service re-attempts to get the full-text.
// So in order to get the number of inspected records, we want the distinct number, which at some point it will remain stable, even though the Service will try again and again some records.
// Before all the records are inspected, this endpoint will report all the inspected records MINUS the duplicate records which come straight from the "publication" table.
logger.info("Received a \"getNumberOfRecordsInspected\" request.");
final String getInspectedRecordsNumberQuery = "select count(dist.id) from (select distinct id, original_url from " + ImpalaConnector.databaseName + ".attempt) as dist";
try {
Object result = jdbcTemplate.queryForObject(getInspectedRecordsNumberQuery, Integer.class);
if ( result != null ) {
int numOfInspectedRecords = (int) result;
logger.info("Number of inspected records from the database \"" + ImpalaConnector.databaseName + "\" is " + numOfInspectedRecords);
return new ResponseEntity<>(numOfInspectedRecords, HttpStatus.OK);
} else
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("The inspected records' number could not be retrieved from the database \"" + ImpalaConnector.databaseName + "\" using the getInspectedRecordsNumberQuery: " + getInspectedRecordsNumberQuery);
} catch (EmptyResultDataAccessException erdae) {
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("The inspected records' number could not be retrieved from the database \"" + ImpalaConnector.databaseName + "\" using the getInspectedRecordsNumberQuery: " + getInspectedRecordsNumberQuery);
} catch (Exception e) {
String errorMsg = "Problem when executing \"getInspectedRecordsNumberQuery\": " + getInspectedRecordsNumberQuery;
logger.error(errorMsg, e);
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorMsg);
// We may get a "Class Cast Exception", in case the Impala returns a non-integer value.
}
}
private ResponseEntity<?> getNumberOfPayloads(String getPayloadsNumberQuery, String extraMsg) {
try {
Object result = jdbcTemplate.queryForObject(getPayloadsNumberQuery, Integer.class);
if ( result != null ) {
int numOfPayloads = (int) result;
logger.info("Number of payloads " + extraMsg + " in the database \"" + ImpalaConnector.databaseName + "\" is " + numOfPayloads);
return new ResponseEntity<>(numOfPayloads, HttpStatus.OK);
} else
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("The payloads' number could not be retrieved from the database \"" + ImpalaConnector.databaseName + "\" using the getPayloadsNumberQuery: " + getPayloadsNumberQuery);
} catch (EmptyResultDataAccessException erdae) {
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("The payloads' number could not be retrieved from the database \"" + ImpalaConnector.databaseName + "\" using the getPayloadsNumberQuery: " + getPayloadsNumberQuery);
} catch (Exception e) {
String errorMsg = "Problem when executing \"getPayloadsNumberQuery\": " + getPayloadsNumberQuery;
logger.error(errorMsg, e);
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorMsg);
// We may get a "Class Cast Exception", in case the Impala returns a non-integer value.
}
return statsService.getNumberOfRecordsInspected();
}
}

70
src/main/java/eu/openaire/urls_controller/services/StatsService.java Normal file
View File

@ -0,0 +1,70 @@
package eu.openaire.urls_controller.services;
import eu.openaire.urls_controller.configuration.ImpalaConnector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.dao.EmptyResultDataAccessException;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
@Service
public class StatsService {
private static final Logger logger = LoggerFactory.getLogger(StatsService.class);
@Autowired
private JdbcTemplate jdbcTemplate;
public ResponseEntity<?> getNumberOfPayloads(String getPayloadsNumberQuery, String extraMsg) {
try {
Object result = jdbcTemplate.queryForObject(getPayloadsNumberQuery, Integer.class);
if ( result != null ) {
int numOfPayloads = (int) result;
logger.info("Number of payloads " + extraMsg + " in the database \"" + ImpalaConnector.databaseName + "\" is " + numOfPayloads);
return new ResponseEntity<>(numOfPayloads, HttpStatus.OK);
} else
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("The payloads' number could not be retrieved from the database \"" + ImpalaConnector.databaseName + "\" using the getPayloadsNumberQuery: " + getPayloadsNumberQuery);
} catch (EmptyResultDataAccessException erdae) {
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("The payloads' number could not be retrieved from the database \"" + ImpalaConnector.databaseName + "\" using the getPayloadsNumberQuery: " + getPayloadsNumberQuery);
} catch (Exception e) {
String errorMsg = "Problem when executing \"getPayloadsNumberQuery\": " + getPayloadsNumberQuery;
logger.error(errorMsg, e);
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorMsg);
// We may get a "Class Cast Exception", in case the Impala returns a non-integer value.
}
}
public ResponseEntity<?> getNumberOfRecordsInspected()
{
// Note that until all the records are inspected, the "attempt" table contains all the inspected records PLUS very few duplicates (id-url) which come from the publications-database.
// After all the records are inspected, it contains duplicate records of more and more id-urls, as time goes one, since for every eligible record the Service re-attempts to get the full-text.
// So in order to get the number of inspected records, we want the distinct number, which at some point it will remain stable, even though the Service will try again and again some records.
// Before all the records are inspected, this endpoint will report all the inspected records MINUS the duplicate records which come straight from the "publication" table.
logger.info("Received a \"getNumberOfRecordsInspected\" request.");
final String getInspectedRecordsNumberQuery = "select count(dist.id) from (select distinct id, original_url from " + ImpalaConnector.databaseName + ".attempt) as dist";
try {
Object result = jdbcTemplate.queryForObject(getInspectedRecordsNumberQuery, Integer.class);
if ( result != null ) {
int numOfInspectedRecords = (int) result;
logger.info("Number of inspected records from the database \"" + ImpalaConnector.databaseName + "\" is " + numOfInspectedRecords);
return new ResponseEntity<>(numOfInspectedRecords, HttpStatus.OK);
} else
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("The inspected records' number could not be retrieved from the database \"" + ImpalaConnector.databaseName + "\" using the getInspectedRecordsNumberQuery: " + getInspectedRecordsNumberQuery);
} catch (EmptyResultDataAccessException erdae) {
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("The inspected records' number could not be retrieved from the database \"" + ImpalaConnector.databaseName + "\" using the getInspectedRecordsNumberQuery: " + getInspectedRecordsNumberQuery);
} catch (Exception e) {
String errorMsg = "Problem when executing \"getInspectedRecordsNumberQuery\": " + getInspectedRecordsNumberQuery;
logger.error(errorMsg, e);
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorMsg);
// We may get a "Class Cast Exception", in case the Impala returns a non-integer value.
}
}
}