diff --git a/src/main/java/eu/openaire/urls_controller/controllers/UrlController.java b/src/main/java/eu/openaire/urls_controller/controllers/UrlsController.java similarity index 98% rename from src/main/java/eu/openaire/urls_controller/controllers/UrlController.java rename to src/main/java/eu/openaire/urls_controller/controllers/UrlsController.java index 6acf104..ef1a3cb 100644 --- a/src/main/java/eu/openaire/urls_controller/controllers/UrlController.java +++ b/src/main/java/eu/openaire/urls_controller/controllers/UrlsController.java @@ -18,9 +18,9 @@ import java.util.regex.Pattern; @RestController @RequestMapping("/urls") -public class UrlController { +public class UrlsController { - private static final Logger logger = LoggerFactory.getLogger(UrlController.class); + private static final Logger logger = LoggerFactory.getLogger(UrlsController.class); @Autowired private UrlsService urlsService; diff --git a/src/main/java/eu/openaire/urls_controller/services/StatsService.java b/src/main/java/eu/openaire/urls_controller/services/StatsService.java index 61e6365..e0be0ea 100644 --- a/src/main/java/eu/openaire/urls_controller/services/StatsService.java +++ b/src/main/java/eu/openaire/urls_controller/services/StatsService.java @@ -1,74 +1,11 @@ package eu.openaire.urls_controller.services; -import eu.openaire.urls_controller.configuration.ImpalaConnector; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.dao.EmptyResultDataAccessException; -import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.stereotype.Service; -@Service -public class StatsService { +public interface StatsService { - private static final Logger logger = LoggerFactory.getLogger(StatsService.class); - - @Autowired - private JdbcTemplate jdbcTemplate; - - - public ResponseEntity getNumberOfPayloads(String getPayloadsNumberQuery, String extraMsg) { - try { - Object result = jdbcTemplate.queryForObject(getPayloadsNumberQuery, Integer.class); - if ( result != null ) { - int numOfPayloads = (int) result; - logger.info("Number of payloads " + extraMsg + " in the database \"" + ImpalaConnector.databaseName + "\" is " + numOfPayloads); - return new ResponseEntity<>(numOfPayloads, HttpStatus.OK); - } else - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("The payloads' number could not be retrieved from the database \"" + ImpalaConnector.databaseName + "\" using the getPayloadsNumberQuery: " + getPayloadsNumberQuery); - } catch (EmptyResultDataAccessException erdae) { - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("The payloads' number could not be retrieved from the database \"" + ImpalaConnector.databaseName + "\" using the getPayloadsNumberQuery: " + getPayloadsNumberQuery); - } catch (Exception e) { - String errorMsg = "Problem when executing \"getPayloadsNumberQuery\": " + getPayloadsNumberQuery; - logger.error(errorMsg, e); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorMsg); - // We may get a "Class Cast Exception", in case the Impala returns a non-integer value. - } - } - - - public ResponseEntity getNumberOfRecordsInspected() - { - // Note that until all the records are inspected, the "attempt" table contains all the inspected records PLUS very few duplicates (id-url) which come from the publications-database. - // After all the records are inspected, it contains duplicate records of more and more id-urls, as time goes on, since for every eligible record the Service re-attempts to get the full-text. - // So in order to get the number of inspected records, we want the distinct number, which at some point it will remain stable, even though the Service will try again and again some records. - // Before all the records are inspected, this endpoint will report all the inspected records MINUS the duplicate records which come straight from the "publication" table. - - logger.info("Received a \"getNumberOfRecordsInspected\" request."); - - final String getInspectedRecordsNumberQuery = "select count(dist.id) from (select distinct id, original_url from " + ImpalaConnector.databaseName + ".attempt) as dist"; - try { - Object result = jdbcTemplate.queryForObject(getInspectedRecordsNumberQuery, Integer.class); - if ( result != null ) { - int numOfInspectedRecords = (int) result; - logger.info("Number of inspected records from the database \"" + ImpalaConnector.databaseName + "\" is " + numOfInspectedRecords); - return new ResponseEntity<>(numOfInspectedRecords, HttpStatus.OK); - } else - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("The inspected records' number could not be retrieved from the database \"" + ImpalaConnector.databaseName + "\" using the getInspectedRecordsNumberQuery: " + getInspectedRecordsNumberQuery); - } catch (EmptyResultDataAccessException erdae) { - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("The inspected records' number could not be retrieved from the database \"" + ImpalaConnector.databaseName + "\" using the getInspectedRecordsNumberQuery: " + getInspectedRecordsNumberQuery); - } catch (Exception e) { - String errorMsg = "Problem when executing \"getInspectedRecordsNumberQuery\": " + getInspectedRecordsNumberQuery; - logger.error(errorMsg, e); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorMsg); - // We may get a "Class Cast Exception", in case the Impala returns a non-integer value. - } - } - - // To get the human-friendly timestamp format from the BigInt in the database: - // select from_timestamp(CAST(CAST(`date` as decimal(30,0))/1000 AS timestamp), "yyyy-MM-dd HH:mm:ss.SSS") from payload + ResponseEntity getNumberOfPayloads(String getPayloadsNumberQuery, String extraMsg); + ResponseEntity getNumberOfRecordsInspected(); } diff --git a/src/main/java/eu/openaire/urls_controller/services/StatsServiceImpl.java b/src/main/java/eu/openaire/urls_controller/services/StatsServiceImpl.java new file mode 100644 index 0000000..c8c82b6 --- /dev/null +++ b/src/main/java/eu/openaire/urls_controller/services/StatsServiceImpl.java @@ -0,0 +1,73 @@ +package eu.openaire.urls_controller.services; + +import eu.openaire.urls_controller.configuration.ImpalaConnector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.dao.EmptyResultDataAccessException; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Service; + +@Service +public class StatsServiceImpl implements StatsService { + + private static final Logger logger = LoggerFactory.getLogger(StatsServiceImpl.class); + + @Autowired + private JdbcTemplate jdbcTemplate; + + + public ResponseEntity getNumberOfPayloads(String getPayloadsNumberQuery, String extraMsg) { + try { + Object result = jdbcTemplate.queryForObject(getPayloadsNumberQuery, Integer.class); + if ( result != null ) { + int numOfPayloads = (int) result; + logger.info("Number of payloads " + extraMsg + " in the database \"" + ImpalaConnector.databaseName + "\" is " + numOfPayloads); + return new ResponseEntity<>(numOfPayloads, HttpStatus.OK); + } else + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("The payloads' number could not be retrieved from the database \"" + ImpalaConnector.databaseName + "\" using the getPayloadsNumberQuery: " + getPayloadsNumberQuery); + } catch (EmptyResultDataAccessException erdae) { + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("The payloads' number could not be retrieved from the database \"" + ImpalaConnector.databaseName + "\" using the getPayloadsNumberQuery: " + getPayloadsNumberQuery); + } catch (Exception e) { + String errorMsg = "Problem when executing \"getPayloadsNumberQuery\": " + getPayloadsNumberQuery; + logger.error(errorMsg, e); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorMsg); + // We may get a "Class Cast Exception", in case the Impala returns a non-integer value. + } + } + + + public ResponseEntity getNumberOfRecordsInspected() + { + // Note that until all the records are inspected, the "attempt" table contains all the inspected records PLUS very few duplicates (id-url) which come from the publications-database. + // After all the records are inspected, it contains duplicate records of more and more id-urls, as time goes on, since for every eligible record the Service re-attempts to get the full-text. + // So in order to get the number of inspected records, we want the distinct number, which at some point it will remain stable, even though the Service will try again and again some records. + // Before all the records are inspected, this endpoint will report all the inspected records MINUS the duplicate records which come straight from the "publication" table. + + logger.info("Received a \"getNumberOfRecordsInspected\" request."); + + final String getInspectedRecordsNumberQuery = "select count(dist.id) from (select distinct id, original_url from " + ImpalaConnector.databaseName + ".attempt) as dist"; + try { + Object result = jdbcTemplate.queryForObject(getInspectedRecordsNumberQuery, Integer.class); + if ( result != null ) { + int numOfInspectedRecords = (int) result; + logger.info("Number of inspected records from the database \"" + ImpalaConnector.databaseName + "\" is " + numOfInspectedRecords); + return new ResponseEntity<>(numOfInspectedRecords, HttpStatus.OK); + } else + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("The inspected records' number could not be retrieved from the database \"" + ImpalaConnector.databaseName + "\" using the getInspectedRecordsNumberQuery: " + getInspectedRecordsNumberQuery); + } catch (EmptyResultDataAccessException erdae) { + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("The inspected records' number could not be retrieved from the database \"" + ImpalaConnector.databaseName + "\" using the getInspectedRecordsNumberQuery: " + getInspectedRecordsNumberQuery); + } catch (Exception e) { + String errorMsg = "Problem when executing \"getInspectedRecordsNumberQuery\": " + getInspectedRecordsNumberQuery; + logger.error(errorMsg, e); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorMsg); + // We may get a "Class Cast Exception", in case the Impala returns a non-integer value. + } + } + + // To get the human-friendly timestamp format from the BigInt in the database: + // select from_timestamp(CAST(CAST(`date` as decimal(30,0))/1000 AS timestamp), "yyyy-MM-dd HH:mm:ss.SSS") from payload + +}