lsmyrnaios
/
pdf-aggregation-statistics
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

- Make the "cacheUpdateInterval" configurable. 

- During boot, wait 3 minutes for the Datasource-pool to get fully initialized before "hitting" the DB to update the cache.
- Improve logging during cache-update.
This commit is contained in:
Lampros Smyrnaios 2023-10-23 14:53:22 +03:00
parent cbaa1cc621
commit 0ce5c98a7e
3 changed files with 9 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/eu/openaire/pdf_aggregation_statistics/components/SchedulingTasks.java
View File

@ -21,8 +21,8 @@ public class SchedulingTasks {
public static boolean runningFirstTime = true;
@Scheduled(initialDelay = 1, fixedDelay = 21_600_000) // Run right after initialization and then every 6 hours.
//@Scheduled(initialDelay = 1, fixedDelay = 300_000) // Just for testing: Run right after initialization and then every 5 mins.
@Scheduled(initialDelay = 180_000, fixedDelayString = "${spring.application.cacheUpdateInterval}") // Run 3 mins after initialization and then every 6 hours.
//@Scheduled(initialDelay = 180_000, fixedDelay = 300_000) // Just for testing: Run 3 mins after initialization and then every 5 mins.
public void gatherPayloadsPerDatasource()
{
// Request the number of payloads for each datasource and keep them in a ConcurrentHashMap,

8
src/main/java/eu/openaire/pdf_aggregation_statistics/services/StatsServiceImpl.java
View File

@ -29,9 +29,12 @@ public class StatsServiceImpl implements StatsService {
public static final ConcurrentHashMap<String, Integer> datasourcesWithNumOfPayloads = new ConcurrentHashMap<>(105_000); // The number of datasources is around 10_000.
private static final int maxRetries = 10;
public boolean gatherNumberOfPayloadsPerDatasource(int retryCount)
{
if ( retryCount > 10 ) {
if ( retryCount > maxRetries ) {
logger.error("Could not find the requested payload-type table in an non-merging state, after " + (retryCount -1) + " retries!");
return false;
}
@ -45,7 +48,7 @@ public class StatsServiceImpl implements StatsService {
if ( logger.isTraceEnabled() )
logger.trace("getNumberOfPayloadsPerDatasourceQuery:\n" + getNumberOfPayloadsPerDatasourceQuery);
logger.info("Going to " + (SchedulingTasks.runningFirstTime ? "populate" : "update") + " the \"datasourcesWithNumOfPayloads\" map.");
logger.info("Going to " + (SchedulingTasks.runningFirstTime ? "populate" : "update") + " the \"datasourcesWithNumOfPayloads\" map." + ((retryCount > 0) ? (" Retry " + retryCount + " out of " + maxRetries + ".") : ""));
final int[] countUpdatedDatasources = {0};
final int[] countNewDatasources = {0};
final int[] countAddedPayloads = {0};
@ -79,6 +82,7 @@ public class StatsServiceImpl implements StatsService {
} catch (Exception e) {
String exMsg = e.getMessage();
if ( (exMsg != null) && (exMsg.contains("Could not resolve table reference") || exMsg.contains("Failed to open HDFS file")) ) {
logger.info("The tables are probably in a merging state. Will wait 2 minutes and try again.");
sleep2mins(); // The tables may be under merging at the moment, so sleep a bit and try again.
return gatherNumberOfPayloadsPerDatasource(++retryCount);
} else {

1
src/main/resources/application.yml
View File

@ -9,6 +9,7 @@ database-name: pdfaggregation_i
spring:
application:
name: PDF_Aggregation_Statistics
cacheUpdateInterval: 21_600_000 # In milliseconds (6 hours).
datasource:
driver-class-name: com.cloudera.impala.jdbc41.Driver
url: