From d20c9a7d2e831c5789376d8fc074e3dd8842c7c8 Mon Sep 17 00:00:00 2001 From: LSmyrnaios Date: Mon, 27 Nov 2023 18:19:53 +0200 Subject: [PATCH] - Show the original exception thrown by the background-job, not the one thrown in the main-thread, which is useless, except from its message. - Reduce the interval for deleting the unhandled assignments to once every 3 days. - Set the upcoming version. - Update dependencies. --- build.gradle | 16 +++++++------- .../components/ScheduledTasks.java | 22 ++++++++++++------- .../urls_controller/util/GenericUtils.java | 2 +- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/build.gradle b/build.gradle index b6b611a..85271a0 100644 --- a/build.gradle +++ b/build.gradle @@ -1,12 +1,12 @@ plugins { - id 'org.springframework.boot' version '2.7.17' - id 'io.spring.dependency-management' version '1.1.3' + id 'org.springframework.boot' version '2.7.18' + id 'io.spring.dependency-management' version '1.1.4' id 'java' } java { group = 'eu.openaire.urls_controller' - version = '2.6.0-SNAPSHOT' + version = '2.7.0-SNAPSHOT' sourceCompatibility = JavaVersion.VERSION_1_8 } @@ -46,15 +46,15 @@ dependencies { implementation group: 'com.google.guava', name: 'guava', version: '32.1.3-jre' // https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 - implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.13.0' + implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.14.0' // https://mvnrepository.com/artifact/org.apache.commons/commons-compress - implementation("org.apache.commons:commons-compress:1.24.0") { + implementation("org.apache.commons:commons-compress:1.25.0") { exclude group: 'com.github.luben', module: 'zstd-jni' } - implementation 'com.github.luben:zstd-jni:1.5.5-6' // Even though this is part of the above dependency, the Apache commons rarely updates it, while the zstd team makes improvements very often. + implementation 'com.github.luben:zstd-jni:1.5.5-10' // Even though this is part of the above dependency, the Apache commons rarely updates it, while the zstd team makes improvements very often. - implementation 'io.minio:minio:8.5.6' + implementation 'io.minio:minio:8.5.7' // https://mvnrepository.com/artifact/com.cloudera.impala/jdbc implementation("com.cloudera.impala:jdbc:2.5.31") { @@ -120,7 +120,7 @@ dependencies { // https://mvnrepository.com/artifact/io.micrometer/micrometer-registry-prometheus - runtimeOnly 'io.micrometer:micrometer-registry-prometheus:1.11.5' + runtimeOnly 'io.micrometer:micrometer-registry-prometheus:1.12.0' testImplementation 'org.springframework.security:spring-security-test' testImplementation "org.springframework.boot:spring-boot-starter-test" diff --git a/src/main/java/eu/openaire/urls_controller/components/ScheduledTasks.java b/src/main/java/eu/openaire/urls_controller/components/ScheduledTasks.java index 8dc5760..24ff7cb 100644 --- a/src/main/java/eu/openaire/urls_controller/components/ScheduledTasks.java +++ b/src/main/java/eu/openaire/urls_controller/components/ScheduledTasks.java @@ -110,9 +110,14 @@ public class ScheduledTasks { future = UrlsController.futuresOfBackgroundTasks.get(i); if ( ! future.get() ) // Get and see if an exception is thrown. This blocks the current thread, until the task of the future has finished. numFailedTasks ++; - } catch (ExecutionException ee) { - String stackTraceMessage = GenericUtils.getSelectiveStackTrace(ee, null, 15); // These can be serious errors like an "out of memory exception" (Java HEAP). - logger.error("Background task_" + i + " failed with: " + ee.getMessage() + GenericUtils.endOfLine + stackTraceMessage); + } catch (ExecutionException ee) { // These can be serious errors like an "out of memory exception" (Java HEAP). + // The stacktrace of the "ExecutionException" is the one of thi code and not the code which ran inside the background-task. Try to get the cause. + Throwable throwable = ee.getCause(); + if ( throwable == null ) { + logger.warn("No cause was retrieved for the \"ExecutionException\"!"); + throwable = ee; + } + logger.error(GenericUtils.getSelectiveStackTrace(throwable, "Background task_" + i + " failed with: " + throwable.getMessage(), 30)); numFailedTasks ++; } catch (CancellationException ce) { logger.error("Background task_" + i + " was cancelled: " + ce.getMessage()); @@ -231,21 +236,20 @@ public class ScheduledTasks { inspectWorkerReportsAndTakeAction(ActionForWorkerReports.delete_old); } - private static final double daysToWaitBeforeDeletion = 7.0; - @Scheduled(initialDelay = 604_800_000, fixedDelay = 604_800_000) // Run every 7 days. - //@Scheduled(initialDelay = 1_200_000, fixedDelay = 1_200_000) // Just for testing (every 1200 secs). + @Scheduled(initialDelay = 259_200_000, fixedDelay = 259_200_000) // Run every 3 days. 3 days after startup. + //@Scheduled(initialDelay = 1_200_000, fixedDelay = 1_200_000) // Just for testing (every 1200 secs). public void checkAndDeleteUnhandledAssignments() { - // Remove the assignments having a "date" older than 7 days. + // Remove the assignments having a "date" older than 3 days. // For some reason, sometimes, the worker cannot receive the assignments in time. // In this case, no job is created for those assignments nd no workerReport gets stored in storage. // The assignments just remain in the table, and the urls cannot be rechecked. Calendar calendar = Calendar.getInstance(); - calendar.add(Calendar.DAY_OF_MONTH, - (int) daysToWaitBeforeDeletion); // Subtract from current Date. + calendar.add(Calendar.DAY_OF_MONTH, - 3); // Subtract from current Date. DatabaseConnector.databaseLock.lock(); urlsService.deleteAssignmentsWithOlderDate(calendar.getTimeInMillis()); // Any error-log is written inside. @@ -312,6 +316,8 @@ public class ScheduledTasks { private static StringBuilder jsonStringBuilder = null; private static final int daysDivisor = (1000 * 60 * 60 * 24); // In order to get the time-difference in days. We divide with: /1000 to get seconds, /60 to get minutes, /60 to get hours and /24 to get days. + private static final double daysToWaitBeforeDeletion = 7.0; + private void inspectWorkerReportsAndTakeAction(ActionForWorkerReports actionForWorkerReports) { diff --git a/src/main/java/eu/openaire/urls_controller/util/GenericUtils.java b/src/main/java/eu/openaire/urls_controller/util/GenericUtils.java index e77ca78..558cb8d 100644 --- a/src/main/java/eu/openaire/urls_controller/util/GenericUtils.java +++ b/src/main/java/eu/openaire/urls_controller/util/GenericUtils.java @@ -27,7 +27,7 @@ public class GenericUtils { StackTraceElement[] stels = thr.getStackTrace(); StringBuilder sb = new StringBuilder(numOfLines *100); if ( initialMessage != null ) - sb.append(initialMessage).append(" Stacktrace:").append(GenericUtils.endOfLine); // This StringBuilder is thread-safe as a local-variable. + sb.append(initialMessage).append(" | Stacktrace:").append(GenericUtils.endOfLine); // This StringBuilder is thread-safe as a local-variable. for ( int i = 0; (i < stels.length) && (i <= numOfLines); ++i ) { sb.append(stels[i]); if (i < numOfLines) sb.append(GenericUtils.endOfLine);