michal.politowski
/
UrlsController
forked from lsmyrnaios/UrlsController
1
0
Fork 0
Code Pull Requests Activity

- Show the original exception thrown by the background-job, not the one thrown in the main-thread, which is useless, except from its message. 

- Reduce the interval for deleting the unhandled assignments to once every 3 days.
- Set the upcoming version.
- Update dependencies.
This commit is contained in:
Lampros Smyrnaios 2023-11-27 18:19:53 +02:00
parent 7f789b8ad0
commit d20c9a7d2e
3 changed files with 23 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
build.gradle
View File

@ -1,12 +1,12 @@
plugins {
id 'org.springframework.boot' version '2.7.17'
id 'io.spring.dependency-management' version '1.1.3'
id 'org.springframework.boot' version '2.7.18'
id 'io.spring.dependency-management' version '1.1.4'
id 'java'
}
java {
group = 'eu.openaire.urls_controller'
version = '2.6.0-SNAPSHOT'
version = '2.7.0-SNAPSHOT'
sourceCompatibility = JavaVersion.VERSION_1_8
}
@ -46,15 +46,15 @@ dependencies {
implementation group: 'com.google.guava', name: 'guava', version: '32.1.3-jre'
// https://mvnrepository.com/artifact/org.apache.commons/commons-lang3
implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.13.0'
implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.14.0'
// https://mvnrepository.com/artifact/org.apache.commons/commons-compress
implementation("org.apache.commons:commons-compress:1.24.0") {
implementation("org.apache.commons:commons-compress:1.25.0") {
exclude group: 'com.github.luben', module: 'zstd-jni'
}
implementation 'com.github.luben:zstd-jni:1.5.5-6' // Even though this is part of the above dependency, the Apache commons rarely updates it, while the zstd team makes improvements very often.
implementation 'com.github.luben:zstd-jni:1.5.5-10' // Even though this is part of the above dependency, the Apache commons rarely updates it, while the zstd team makes improvements very often.
implementation 'io.minio:minio:8.5.6'
implementation 'io.minio:minio:8.5.7'
// https://mvnrepository.com/artifact/com.cloudera.impala/jdbc
implementation("com.cloudera.impala:jdbc:2.5.31") {
@ -120,7 +120,7 @@ dependencies {
// https://mvnrepository.com/artifact/io.micrometer/micrometer-registry-prometheus
runtimeOnly 'io.micrometer:micrometer-registry-prometheus:1.11.5'
runtimeOnly 'io.micrometer:micrometer-registry-prometheus:1.12.0'
testImplementation 'org.springframework.security:spring-security-test'
testImplementation "org.springframework.boot:spring-boot-starter-test"

22
src/main/java/eu/openaire/urls_controller/components/ScheduledTasks.java
View File

@ -110,9 +110,14 @@ public class ScheduledTasks {
future = UrlsController.futuresOfBackgroundTasks.get(i);
if ( ! future.get() ) // Get and see if an exception is thrown. This blocks the current thread, until the task of the future has finished.
numFailedTasks ++;
} catch (ExecutionException ee) {
String stackTraceMessage = GenericUtils.getSelectiveStackTrace(ee, null, 15); // These can be serious errors like an "out of memory exception" (Java HEAP).
logger.error("Background task_" + i + " failed with: " + ee.getMessage() + GenericUtils.endOfLine + stackTraceMessage);
} catch (ExecutionException ee) { // These can be serious errors like an "out of memory exception" (Java HEAP).
// The stacktrace of the "ExecutionException" is the one of thi code and not the code which ran inside the background-task. Try to get the cause.
Throwable throwable = ee.getCause();
if ( throwable == null ) {
logger.warn("No cause was retrieved for the \"ExecutionException\"!");
throwable = ee;
}
logger.error(GenericUtils.getSelectiveStackTrace(throwable, "Background task_" + i + " failed with: " + throwable.getMessage(), 30));
numFailedTasks ++;
} catch (CancellationException ce) {
logger.error("Background task_" + i + " was cancelled: " + ce.getMessage());
@ -231,21 +236,20 @@ public class ScheduledTasks {
inspectWorkerReportsAndTakeAction(ActionForWorkerReports.delete_old);
}
private static final double daysToWaitBeforeDeletion = 7.0;
@Scheduled(initialDelay = 604_800_000, fixedDelay = 604_800_000) // Run every 7 days.
//@Scheduled(initialDelay = 1_200_000, fixedDelay = 1_200_000) // Just for testing (every 1200 secs).
@Scheduled(initialDelay = 259_200_000, fixedDelay = 259_200_000) // Run every 3 days. 3 days after startup.
//@Scheduled(initialDelay = 1_200_000, fixedDelay = 1_200_000) // Just for testing (every 1200 secs).
public void checkAndDeleteUnhandledAssignments()
{
// Remove the assignments having a "date" older than 7 days.
// Remove the assignments having a "date" older than 3 days.
// For some reason, sometimes, the worker cannot receive the assignments in time.
// In this case, no job is created for those assignments nd no workerReport gets stored in storage.
// The assignments just remain in the table, and the urls cannot be rechecked.
Calendar calendar = Calendar.getInstance();
calendar.add(Calendar.DAY_OF_MONTH, - (int) daysToWaitBeforeDeletion); // Subtract <daysToWaitBeforeDeletion> from current Date.
calendar.add(Calendar.DAY_OF_MONTH, - 3); // Subtract <daysToWaitBeforeDeletion> from current Date.
DatabaseConnector.databaseLock.lock();
urlsService.deleteAssignmentsWithOlderDate(calendar.getTimeInMillis()); // Any error-log is written inside.
@ -312,6 +316,8 @@ public class ScheduledTasks {
private static StringBuilder jsonStringBuilder = null;
private static final int daysDivisor = (1000 * 60 * 60 * 24); // In order to get the time-difference in days. We divide with: /1000 to get seconds, /60 to get minutes, /60 to get hours and /24 to get days.
private static final double daysToWaitBeforeDeletion = 7.0;
private void inspectWorkerReportsAndTakeAction(ActionForWorkerReports actionForWorkerReports)
{

2
src/main/java/eu/openaire/urls_controller/util/GenericUtils.java
View File

@ -27,7 +27,7 @@ public class GenericUtils {
StackTraceElement[] stels = thr.getStackTrace();
StringBuilder sb = new StringBuilder(numOfLines *100);
if ( initialMessage != null )
sb.append(initialMessage).append(" Stacktrace:").append(GenericUtils.endOfLine); // This StringBuilder is thread-safe as a local-variable.
sb.append(initialMessage).append(" | Stacktrace:").append(GenericUtils.endOfLine); // This StringBuilder is thread-safe as a local-variable.
for ( int i = 0; (i < stels.length) && (i <= numOfLines); ++i ) {
sb.append(stels[i]);
if (i < numOfLines) sb.append(GenericUtils.endOfLine);