From f61cae41a107b639771188b74f0df956154e56b5 Mon Sep 17 00:00:00 2001
From: LSmyrnaios <lsmyrnaios@gmail.com>
Date: Wed, 13 Mar 2024 12:15:59 +0200
Subject: [PATCH] - Try to get the cause of the exception of the callable-tasks
 which handle the bulk-import of fileSegments. - Fix not counting the
 failedSegments when an exception was thrown. - Code polishing.

---
 .../services/BulkImportServiceImpl.java          | 16 ++++++++++++----
 .../urls_controller/util/GenericUtils.java       |  2 +-
 .../urls_controller/util/ParquetFileUtils.java   |  2 +-
 src/main/resources/application.yml               |  2 +-
 4 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/src/main/java/eu/openaire/urls_controller/services/BulkImportServiceImpl.java b/src/main/java/eu/openaire/urls_controller/services/BulkImportServiceImpl.java
index 1bcbe64..056a722 100644
--- a/src/main/java/eu/openaire/urls_controller/services/BulkImportServiceImpl.java
+++ b/src/main/java/eu/openaire/urls_controller/services/BulkImportServiceImpl.java
@@ -163,11 +163,19 @@ public class BulkImportServiceImpl implements BulkImportService {
                         numFailedSegments++;
                     // In case all the files failed to be bulk-imported, then we will detect it in the "numSuccessfulSegments"-check later.
                     // The failed-to-be-imported files, will not be deleted, even if the user specifies that he wants to delete the directory.
-                } catch (ExecutionException ee) {
-                    String stackTraceMessage = GenericUtils.getSelectiveStackTrace(ee, null, 15);	// These can be serious errors like an "out of memory exception" (Java HEAP).
-                    logger.error("Task_" + (i+1) + " failed with: " + ee.getMessage() + additionalLoggingMsg + GenericUtils.endOfLine + stackTraceMessage);
+                } catch (ExecutionException ee) {   // These can be serious errors like an "out of memory exception" (Java HEAP).
+                    // The stacktrace of the "ExecutionException" is the one of the current code and not the code which ran inside the background-task. Try to get the cause.
+                    Throwable throwable = ee.getCause();
+                    if ( throwable == null ) {
+                        logger.warn("No cause was retrieved for the \"ExecutionException\"!");
+                        throwable = ee;
+                    }
+                    String stackTraceMessage = GenericUtils.getSelectiveStackTrace(throwable, "Task_" + i + " failed with: " + throwable.getMessage() + additionalLoggingMsg, 15);
+                    numFailedSegments ++;
+                    logger.error("Task_" + i + " failed with: " + stackTraceMessage);
                 } catch (CancellationException ce) {
-                    logger.error("Task_" + (i+1) + " was cancelled: " + ce.getMessage() + additionalLoggingMsg);
+                    numFailedSegments ++;
+                    logger.error("Task_" + i + " was cancelled: " + ce.getMessage() + additionalLoggingMsg);
                 } catch (IndexOutOfBoundsException ioobe) {
                     logger.error("IOOBE for task_" + i + " in the futures-list! " + ioobe.getMessage() + additionalLoggingMsg);
                 }
diff --git a/src/main/java/eu/openaire/urls_controller/util/GenericUtils.java b/src/main/java/eu/openaire/urls_controller/util/GenericUtils.java
index 558cb8d..876892e 100644
--- a/src/main/java/eu/openaire/urls_controller/util/GenericUtils.java
+++ b/src/main/java/eu/openaire/urls_controller/util/GenericUtils.java
@@ -27,7 +27,7 @@ public class GenericUtils {
         StackTraceElement[] stels = thr.getStackTrace();
         StringBuilder sb = new StringBuilder(numOfLines *100);
         if ( initialMessage != null )
-            sb.append(initialMessage).append(" | Stacktrace:").append(GenericUtils.endOfLine);	// This StringBuilder is thread-safe as a local-variable.
+            sb.append(initialMessage).append(GenericUtils.endOfLine).append("Stacktrace:").append(GenericUtils.endOfLine);	// This StringBuilder is thread-safe as a local-variable.
         for ( int i = 0; (i < stels.length) && (i <= numOfLines); ++i ) {
             sb.append(stels[i]);
             if (i < numOfLines) sb.append(GenericUtils.endOfLine);
diff --git a/src/main/java/eu/openaire/urls_controller/util/ParquetFileUtils.java b/src/main/java/eu/openaire/urls_controller/util/ParquetFileUtils.java
index 617fd5b..c2a5db7 100644
--- a/src/main/java/eu/openaire/urls_controller/util/ParquetFileUtils.java
+++ b/src/main/java/eu/openaire/urls_controller/util/ParquetFileUtils.java
@@ -322,7 +322,7 @@ public class ParquetFileUtils {
         }
 
         if ( numPayloadsInsideUrlReports == 0 )
-            return true;    // This urlsRerports-sublist does not have any payloads inside to use. That's fine.
+            return true;    // This urlsReports-sublist does not have any payloads inside to use. That's fine.
 
         int recordsSize = recordList.size();
         if ( recordsSize == 0 ) {
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index 5ec1962..1ab2856 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -15,7 +15,7 @@ services:
 
             db:
                 initialDatabaseName: pdfaggregation_i
-                testDatabaseName: pdfaggregationdatabase_payloads_view_test
+                testDatabaseName: pdfaggregationdatabase_test_new
 
             assignmentLimit: 10000
             maxAttemptsPerRecord: 3