- Try to get the cause of the exception of the callable-tasks which handle the bulk-import of fileSegments.

- Fix not counting the failedSegments when an exception was thrown.
- Code polishing.

src/main/java/eu/openaire/urls_controller/services/BulkImportServiceImpl.java

@@ -163,11 +163,19 @@ public class BulkImportServiceImpl implements BulkImportService {
                         numFailedSegments++;
                     // In case all the files failed to be bulk-imported, then we will detect it in the "numSuccessfulSegments"-check later.
                     // The failed-to-be-imported files, will not be deleted, even if the user specifies that he wants to delete the directory.
-                } catch (ExecutionException ee) {
-                    String stackTraceMessage = GenericUtils.getSelectiveStackTrace(ee, null, 15);	// These can be serious errors like an "out of memory exception" (Java HEAP).
-                    logger.error("Task_" + (i+1) + " failed with: " + ee.getMessage() + additionalLoggingMsg + GenericUtils.endOfLine + stackTraceMessage);
+                } catch (ExecutionException ee) {   // These can be serious errors like an "out of memory exception" (Java HEAP).
+                    // The stacktrace of the "ExecutionException" is the one of the current code and not the code which ran inside the background-task. Try to get the cause.
+                    Throwable throwable = ee.getCause();
+                    if ( throwable == null ) {
+                        logger.warn("No cause was retrieved for the \"ExecutionException\"!");
+                        throwable = ee;
+                    }
+                    String stackTraceMessage = GenericUtils.getSelectiveStackTrace(throwable, "Task_" + i + " failed with: " + throwable.getMessage() + additionalLoggingMsg, 15);
+                    numFailedSegments ++;
+                    logger.error("Task_" + i + " failed with: " + stackTraceMessage);
                 } catch (CancellationException ce) {
-                    logger.error("Task_" + (i+1) + " was cancelled: " + ce.getMessage() + additionalLoggingMsg);
+                    numFailedSegments ++;
+                    logger.error("Task_" + i + " was cancelled: " + ce.getMessage() + additionalLoggingMsg);
                 } catch (IndexOutOfBoundsException ioobe) {
                     logger.error("IOOBE for task_" + i + " in the futures-list! " + ioobe.getMessage() + additionalLoggingMsg);
                 }

src/main/java/eu/openaire/urls_controller/util/GenericUtils.java

@@ -27,7 +27,7 @@ public class GenericUtils {
         StackTraceElement[] stels = thr.getStackTrace();
         StringBuilder sb = new StringBuilder(numOfLines *100);
         if ( initialMessage != null )
-            sb.append(initialMessage).append(" | Stacktrace:").append(GenericUtils.endOfLine);	// This StringBuilder is thread-safe as a local-variable.
+            sb.append(initialMessage).append(GenericUtils.endOfLine).append("Stacktrace:").append(GenericUtils.endOfLine);	// This StringBuilder is thread-safe as a local-variable.
         for ( int i = 0; (i < stels.length) && (i <= numOfLines); ++i ) {
             sb.append(stels[i]);
             if (i < numOfLines) sb.append(GenericUtils.endOfLine);

src/main/java/eu/openaire/urls_controller/util/ParquetFileUtils.java

@@ -322,7 +322,7 @@ public class ParquetFileUtils {
         }
 
         if ( numPayloadsInsideUrlReports == 0 )
-            return true;    // This urlsRerports-sublist does not have any payloads inside to use. That's fine.
+            return true;    // This urlsReports-sublist does not have any payloads inside to use. That's fine.
 
         int recordsSize = recordList.size();
         if ( recordsSize == 0 ) {

src/main/resources/application.yml

@@ -15,7 +15,7 @@ services:
 
             db:
                 initialDatabaseName: pdfaggregation_i
-                testDatabaseName: pdfaggregationdatabase_payloads_view_test
+                testDatabaseName: pdfaggregationdatabase_test_new
 
             assignmentLimit: 10000
             maxAttemptsPerRecord: 3