From 39c36f9e662ca77db5b97422c541ab36b412177a Mon Sep 17 00:00:00 2001 From: LSmyrnaios Date: Wed, 1 May 2024 01:29:25 +0300 Subject: [PATCH] - Resolve a concurrency issue, by enforcing synchronization on the "BulkImportReport.getJsonReport()" method. - Increase the number of stacktrace-lines to 20, for bulkImport-segment-failures. - Improve "GenericUtils.getSelectiveStackTrace()". --- .../urls_controller/models/BulkImportReport.java | 5 ++++- .../services/BulkImportServiceImpl.java | 2 +- .../eu/openaire/urls_controller/util/FileUtils.java | 3 +++ .../openaire/urls_controller/util/GenericUtils.java | 11 ++++++----- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/main/java/eu/openaire/urls_controller/models/BulkImportReport.java b/src/main/java/eu/openaire/urls_controller/models/BulkImportReport.java index 83c3274..60420e3 100644 --- a/src/main/java/eu/openaire/urls_controller/models/BulkImportReport.java +++ b/src/main/java/eu/openaire/urls_controller/models/BulkImportReport.java @@ -46,7 +46,10 @@ public class BulkImportReport { eventsMultimap.put(GenericUtils.getReadableCurrentTimeAndZone(), event); // This is synchronized. } - public String getJsonReport() + /** + * Synchronize it to avoid concurrency issues when concurrent calls are made to the same bulkImport-Report object. + * */ + public synchronized String getJsonReport() { //Convert the LinkedHashMultiMap to Map>, since Gson cannot serialize Multimaps. eventsMap = eventsMultimap.asMap(); diff --git a/src/main/java/eu/openaire/urls_controller/services/BulkImportServiceImpl.java b/src/main/java/eu/openaire/urls_controller/services/BulkImportServiceImpl.java index 0725d5f..db8ef43 100644 --- a/src/main/java/eu/openaire/urls_controller/services/BulkImportServiceImpl.java +++ b/src/main/java/eu/openaire/urls_controller/services/BulkImportServiceImpl.java @@ -157,7 +157,7 @@ public class BulkImportServiceImpl implements BulkImportService { } catch (ExecutionException ee) { // These can be serious errors like an "out of memory exception" (Java HEAP). numFailedSegments ++; numAllFailedFiles += subLists.get(i).size(); // We assume all files of this segment failed, as all are passed through the same parts of code, so any serious exception should arise from the 1st files being processed and the rest of the files wil be skipped.. - logger.error(GenericUtils.getSelectedStackTraceForCausedException(ee, "Task_" + i + " failed with: ", additionalLoggingMsg, 15)); + logger.error(GenericUtils.getSelectedStackTraceForCausedException(ee, "Task_" + i + " failed with: ", additionalLoggingMsg, 20)); bulkImportReport.addEvent("Segment_" + i + " failed with: " + ee.getCause().getMessage()); } catch (CancellationException ce) { numFailedSegments ++; diff --git a/src/main/java/eu/openaire/urls_controller/util/FileUtils.java b/src/main/java/eu/openaire/urls_controller/util/FileUtils.java index 20b3f5a..68e0493 100644 --- a/src/main/java/eu/openaire/urls_controller/util/FileUtils.java +++ b/src/main/java/eu/openaire/urls_controller/util/FileUtils.java @@ -816,6 +816,9 @@ public class FileUtils { if ( shouldLockThreads ) // In case multiple threads write to the same file. for ex. during the bulk-import procedure. fileAccessLock.lock(); + // TODO - Make this method to be synchronized be specific file, not in general. + // TODO - NOW: Multiple bulkImport procedures (with diff DIRs), are blocked while writing to DIFFERENT files.. + try ( BufferedWriter bufferedWriter = new BufferedWriter(Files.newBufferedWriter(Paths.get(fileFullPath)), halfMb) ) { bufferedWriter.write(stringToWrite); // This will overwrite the file. If the new string is smaller, then it does not matter. diff --git a/src/main/java/eu/openaire/urls_controller/util/GenericUtils.java b/src/main/java/eu/openaire/urls_controller/util/GenericUtils.java index 4bc98fd..dbfb198 100644 --- a/src/main/java/eu/openaire/urls_controller/util/GenericUtils.java +++ b/src/main/java/eu/openaire/urls_controller/util/GenericUtils.java @@ -13,6 +13,7 @@ public class GenericUtils { public static final String endOfLine = "\n"; + public static final String tab = "\t"; private static final SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS z"); @@ -46,11 +47,11 @@ public class GenericUtils { StackTraceElement[] stels = thr.getStackTrace(); StringBuilder sb = new StringBuilder(numOfLines *100); // This StringBuilder is thread-safe as a local-variable. if ( initialMessage != null ) - sb.append(initialMessage).append(GenericUtils.endOfLine); - sb.append("Stacktrace:").append(GenericUtils.endOfLine); - for ( int i = 0; (i < stels.length) && (i <= numOfLines); ++i ) { - sb.append(stels[i]); - if (i < numOfLines) sb.append(GenericUtils.endOfLine); + sb.append(initialMessage).append(endOfLine); + sb.append("Stacktrace:").append(endOfLine); + for ( int i = 0; (i < stels.length) && (i < numOfLines); ++i ) { + sb.append(tab).append(stels[i]); + if (i < (numOfLines -1)) sb.append(endOfLine); } return sb.toString(); }