- Make sure the "UTF_8" charset is used, when we get a message from the response-body.
- Improve some log-messages.
This commit is contained in:
parent
bdf834c439
commit
856c62887d
|
@ -104,7 +104,7 @@ public class ScheduledTasks {
|
||||||
|
|
||||||
List<Future<Boolean>> futuresToDelete = new ArrayList<>(sizeOfFutures);
|
List<Future<Boolean>> futuresToDelete = new ArrayList<>(sizeOfFutures);
|
||||||
|
|
||||||
for ( int i=0; i < sizeOfFutures; ++i ) {
|
for ( int i=0; i < sizeOfFutures; ++i ) { // Check the futures up to the current "sizeOfFutures". The size of the list may increase while we check, but here we wil check only the first e.g. 10 futures.
|
||||||
Future<Boolean> future = null;
|
Future<Boolean> future = null;
|
||||||
try {
|
try {
|
||||||
future = UrlsController.futuresOfBackgroundTasks.get(i);
|
future = UrlsController.futuresOfBackgroundTasks.get(i);
|
||||||
|
@ -112,16 +112,16 @@ public class ScheduledTasks {
|
||||||
numFailedTasks ++;
|
numFailedTasks ++;
|
||||||
} catch (ExecutionException ee) {
|
} catch (ExecutionException ee) {
|
||||||
String stackTraceMessage = GenericUtils.getSelectiveStackTrace(ee, null, 15); // These can be serious errors like an "out of memory exception" (Java HEAP).
|
String stackTraceMessage = GenericUtils.getSelectiveStackTrace(ee, null, 15); // These can be serious errors like an "out of memory exception" (Java HEAP).
|
||||||
logger.error("Task_" + i + " failed with: " + ee.getMessage() + GenericUtils.endOfLine + stackTraceMessage);
|
logger.error("Background task_" + i + " failed with: " + ee.getMessage() + GenericUtils.endOfLine + stackTraceMessage);
|
||||||
numFailedTasks ++;
|
numFailedTasks ++;
|
||||||
} catch (CancellationException ce) {
|
} catch (CancellationException ce) {
|
||||||
logger.error("Task_" + i + " was cancelled: " + ce.getMessage());
|
logger.error("Background task_" + i + " was cancelled: " + ce.getMessage());
|
||||||
numFailedTasks ++;
|
numFailedTasks ++;
|
||||||
} catch (InterruptedException ie) {
|
} catch (InterruptedException ie) {
|
||||||
logger.error("Task_" + i + " was interrupted: " + ie.getMessage());
|
logger.error("Background task_" + i + " was interrupted: " + ie.getMessage());
|
||||||
numFailedTasks ++;
|
numFailedTasks ++;
|
||||||
} catch (IndexOutOfBoundsException ioobe) {
|
} catch (IndexOutOfBoundsException ioobe) {
|
||||||
logger.error("IOOBE for task_" + i + " in the futures-list! " + ioobe.getMessage());
|
logger.error("IOOBE for background task_" + i + " in the futures-list! " + ioobe.getMessage());
|
||||||
// Only here, the "future" will be null.
|
// Only here, the "future" will be null.
|
||||||
} finally {
|
} finally {
|
||||||
if ( future != null )
|
if ( future != null )
|
||||||
|
@ -133,9 +133,9 @@ public class ScheduledTasks {
|
||||||
UrlsController.futuresOfBackgroundTasks.remove(future); // We do not need it anymore. This is the safest way to delete them without removing newly added futures as well.
|
UrlsController.futuresOfBackgroundTasks.remove(future); // We do not need it anymore. This is the safest way to delete them without removing newly added futures as well.
|
||||||
|
|
||||||
if ( numFailedTasks > 0 )
|
if ( numFailedTasks > 0 )
|
||||||
logger.warn(numFailedTasks + " out of " + sizeOfFutures + " tasks have failed!");
|
logger.warn(numFailedTasks + " out of " + sizeOfFutures + " background tasks have failed!");
|
||||||
else
|
else
|
||||||
logger.debug("All of the " + sizeOfFutures + " tasks have succeeded.");
|
logger.debug("All of the " + sizeOfFutures + " background tasks have succeeded.");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -189,7 +189,7 @@ public class ScheduledTasks {
|
||||||
|
|
||||||
if ( UrlsController.numOfWorkers.get() == 0 ) {
|
if ( UrlsController.numOfWorkers.get() == 0 ) {
|
||||||
long timeToWait = (isTestEnvironment ? 1_200_000 : 43_200_000); // 10 mins | 12 hours
|
long timeToWait = (isTestEnvironment ? 1_200_000 : 43_200_000); // 10 mins | 12 hours
|
||||||
logger.warn("None of the workers have participated in the service yet. Will wait " + ((timeToWait /1000) /60) + " minutes and try again..");
|
logger.warn("None of the workers is participating in the service, at the moment. Will wait " + ((timeToWait /1000) /60) + " minutes and try again..");
|
||||||
try {
|
try {
|
||||||
Thread.sleep(timeToWait);
|
Thread.sleep(timeToWait);
|
||||||
} catch (InterruptedException ie) {
|
} catch (InterruptedException ie) {
|
||||||
|
|
|
@ -190,7 +190,7 @@ public class BulkImportController {
|
||||||
String bulkImportReportID = provenance + "/" + relativeBulkImportDir + "report_" + GenericUtils.getRandomNumber(10000, 99999);
|
String bulkImportReportID = provenance + "/" + relativeBulkImportDir + "report_" + GenericUtils.getRandomNumber(10000, 99999);
|
||||||
String bulkImportReportFullPath = this.bulkImportReportLocation + bulkImportReportID + ".json";
|
String bulkImportReportFullPath = this.bulkImportReportLocation + bulkImportReportID + ".json";
|
||||||
|
|
||||||
String msg = "The bulkImportFullTexts request for " + provenance + " procedure and bulkImportDir: " + givenBulkDir + " was accepted and will be scheduled for execution. "
|
String msg = "The bulkImportFullTexts request for " + provenance + " procedure and bulkImportDir: " + givenBulkDir + " was accepted and will be submitted for execution. "
|
||||||
+ (shouldDeleteFilesOnFinish ? "The successfully imported files will be deleted." : "All files will remain inside the directory after processing.")
|
+ (shouldDeleteFilesOnFinish ? "The successfully imported files will be deleted." : "All files will remain inside the directory after processing.")
|
||||||
+ " You can request a report at any moment, using the reportID.";
|
+ " You can request a report at any moment, using the reportID.";
|
||||||
|
|
||||||
|
|
|
@ -196,7 +196,7 @@ public class UrlsController {
|
||||||
return ResponseEntity.internalServerError().body(errorMsg);
|
return ResponseEntity.internalServerError().body(errorMsg);
|
||||||
}
|
}
|
||||||
|
|
||||||
String msg = "The 'addWorkerReport' request for worker with id: '" + curWorkerId + "' and assignments_" + curReportAssignmentsCounter + " , was accepted and will be scheduled for execution.";
|
String msg = "The 'addWorkerReport' request for worker with id: '" + curWorkerId + "' and assignments_" + curReportAssignmentsCounter + " , was accepted and submitted for execution.";
|
||||||
logger.info(msg);
|
logger.info(msg);
|
||||||
return ResponseEntity.ok().body(msg);
|
return ResponseEntity.ok().body(msg);
|
||||||
}
|
}
|
||||||
|
|
|
@ -413,7 +413,7 @@ public class UrlsServiceImpl implements UrlsService {
|
||||||
// For every "numOfWorkers" assignment-batches that go to workers, we merge the tables, once a workerReport comes in.
|
// For every "numOfWorkers" assignment-batches that go to workers, we merge the tables, once a workerReport comes in.
|
||||||
// After the first few increases of "assignmentsBatchCounter" until all workers get assignment-batches,
|
// After the first few increases of "assignmentsBatchCounter" until all workers get assignment-batches,
|
||||||
// there will always be a time when the counter will be just before the "golden-value" and then one workerReport has to be processed here and the counter will be incremented by one and signal the merging-time.
|
// there will always be a time when the counter will be just before the "golden-value" and then one workerReport has to be processed here and the counter will be incremented by one and signal the merging-time.
|
||||||
if ( (currentNumOfWorkerReportsProcessed % UrlsController.numOfWorkers.get()) == 0 ) // The workersNum should not be zero! If a "division by zero" exception is thrown below, then there's a big bug somewhere in the design.
|
if ( (currentNumOfWorkerReportsProcessed % UrlsController.numOfWorkers.get()) == 0 ) // The workersNum will not be zero!
|
||||||
if ( ! mergeWorkerRelatedTables(curWorkerId, curReportAssignmentsCounter, hasAttemptParquetFileProblem, hasPayloadParquetFileProblem) )
|
if ( ! mergeWorkerRelatedTables(curWorkerId, curReportAssignmentsCounter, hasAttemptParquetFileProblem, hasPayloadParquetFileProblem) )
|
||||||
// The "postReportResultToWorker()" was called inside.
|
// The "postReportResultToWorker()" was called inside.
|
||||||
return false;
|
return false;
|
||||||
|
|
|
@ -23,6 +23,7 @@ import java.io.*;
|
||||||
import java.net.ConnectException;
|
import java.net.ConnectException;
|
||||||
import java.net.HttpURLConnection;
|
import java.net.HttpURLConnection;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
import java.nio.file.Paths;
|
import java.nio.file.Paths;
|
||||||
|
@ -624,7 +625,7 @@ public class FileUtils {
|
||||||
public String getMessageFromResponseBody(HttpURLConnection conn, boolean isError)
|
public String getMessageFromResponseBody(HttpURLConnection conn, boolean isError)
|
||||||
{
|
{
|
||||||
final StringBuilder msgStrB = new StringBuilder(500);
|
final StringBuilder msgStrB = new StringBuilder(500);
|
||||||
try ( BufferedReader br = new BufferedReader(new InputStreamReader((isError ? conn.getErrorStream() : conn.getInputStream()))) ) {
|
try ( BufferedReader br = new BufferedReader(new InputStreamReader((isError ? conn.getErrorStream() : conn.getInputStream()), StandardCharsets.UTF_8)) ) {
|
||||||
String inputLine;
|
String inputLine;
|
||||||
while ( (inputLine = br.readLine()) != null )
|
while ( (inputLine = br.readLine()) != null )
|
||||||
{
|
{
|
||||||
|
|
Loading…
Reference in New Issue