michal.politowski
/
UrlsController
forked from lsmyrnaios/UrlsController
1
0
Fork 0
Code Pull Requests Activity

- Make sure the "UTF_8" charset is used, when we get a message from the response-body. 

- Improve some log-messages.
This commit is contained in:
Lampros Smyrnaios 2023-10-26 11:44:23 +03:00
parent bdf834c439
commit 856c62887d
5 changed files with 13 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/main/java/eu/openaire/urls_controller/components/ScheduledTasks.java
View File

@ -104,7 +104,7 @@ public class ScheduledTasks {
List<Future<Boolean>> futuresToDelete = new ArrayList<>(sizeOfFutures);
for ( int i=0; i < sizeOfFutures; ++i ) {
for ( int i=0; i < sizeOfFutures; ++i ) { // Check the futures up to the current "sizeOfFutures". The size of the list may increase while we check, but here we wil check only the first e.g. 10 futures.
Future<Boolean> future = null;
try {
future = UrlsController.futuresOfBackgroundTasks.get(i);
@ -112,16 +112,16 @@ public class ScheduledTasks {
numFailedTasks ++;
} catch (ExecutionException ee) {
String stackTraceMessage = GenericUtils.getSelectiveStackTrace(ee, null, 15); // These can be serious errors like an "out of memory exception" (Java HEAP).
logger.error("Task_" + i + " failed with: " + ee.getMessage() + GenericUtils.endOfLine + stackTraceMessage);
logger.error("Background task_" + i + " failed with: " + ee.getMessage() + GenericUtils.endOfLine + stackTraceMessage);
numFailedTasks ++;
} catch (CancellationException ce) {
logger.error("Task_" + i + " was cancelled: " + ce.getMessage());
logger.error("Background task_" + i + " was cancelled: " + ce.getMessage());
numFailedTasks ++;
} catch (InterruptedException ie) {
logger.error("Task_" + i + " was interrupted: " + ie.getMessage());
logger.error("Background task_" + i + " was interrupted: " + ie.getMessage());
numFailedTasks ++;
} catch (IndexOutOfBoundsException ioobe) {
logger.error("IOOBE for task_" + i + " in the futures-list! " + ioobe.getMessage());
logger.error("IOOBE for background task_" + i + " in the futures-list! " + ioobe.getMessage());
// Only here, the "future" will be null.
} finally {
if ( future != null )
@ -133,9 +133,9 @@ public class ScheduledTasks {
UrlsController.futuresOfBackgroundTasks.remove(future); // We do not need it anymore. This is the safest way to delete them without removing newly added futures as well.
if ( numFailedTasks > 0 )
logger.warn(numFailedTasks + " out of " + sizeOfFutures + " tasks have failed!");
logger.warn(numFailedTasks + " out of " + sizeOfFutures + " background tasks have failed!");
else
logger.debug("All of the " + sizeOfFutures + " tasks have succeeded.");
logger.debug("All of the " + sizeOfFutures + " background tasks have succeeded.");
}
@ -189,7 +189,7 @@ public class ScheduledTasks {
if ( UrlsController.numOfWorkers.get() == 0 ) {
long timeToWait = (isTestEnvironment ? 1_200_000 : 43_200_000); // 10 mins | 12 hours
logger.warn("None of the workers have participated in the service yet. Will wait " + ((timeToWait /1000) /60) + " minutes and try again..");
logger.warn("None of the workers is participating in the service, at the moment. Will wait " + ((timeToWait /1000) /60) + " minutes and try again..");
try {
Thread.sleep(timeToWait);
} catch (InterruptedException ie) {

2
src/main/java/eu/openaire/urls_controller/controllers/BulkImportController.java
View File

@ -190,7 +190,7 @@ public class BulkImportController {
String bulkImportReportID = provenance + "/" + relativeBulkImportDir + "report_" + GenericUtils.getRandomNumber(10000, 99999);
String bulkImportReportFullPath = this.bulkImportReportLocation + bulkImportReportID + ".json";
String msg = "The bulkImportFullTexts request for " + provenance + " procedure and bulkImportDir: " + givenBulkDir + " was accepted and will be scheduled for execution. "
String msg = "The bulkImportFullTexts request for " + provenance + " procedure and bulkImportDir: " + givenBulkDir + " was accepted and will be submitted for execution. "
+ (shouldDeleteFilesOnFinish ? "The successfully imported files will be deleted." : "All files will remain inside the directory after processing.")
+ " You can request a report at any moment, using the reportID.";

2
src/main/java/eu/openaire/urls_controller/controllers/UrlsController.java
View File

@ -196,7 +196,7 @@ public class UrlsController {
return ResponseEntity.internalServerError().body(errorMsg);
}
String msg = "The 'addWorkerReport' request for worker with id: '" + curWorkerId + "' and assignments_" + curReportAssignmentsCounter + " , was accepted and will be scheduled for execution.";
String msg = "The 'addWorkerReport' request for worker with id: '" + curWorkerId + "' and assignments_" + curReportAssignmentsCounter + " , was accepted and submitted for execution.";
logger.info(msg);
return ResponseEntity.ok().body(msg);
}

2
src/main/java/eu/openaire/urls_controller/services/UrlsServiceImpl.java
View File

@ -413,7 +413,7 @@ public class UrlsServiceImpl implements UrlsService {
// For every "numOfWorkers" assignment-batches that go to workers, we merge the tables, once a workerReport comes in.
// After the first few increases of "assignmentsBatchCounter" until all workers get assignment-batches,
// there will always be a time when the counter will be just before the "golden-value" and then one workerReport has to be processed here and the counter will be incremented by one and signal the merging-time.
if ( (currentNumOfWorkerReportsProcessed % UrlsController.numOfWorkers.get()) == 0 ) // The workersNum should not be zero! If a "division by zero" exception is thrown below, then there's a big bug somewhere in the design.
if ( (currentNumOfWorkerReportsProcessed % UrlsController.numOfWorkers.get()) == 0 ) // The workersNum will not be zero!
if ( ! mergeWorkerRelatedTables(curWorkerId, curReportAssignmentsCounter, hasAttemptParquetFileProblem, hasPayloadParquetFileProblem) )
// The "postReportResultToWorker()" was called inside.
return false;

3
src/main/java/eu/openaire/urls_controller/util/FileUtils.java
View File

@ -23,6 +23,7 @@ import java.io.*;
import java.net.ConnectException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@ -624,7 +625,7 @@ public class FileUtils {
public String getMessageFromResponseBody(HttpURLConnection conn, boolean isError)
{
final StringBuilder msgStrB = new StringBuilder(500);
try ( BufferedReader br = new BufferedReader(new InputStreamReader((isError ? conn.getErrorStream() : conn.getInputStream()))) ) {
try ( BufferedReader br = new BufferedReader(new InputStreamReader((isError ? conn.getErrorStream() : conn.getInputStream()), StandardCharsets.UTF_8)) ) {
String inputLine;
while ( (inputLine = br.readLine()) != null )
{