lsmyrnaios
/
UrlsController
1
0
Fork 2
Code Issues Pull Requests Releases 18 Wiki Activity

Improve error-handling in "BulkImportReport.getJsonReport()" and "FileUtils.writeToFile()".

This commit is contained in:
Lampros Smyrnaios 2024-05-30 11:52:04 +03:00
parent d7697ef3f8
commit edf064616a
4 changed files with 20 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build.gradle
View File

@ -6,7 +6,7 @@ plugins {
java {
group = 'eu.openaire.urls_controller'
version = '2.7.0-SNAPSHOT'
version = '2.7.3-SNAPSHOT'
sourceCompatibility = JavaVersion.VERSION_1_8
}

20
src/main/java/eu/openaire/urls_controller/models/BulkImportReport.java
View File

@ -7,6 +7,8 @@ import com.google.common.collect.Multimaps;
import com.google.common.collect.SetMultimap;
import com.google.gson.Gson;
import eu.openaire.urls_controller.util.GenericUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Collection;
import java.util.HashMap;
@ -18,6 +20,8 @@ import java.util.concurrent.locks.ReentrantLock;
@JsonInclude(JsonInclude.Include.NON_NULL)
public class BulkImportReport {
private static final Logger logger = LoggerFactory.getLogger(BulkImportReport.class);
private static final Gson gson = new Gson(); // This is "transient" by default. It won't be included in any json object.
@JsonProperty
@ -56,12 +60,18 @@ public class BulkImportReport {
* */
public String getJsonReport()
{
String reportToReturn = null;
reportLock.lock();
//Convert the LinkedHashMultiMap<String, String> to Map<String, Collection<String>>, since Gson cannot serialize Multimaps.
eventsMap = new HashMap<>(eventsMultimap.asMap()); // Make sure we use a clone of the original data, in order to avoid any exception in the "gson.toJson()" method, when at the same time another thread modifies the "eventsMultimap".
String reportToReturn = gson.toJson(this, BulkImportReport.class);
reportLock.unlock();
return reportToReturn;
try {
//Convert the LinkedHashMultiMap<String, String> to Map<String, Collection<String>>, since Gson cannot serialize Multimaps.
eventsMap = new HashMap<>(eventsMultimap.asMap()); // Make sure we use a clone of the original data, in order to avoid any exception in the "gson.toJson()" method, when at the same time another thread modifies the "eventsMultimap".
reportToReturn = gson.toJson(this, BulkImportReport.class);
} catch (Exception e) {
logger.error("Problem when producing the JSON-string with the BulkImportReport! | reportID: '" + reportID + "'", e);
} finally {
reportLock.unlock();
}
return reportToReturn; // It may be null.
}
public String getProvenance() {

3
src/main/java/eu/openaire/urls_controller/util/FileUtils.java
View File

@ -459,6 +459,9 @@ public class FileUtils {
public String writeToFile(String fileFullPath, String stringToWrite, boolean shouldLockThreads)
{
if ( stringToWrite == null )
return "The string to write to file '" + fileFullPath + "' is null!";
if ( shouldLockThreads ) // In case multiple threads write to the same file. for ex. during the bulk-import procedure.
fileAccessLock.lock();

3
src/main/java/eu/openaire/urls_controller/util/FilesHandler.java
View File

@ -98,7 +98,7 @@ public class FilesHandler {
throw new RuntimeException(); // Avoid any other batches.
} else if ( statusCode != 200 ) {
String errMsg = fileUtils.getMessageFromResponseBody(conn, true);
logger.warn("HTTP-" + statusCode + ": " + errMsg + "\n\nProblem when requesting the ZstdFile of batch_" + batchNum + " from the Worker with ID \"" + workerId + "\" and requestUrl: " + requestUrl);
logger.warn("HTTP-" + statusCode + ": " + errMsg + "\nProblem when requesting the ZstdFile of batch_" + batchNum + " from the Worker with ID \"" + workerId + "\" and requestUrl: " + requestUrl);
if ( ((statusCode >= 500) && (statusCode <= 599))
|| ((statusCode == 400) && ((errMsg != null) && errMsg.contains("The base directory for assignments_" + assignmentsBatchCounter + " was not found"))) )
throw new RuntimeException(); // Throw an exception to indicate that the Worker has problems and all remaining batches will fail as well.
@ -218,7 +218,6 @@ public class FilesHandler {
String baseUrl = "http://" + workerIp + ":" + workerPort + "/api/full-texts/getFullTexts/" + assignmentsBatchCounter + "/" + numOfBatches + "/";
// TODO - The worker should send the port in which it accepts requests, along with the current request.
// TODO - The least we have to do it to expose the port-assignment somewhere more obvious like inside the "application.yml" file.
String curAssignmentsBaseLocation = baseFilesLocation + "assignments_" + assignmentsBatchCounter + File.separator;
// Note: the "curAssignmentsBaseLocation"-directory will be created once the first batch subdirectory is called for creation.