From fd15372fd66c2c1d73fcaedc3741f80b760f1a86 Mon Sep 17 00:00:00 2001 From: LSmyrnaios Date: Wed, 3 May 2023 13:30:29 +0300 Subject: [PATCH] Add error-checks for retrieving the status-code from HttpUrlConnections. --- .../java/eu/openaire/urls_controller/util/FileUtils.java | 5 ++++- .../openaire/urls_controller/util/ParquetFileUtils.java | 8 ++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/main/java/eu/openaire/urls_controller/util/FileUtils.java b/src/main/java/eu/openaire/urls_controller/util/FileUtils.java index 7accd4f..f5f79a8 100644 --- a/src/main/java/eu/openaire/urls_controller/util/FileUtils.java +++ b/src/main/java/eu/openaire/urls_controller/util/FileUtils.java @@ -329,7 +329,10 @@ public class FileUtils { conn.setRequestProperty("User-Agent", "UrlsController"); conn.connect(); int statusCode = conn.getResponseCode(); - if ( statusCode != 200 ) { + if ( statusCode == -1 ) { + logger.warn("Problem when getting the \"status-code\" for url: " + requestUrl); + throw new RuntimeException(); + } else if ( statusCode != 200 ) { logger.warn("HTTP-" + statusCode + ": " + getMessageFromResponseBody(conn, true) + "\n\nProblem when requesting the ZstdFile of batch_" + batchNum + " from the Worker with ID \"" + workerId + "\" and requestUrl: " + requestUrl); if ( (statusCode >= 500) && (statusCode <= 599) ) throw new RuntimeException(); // Throw an exception to indicate that the Worker has problems and all remaining batches will fail as well. diff --git a/src/main/java/eu/openaire/urls_controller/util/ParquetFileUtils.java b/src/main/java/eu/openaire/urls_controller/util/ParquetFileUtils.java index 8634f82..98a5529 100644 --- a/src/main/java/eu/openaire/urls_controller/util/ParquetFileUtils.java +++ b/src/main/java/eu/openaire/urls_controller/util/ParquetFileUtils.java @@ -585,16 +585,20 @@ public class ParquetFileUtils { conn.setInstanceFollowRedirects(false); // We will handle the redirection ourselves. conn.connect(); int statusCode = conn.getResponseCode(); - if ( statusCode != 200 ) { + if ( statusCode == -1 ) { + logger.error("Problem when getting the \"status-code\" for url: " + createDirectoryUrl); + return false; + } + else if ( statusCode != 200 ) { String errorMsg = "We expected a \"200 OK\" response, but got: \"" + statusCode + "\" instead, for url: " + createDirectoryUrl; logger.error(errorMsg + "\n\n" + fileUtils.getMessageFromResponseBody(conn, true)); return false; } + logger.trace("Creation was successful for hdfs-dir-url: " + createDirectoryUrl + "\n" + fileUtils.getMessageFromResponseBody(conn, false)); } catch (Exception e) { logger.error("", e); return false; } - return true; }