lsmyrnaios
/
UrlsController
1
0
Fork 2
Code Issues Pull Requests Releases 18 Wiki Activity

Add error-checks for retrieving the status-code from HttpUrlConnections.

This commit is contained in:
Lampros Smyrnaios 2023-05-03 13:30:29 +03:00
parent 49662319a1
commit fd15372fd6
2 changed files with 10 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/main/java/eu/openaire/urls_controller/util/FileUtils.java
View File

@ -329,7 +329,10 @@ public class FileUtils {
conn.setRequestProperty("User-Agent", "UrlsController"); conn.setRequestProperty("User-Agent", "UrlsController");
conn.connect(); conn.connect();
int statusCode = conn.getResponseCode(); int statusCode = conn.getResponseCode();
if ( statusCode != 200 ) { if ( statusCode == -1 ) {
logger.warn("Problem when getting the \"status-code\" for url: " + requestUrl);
throw new RuntimeException();
} else if ( statusCode != 200 ) {
logger.warn("HTTP-" + statusCode + ": " + getMessageFromResponseBody(conn, true) + "\n\nProblem when requesting the ZstdFile of batch_" + batchNum + " from the Worker with ID \"" + workerId + "\" and requestUrl: " + requestUrl); logger.warn("HTTP-" + statusCode + ": " + getMessageFromResponseBody(conn, true) + "\n\nProblem when requesting the ZstdFile of batch_" + batchNum + " from the Worker with ID \"" + workerId + "\" and requestUrl: " + requestUrl);
if ( (statusCode >= 500) && (statusCode <= 599) ) if ( (statusCode >= 500) && (statusCode <= 599) )
throw new RuntimeException(); // Throw an exception to indicate that the Worker has problems and all remaining batches will fail as well. throw new RuntimeException(); // Throw an exception to indicate that the Worker has problems and all remaining batches will fail as well.

8
src/main/java/eu/openaire/urls_controller/util/ParquetFileUtils.java
View File

@ -585,16 +585,20 @@ public class ParquetFileUtils {
conn.setInstanceFollowRedirects(false); // We will handle the redirection ourselves. conn.setInstanceFollowRedirects(false); // We will handle the redirection ourselves.
conn.connect(); conn.connect();
int statusCode = conn.getResponseCode(); int statusCode = conn.getResponseCode();
if ( statusCode != 200 ) { if ( statusCode == -1 ) {
logger.error("Problem when getting the \"status-code\" for url: " + createDirectoryUrl);
return false;
}
else if ( statusCode != 200 ) {
String errorMsg = "We expected a \"200 OK\" response, but got: \"" + statusCode + "\" instead, for url: " + createDirectoryUrl; String errorMsg = "We expected a \"200 OK\" response, but got: \"" + statusCode + "\" instead, for url: " + createDirectoryUrl;
logger.error(errorMsg + "\n\n" + fileUtils.getMessageFromResponseBody(conn, true)); logger.error(errorMsg + "\n\n" + fileUtils.getMessageFromResponseBody(conn, true));
return false; return false;
} }
logger.trace("Creation was successful for hdfs-dir-url: " + createDirectoryUrl + "\n" + fileUtils.getMessageFromResponseBody(conn, false));
} catch (Exception e) { } catch (Exception e) {
logger.error("", e); logger.error("", e);
return false; return false;
} }
return true; return true;
} }