sabeel
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

HttpClient used in metadata collection retries also on 404

This commit is contained in:
Claudio Atzori 2021-07-05 18:04:30 +02:00
parent 350a0823bd
commit 70ded407bb
1 changed files with 2 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/HttpConnector2.java
View File

@ -131,18 +131,9 @@ public class HttpConnector2 {
} }
return attemptDownload(newUrl, retryNumber + 1, report); return attemptDownload(newUrl, retryNumber + 1, report);
} }
if (is4xx(urlConn.getResponseCode())) { if (is4xx(urlConn.getResponseCode()) || is5xx(urlConn.getResponseCode())) {
// CLIENT ERROR, DO NOT RETRY
report
.put(
REPORT_PREFIX + urlConn.getResponseCode(),
String
.format(
"%s error: %s", requestUrl, urlConn.getResponseMessage()));
throw new CollectorException("4xx error: request will not be repeated. " + report);
}
if (is5xx(urlConn.getResponseCode())) {
switch (urlConn.getResponseCode()) { switch (urlConn.getResponseCode()) {
case HttpURLConnection.HTTP_NOT_FOUND:
case HttpURLConnection.HTTP_BAD_GATEWAY: case HttpURLConnection.HTTP_BAD_GATEWAY:
case HttpURLConnection.HTTP_UNAVAILABLE: case HttpURLConnection.HTTP_UNAVAILABLE:
case HttpURLConnection.HTTP_GATEWAY_TIMEOUT: case HttpURLConnection.HTTP_GATEWAY_TIMEOUT: