From 4d68ecad58d8991e1a12fa15c03325aa7cd4a2d1 Mon Sep 17 00:00:00 2001 From: Alessia Bardi Date: Fri, 5 Jun 2020 16:19:54 +0200 Subject: [PATCH] the job fails if the endpoint returns client errors 4xx --- .../workflows/nodes/PublishGraphDBJobNode.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/PublishGraphDBJobNode.java b/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/PublishGraphDBJobNode.java index fce2722..e94a11e 100644 --- a/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/PublishGraphDBJobNode.java +++ b/dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/PublishGraphDBJobNode.java @@ -86,7 +86,7 @@ public class PublishGraphDBJobNode extends AsyncJobNode { log.info("drop datasourceApis partition info completed"); break; default: - log.error("error dropping datasourceApis partition info " + responseDDAPIPOST.getStatusLine().getStatusCode() + ": " + responseDDAPIPOST.getStatusLine().getReasonPhrase()); + log.error("error dropping datasourceApis partition info " + statusCode + ": " + responseDDAPIPOST.getStatusLine().getReasonPhrase()); break; } } catch (ConnectException ce) { @@ -102,7 +102,9 @@ public class PublishGraphDBJobNode extends AsyncJobNode { //need to slow down the producer to avoid OOM errors due to many tasks in the queue of the executor //see for example here: https://stackoverflow.com/questions/42108351/executorservice-giving-out-of-memory-error //let's stop and wait after submission of nLatch tasks + boolean forceExit = false; for (String record : getResultSetClient().iter(rsIn, String.class)) { + if(forceExit) break; countAll++; if(partial == nTasks) { log.debug("Waiting for tasks to complete before resubmitting to executor (countAll = "+countAll+") . . . "); @@ -110,6 +112,10 @@ public class PublishGraphDBJobNode extends AsyncJobNode { long startWait = System.currentTimeMillis(); for(Future res : resList){ if(res.get() == 200) countOk++; + if(res.get() == 400 || res.get() == 401 || res.get() == 402 | res.get() == 403 || res.get() == 404){ + executorService.shutdownNow(); + throw new MSROException("Client error "+ res.get()); + } } resList.clear(); partial = 0;