D-Net
/
AriadnePlus
4
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

the job fails if the endpoint returns client errors 4xx

This commit is contained in:
Alessia Bardi 2020-06-05 16:19:54 +02:00
parent 30410e8cca
commit 4d68ecad58
1 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
dnet-ariadneplus/src/main/java/eu/dnetlib/ariadneplus/workflows/nodes/PublishGraphDBJobNode.java
View File

@ -86,7 +86,7 @@ public class PublishGraphDBJobNode extends AsyncJobNode {
log.info("drop datasourceApis partition info completed"); log.info("drop datasourceApis partition info completed");
break; break;
default: default:
log.error("error dropping datasourceApis partition info " + responseDDAPIPOST.getStatusLine().getStatusCode() + ": " + responseDDAPIPOST.getStatusLine().getReasonPhrase()); log.error("error dropping datasourceApis partition info " + statusCode + ": " + responseDDAPIPOST.getStatusLine().getReasonPhrase());
break; break;
} }
} catch (ConnectException ce) { } catch (ConnectException ce) {
@ -102,7 +102,9 @@ public class PublishGraphDBJobNode extends AsyncJobNode {
//need to slow down the producer to avoid OOM errors due to many tasks in the queue of the executor //need to slow down the producer to avoid OOM errors due to many tasks in the queue of the executor
//see for example here: https://stackoverflow.com/questions/42108351/executorservice-giving-out-of-memory-error //see for example here: https://stackoverflow.com/questions/42108351/executorservice-giving-out-of-memory-error
//let's stop and wait after submission of nLatch tasks //let's stop and wait after submission of nLatch tasks
boolean forceExit = false;
for (String record : getResultSetClient().iter(rsIn, String.class)) { for (String record : getResultSetClient().iter(rsIn, String.class)) {
if(forceExit) break;
countAll++; countAll++;
if(partial == nTasks) { if(partial == nTasks) {
log.debug("Waiting for tasks to complete before resubmitting to executor (countAll = "+countAll+") . . . "); log.debug("Waiting for tasks to complete before resubmitting to executor (countAll = "+countAll+") . . . ");
@ -110,6 +112,10 @@ public class PublishGraphDBJobNode extends AsyncJobNode {
long startWait = System.currentTimeMillis(); long startWait = System.currentTimeMillis();
for(Future<Integer> res : resList){ for(Future<Integer> res : resList){
if(res.get() == 200) countOk++; if(res.get() == 200) countOk++;
if(res.get() == 400 || res.get() == 401 || res.get() == 402 | res.get() == 403 || res.get() == 404){
executorService.shutdownNow();
throw new MSROException("Client error "+ res.get());
}
} }
resList.clear(); resList.clear();
partial = 0; partial = 0;