D-Net
/
AriadnePlus
4
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

added retry loop on HTTPQueryEvaluationException (Heap space almost full)

This commit is contained in:
Enrico Ottonello 2020-10-10 00:55:26 +02:00
parent f3c93400a3
commit fe614ce2ed
1 changed files with 30 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
dnet-ariadneplus-graphdb-publisher/src/main/java/eu/dnetlib/ariadneplus/reader/RunSPARQLQueryService.java
View File

@ -7,6 +7,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.eclipse.rdf4j.model.Model;
import org.eclipse.rdf4j.query.*;
import org.eclipse.rdf4j.repository.http.HTTPQueryEvaluationException;
import org.eclipse.rdf4j.repository.Repository;
import org.eclipse.rdf4j.repository.RepositoryConnection;
import org.eclipse.rdf4j.repository.manager.RemoteRepositoryManager;
@ -70,17 +71,35 @@ public class RunSPARQLQueryService {
final List<Integer> successCodesCount = Arrays.asList(new Integer(0));
final List<Integer> counter = Arrays.asList(new Integer(0));
recordIds.forEach(recordId -> {
log.debug(recordId+" >");
int waitAmount=1;
int operationResult = executeQueryGraph(selectQueryTemplate, recordId, isCollection);
log.debug(" "+operationResult);
if (operationResult!=200) {
log.error(recordId + " error_code: "+ operationResult);
if (isCollection) {
throw new RuntimeException("Collection indexing problem - errcode: "+operationResult);
if (operationResult==-5) {
do {
// let's wait if heap space decreases
try {
log.warn("Waiting more free space on heap for " + waitAmount + " seconds ...");
Thread.sleep(waitAmount * 1000);
waitAmount = waitAmount*2;
} catch (InterruptedException ie) {
log.error(ie);
}
int retryResult = 0;
retryResult = executeQueryGraph(selectQueryTemplate, recordId, isCollection);
log.debug("retryResult: " + retryResult);
} while (retryResult!=200);
}
if (operationResult!=200) {
int currentErrorsCount = errorCodesCount.get(0).intValue();
currentErrorsCount += 1;
errorCodesCount.set(0, new Integer(currentErrorsCount));
}
else {
int currentSuccessCount = successCodesCount.get(0).intValue();
currentSuccessCount+=1;
successCodesCount.set(0, new Integer(currentSuccessCount));
}
int currentErrorsCount = errorCodesCount.get(0).intValue();
currentErrorsCount+=1;
errorCodesCount.set(0, new Integer(currentErrorsCount));
}
else {
int currentSuccessCount = successCodesCount.get(0).intValue();
@ -140,7 +159,10 @@ public class RunSPARQLQueryService {
}
resourceManager.manage(parser);
return bulkUpload.index(resourceManager, isCollection);
} catch(Exception e){
} catch (HTTPQueryEvaluationException qe) {
log.error(qe);
return -5;
} catch(Exception e){
log.error(e);
return -1;
} finally{