- Fix a bug, which caused multiple workers to get assigned the same batch-counter, while the assignment-tasks where different.
- Set a max-size limit to the amount of space the logs can use. Over that size, the older logs will be deleted.
- Show the error-message returned from the Worker, when a getFullTexts-request fails.
- Improve some log-messages.
- Update dependencies.
- Code cleanup.
logger.info("Sending batch-assignments_"+assignmentsBatchCounter.incrementAndGet()+" with "+assignmentsSize+" assignments to worker with ID: "+workerId+".");
logger.info("Received the WorkerReport for batch-assignments_"+workerReport.getAssignmentRequestCounter()+", from the worker with id: "+workerReport.getWorkerId()+". It contains "+urlReports.size()+" urlReports. Going to request the fullTexts from the Worker and insert the UrlReports into the database.");
logger.info("Received the WorkerReport for batch-assignments_"+curReportAssignments+", from the worker with id: "+workerReport.getWorkerId()+". It contains "+urlReports.size()+" urlReports. Going to request the fullTexts from the Worker and insert the UrlReports into the database.");
// Before continuing with inserts, take and upload the fullTexts from the Worker. Also, update the file-"location".
// Before continuing with inserts, take and upload the fullTexts from the Worker. Also, update the file-"location".
logger.error("Failed to get and/or upload the fullTexts for assignments_"+assignmentsBatchCounter);
logger.error("Failed to get and/or upload the fullTexts for assignments_"+curReportAssignments);
// The docUrls were still found! Just update ALL the fileLocations. sizes and hashes, to show that the files are not available and continue with writing the attempts and the Payloads.
// The docUrls were still found! Just update ALL the fileLocations. sizes and hashes, to show that the files are not available and continue with writing the attempts and the Payloads.
logger.error("Problem when executing the \"insertIntoAttemptBaseQuery\":\n"+tempFullQueryString+"\n" +sqle.getMessage()+"\n\n");
logger.error("Problem when executing the \"insertIntoAttemptBaseQuery\": " +sqle.getMessage()+"\n\n");
}
}
}//end for-loop
}//end for-loop
@ -488,9 +487,9 @@ public class UrlController {
if(scanner!=null)// Check if the initial value is null.
if(scanner!=null)// Check if the initial value is null.
scanner.close();
scanner.close();
logger.info("Sending batch_"+assignmentsBatchCounter.incrementAndGet()+" with "+assignments.size()+" assignments ("+FileUtils.duplicateIdUrlEntries.get()+" more assignments were discarded as duplicates), to worker with ID: "+workerId);
logger.info("Sending batch_"+curAssignmentsBatchCounter+" with "+assignments.size()+" assignments ("+FileUtils.duplicateIdUrlEntries.get()+" more assignments were discarded as duplicates), to worker with ID: "+workerId);
logger.info("Finished uploading "+numUploadedFiles+" full-texts of assignments_"+assignmentsBatchCounter+" on S3-ObjectStore.");
logger.info("Finished uploading "+numUploadedFiles+" full-texts of assignments_"+assignmentsBatchCounter+", batch_" +batchCounter+" on S3-ObjectStore.");
}catch(Exceptione){
}catch(Exceptione){
logger.error("Could not extract and upload the full-texts for batch_"+i+" of assignments_"+assignmentsBatchCounter+"\n"+e.getMessage(),e);// It shows the response body (after Spring v.2.5.6).
logger.error("Could not extract and upload the full-texts for batch_"+batchCounter+" of assignments_"+assignmentsBatchCounter+"\n"+e.getMessage(),e);// It shows the response body (after Spring v.2.5.6).
logger.info("Going to request the batch_"+batchNum+" (out of "+totalBatches+") with "+fileNamesForCurBatch.size()+" fullTexts, of assignments_"+assignmentsBatchCounter+" from the Worker with ID \""+workerId+"\" and baseRequestUrl: "+baseUrl+"[fileNames]");
logger.info("Going to request the batch_"+batchNum+" (out of "+totalBatches+") with "+fileNamesForCurBatch.size()+" fullTexts, of assignments_"+assignmentsBatchCounter+" from the Worker with ID \""+workerId+"\" and baseRequestUrl: "+baseUrl+"[fileNames]");
HttpURLConnectionconn=null;
HttpURLConnectionconn=null;
try{
try{
@ -291,7 +288,7 @@ public class FileUtils {
conn.connect();
conn.connect();
intstatusCode=conn.getResponseCode();
intstatusCode=conn.getResponseCode();
if(statusCode!=200){
if(statusCode!=200){
logger.warn("HTTP-"+statusCode+": Problem with when requesting the ZipFile of batch_" +batchNum+" from the Worker with ID \""+workerId+"\" and requestUrl: "+requestUrl);
logger.warn("HTTP-"+statusCode+": " +getErrorMessageFromResponseBody(conn)+"\nProblem when requesting the ZipFile of batch_" +batchNum+" from the Worker with ID \""+workerId+"\" and requestUrl: "+requestUrl);
return(errorMsgStrB.length()!=0)?errorMsgStrB.toString():null;// Make sure we return a "null" on empty string, to better handle the case in the caller-function.
}catch(IOExceptionioe){
logger.error("IOException when retrieving the error-message: "+ioe.getMessage());
returnnull;
}catch(Exceptione){
logger.error("Could not extract the errorMessage!",e);