lsmyrnaios
/
UrlsWorker
1
0
Fork 1
Code Issues Pull Requests Releases 18 Wiki Activity

Update the "addReportResultToWorker"-endpoint to check if the given "assignmentsCounter" was handled by that worker, without considering the related full-texts directory, since that may have been deleted in the meantime.

This commit is contained in:
Lampros Smyrnaios 2023-08-31 17:52:52 +03:00
parent b579296ada
commit e85282d35b
2 changed files with 12 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java
View File

@ -31,6 +31,7 @@ import java.nio.file.Paths;
import java.time.Duration; import java.time.Duration;
import java.time.Instant; import java.time.Instant;
import java.util.*; import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantLock;
@ -226,7 +227,7 @@ public class AssignmentsHandler {
} }
public static HashSet<Long> assignmentsNumsHandled = new HashSet<>(); public static final Set<Long> handledAssignmentsCounters = Collections.newSetFromMap(new ConcurrentHashMap<Long, Boolean>());
/** /**
@ -250,7 +251,7 @@ public class AssignmentsHandler {
int responseCode = responseEntity.getStatusCodeValue(); int responseCode = responseEntity.getStatusCodeValue();
if ( responseCode == HttpStatus.OK.value() ) { if ( responseCode == HttpStatus.OK.value() ) {
logger.info("The submission of the WorkerReport of assignments_" + assignmentRequestCounter + " to the Controller, was successful."); logger.info("The submission of the WorkerReport of assignments_" + assignmentRequestCounter + " to the Controller, was successful.");
assignmentsNumsHandled.add(assignmentRequestCounter); handledAssignmentsCounters.add(assignmentRequestCounter);
return true; return true;
} else { // This does not include HTTP-5XX errors. For them an "HttpServerErrorException" is thrown. } else { // This does not include HTTP-5XX errors. For them an "HttpServerErrorException" is thrown.
logger.error("HTTP-Connection problem with the submission of the WorkerReport of assignments_" + assignmentRequestCounter + " to the Controller! Error-code was: " + responseCode); logger.error("HTTP-Connection problem with the submission of the WorkerReport of assignments_" + assignmentRequestCounter + " to the Controller! Error-code was: " + responseCode);

12
src/main/java/eu/openaire/urls_worker/controllers/GeneralController.java
View File

@ -94,23 +94,27 @@ public class GeneralController {
@GetMapping("getHandledAssignmentsCounts") @GetMapping("getHandledAssignmentsCounts")
public ResponseEntity<?> getHandledAssignmentsCounts() public ResponseEntity<?> getHandledAssignmentsCounts()
{ {
return ResponseEntity.ok(AssignmentsHandler.assignmentsNumsHandled); return ResponseEntity.ok(AssignmentsHandler.handledAssignmentsCounters);
} }
@PostMapping("addReportResultToWorker/{assignmentsCounter}") @PostMapping("addReportResultToWorker/{assignmentsCounter}")
public ResponseEntity<?> addReportResultToWorker(@PathVariable long assignmentsCounter, @RequestBody(required=false) String errorMsg) public ResponseEntity<?> addReportResultToWorker(@PathVariable long assignmentsCounter, @RequestBody(required=false) String errorMsg)
{ {
String directoryPath = PublicationsRetrieverPlugin.assignmentsBasePath + "assignments_" + assignmentsCounter + "_fullTexts"; if ( ! AssignmentsHandler.handledAssignmentsCounters.contains(assignmentsCounter) ) {
File dir = new File(directoryPath);
if ( ! dir.isDirectory() ) {
logger.error("The \"addReportResultToWorker\"-endpoint was called for an unknown \"assignmentsCounter\": " + assignmentsCounter); logger.error("The \"addReportResultToWorker\"-endpoint was called for an unknown \"assignmentsCounter\": " + assignmentsCounter);
return ResponseEntity.notFound().build(); return ResponseEntity.notFound().build();
} }
if ( errorMsg == null ) { if ( errorMsg == null ) {
logger.info("The Controller successfully handled the WorkerReport, for assignments_" + assignmentsCounter + ". The worker-report and all full-text files associated with it, will be deleted."); logger.info("The Controller successfully handled the WorkerReport, for assignments_" + assignmentsCounter + ". The worker-report and all full-text files associated with it, will be deleted.");
String directoryPath = PublicationsRetrieverPlugin.assignmentsBasePath + "assignments_" + assignmentsCounter + "_fullTexts";
File dir = new File(directoryPath);
if ( dir.isDirectory() )
FullTextsController.deleteAssignmentsDirectory(assignmentsCounter, dir); FullTextsController.deleteAssignmentsDirectory(assignmentsCounter, dir);
else
logger.warn("The full-texts directory \"" + directoryPath + "\" has already been deleted by the scheduler.");
FullTextsController.deleteFile(this.workerReportsDirPath + this.workerId + "_assignments_" + assignmentsCounter + "_report.json"); FullTextsController.deleteFile(this.workerReportsDirPath + this.workerId + "_assignments_" + assignmentsCounter + "_report.json");
} else } else
logger.error("The Controller failed to handle the WorkerReport, for assignments_" + assignmentsCounter + ". The error is:\n" + errorMsg); logger.error("The Controller failed to handle the WorkerReport, for assignments_" + assignmentsCounter + ". The error is:\n" + errorMsg);