lsmyrnaios
/
UrlsWorker
1
0
Fork 1
Code Issues Pull Requests Releases 17 Wiki Activity

Make sure the "already-retrieved" fulltexts are deleted as early as possible. These are the files which have their hash been added in the Database sometime in the past and the Controller has not requested them now.

This commit is contained in:
Lampros Smyrnaios 2024-12-04 00:59:00 +02:00
parent bb521cc493
commit fa7b2da926
1 changed files with 13 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java
View File

@ -75,8 +75,20 @@ public class FullTextsController {
return ResponseEntity.internalServerError().body(errorMsg);
}
if ( batchCounter == totalBatches )
if ( batchCounter == totalBatches ) {
logger.debug("Will return the " + ((totalBatches > 1) ? "last" : "only one") + " batch (" + batchCounter + ") of assignments_" + assignmentsCounter + " to the Controller.");
// Delete any leftover files in the directory, as these are duplicates which have already been collected in the past and will not be requested by the Controller.
// There is the rare case where some files which the Controller requests, may fail to be added to the .tar archive. In this case, though, they get deleted before the program reaches this line.
File[] files = curAssignmentsDir.listFiles();
if ( files != null ) {
for ( File file : files ) {
// Delete every file except the ".zstd" one which will be sent to the Controller.
String fileLocation = file.getAbsolutePath();
if ( !fileLocation.endsWith(".zstd") )
deleteFile(fileLocation);
}
} // Else, the whole directory will bne deleted at a scheduled time.
}
String zstdName = zstdFile.getName();
String zstdTarFileFullPath = currentAssignmentsBaseFullTextsPath + zstdName;