package eu.openaire.urls_worker.controllers; import eu.openaire.urls_worker.components.plugins.PublicationsRetrieverPlugin; import eu.openaire.urls_worker.services.FileStorageService; import eu.openaire.urls_worker.util.FilesCompressor; import org.apache.commons.io.FileDeleteStrategy; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.InputStreamResource; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import java.io.BufferedInputStream; import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; import java.util.List; @RestController @RequestMapping("full-texts/") public class FullTextsController { private static final Logger logger = LoggerFactory.getLogger(GeneralController.class); @Autowired private FileStorageService fileStorageService; @GetMapping("getFullTexts/{assignmentsCounter:[\\d]+}/{totalBatches:[\\d]+}/{batchCounter:[\\d]+}/{fileNamesWithExtensions}") public Object getFullTexts(@PathVariable long assignmentsCounter, @PathVariable int totalBatches, @PathVariable int batchCounter, @PathVariable List fileNamesWithExtensions) { int fileNamesListNum = fileNamesWithExtensions.size(); if ( (fileNamesListNum == 1) && (fileNamesWithExtensions.get(0).length() == 0) ) { // In case the last "/" in the url was given (without any files following), then this list will not be empty, but have one empty item instead. // In case the url does not end in "/", then Spring will automatically return an "HTTP-BadRequest". String errorMsg = "An empty \"fileNamesWithExtensions\" list was given from assignments_" + assignmentsCounter + ", for batch_" + batchCounter; logger.error(errorMsg); return ResponseEntity.badRequest().body(errorMsg); } if ( totalBatches == 0 ) { String errorMsg = "The given \"totalBatches\" (" + totalBatches + ") was < 0 >!"; logger.error(errorMsg); return ResponseEntity.badRequest().body(errorMsg); } else if ( batchCounter > totalBatches ) { String errorMsg = "The given \"batchCounter\" (" + batchCounter + ") is greater than the \"totalBatches\" (" + totalBatches + ")!"; logger.error(errorMsg); return ResponseEntity.badRequest().body(errorMsg); } logger.info("Received a \"getFullTexts\" request for returning a tar-file containing " + fileNamesListNum + " full-texts, from assignments_" + assignmentsCounter + ", for batch_" + batchCounter + " (out of " + totalBatches + ")."); String currentAssignmentsBaseFullTextsPath = fileStorageService.assignmentsBaseLocation + "assignments_" + assignmentsCounter + "_fullTexts" + File.separator; if ( ! (new File(currentAssignmentsBaseFullTextsPath).isDirectory()) ) { String errorMsg = "The base directory for assignments_" + assignmentsCounter + " was not found: " + currentAssignmentsBaseFullTextsPath; logger.error(errorMsg); return ResponseEntity.badRequest().body(errorMsg); } File zstdFile = FilesCompressor.compressMultipleFilesIntoOne(assignmentsCounter, batchCounter, fileNamesWithExtensions, currentAssignmentsBaseFullTextsPath); if ( zstdFile == null ) { String errorMsg = "Failed to create the zstd file for \"batchCounter\"-" + batchCounter; logger.error(errorMsg); return ResponseEntity.internalServerError().body(errorMsg); // The related files will be deleted later, upon completing the Worker-report process, in "AssignmentsHandler.postWorkerReport()". } if ( batchCounter == totalBatches ) logger.debug("Will return the " + ((totalBatches > 1) ? "last" : "only one") + " batch (" + batchCounter + ") of assignments_" + assignmentsCounter + " to the Controller."); String zstdName = zstdFile.getName(); String zstdTarFileFullPath = currentAssignmentsBaseFullTextsPath + zstdName; try { return ResponseEntity.ok() .contentType(MediaType.APPLICATION_OCTET_STREAM) .header(HttpHeaders.CONTENT_DISPOSITION, "inline; filename=\"" + zstdName + "\"") .body(new InputStreamResource(new BufferedInputStream(Files.newInputStream(Paths.get(zstdTarFileFullPath)), FilesCompressor.bufferSize))); } catch (Exception e) { String errorMsg = "Could not load the FileInputStream of the zstd-tar-file \"" + zstdTarFileFullPath + "\"!"; logger.error(errorMsg, e); return ResponseEntity.internalServerError().body(errorMsg); } finally { // In some cases, the full-texts might be too large and their total number too, // so if we leave them be, and wait for all batches to finish, we may get a "java.io.IOException: No space left on device" error. deleteFulltextBatchFiles(currentAssignmentsBaseFullTextsPath, assignmentsCounter, batchCounter, fileNamesWithExtensions); // The ".tar.zstd" file of this batch, for which we pass a steam to the Controller, will be deleted after the next batch, or after all batches are transferred and handles by the Controller. } // The related fulltext and (zstd-)tar files will be deleted in "AssignmentsHandler.postWorkerReport()", after the Controller has finished transferring them. They will be deleted even in case of a Controller-error. // In case of an error and file-deletion, the related id-url records will just be re-processed in the future by some (maybe different) Worker. } @GetMapping("getFullText/{assignmentsCounter:[\\d]+}/{fileNameWithExtension:[\\w_:]+.[\\w]{2,10}}") public ResponseEntity getFullText(@PathVariable long assignmentsCounter, @PathVariable String fileNameWithExtension) { logger.info("Received a \"getFullText\" request."); String fullTextFileFullPath = fileStorageService.assignmentsBaseLocation + "assignments_" + assignmentsCounter + "_fullTexts" + File.separator + fileNameWithExtension; File file = new File(fullTextFileFullPath); if ( !file.isFile() ) { logger.error("The file \"" + fullTextFileFullPath + "\" does not exist!"); return ResponseEntity.notFound().build(); } try { return ResponseEntity.ok() .contentType(MediaType.APPLICATION_OCTET_STREAM) .header(HttpHeaders.CONTENT_DISPOSITION, "inline; filename=\"" + file.getName() + "\"") .body(new InputStreamResource(new BufferedInputStream(Files.newInputStream(Paths.get(fullTextFileFullPath)), FilesCompressor.bufferSize))); } catch (Exception e) { String errorMsg = "Could not load the FileInputStream of the full-text-file \"" + fullTextFileFullPath + "\"!"; logger.error(errorMsg, e); return ResponseEntity.internalServerError().body(errorMsg); } } public static boolean deleteDirectory(long curAssignments) { String directoryPath = PublicationsRetrieverPlugin.assignmentsBasePath; if ( curAssignments != -1 ) { directoryPath += "assignments_" + curAssignments + "_fullTexts" + File.separator; logger.debug("Going to delete the files inside the directory of assignments_" + curAssignments); } else logger.debug("Going to delete the parent directory: " + directoryPath); try { FileUtils.deleteDirectory(new File(directoryPath)); return true; } catch (IOException e) { logger.error("The following directory could not be deleted: " + directoryPath, e); return false; } catch (IllegalArgumentException iae) { logger.error("This assignments-dir does not exist: " + directoryPath); return false; } } public static void deleteFulltextBatchFiles(String assignmentsBatchDir, long assignmentsCounter, long fulltextsBatch, List filenames) { // We will delete all the files related to the given fulltexts-batch, along with the created tar and zstd files. for ( String fileName : filenames ) deleteFile(assignmentsBatchDir + fileName); // Now let's delete the ".tar" and ".tar.zstd" files as well. String partialNonBatchFileName = assignmentsBatchDir + "assignments_" + assignmentsCounter + "_full-texts_"; deleteFile(partialNonBatchFileName + fulltextsBatch + ".tar"); // The ".tar.zstd" file of this batch will be deleted by the next batch or in the end of these assignments. // Now we will delete the zstd file of the previous assignments. if ( fulltextsBatch >= 2 ) deleteFile(partialNonBatchFileName + (fulltextsBatch -1) + ".tar.zstd"); // We do not use a fulltexts-batch directory, since even if it makes the deletion faster, it will make the full-texts delivery to the controller slower, // since we will need to move the requested full-texts to that directory before continuing with tarring and compressing the files and sending them over to the Controller. // Also, we cannot pre-create such directories, since it will add complexity in the download process and also some of the full-texts may not be requested by the Controller (because of duplicates). } public static boolean deleteFile(String fileFullPathString) { try { FileDeleteStrategy.FORCE.delete(new File(fileFullPathString)); } catch (IOException e) { logger.error("Error when deleting the file: " + fileFullPathString); return false; } return true; } }