From 839a797124c100da4e0881d6771db2e026e76735 Mon Sep 17 00:00:00 2001 From: LSmyrnaios Date: Wed, 8 Mar 2023 13:12:17 +0200 Subject: [PATCH] - Improve performance of full-texts transferring to the Controller, by preloading some bytes for faster response to the Controller's read requests. - Optimize directories-creation process by eliminating the additive check for existence, as that check already takes place inside the "mkdirs()" method. - Remove the obsolete code which in case the specific assignments' subdirectory failed to be created, then a different base-dir was used instead. Since the user-defined baseDir is already been successfully created upon initialization, any problem on creating subdirectories inside that base-directory will most likely persist even when changing the base directory. Additionally, even if the subdirectory with the changed base-directory succeeded, the "FullTextsController.getFullTexts()" method would not use it, resulting in errors. - Code polishing. --- .../components/AssignmentsHandler.java | 5 ----- .../plugins/PublicationsRetrieverPlugin.java | 17 +++++------------ .../controllers/FullTextsController.java | 11 ++++++----- .../services/FileStorageService.java | 3 ++- .../urls_worker/util/FilesCompressor.java | 2 +- 5 files changed, 14 insertions(+), 24 deletions(-) diff --git a/src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java b/src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java index e555449..9245e94 100644 --- a/src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java +++ b/src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java @@ -304,9 +304,6 @@ public class AssignmentsHandler { spacedOutAssignments.add(nextAssignment); } - // The HashMultimap is no longer needed. - domainsWithAssignments.clear(); - if ( sb != null ) logger.debug("After change:\n" + sb.toString()); @@ -343,9 +340,7 @@ public class AssignmentsHandler { } else { nextAssignment = assignmentsOfCurrentDomain.toArray()[0]; result.put(nextAssignment, domainsCounter); - domainsWithAssignments.remove(currentDomain, nextAssignment); - if ( sb != null ) sb.append(currentDomain).append("\n"); // DEBUG! } diff --git a/src/main/java/eu/openaire/urls_worker/components/plugins/PublicationsRetrieverPlugin.java b/src/main/java/eu/openaire/urls_worker/components/plugins/PublicationsRetrieverPlugin.java index b546981..2fba042 100644 --- a/src/main/java/eu/openaire/urls_worker/components/plugins/PublicationsRetrieverPlugin.java +++ b/src/main/java/eu/openaire/urls_worker/components/plugins/PublicationsRetrieverPlugin.java @@ -75,23 +75,16 @@ public class PublicationsRetrieverPlugin { public static void processAssignments(Long assignmentRequestCounter, Collection assignments) throws RuntimeException { + // At this point, the "assignmentsBasePath"-directory has already been successfully created. + String currentAssignmentsSubDir = "assignments_" + assignmentRequestCounter + "_fullTexts" + File.separator; FileUtils.storeDocFilesDir = assignmentsBasePath + currentAssignmentsSubDir; // It needs the last separator, because of how the docFiles are named and stored. File curAssignmentsDirs = new File(FileUtils.storeDocFilesDir); try { - if ( !curAssignmentsDirs.exists() ) { - if ( !curAssignmentsDirs.mkdirs() ) { // Create the directories. - String workingDir = System.getProperty("user.dir") + File.separator; - logger.error("Could not create the \"assignments_fullTexts directories\": \"" + FileUtils.storeDocFilesDir + "\". Using the \"workingDir\", as the base-dir instead (" + workingDir + ")."); - FileUtils.storeDocFilesDir = (workingDir + currentAssignmentsSubDir); - if ( ! (new File(FileUtils.storeDocFilesDir)).mkdirs() ) { // Create the alternative directories. - logger.error("Could not create the directory where the downloaded files will be stored!"); - System.exit(-40); - } - } - } - } catch (Exception e) { + if ( !curAssignmentsDirs.mkdirs() ) // Create the subdirectory. + logger.error("Could not create the \"assignments_" + assignmentRequestCounter + "_fullTexts\" directories: \"" + FileUtils.storeDocFilesDir + "\"!"); + } catch (Exception e) { // Mainly a SecurityException. String errorMsg = "Failed to create the full-texts directory for assignments_" + assignmentRequestCounter; logger.error(errorMsg, e); throw new RuntimeException(errorMsg + ": " + e.getMessage()); diff --git a/src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java b/src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java index 841a56d..67d5b57 100644 --- a/src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java +++ b/src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java @@ -17,6 +17,7 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import java.io.BufferedInputStream; import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -34,8 +35,8 @@ public class FullTextsController { @GetMapping("getFullTexts/{assignmentsCounter:[\\d]+}/{totalBatches:[\\d]+}/{batchCounter:[\\d]+}/{fileNamesWithExtensions}") - public Object getFullTexts(@PathVariable long assignmentsCounter, @PathVariable int totalBatches, @PathVariable int batchCounter, @PathVariable List fileNamesWithExtensions) { - + public Object getFullTexts(@PathVariable long assignmentsCounter, @PathVariable int totalBatches, @PathVariable int batchCounter, @PathVariable List fileNamesWithExtensions) + { int fileNamesListNum = fileNamesWithExtensions.size(); if ( (fileNamesListNum == 1) && (fileNamesWithExtensions.get(0).length() == 0) ) { // In case the last "/" in the url was given (without any files following), then this list will not be empty, but have one empty item instead. // In case the url does not end in "/", then Spring will automatically return an "HTTP-BadRequest". @@ -74,7 +75,7 @@ public class FullTextsController { } if ( batchCounter == totalBatches ) - logger.debug("Will return the " + ((totalBatches > 1) ? "last" : "only one") + " batch (" + batchCounter + ") of Assignments_" + assignmentsCounter + " to the Controller."); + logger.debug("Will return the " + ((totalBatches > 1) ? "last" : "only one") + " batch (" + batchCounter + ") of assignments_" + assignmentsCounter + " to the Controller."); String zstdName = zstdFile.getName(); String zstdTarFileFullPath = currentAssignmentsBaseFullTextsPath + zstdName; @@ -82,7 +83,7 @@ public class FullTextsController { return ResponseEntity.ok() .contentType(MediaType.APPLICATION_OCTET_STREAM) .header(HttpHeaders.CONTENT_DISPOSITION, "inline; filename=\"" + zstdName + "\"") - .body(new InputStreamResource(Files.newInputStream(Paths.get(zstdTarFileFullPath)))); + .body(new InputStreamResource(new BufferedInputStream(Files.newInputStream(Paths.get(zstdTarFileFullPath)), FilesCompressor.tenMb))); } catch (Exception e) { String errorMsg = "Could not load the FileInputStream of the zstd-tar-file \"" + zstdTarFileFullPath + "\"!"; logger.error(errorMsg, e); @@ -114,7 +115,7 @@ public class FullTextsController { return ResponseEntity.ok() .contentType(MediaType.APPLICATION_OCTET_STREAM) .header(HttpHeaders.CONTENT_DISPOSITION, "inline; filename=\"" + file.getName() + "\"") - .body(new InputStreamResource(Files.newInputStream(Paths.get(fullTextFileFullPath)))); + .body(new InputStreamResource(new BufferedInputStream(Files.newInputStream(Paths.get(fullTextFileFullPath)), FilesCompressor.tenMb))); } catch (Exception e) { String errorMsg = "Could not load the FileInputStream of the full-text-file \"" + fullTextFileFullPath + "\"!"; logger.error(errorMsg, e); diff --git a/src/main/java/eu/openaire/urls_worker/services/FileStorageService.java b/src/main/java/eu/openaire/urls_worker/services/FileStorageService.java index 66e5662..281edda 100644 --- a/src/main/java/eu/openaire/urls_worker/services/FileStorageService.java +++ b/src/main/java/eu/openaire/urls_worker/services/FileStorageService.java @@ -31,10 +31,11 @@ public class FileStorageService { if ( !this.assignmentsBaseLocation.endsWith(File.separator) ) this.assignmentsBaseLocation += File.separator; + // Create the base-directory. try { Files.createDirectories(Paths.get(this.assignmentsBaseLocation)); } catch (Exception e) { - logger.error("Could not create the directory where the downloaded files will be stored!", e); + logger.error("Could not create the base-directory where the downloaded files will be stored!", e); System.exit(-10); } } diff --git a/src/main/java/eu/openaire/urls_worker/util/FilesCompressor.java b/src/main/java/eu/openaire/urls_worker/util/FilesCompressor.java index 45daeee..6a5fc09 100644 --- a/src/main/java/eu/openaire/urls_worker/util/FilesCompressor.java +++ b/src/main/java/eu/openaire/urls_worker/util/FilesCompressor.java @@ -19,7 +19,7 @@ public class FilesCompressor { private static final Logger logger = LoggerFactory.getLogger(FilesCompressor.class); - static final int tenMb = (10 * 1_048_576); + public static final int tenMb = (10 * 1_048_576); public static File compressMultipleFilesIntoOne(long assignmentsCounter, int tarBatchCounter, List filesToCompress, String baseDirectory)