lsmyrnaios
/
UrlsWorker
1
0
Fork 1
Code Issues Pull Requests Releases 12 Wiki Activity

- Improve performance of full-texts transferring to the Controller, by preloading some bytes for faster response to the Controller's read requests. 

- Optimize directories-creation process by eliminating the additive check for existence, as that check already takes place inside the "mkdirs()" method.
- Remove the obsolete code which in case the specific assignments' subdirectory failed to be created, then a different base-dir was used instead. Since the user-defined baseDir is already been successfully created upon initialization, any problem on creating subdirectories inside that base-directory will most likely persist even when changing the base directory. Additionally, even if the subdirectory with the changed base-directory succeeded, the "FullTextsController.getFullTexts()" method would not use it, resulting in errors.
- Code polishing.
This commit is contained in:
Lampros Smyrnaios 2023-03-08 13:12:17 +02:00
parent 4da54e7a7d
commit 839a797124
5 changed files with 14 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/main/java/eu/openaire/urls_worker/components/AssignmentsHandler.java
View File

@ -304,9 +304,6 @@ public class AssignmentsHandler {
spacedOutAssignments.add(nextAssignment);
}
// The HashMultimap is no longer needed.
domainsWithAssignments.clear();
if ( sb != null )
logger.debug("After change:\n" + sb.toString());
@ -343,9 +340,7 @@ public class AssignmentsHandler {
} else {
nextAssignment = assignmentsOfCurrentDomain.toArray()[0];
result.put(nextAssignment, domainsCounter);
domainsWithAssignments.remove(currentDomain, nextAssignment);
if ( sb != null )
sb.append(currentDomain).append("\n"); // DEBUG!
}

17
src/main/java/eu/openaire/urls_worker/components/plugins/PublicationsRetrieverPlugin.java
View File

@ -75,23 +75,16 @@ public class PublicationsRetrieverPlugin {
public static void processAssignments(Long assignmentRequestCounter, Collection<Assignment> assignments) throws RuntimeException
{
// At this point, the "assignmentsBasePath"-directory has already been successfully created.
String currentAssignmentsSubDir = "assignments_" + assignmentRequestCounter + "_fullTexts" + File.separator;
FileUtils.storeDocFilesDir = assignmentsBasePath + currentAssignmentsSubDir; // It needs the last separator, because of how the docFiles are named and stored.
File curAssignmentsDirs = new File(FileUtils.storeDocFilesDir);
try {
if ( !curAssignmentsDirs.exists() ) {
if ( !curAssignmentsDirs.mkdirs() ) { // Create the directories.
String workingDir = System.getProperty("user.dir") + File.separator;
logger.error("Could not create the \"assignments_fullTexts directories\": \"" + FileUtils.storeDocFilesDir + "\". Using the \"workingDir\", as the base-dir instead (" + workingDir + ").");
FileUtils.storeDocFilesDir = (workingDir + currentAssignmentsSubDir);
if ( ! (new File(FileUtils.storeDocFilesDir)).mkdirs() ) { // Create the alternative directories.
logger.error("Could not create the directory where the downloaded files will be stored!");
System.exit(-40);
}
}
}
} catch (Exception e) {
if ( !curAssignmentsDirs.mkdirs() ) // Create the subdirectory.
logger.error("Could not create the \"assignments_" + assignmentRequestCounter + "_fullTexts\" directories: \"" + FileUtils.storeDocFilesDir + "\"!");
} catch (Exception e) { // Mainly a SecurityException.
String errorMsg = "Failed to create the full-texts directory for assignments_" + assignmentRequestCounter;
logger.error(errorMsg, e);
throw new RuntimeException(errorMsg + ": " + e.getMessage());

11
src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java
View File

@ -17,6 +17,7 @@ import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
@ -34,8 +35,8 @@ public class FullTextsController {
@GetMapping("getFullTexts/{assignmentsCounter:[\\d]+}/{totalBatches:[\\d]+}/{batchCounter:[\\d]+}/{fileNamesWithExtensions}")
public Object getFullTexts(@PathVariable long assignmentsCounter, @PathVariable int totalBatches, @PathVariable int batchCounter, @PathVariable List<String> fileNamesWithExtensions) {
public Object getFullTexts(@PathVariable long assignmentsCounter, @PathVariable int totalBatches, @PathVariable int batchCounter, @PathVariable List<String> fileNamesWithExtensions)
{
int fileNamesListNum = fileNamesWithExtensions.size();
if ( (fileNamesListNum == 1) && (fileNamesWithExtensions.get(0).length() == 0) ) { // In case the last "/" in the url was given (without any files following), then this list will not be empty, but have one empty item instead.
// In case the url does not end in "/", then Spring will automatically return an "HTTP-BadRequest".
@ -74,7 +75,7 @@ public class FullTextsController {
}
if ( batchCounter == totalBatches )
logger.debug("Will return the " + ((totalBatches > 1) ? "last" : "only one") + " batch (" + batchCounter + ") of Assignments_" + assignmentsCounter + " to the Controller.");
logger.debug("Will return the " + ((totalBatches > 1) ? "last" : "only one") + " batch (" + batchCounter + ") of assignments_" + assignmentsCounter + " to the Controller.");
String zstdName = zstdFile.getName();
String zstdTarFileFullPath = currentAssignmentsBaseFullTextsPath + zstdName;
@ -82,7 +83,7 @@ public class FullTextsController {
return ResponseEntity.ok()
.contentType(MediaType.APPLICATION_OCTET_STREAM)
.header(HttpHeaders.CONTENT_DISPOSITION, "inline; filename=\"" + zstdName + "\"")
.body(new InputStreamResource(Files.newInputStream(Paths.get(zstdTarFileFullPath))));
.body(new InputStreamResource(new BufferedInputStream(Files.newInputStream(Paths.get(zstdTarFileFullPath)), FilesCompressor.tenMb)));
} catch (Exception e) {
String errorMsg = "Could not load the FileInputStream of the zstd-tar-file \"" + zstdTarFileFullPath + "\"!";
logger.error(errorMsg, e);
@ -114,7 +115,7 @@ public class FullTextsController {
return ResponseEntity.ok()
.contentType(MediaType.APPLICATION_OCTET_STREAM)
.header(HttpHeaders.CONTENT_DISPOSITION, "inline; filename=\"" + file.getName() + "\"")
.body(new InputStreamResource(Files.newInputStream(Paths.get(fullTextFileFullPath))));
.body(new InputStreamResource(new BufferedInputStream(Files.newInputStream(Paths.get(fullTextFileFullPath)), FilesCompressor.tenMb)));
} catch (Exception e) {
String errorMsg = "Could not load the FileInputStream of the full-text-file \"" + fullTextFileFullPath + "\"!";
logger.error(errorMsg, e);

3
src/main/java/eu/openaire/urls_worker/services/FileStorageService.java
View File

@ -31,10 +31,11 @@ public class FileStorageService {
if ( !this.assignmentsBaseLocation.endsWith(File.separator) )
this.assignmentsBaseLocation += File.separator;
// Create the base-directory.
try {
Files.createDirectories(Paths.get(this.assignmentsBaseLocation));
} catch (Exception e) {
logger.error("Could not create the directory where the downloaded files will be stored!", e);
logger.error("Could not create the base-directory where the downloaded files will be stored!", e);
System.exit(-10);
}
}

2
src/main/java/eu/openaire/urls_worker/util/FilesCompressor.java
View File

@ -19,7 +19,7 @@ public class FilesCompressor {
private static final Logger logger = LoggerFactory.getLogger(FilesCompressor.class);
static final int tenMb = (10 * 1_048_576);
public static final int tenMb = (10 * 1_048_576);
public static File compressMultipleFilesIntoOne(long assignmentsCounter, int tarBatchCounter, List<String> filesToCompress, String baseDirectory)