diff --git a/src/main/java/eu/openaire/urls_worker/components/plugins/PublicationsRetrieverPlugin.java b/src/main/java/eu/openaire/urls_worker/components/plugins/PublicationsRetrieverPlugin.java index d20319f..b546981 100644 --- a/src/main/java/eu/openaire/urls_worker/components/plugins/PublicationsRetrieverPlugin.java +++ b/src/main/java/eu/openaire/urls_worker/components/plugins/PublicationsRetrieverPlugin.java @@ -37,13 +37,12 @@ public class PublicationsRetrieverPlugin { private static final Logger logger = LoggerFactory.getLogger(PublicationsRetrieverPlugin.class); - public static String assignmentsBasePath; private static CookieStore cookieStore = null; - public PublicationsRetrieverPlugin(@Value("${info.maxAssignmentsLimitPerBatch}") int maxAssignmentsLimitPerBatch) { + public PublicationsRetrieverPlugin(@Value("${info.maxAssignmentsLimitPerBatch}") int maxAssignmentsLimitPerBatch, FileStorageService fileStorageService) { // Specify some configurations LoaderAndChecker.retrieveDocuments = true; LoaderAndChecker.retrieveDatasets = false; @@ -53,9 +52,7 @@ public class PublicationsRetrieverPlugin { PublicationsRetriever.targetUrlType = "docUrl"; FileUtils.jsonBatchSize = maxAssignmentsLimitPerBatch; - assignmentsBasePath = FileStorageService.assignmentsLocation.toString(); - if ( !assignmentsBasePath.endsWith(File.separator) ) - assignmentsBasePath += File.separator; + assignmentsBasePath = fileStorageService.assignmentsBaseLocation; ConnSupportUtils.shouldBlockMost5XXDomains = false; // If this is "true", all but the "503" will be blocked. Otherwise, only the "511" will be blocked. LoaderAndChecker.setCouldRetryRegex(); @@ -78,15 +75,20 @@ public class PublicationsRetrieverPlugin { public static void processAssignments(Long assignmentRequestCounter, Collection assignments) throws RuntimeException { - FileUtils.storeDocFilesDir = assignmentsBasePath + "assignments_" + assignmentRequestCounter + "_fullTexts" + File.separator; // It needs the last separator, because of how the docFiles are named and stored. + String currentAssignmentsSubDir = "assignments_" + assignmentRequestCounter + "_fullTexts" + File.separator; + FileUtils.storeDocFilesDir = assignmentsBasePath + currentAssignmentsSubDir; // It needs the last separator, because of how the docFiles are named and stored. File curAssignmentsDirs = new File(FileUtils.storeDocFilesDir); try { if ( !curAssignmentsDirs.exists() ) { if ( !curAssignmentsDirs.mkdirs() ) { // Create the directories. String workingDir = System.getProperty("user.dir") + File.separator; - logger.error("Could not create the \"assignments_fullTexts directories\": \"" + FileUtils.storeDocFilesDir + "\". Using the \"workingDir\" instead (" + workingDir + ")."); - FileUtils.storeDocFilesDir = assignmentsBasePath = workingDir; + logger.error("Could not create the \"assignments_fullTexts directories\": \"" + FileUtils.storeDocFilesDir + "\". Using the \"workingDir\", as the base-dir instead (" + workingDir + ")."); + FileUtils.storeDocFilesDir = (workingDir + currentAssignmentsSubDir); + if ( ! (new File(FileUtils.storeDocFilesDir)).mkdirs() ) { // Create the alternative directories. + logger.error("Could not create the directory where the downloaded files will be stored!"); + System.exit(-40); + } } } } catch (Exception e) { diff --git a/src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java b/src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java index a285cb1..841a56d 100644 --- a/src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java +++ b/src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java @@ -7,6 +7,7 @@ import org.apache.commons.io.FileDeleteStrategy; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.InputStreamResource; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; @@ -28,12 +29,8 @@ public class FullTextsController { private static final Logger logger = LoggerFactory.getLogger(GeneralController.class); - public static String assignmentsBaseDir = null; - - - public FullTextsController() { - assignmentsBaseDir = FileStorageService.assignmentsLocation.toString() + File.separator; - } + @Autowired + private FileStorageService fileStorageService; @GetMapping("getFullTexts/{assignmentsCounter:[\\d]+}/{totalBatches:[\\d]+}/{batchCounter:[\\d]+}/{fileNamesWithExtensions}") @@ -60,7 +57,7 @@ public class FullTextsController { logger.info("Received a \"getFullTexts\" request for returning a tar-file containing " + fileNamesListNum + " full-texts, from assignments_" + assignmentsCounter + ", for batch_" + batchCounter + " (out of " + totalBatches + ")."); - String currentAssignmentsBaseFullTextsPath = assignmentsBaseDir + "assignments_" + assignmentsCounter + "_fullTexts" + File.separator; + String currentAssignmentsBaseFullTextsPath = fileStorageService.assignmentsBaseLocation + "assignments_" + assignmentsCounter + "_fullTexts" + File.separator; if ( ! (new File(currentAssignmentsBaseFullTextsPath).isDirectory()) ) { String errorMsg = "The base directory for assignments_" + assignmentsCounter + " was not found: " + currentAssignmentsBaseFullTextsPath; @@ -106,7 +103,7 @@ public class FullTextsController { public ResponseEntity getFullText(@PathVariable long assignmentsCounter, @PathVariable String fileNameWithExtension) { logger.info("Received a \"getFullText\" request."); - String fullTextFileFullPath = assignmentsBaseDir + "assignments_" + assignmentsCounter + "_fullTexts" + File.separator + fileNameWithExtension; + String fullTextFileFullPath = fileStorageService.assignmentsBaseLocation + "assignments_" + assignmentsCounter + "_fullTexts" + File.separator + fileNameWithExtension; File file = new File(fullTextFileFullPath); if ( !file.isFile() ) { logger.error("The file \"" + fullTextFileFullPath + "\" does not exist!"); diff --git a/src/main/java/eu/openaire/urls_worker/services/FileStorageService.java b/src/main/java/eu/openaire/urls_worker/services/FileStorageService.java index bf7817f..66e5662 100644 --- a/src/main/java/eu/openaire/urls_worker/services/FileStorageService.java +++ b/src/main/java/eu/openaire/urls_worker/services/FileStorageService.java @@ -1,17 +1,15 @@ package eu.openaire.urls_worker.services; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import java.io.File; -import java.io.FileReader; -import java.io.IOException; import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Properties; @Service @@ -19,37 +17,25 @@ public class FileStorageService { private static final Logger logger = LoggerFactory.getLogger(FileStorageService.class); - public static Path assignmentsLocation = null; - - static { - String springPropertiesFile = System.getProperty("user.dir") + File.separator + "src" + File.separator + "main" + File.separator + "resources" + File.separator + "application.properties"; - FileReader fReader = null; - try { - fReader = new FileReader(springPropertiesFile); - Properties props = new Properties(); - props.load(fReader); // Load jdbc related properties. - String assignmentsDir = props.getProperty("file.assignments-dir"); - assignmentsLocation = Paths.get(assignmentsDir).toAbsolutePath().normalize(); - } catch (java.io.FileNotFoundException fnfe) { - logger.error("The properties file was not found!", fnfe); - System.exit(-10); - } catch (IOException ioe) { - logger.error("I/O error when reading the properties file!", ioe); - System.exit(-11); - } catch (Exception e) { - logger.error("", e); - System.exit(-12); - } - } + public String assignmentsBaseLocation = null; @Autowired - public FileStorageService() { + public FileStorageService(@Value("${file.assignments-dir}") String assignmentsBaseLocation) { + this.assignmentsBaseLocation = assignmentsBaseLocation; + + // In case the user-defined storageDir starts with "./", then replace that part with the actual user.dir", in order to have valid storage-locations for fileName-extraction in the Controller, even if the files are correctly downloaded there. + if ( this.assignmentsBaseLocation.startsWith("." + File.separator) ) + this.assignmentsBaseLocation = ((System.getProperty("user.dir") + File.separator) + StringUtils.replace(this.assignmentsBaseLocation, ("." + File.separator), "", 1)); + + if ( !this.assignmentsBaseLocation.endsWith(File.separator) ) + this.assignmentsBaseLocation += File.separator; + try { - Files.createDirectories(assignmentsLocation); + Files.createDirectories(Paths.get(this.assignmentsBaseLocation)); } catch (Exception e) { - logger.error("Could not create the directory where the downloaded files will be stored.", e); - System.exit(-13); + logger.error("Could not create the directory where the downloaded files will be stored!", e); + System.exit(-10); } }