- Refactor and Spring-ify the File-storage initialization process.
- Fix the problematic file-storage-path (it could not be used when the Controller was requesting the full-texts), which was produced when the user-defined path could not be created.
This commit is contained in:
parent
ba989484e4
commit
ec09ecc7ff
|
@ -37,13 +37,12 @@ public class PublicationsRetrieverPlugin {
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(PublicationsRetrieverPlugin.class);
|
private static final Logger logger = LoggerFactory.getLogger(PublicationsRetrieverPlugin.class);
|
||||||
|
|
||||||
|
|
||||||
public static String assignmentsBasePath;
|
public static String assignmentsBasePath;
|
||||||
|
|
||||||
private static CookieStore cookieStore = null;
|
private static CookieStore cookieStore = null;
|
||||||
|
|
||||||
|
|
||||||
public PublicationsRetrieverPlugin(@Value("${info.maxAssignmentsLimitPerBatch}") int maxAssignmentsLimitPerBatch) {
|
public PublicationsRetrieverPlugin(@Value("${info.maxAssignmentsLimitPerBatch}") int maxAssignmentsLimitPerBatch, FileStorageService fileStorageService) {
|
||||||
// Specify some configurations
|
// Specify some configurations
|
||||||
LoaderAndChecker.retrieveDocuments = true;
|
LoaderAndChecker.retrieveDocuments = true;
|
||||||
LoaderAndChecker.retrieveDatasets = false;
|
LoaderAndChecker.retrieveDatasets = false;
|
||||||
|
@ -53,9 +52,7 @@ public class PublicationsRetrieverPlugin {
|
||||||
PublicationsRetriever.targetUrlType = "docUrl";
|
PublicationsRetriever.targetUrlType = "docUrl";
|
||||||
FileUtils.jsonBatchSize = maxAssignmentsLimitPerBatch;
|
FileUtils.jsonBatchSize = maxAssignmentsLimitPerBatch;
|
||||||
|
|
||||||
assignmentsBasePath = FileStorageService.assignmentsLocation.toString();
|
assignmentsBasePath = fileStorageService.assignmentsBaseLocation;
|
||||||
if ( !assignmentsBasePath.endsWith(File.separator) )
|
|
||||||
assignmentsBasePath += File.separator;
|
|
||||||
|
|
||||||
ConnSupportUtils.shouldBlockMost5XXDomains = false; // If this is "true", all but the "503" will be blocked. Otherwise, only the "511" will be blocked.
|
ConnSupportUtils.shouldBlockMost5XXDomains = false; // If this is "true", all but the "503" will be blocked. Otherwise, only the "511" will be blocked.
|
||||||
LoaderAndChecker.setCouldRetryRegex();
|
LoaderAndChecker.setCouldRetryRegex();
|
||||||
|
@ -78,15 +75,20 @@ public class PublicationsRetrieverPlugin {
|
||||||
|
|
||||||
public static void processAssignments(Long assignmentRequestCounter, Collection<Assignment> assignments) throws RuntimeException
|
public static void processAssignments(Long assignmentRequestCounter, Collection<Assignment> assignments) throws RuntimeException
|
||||||
{
|
{
|
||||||
FileUtils.storeDocFilesDir = assignmentsBasePath + "assignments_" + assignmentRequestCounter + "_fullTexts" + File.separator; // It needs the last separator, because of how the docFiles are named and stored.
|
String currentAssignmentsSubDir = "assignments_" + assignmentRequestCounter + "_fullTexts" + File.separator;
|
||||||
|
FileUtils.storeDocFilesDir = assignmentsBasePath + currentAssignmentsSubDir; // It needs the last separator, because of how the docFiles are named and stored.
|
||||||
|
|
||||||
File curAssignmentsDirs = new File(FileUtils.storeDocFilesDir);
|
File curAssignmentsDirs = new File(FileUtils.storeDocFilesDir);
|
||||||
try {
|
try {
|
||||||
if ( !curAssignmentsDirs.exists() ) {
|
if ( !curAssignmentsDirs.exists() ) {
|
||||||
if ( !curAssignmentsDirs.mkdirs() ) { // Create the directories.
|
if ( !curAssignmentsDirs.mkdirs() ) { // Create the directories.
|
||||||
String workingDir = System.getProperty("user.dir") + File.separator;
|
String workingDir = System.getProperty("user.dir") + File.separator;
|
||||||
logger.error("Could not create the \"assignments_fullTexts directories\": \"" + FileUtils.storeDocFilesDir + "\". Using the \"workingDir\" instead (" + workingDir + ").");
|
logger.error("Could not create the \"assignments_fullTexts directories\": \"" + FileUtils.storeDocFilesDir + "\". Using the \"workingDir\", as the base-dir instead (" + workingDir + ").");
|
||||||
FileUtils.storeDocFilesDir = assignmentsBasePath = workingDir;
|
FileUtils.storeDocFilesDir = (workingDir + currentAssignmentsSubDir);
|
||||||
|
if ( ! (new File(FileUtils.storeDocFilesDir)).mkdirs() ) { // Create the alternative directories.
|
||||||
|
logger.error("Could not create the directory where the downloaded files will be stored!");
|
||||||
|
System.exit(-40);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
|
|
|
@ -7,6 +7,7 @@ import org.apache.commons.io.FileDeleteStrategy;
|
||||||
import org.apache.commons.io.FileUtils;
|
import org.apache.commons.io.FileUtils;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.core.io.InputStreamResource;
|
import org.springframework.core.io.InputStreamResource;
|
||||||
import org.springframework.http.HttpHeaders;
|
import org.springframework.http.HttpHeaders;
|
||||||
import org.springframework.http.MediaType;
|
import org.springframework.http.MediaType;
|
||||||
|
@ -28,12 +29,8 @@ public class FullTextsController {
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(GeneralController.class);
|
private static final Logger logger = LoggerFactory.getLogger(GeneralController.class);
|
||||||
|
|
||||||
public static String assignmentsBaseDir = null;
|
@Autowired
|
||||||
|
private FileStorageService fileStorageService;
|
||||||
|
|
||||||
public FullTextsController() {
|
|
||||||
assignmentsBaseDir = FileStorageService.assignmentsLocation.toString() + File.separator;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@GetMapping("getFullTexts/{assignmentsCounter:[\\d]+}/{totalBatches:[\\d]+}/{batchCounter:[\\d]+}/{fileNamesWithExtensions}")
|
@GetMapping("getFullTexts/{assignmentsCounter:[\\d]+}/{totalBatches:[\\d]+}/{batchCounter:[\\d]+}/{fileNamesWithExtensions}")
|
||||||
|
@ -60,7 +57,7 @@ public class FullTextsController {
|
||||||
|
|
||||||
logger.info("Received a \"getFullTexts\" request for returning a tar-file containing " + fileNamesListNum + " full-texts, from assignments_" + assignmentsCounter + ", for batch_" + batchCounter + " (out of " + totalBatches + ").");
|
logger.info("Received a \"getFullTexts\" request for returning a tar-file containing " + fileNamesListNum + " full-texts, from assignments_" + assignmentsCounter + ", for batch_" + batchCounter + " (out of " + totalBatches + ").");
|
||||||
|
|
||||||
String currentAssignmentsBaseFullTextsPath = assignmentsBaseDir + "assignments_" + assignmentsCounter + "_fullTexts" + File.separator;
|
String currentAssignmentsBaseFullTextsPath = fileStorageService.assignmentsBaseLocation + "assignments_" + assignmentsCounter + "_fullTexts" + File.separator;
|
||||||
|
|
||||||
if ( ! (new File(currentAssignmentsBaseFullTextsPath).isDirectory()) ) {
|
if ( ! (new File(currentAssignmentsBaseFullTextsPath).isDirectory()) ) {
|
||||||
String errorMsg = "The base directory for assignments_" + assignmentsCounter + " was not found: " + currentAssignmentsBaseFullTextsPath;
|
String errorMsg = "The base directory for assignments_" + assignmentsCounter + " was not found: " + currentAssignmentsBaseFullTextsPath;
|
||||||
|
@ -106,7 +103,7 @@ public class FullTextsController {
|
||||||
public ResponseEntity<?> getFullText(@PathVariable long assignmentsCounter, @PathVariable String fileNameWithExtension) {
|
public ResponseEntity<?> getFullText(@PathVariable long assignmentsCounter, @PathVariable String fileNameWithExtension) {
|
||||||
|
|
||||||
logger.info("Received a \"getFullText\" request.");
|
logger.info("Received a \"getFullText\" request.");
|
||||||
String fullTextFileFullPath = assignmentsBaseDir + "assignments_" + assignmentsCounter + "_fullTexts" + File.separator + fileNameWithExtension;
|
String fullTextFileFullPath = fileStorageService.assignmentsBaseLocation + "assignments_" + assignmentsCounter + "_fullTexts" + File.separator + fileNameWithExtension;
|
||||||
File file = new File(fullTextFileFullPath);
|
File file = new File(fullTextFileFullPath);
|
||||||
if ( !file.isFile() ) {
|
if ( !file.isFile() ) {
|
||||||
logger.error("The file \"" + fullTextFileFullPath + "\" does not exist!");
|
logger.error("The file \"" + fullTextFileFullPath + "\" does not exist!");
|
||||||
|
|
|
@ -1,17 +1,15 @@
|
||||||
package eu.openaire.urls_worker.services;
|
package eu.openaire.urls_worker.services;
|
||||||
|
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.FileReader;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
import java.nio.file.Path;
|
|
||||||
import java.nio.file.Paths;
|
import java.nio.file.Paths;
|
||||||
import java.util.Properties;
|
|
||||||
|
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
|
@ -19,37 +17,25 @@ public class FileStorageService {
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(FileStorageService.class);
|
private static final Logger logger = LoggerFactory.getLogger(FileStorageService.class);
|
||||||
|
|
||||||
public static Path assignmentsLocation = null;
|
public String assignmentsBaseLocation = null;
|
||||||
|
|
||||||
static {
|
|
||||||
String springPropertiesFile = System.getProperty("user.dir") + File.separator + "src" + File.separator + "main" + File.separator + "resources" + File.separator + "application.properties";
|
|
||||||
FileReader fReader = null;
|
|
||||||
try {
|
|
||||||
fReader = new FileReader(springPropertiesFile);
|
|
||||||
Properties props = new Properties();
|
|
||||||
props.load(fReader); // Load jdbc related properties.
|
|
||||||
String assignmentsDir = props.getProperty("file.assignments-dir");
|
|
||||||
assignmentsLocation = Paths.get(assignmentsDir).toAbsolutePath().normalize();
|
|
||||||
} catch (java.io.FileNotFoundException fnfe) {
|
|
||||||
logger.error("The properties file was not found!", fnfe);
|
|
||||||
System.exit(-10);
|
|
||||||
} catch (IOException ioe) {
|
|
||||||
logger.error("I/O error when reading the properties file!", ioe);
|
|
||||||
System.exit(-11);
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.error("", e);
|
|
||||||
System.exit(-12);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
public FileStorageService() {
|
public FileStorageService(@Value("${file.assignments-dir}") String assignmentsBaseLocation) {
|
||||||
|
this.assignmentsBaseLocation = assignmentsBaseLocation;
|
||||||
|
|
||||||
|
// In case the user-defined storageDir starts with "./", then replace that part with the actual user.dir", in order to have valid storage-locations for fileName-extraction in the Controller, even if the files are correctly downloaded there.
|
||||||
|
if ( this.assignmentsBaseLocation.startsWith("." + File.separator) )
|
||||||
|
this.assignmentsBaseLocation = ((System.getProperty("user.dir") + File.separator) + StringUtils.replace(this.assignmentsBaseLocation, ("." + File.separator), "", 1));
|
||||||
|
|
||||||
|
if ( !this.assignmentsBaseLocation.endsWith(File.separator) )
|
||||||
|
this.assignmentsBaseLocation += File.separator;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
Files.createDirectories(assignmentsLocation);
|
Files.createDirectories(Paths.get(this.assignmentsBaseLocation));
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
logger.error("Could not create the directory where the downloaded files will be stored.", e);
|
logger.error("Could not create the directory where the downloaded files will be stored!", e);
|
||||||
System.exit(-13);
|
System.exit(-10);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue