lsmyrnaios
/
UrlsWorker
1
0
Fork 1
Code Issues Pull Requests Releases 12 Wiki Activity

- Refactor and Spring-ify the File-storage initialization process. 

- Fix the problematic file-storage-path (it could not be used when the Controller was requesting the full-texts), which was produced when the user-defined path could not be created.
This commit is contained in:
Lampros Smyrnaios 2023-03-07 16:21:32 +02:00
parent ba989484e4
commit ec09ecc7ff
3 changed files with 31 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/main/java/eu/openaire/urls_worker/components/plugins/PublicationsRetrieverPlugin.java
View File

@ -37,13 +37,12 @@ public class PublicationsRetrieverPlugin {
private static final Logger logger = LoggerFactory.getLogger(PublicationsRetrieverPlugin.class);
public static String assignmentsBasePath;
private static CookieStore cookieStore = null;
public PublicationsRetrieverPlugin(@Value("${info.maxAssignmentsLimitPerBatch}") int maxAssignmentsLimitPerBatch) {
public PublicationsRetrieverPlugin(@Value("${info.maxAssignmentsLimitPerBatch}") int maxAssignmentsLimitPerBatch, FileStorageService fileStorageService) {
// Specify some configurations
LoaderAndChecker.retrieveDocuments = true;
LoaderAndChecker.retrieveDatasets = false;
@ -53,9 +52,7 @@ public class PublicationsRetrieverPlugin {
PublicationsRetriever.targetUrlType = "docUrl";
FileUtils.jsonBatchSize = maxAssignmentsLimitPerBatch;
assignmentsBasePath = FileStorageService.assignmentsLocation.toString();
if ( !assignmentsBasePath.endsWith(File.separator) )
assignmentsBasePath += File.separator;
assignmentsBasePath = fileStorageService.assignmentsBaseLocation;
ConnSupportUtils.shouldBlockMost5XXDomains = false; // If this is "true", all but the "503" will be blocked. Otherwise, only the "511" will be blocked.
LoaderAndChecker.setCouldRetryRegex();
@ -78,15 +75,20 @@ public class PublicationsRetrieverPlugin {
public static void processAssignments(Long assignmentRequestCounter, Collection<Assignment> assignments) throws RuntimeException
{
FileUtils.storeDocFilesDir = assignmentsBasePath + "assignments_" + assignmentRequestCounter + "_fullTexts" + File.separator; // It needs the last separator, because of how the docFiles are named and stored.
String currentAssignmentsSubDir = "assignments_" + assignmentRequestCounter + "_fullTexts" + File.separator;
FileUtils.storeDocFilesDir = assignmentsBasePath + currentAssignmentsSubDir; // It needs the last separator, because of how the docFiles are named and stored.
File curAssignmentsDirs = new File(FileUtils.storeDocFilesDir);
try {
if ( !curAssignmentsDirs.exists() ) {
if ( !curAssignmentsDirs.mkdirs() ) { // Create the directories.
String workingDir = System.getProperty("user.dir") + File.separator;
logger.error("Could not create the \"assignments_fullTexts directories\": \"" + FileUtils.storeDocFilesDir + "\". Using the \"workingDir\" instead (" + workingDir + ").");
FileUtils.storeDocFilesDir = assignmentsBasePath = workingDir;
logger.error("Could not create the \"assignments_fullTexts directories\": \"" + FileUtils.storeDocFilesDir + "\". Using the \"workingDir\", as the base-dir instead (" + workingDir + ").");
FileUtils.storeDocFilesDir = (workingDir + currentAssignmentsSubDir);
if ( ! (new File(FileUtils.storeDocFilesDir)).mkdirs() ) { // Create the alternative directories.
logger.error("Could not create the directory where the downloaded files will be stored!");
System.exit(-40);
}
}
}
} catch (Exception e) {

13
src/main/java/eu/openaire/urls_worker/controllers/FullTextsController.java
View File

@ -7,6 +7,7 @@ import org.apache.commons.io.FileDeleteStrategy;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.InputStreamResource;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
@ -28,12 +29,8 @@ public class FullTextsController {
private static final Logger logger = LoggerFactory.getLogger(GeneralController.class);
public static String assignmentsBaseDir = null;
public FullTextsController() {
assignmentsBaseDir = FileStorageService.assignmentsLocation.toString() + File.separator;
}
@Autowired
private FileStorageService fileStorageService;
@GetMapping("getFullTexts/{assignmentsCounter:[\\d]+}/{totalBatches:[\\d]+}/{batchCounter:[\\d]+}/{fileNamesWithExtensions}")
@ -60,7 +57,7 @@ public class FullTextsController {
logger.info("Received a \"getFullTexts\" request for returning a tar-file containing " + fileNamesListNum + " full-texts, from assignments_" + assignmentsCounter + ", for batch_" + batchCounter + " (out of " + totalBatches + ").");
String currentAssignmentsBaseFullTextsPath = assignmentsBaseDir + "assignments_" + assignmentsCounter + "_fullTexts" + File.separator;
String currentAssignmentsBaseFullTextsPath = fileStorageService.assignmentsBaseLocation + "assignments_" + assignmentsCounter + "_fullTexts" + File.separator;
if ( ! (new File(currentAssignmentsBaseFullTextsPath).isDirectory()) ) {
String errorMsg = "The base directory for assignments_" + assignmentsCounter + " was not found: " + currentAssignmentsBaseFullTextsPath;
@ -106,7 +103,7 @@ public class FullTextsController {
public ResponseEntity<?> getFullText(@PathVariable long assignmentsCounter, @PathVariable String fileNameWithExtension) {
logger.info("Received a \"getFullText\" request.");
String fullTextFileFullPath = assignmentsBaseDir + "assignments_" + assignmentsCounter + "_fullTexts" + File.separator + fileNameWithExtension;
String fullTextFileFullPath = fileStorageService.assignmentsBaseLocation + "assignments_" + assignmentsCounter + "_fullTexts" + File.separator + fileNameWithExtension;
File file = new File(fullTextFileFullPath);
if ( !file.isFile() ) {
logger.error("The file \"" + fullTextFileFullPath + "\" does not exist!");

46
src/main/java/eu/openaire/urls_worker/services/FileStorageService.java
View File

@ -1,17 +1,15 @@
package eu.openaire.urls_worker.services;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Properties;
@Service
@ -19,37 +17,25 @@ public class FileStorageService {
private static final Logger logger = LoggerFactory.getLogger(FileStorageService.class);
public static Path assignmentsLocation = null;
static {
String springPropertiesFile = System.getProperty("user.dir") + File.separator + "src" + File.separator + "main" + File.separator + "resources" + File.separator + "application.properties";
FileReader fReader = null;
try {
fReader = new FileReader(springPropertiesFile);
Properties props = new Properties();
props.load(fReader); // Load jdbc related properties.
String assignmentsDir = props.getProperty("file.assignments-dir");
assignmentsLocation = Paths.get(assignmentsDir).toAbsolutePath().normalize();
} catch (java.io.FileNotFoundException fnfe) {
logger.error("The properties file was not found!", fnfe);
System.exit(-10);
} catch (IOException ioe) {
logger.error("I/O error when reading the properties file!", ioe);
System.exit(-11);
} catch (Exception e) {
logger.error("", e);
System.exit(-12);
}
}
public String assignmentsBaseLocation = null;
@Autowired
public FileStorageService() {
public FileStorageService(@Value("${file.assignments-dir}") String assignmentsBaseLocation) {
this.assignmentsBaseLocation = assignmentsBaseLocation;
// In case the user-defined storageDir starts with "./", then replace that part with the actual user.dir", in order to have valid storage-locations for fileName-extraction in the Controller, even if the files are correctly downloaded there.
if ( this.assignmentsBaseLocation.startsWith("." + File.separator) )
this.assignmentsBaseLocation = ((System.getProperty("user.dir") + File.separator) + StringUtils.replace(this.assignmentsBaseLocation, ("." + File.separator), "", 1));
if ( !this.assignmentsBaseLocation.endsWith(File.separator) )
this.assignmentsBaseLocation += File.separator;
try {
Files.createDirectories(assignmentsLocation);
Files.createDirectories(Paths.get(this.assignmentsBaseLocation));
} catch (Exception e) {
logger.error("Could not create the directory where the downloaded files will be stored.", e);
System.exit(-13);
logger.error("Could not create the directory where the downloaded files will be stored!", e);
System.exit(-10);
}
}