michal.politowski
/
UrlsController
forked from lsmyrnaios/UrlsController
1
0
Fork 0
Code Pull Requests Activity

Refactor the files' storage location.

This commit is contained in:
Lampros Smyrnaios 2022-12-15 18:29:51 +02:00
parent e51ee9dd27
commit 9cdbbdea67
3 changed files with 10 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/eu/openaire/urls_controller/controllers/UrlController.java
View File

@ -334,7 +334,7 @@ public class UrlController {
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorMsg);
} finally {
logger.debug("Deleting directory: " + currentParquetPath);
fileUtils.deleteDirectory(new File(currentParquetPath));
FileUtils.deleteDirectory(new File(currentParquetPath));
}
logger.debug("Going to merge the parquet files for the tables which were altered.");

14
src/main/java/eu/openaire/urls_controller/util/FileUtils.java
View File

@ -82,8 +82,8 @@ public class FileUtils {
}
@Value("${services.pdfaggregation.controller.baseTargetLocation}")
private String baseTargetLocation;
@Value("${services.pdfaggregation.controller.baseFilesLocation}")
public static String baseFilesLocation;
public static DecimalFormat df = new DecimalFormat("0.00");
@ -193,7 +193,7 @@ public class FileUtils {
// Check if one full text is left out because of the division. Put it int the last batch.
String baseUrl = "http://" + remoteAddr + ":1881/api/full-texts/getFullTexts/" + assignmentsBatchCounter + "/" + numOfBatches + "/";
String curAssignmentsBaseLocation = baseTargetLocation + "assignments_" + assignmentsBatchCounter + File.separator;
String curAssignmentsBaseLocation = baseFilesLocation + "assignments_" + assignmentsBatchCounter + File.separator;
File curAssignmentsBaseDir = new File(curAssignmentsBaseLocation);
int failedBatches = 0;
@ -490,16 +490,16 @@ public class FileUtils {
}
public boolean deleteDirectory(File curBatchDir)
public static boolean deleteDirectory(File directory)
{
try {
org.apache.commons.io.FileUtils.deleteDirectory(curBatchDir);
org.apache.commons.io.FileUtils.deleteDirectory(directory);
return true;
} catch (IOException e) {
logger.error("The following directory could not be deleted: " + curBatchDir.getName(), e);
logger.error("The following directory could not be deleted: " + directory.getName(), e);
return false;
} catch (IllegalArgumentException iae) {
logger.error("This batch-dir does not exist: " + curBatchDir.getName());
logger.error("This batch-dir does not exist: " + directory.getName());
return false;
}
}

4
src/main/resources/application.properties
View File

@ -14,7 +14,7 @@ services.pdfaggregation.controller.isTestEnvironment = false
services.pdfaggregation.controller.db.initialDatabaseName = pdfaggregation_i
services.pdfaggregation.controller.db.testDatabaseName = pdfaggregationdatabase_new_s3_names
services.pdfaggregation.controller.baseTargetLocation = /tmp/
services.pdfaggregation.controller.baseFilesLocation = tmp/
services.pdfaggregation.controller.maxAttemptsPerRecord = 3
services.pdfaggregation.controller.assignmentLimit = 10000
@ -60,7 +60,7 @@ hdfs.httpAuth=
hdfs.userName=
hdfs.password=
output.parquetLocalDirectoryPath=parquetFiles/
output.parquetLocalDirectoryPath=${services.pdfaggregation.controller.baseFilesLocation}parquetFiles/
hdfs.parquetRemoteBaseDirectoryPath=/tmp/parquet_uploads/
## MULTIPART (MultipartProperties)