michal.politowski
/
UrlsController
forked from lsmyrnaios/UrlsController
1
0
Fork 0
Code Pull Requests Activity

Refactor the files' storage location.

This commit is contained in:
Lampros Smyrnaios 2022-12-15 18:29:51 +02:00
parent e51ee9dd27
commit 9cdbbdea67
3 changed files with 10 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/eu/openaire/urls_controller/controllers/UrlController.java
View File

@ -334,7 +334,7 @@ public class UrlController {
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorMsg); return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorMsg);
} finally { } finally {
logger.debug("Deleting directory: " + currentParquetPath); logger.debug("Deleting directory: " + currentParquetPath);
fileUtils.deleteDirectory(new File(currentParquetPath)); FileUtils.deleteDirectory(new File(currentParquetPath));
} }
logger.debug("Going to merge the parquet files for the tables which were altered."); logger.debug("Going to merge the parquet files for the tables which were altered.");

14
src/main/java/eu/openaire/urls_controller/util/FileUtils.java
View File

@ -82,8 +82,8 @@ public class FileUtils {
} }
@Value("${services.pdfaggregation.controller.baseTargetLocation}") @Value("${services.pdfaggregation.controller.baseFilesLocation}")
private String baseTargetLocation; public static String baseFilesLocation;
public static DecimalFormat df = new DecimalFormat("0.00"); public static DecimalFormat df = new DecimalFormat("0.00");
@ -193,7 +193,7 @@ public class FileUtils {
// Check if one full text is left out because of the division. Put it int the last batch. // Check if one full text is left out because of the division. Put it int the last batch.
String baseUrl = "http://" + remoteAddr + ":1881/api/full-texts/getFullTexts/" + assignmentsBatchCounter + "/" + numOfBatches + "/"; String baseUrl = "http://" + remoteAddr + ":1881/api/full-texts/getFullTexts/" + assignmentsBatchCounter + "/" + numOfBatches + "/";
String curAssignmentsBaseLocation = baseTargetLocation + "assignments_" + assignmentsBatchCounter + File.separator; String curAssignmentsBaseLocation = baseFilesLocation + "assignments_" + assignmentsBatchCounter + File.separator;
File curAssignmentsBaseDir = new File(curAssignmentsBaseLocation); File curAssignmentsBaseDir = new File(curAssignmentsBaseLocation);
int failedBatches = 0; int failedBatches = 0;
@ -490,16 +490,16 @@ public class FileUtils {
} }
public boolean deleteDirectory(File curBatchDir) public static boolean deleteDirectory(File directory)
{ {
try { try {
org.apache.commons.io.FileUtils.deleteDirectory(curBatchDir); org.apache.commons.io.FileUtils.deleteDirectory(directory);
return true; return true;
} catch (IOException e) { } catch (IOException e) {
logger.error("The following directory could not be deleted: " + curBatchDir.getName(), e); logger.error("The following directory could not be deleted: " + directory.getName(), e);
return false; return false;
} catch (IllegalArgumentException iae) { } catch (IllegalArgumentException iae) {
logger.error("This batch-dir does not exist: " + curBatchDir.getName()); logger.error("This batch-dir does not exist: " + directory.getName());
return false; return false;
} }
} }

4
src/main/resources/application.properties
View File

@ -14,7 +14,7 @@ services.pdfaggregation.controller.isTestEnvironment = false
services.pdfaggregation.controller.db.initialDatabaseName = pdfaggregation_i services.pdfaggregation.controller.db.initialDatabaseName = pdfaggregation_i
services.pdfaggregation.controller.db.testDatabaseName = pdfaggregationdatabase_new_s3_names services.pdfaggregation.controller.db.testDatabaseName = pdfaggregationdatabase_new_s3_names
services.pdfaggregation.controller.baseTargetLocation = /tmp/ services.pdfaggregation.controller.baseFilesLocation = tmp/
services.pdfaggregation.controller.maxAttemptsPerRecord = 3 services.pdfaggregation.controller.maxAttemptsPerRecord = 3
services.pdfaggregation.controller.assignmentLimit = 10000 services.pdfaggregation.controller.assignmentLimit = 10000
@ -60,7 +60,7 @@ hdfs.httpAuth=
hdfs.userName= hdfs.userName=
hdfs.password= hdfs.password=
output.parquetLocalDirectoryPath=parquetFiles/ output.parquetLocalDirectoryPath=${services.pdfaggregation.controller.baseFilesLocation}parquetFiles/
hdfs.parquetRemoteBaseDirectoryPath=/tmp/parquet_uploads/ hdfs.parquetRemoteBaseDirectoryPath=/tmp/parquet_uploads/
## MULTIPART (MultipartProperties) ## MULTIPART (MultipartProperties)