lsmyrnaios
/
UrlsController
1
0
Fork 1
Code Issues Pull Requests Releases 11 Wiki Activity

Code polishing.

This commit is contained in:
Lampros Smyrnaios 2023-09-05 15:20:03 +03:00
parent 199105f7f1
commit 4014d1eabb
1 changed files with 6 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/main/java/eu/openaire/urls_controller/util/FileUtils.java
View File

@ -181,7 +181,7 @@ public class FileUtils {
} }
public static DecimalFormat df = new DecimalFormat("0.00"); public static final DecimalFormat df = new DecimalFormat("0.00");
// The following regex might be usefull in a future scenario. It extracts the "plain-filename" and "file-ID" and the "file-extension". // The following regex might be usefull in a future scenario. It extracts the "plain-filename" and "file-ID" and the "file-extension".
// Possible full-filenames are: "path1/path2/ID.pdf", "ID2.pdf", "path1/path2/ID(12).pdf", "ID2(25).pdf" // Possible full-filenames are: "path1/path2/ID.pdf", "ID2.pdf", "path1/path2/ID(12).pdf", "ID2(25).pdf"
@ -215,7 +215,7 @@ public class FileUtils {
final String getFileLocationForHashQuery = "select `location` from " + DatabaseConnector.databaseName + ".payload" + (isTestEnvironment ? "_aggregated" : "") + " where `hash` = ? limit 1"; final String getFileLocationForHashQuery = "select `location` from " + DatabaseConnector.databaseName + ".payload" + (isTestEnvironment ? "_aggregated" : "") + " where `hash` = ? limit 1";
final int[] hashArgType = new int[] {Types.VARCHAR}; final int[] hashArgType = new int[] {Types.VARCHAR};
List<Callable<Void>> callableTasks = new ArrayList<>(6); final List<Callable<Void>> callableTasks = new ArrayList<>(6);
for ( UrlReport urlReport : urlReports ) for ( UrlReport urlReport : urlReports )
{ {
@ -388,14 +388,8 @@ public class FileUtils {
} }
// Save and decompress the zstd file. Iterate over the PDFs and upload each one of them and get the S3-Url. // Save and decompress the zstd file. Iterate over the PDFs and upload each one of them and get the S3-Url.
File zstdFile = new File(zstdFileFullPath); return saveArchive(conn, new File(zstdFileFullPath));
if ( ! saveArchive(conn, zstdFile) )
return false;
//logger.trace("The zstd file has been saved: " + zstdFileFullPath); // DEBUG!
// We do not call "conn.disconnect()", since more request are about to be made to the worker, in the near future. // We do not call "conn.disconnect()", since more request are about to be made to the worker, in the near future.
return true;
} }
@ -536,7 +530,7 @@ public class FileUtils {
logger.error("Could not upload the file \"" + fileName + "\" to the S3 ObjectStore!", e); logger.error("Could not upload the file \"" + fileName + "\" to the S3 ObjectStore!", e);
} }
// Else, the record will have its file-data set to "null", in the end of this method. // Else, the record will have its file-data set to "null", in the end of this method.
} }//end filenames-for-loop
//logger.debug("Finished uploading " + numUploadedFiles + " full-texts (out of " + (fileNames.length -2) + " distinct files) from assignments_" + assignmentsBatchCounter + ", batch_" + batchCounter + " on S3-ObjectStore."); //logger.debug("Finished uploading " + numUploadedFiles + " full-texts (out of " + (fileNames.length -2) + " distinct files) from assignments_" + assignmentsBatchCounter + ", batch_" + batchCounter + " on S3-ObjectStore.");
// (fileNames.length -2) --> minus the zstd and the tar files. // (fileNames.length -2) --> minus the zstd and the tar files.
@ -566,7 +560,7 @@ public class FileUtils {
public String getMessageFromResponseBody(HttpURLConnection conn, boolean isError) { public String getMessageFromResponseBody(HttpURLConnection conn, boolean isError) {
final StringBuilder msgStrB = new StringBuilder(500); final StringBuilder msgStrB = new StringBuilder(500);
try ( BufferedReader br = new BufferedReader(new InputStreamReader((isError ? conn.getErrorStream() : conn.getInputStream()))) ) { // Try-with-resources try ( BufferedReader br = new BufferedReader(new InputStreamReader((isError ? conn.getErrorStream() : conn.getInputStream()))) ) {
String inputLine; String inputLine;
while ( (inputLine = br.readLine()) != null ) while ( (inputLine = br.readLine()) != null )
{ {
@ -590,7 +584,7 @@ public class FileUtils {
if ( endingIndex > numAllFullTexts ) // This might be the case, when the "numAllFullTexts" is too small. if ( endingIndex > numAllFullTexts ) // This might be the case, when the "numAllFullTexts" is too small.
endingIndex = numAllFullTexts; endingIndex = numAllFullTexts;
List<String> fileNamesOfCurBatch = new ArrayList<>(numOfFullTextsPerBatch); final List<String> fileNamesOfCurBatch = new ArrayList<>(numOfFullTextsPerBatch);
for ( int i = initialIndex; i < endingIndex; ++i ) { for ( int i = initialIndex; i < endingIndex; ++i ) {
try { try {
fileNamesOfCurBatch.add(allFileNames.get(i)); fileNamesOfCurBatch.add(allFileNames.get(i));