michal.politowski
/
UrlsController
forked from lsmyrnaios/UrlsController
1
0
Fork 0
Code Pull Requests Activity

Code polishing.

This commit is contained in:
Lampros Smyrnaios 2023-09-05 15:20:03 +03:00
parent 199105f7f1
commit 4014d1eabb
1 changed files with 6 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/main/java/eu/openaire/urls_controller/util/FileUtils.java
View File

@ -181,7 +181,7 @@ public class FileUtils {
}
public static DecimalFormat df = new DecimalFormat("0.00");
public static final DecimalFormat df = new DecimalFormat("0.00");
// The following regex might be usefull in a future scenario. It extracts the "plain-filename" and "file-ID" and the "file-extension".
// Possible full-filenames are: "path1/path2/ID.pdf", "ID2.pdf", "path1/path2/ID(12).pdf", "ID2(25).pdf"
@ -215,7 +215,7 @@ public class FileUtils {
final String getFileLocationForHashQuery = "select `location` from " + DatabaseConnector.databaseName + ".payload" + (isTestEnvironment ? "_aggregated" : "") + " where `hash` = ? limit 1";
final int[] hashArgType = new int[] {Types.VARCHAR};
List<Callable<Void>> callableTasks = new ArrayList<>(6);
final List<Callable<Void>> callableTasks = new ArrayList<>(6);
for ( UrlReport urlReport : urlReports )
{
@ -388,14 +388,8 @@ public class FileUtils {
}
// Save and decompress the zstd file. Iterate over the PDFs and upload each one of them and get the S3-Url.
File zstdFile = new File(zstdFileFullPath);
if ( ! saveArchive(conn, zstdFile) )
return false;
//logger.trace("The zstd file has been saved: " + zstdFileFullPath); // DEBUG!
return saveArchive(conn, new File(zstdFileFullPath));
// We do not call "conn.disconnect()", since more request are about to be made to the worker, in the near future.
return true;
}
@ -536,7 +530,7 @@ public class FileUtils {
logger.error("Could not upload the file \"" + fileName + "\" to the S3 ObjectStore!", e);
}
// Else, the record will have its file-data set to "null", in the end of this method.
}
}//end filenames-for-loop
//logger.debug("Finished uploading " + numUploadedFiles + " full-texts (out of " + (fileNames.length -2) + " distinct files) from assignments_" + assignmentsBatchCounter + ", batch_" + batchCounter + " on S3-ObjectStore.");
// (fileNames.length -2) --> minus the zstd and the tar files.
@ -566,7 +560,7 @@ public class FileUtils {
public String getMessageFromResponseBody(HttpURLConnection conn, boolean isError) {
final StringBuilder msgStrB = new StringBuilder(500);
try ( BufferedReader br = new BufferedReader(new InputStreamReader((isError ? conn.getErrorStream() : conn.getInputStream()))) ) { // Try-with-resources
try ( BufferedReader br = new BufferedReader(new InputStreamReader((isError ? conn.getErrorStream() : conn.getInputStream()))) ) {
String inputLine;
while ( (inputLine = br.readLine()) != null )
{
@ -590,7 +584,7 @@ public class FileUtils {
if ( endingIndex > numAllFullTexts ) // This might be the case, when the "numAllFullTexts" is too small.
endingIndex = numAllFullTexts;
List<String> fileNamesOfCurBatch = new ArrayList<>(numOfFullTextsPerBatch);
final List<String> fileNamesOfCurBatch = new ArrayList<>(numOfFullTextsPerBatch);
for ( int i = initialIndex; i < endingIndex; ++i ) {
try {
fileNamesOfCurBatch.add(allFileNames.get(i));