From 6891c467d4562fa73dfda66ac9e2b145d4abbe0e Mon Sep 17 00:00:00 2001
From: LSmyrnaios <lsmyrnaios@gmail.com>
Date: Wed, 13 Sep 2023 15:29:30 +0300
Subject: [PATCH] - Avoid displaying a warning for the "test" HDFS directory,
 when the Controller is running in PROD mode. - Add a missing change for the
 optimization of reading files. - Update dependencies.

---
 build.gradle                                                 | 4 ++--
 .../openaire/urls_controller/components/ScheduledTasks.java  | 2 +-
 .../openaire/urls_controller/services/UrlsServiceImpl.java   | 5 ++++-
 .../eu/openaire/urls_controller/util/ParquetFileUtils.java   | 2 +-
 4 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/build.gradle b/build.gradle
index 9954268..f09e0df 100644
--- a/build.gradle
+++ b/build.gradle
@@ -49,7 +49,7 @@ dependencies {
     implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.13.0'
 
     // https://mvnrepository.com/artifact/org.apache.commons/commons-compress
-    implementation("org.apache.commons:commons-compress:1.23.0") {
+    implementation("org.apache.commons:commons-compress:1.24.0") {
         exclude group: 'com.github.luben', module: 'zstd-jni'
     }
     implementation 'com.github.luben:zstd-jni:1.5.5-5'  // Even though this is part of the above dependency, the Apache commons rarely updates it, while the zstd team makes improvements very often.
@@ -120,7 +120,7 @@ dependencies {
 
 
     // https://mvnrepository.com/artifact/io.micrometer/micrometer-registry-prometheus
-    runtimeOnly 'io.micrometer:micrometer-registry-prometheus:1.11.3'
+    runtimeOnly 'io.micrometer:micrometer-registry-prometheus:1.11.4'
 
     testImplementation 'org.springframework.security:spring-security-test'
     testImplementation "org.springframework.boot:spring-boot-starter-test"
diff --git a/src/main/java/eu/openaire/urls_controller/components/ScheduledTasks.java b/src/main/java/eu/openaire/urls_controller/components/ScheduledTasks.java
index 77467b6..fd6670d 100644
--- a/src/main/java/eu/openaire/urls_controller/components/ScheduledTasks.java
+++ b/src/main/java/eu/openaire/urls_controller/components/ScheduledTasks.java
@@ -378,7 +378,7 @@ public class ScheduledTasks {
         logger.debug("Going to load and parse the workerReport: " + workerReportName);
 
         // Load the file's json content into a "WorkerReport" object.
-        try ( BufferedReader bfRead = new BufferedReader(new FileReader(workerReportFile)) ) {  // The default size is sufficient here.
+        try ( BufferedReader bfRead = new BufferedReader(new FileReader(workerReportFile), FileUtils.halfMb) ) {
             String line;
             while ( (line = bfRead.readLine()) != null )    // The line, without any line-termination-characters.
                 jsonStringBuilder.append(line).append("\n");
diff --git a/src/main/java/eu/openaire/urls_controller/services/UrlsServiceImpl.java b/src/main/java/eu/openaire/urls_controller/services/UrlsServiceImpl.java
index a49f906..da81187 100644
--- a/src/main/java/eu/openaire/urls_controller/services/UrlsServiceImpl.java
+++ b/src/main/java/eu/openaire/urls_controller/services/UrlsServiceImpl.java
@@ -28,7 +28,10 @@ import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
 import java.sql.Timestamp;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.HashMap;
+import java.util.List;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
diff --git a/src/main/java/eu/openaire/urls_controller/util/ParquetFileUtils.java b/src/main/java/eu/openaire/urls_controller/util/ParquetFileUtils.java
index 7e09c1c..311e10a 100644
--- a/src/main/java/eu/openaire/urls_controller/util/ParquetFileUtils.java
+++ b/src/main/java/eu/openaire/urls_controller/util/ParquetFileUtils.java
@@ -558,7 +558,7 @@ public class ParquetFileUtils {
                         foundPayloadsAggregatedDir = true;
                     else if ( dirPath.equals("payloads_bulk_import") )
                         foundPayloadsBulkImportDir = true;
-                    else
+                    else if ( ! dirPath.equals("test") )    // The "test" directory helps with testing the service, without interfering with the production directories.
                         logger.warn("Unknown remote parquet HDFS-directory found: " + dirPath);
                 }
             } catch (JSONException je) {    // In case any of the above "json-keys" was not found.