- Increase the waiting-time before checking the docker containers' status, in order to catch configuration-crashes.

- Code polishing.

installAndRun.sh

@@ -52,9 +52,9 @@ if [[ justInstall -eq 0 ]]; then
     sudo docker --version || handle_error "Docker was not found!" 3
     (sudo mkdir -p "$HOME"/tmp/config && sudo cp ./src/main/resources/application.yml "$HOME"/tmp/config) || true  # This also replaces an existing "application.yml".
     sudo mkdir -p "$HOME"/logs || true
-    (sudo docker compose up --build -d && echo -e "The Urls_Controller, Prometheus and Grafana docker-containers started running.\nWaiting 30 seconds before getting their status.") || handle_error "Could not list docker containers!" 4
+    (sudo docker compose up --build -d && echo -e "The Urls_Controller, Prometheus and Grafana docker-containers started running.\nWaiting 65 seconds before getting their status.") || handle_error "Could not list docker containers!" 4
     # Run in "detached mode" -d (in the background).
-    sleep 30
+    sleep 65
     sudo docker ps -a || handle_error "Could not get the status of docker-containers!" 5
     # Using -a to get the status of failed containers as well.
   fi

src/main/java/eu/openaire/urls_controller/configuration/ImpalaConnector.java

@@ -39,8 +39,8 @@ public class ImpalaConnector {
 
     @PostConstruct
     public void init() {
-        int mb = 1048576;
+        int OneMb = 1048576;
-        logger.info("Max available memory to the Controller: " +  (Runtime.getRuntime().maxMemory() / mb) + " Mb.");
+        logger.info("Max available memory to the Controller: " +  (Runtime.getRuntime().maxMemory() / OneMb) + " Mb.");
         try {
             boolean supportsBatchUpdates = jdbcTemplate.getDataSource().getConnection().getMetaData().supportsBatchUpdates();
             logger.info("The database " + (supportsBatchUpdates ? "supports" : "does not support") + " \"BatchUpdates\"!");

src/main/java/eu/openaire/urls_controller/services/UrlsServiceImpl.java

@@ -211,7 +211,6 @@ public class UrlsServiceImpl implements UrlsService {
 
     public ResponseEntity<?> addWorkerReport(String curWorkerId, long curReportAssignments, List<UrlReport> urlReports, int sizeOfUrlReports, HttpServletRequest request)
     {
-
         // Before continuing with inserts, take and upload the fullTexts from the Worker. Also, update the file-"location".
         FileUtils.UploadFullTextsResponse uploadFullTextsResponse = fileUtils.getAndUploadFullTexts(urlReports, request, curReportAssignments, curWorkerId);
         if ( uploadFullTextsResponse == FileUtils.UploadFullTextsResponse.databaseError ) {
@@ -246,7 +245,7 @@ public class UrlsServiceImpl implements UrlsService {
         boolean hasAttemptParquetFileProblem = false;
         boolean hasPayloadParquetFileProblem = false;
 
-        try {	// Invoke all the tasks and wait for them to finish before moving to the next batch.
+        try {	// Invoke all the tasks and wait for them to finish.
             List<Future<ParquetReport>> futures = insertsExecutor.invokeAll(callableTasks);
 
             SumParquetSuccess sumParquetSuccess = parquetFileUtils.checkParquetFilesSuccess(futures);

src/main/java/eu/openaire/urls_controller/util/ParquetFileUtils.java

@@ -112,12 +112,13 @@ public class ParquetFileUtils {
         else
             this.parquetBaseLocalDirectoryPath = parquetBaseDirectoryPath;
 
+        // Create the local parquet file base directory, if it does not exist.
         java.nio.file.Path parquetDirPath = Paths.get(this.parquetBaseLocalDirectoryPath);
         if ( !Files.isDirectory(parquetDirPath) )
             Files.createDirectories(parquetDirPath);
 
         // Create the remote directories for uploading the parquet-files, if those directories do not exist.
-        // The limited-permissions user in use, does not have permission to acces other users' created directories, so we have to make sure it creates its own.
+        // The limited-permissions user in use, does not have permission to access other users' created directories, so we have to make sure it creates its own.
 
         if ( !hdfsParquetBaseDir.endsWith("/") )
             hdfsParquetBaseDir += "/";
@@ -215,7 +216,7 @@ public class ParquetFileUtils {
 
             try {
                 record = new GenericData.Record(attemptsSchema);
-                record.put("id",payload.getId());
+                record.put("id", payload.getId());
                 record.put("original_url", payload.getOriginal_url());
                 Timestamp timestamp = payload.getTimestamp_acquired();
                 record.put("date", (timestamp != null) ? timestamp.getTime() : System.currentTimeMillis());