From 4dc34429f8ac0e18e9eb114d08e948f4b4de4c76 Mon Sep 17 00:00:00 2001 From: LSmyrnaios Date: Mon, 10 Apr 2023 22:28:53 +0300 Subject: [PATCH] - Increase the waiting-time before checking the docker containers' status, in order to catch configuration-crashes. - Code polishing. --- installAndRun.sh | 4 ++-- .../urls_controller/configuration/ImpalaConnector.java | 4 ++-- .../openaire/urls_controller/services/UrlsServiceImpl.java | 3 +-- .../eu/openaire/urls_controller/util/ParquetFileUtils.java | 5 +++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/installAndRun.sh b/installAndRun.sh index 0e41ab8..5fa9725 100755 --- a/installAndRun.sh +++ b/installAndRun.sh @@ -52,9 +52,9 @@ if [[ justInstall -eq 0 ]]; then sudo docker --version || handle_error "Docker was not found!" 3 (sudo mkdir -p "$HOME"/tmp/config && sudo cp ./src/main/resources/application.yml "$HOME"/tmp/config) || true # This also replaces an existing "application.yml". sudo mkdir -p "$HOME"/logs || true - (sudo docker compose up --build -d && echo -e "The Urls_Controller, Prometheus and Grafana docker-containers started running.\nWaiting 30 seconds before getting their status.") || handle_error "Could not list docker containers!" 4 + (sudo docker compose up --build -d && echo -e "The Urls_Controller, Prometheus and Grafana docker-containers started running.\nWaiting 65 seconds before getting their status.") || handle_error "Could not list docker containers!" 4 # Run in "detached mode" -d (in the background). - sleep 30 + sleep 65 sudo docker ps -a || handle_error "Could not get the status of docker-containers!" 5 # Using -a to get the status of failed containers as well. fi diff --git a/src/main/java/eu/openaire/urls_controller/configuration/ImpalaConnector.java b/src/main/java/eu/openaire/urls_controller/configuration/ImpalaConnector.java index bc580e3..9e562ee 100644 --- a/src/main/java/eu/openaire/urls_controller/configuration/ImpalaConnector.java +++ b/src/main/java/eu/openaire/urls_controller/configuration/ImpalaConnector.java @@ -39,8 +39,8 @@ public class ImpalaConnector { @PostConstruct public void init() { - int mb = 1048576; - logger.info("Max available memory to the Controller: " + (Runtime.getRuntime().maxMemory() / mb) + " Mb."); + int OneMb = 1048576; + logger.info("Max available memory to the Controller: " + (Runtime.getRuntime().maxMemory() / OneMb) + " Mb."); try { boolean supportsBatchUpdates = jdbcTemplate.getDataSource().getConnection().getMetaData().supportsBatchUpdates(); logger.info("The database " + (supportsBatchUpdates ? "supports" : "does not support") + " \"BatchUpdates\"!"); diff --git a/src/main/java/eu/openaire/urls_controller/services/UrlsServiceImpl.java b/src/main/java/eu/openaire/urls_controller/services/UrlsServiceImpl.java index a71e53c..3e0bed3 100644 --- a/src/main/java/eu/openaire/urls_controller/services/UrlsServiceImpl.java +++ b/src/main/java/eu/openaire/urls_controller/services/UrlsServiceImpl.java @@ -211,7 +211,6 @@ public class UrlsServiceImpl implements UrlsService { public ResponseEntity addWorkerReport(String curWorkerId, long curReportAssignments, List urlReports, int sizeOfUrlReports, HttpServletRequest request) { - // Before continuing with inserts, take and upload the fullTexts from the Worker. Also, update the file-"location". FileUtils.UploadFullTextsResponse uploadFullTextsResponse = fileUtils.getAndUploadFullTexts(urlReports, request, curReportAssignments, curWorkerId); if ( uploadFullTextsResponse == FileUtils.UploadFullTextsResponse.databaseError ) { @@ -246,7 +245,7 @@ public class UrlsServiceImpl implements UrlsService { boolean hasAttemptParquetFileProblem = false; boolean hasPayloadParquetFileProblem = false; - try { // Invoke all the tasks and wait for them to finish before moving to the next batch. + try { // Invoke all the tasks and wait for them to finish. List> futures = insertsExecutor.invokeAll(callableTasks); SumParquetSuccess sumParquetSuccess = parquetFileUtils.checkParquetFilesSuccess(futures); diff --git a/src/main/java/eu/openaire/urls_controller/util/ParquetFileUtils.java b/src/main/java/eu/openaire/urls_controller/util/ParquetFileUtils.java index d8c6788..1c37e60 100644 --- a/src/main/java/eu/openaire/urls_controller/util/ParquetFileUtils.java +++ b/src/main/java/eu/openaire/urls_controller/util/ParquetFileUtils.java @@ -112,12 +112,13 @@ public class ParquetFileUtils { else this.parquetBaseLocalDirectoryPath = parquetBaseDirectoryPath; + // Create the local parquet file base directory, if it does not exist. java.nio.file.Path parquetDirPath = Paths.get(this.parquetBaseLocalDirectoryPath); if ( !Files.isDirectory(parquetDirPath) ) Files.createDirectories(parquetDirPath); // Create the remote directories for uploading the parquet-files, if those directories do not exist. - // The limited-permissions user in use, does not have permission to acces other users' created directories, so we have to make sure it creates its own. + // The limited-permissions user in use, does not have permission to access other users' created directories, so we have to make sure it creates its own. if ( !hdfsParquetBaseDir.endsWith("/") ) hdfsParquetBaseDir += "/"; @@ -215,7 +216,7 @@ public class ParquetFileUtils { try { record = new GenericData.Record(attemptsSchema); - record.put("id",payload.getId()); + record.put("id", payload.getId()); record.put("original_url", payload.getOriginal_url()); Timestamp timestamp = payload.getTimestamp_acquired(); record.put("date", (timestamp != null) ? timestamp.getTime() : System.currentTimeMillis());