From 71f6b46130e9199b88e5e91b3396a7f985b8032d Mon Sep 17 00:00:00 2001 From: LSmyrnaios Date: Mon, 14 Feb 2022 12:36:00 +0200 Subject: [PATCH] - In case of an error when creating the "current_assignment" table (e.g out of memory in the backend database server), check for partial-creation and drop it. Also, in any case, before we drop this table, now check if it exists firsts (in general it should always exist, unless the creation results in an error and the table was not created at all). - Fix an error-message. - Update dependencies. - Code cleanup. --- build.gradle | 8 +--- gradle/wrapper/gradle-wrapper.properties | 2 +- installAndRun.sh | 4 +- .../controllers/UrlController.java | 39 ++++++++++--------- 4 files changed, 25 insertions(+), 28 deletions(-) diff --git a/build.gradle b/build.gradle index 52075eb..bfd0f9f 100644 --- a/build.gradle +++ b/build.gradle @@ -45,7 +45,7 @@ dependencies { // https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0' - implementation 'io.minio:minio:8.3.5' + implementation 'io.minio:minio:8.3.6' // https://mvnrepository.com/artifact/com.squareup.okhttp3/okhttp implementation group: 'com.squareup.okhttp3', name: 'okhttp', version: '4.9.3' @@ -69,12 +69,6 @@ tasks.withType(JavaExec) { jvmArgs = ['-Xms512m', '-Xmx8g'] } - -configurations { - // Eliminates slf4j-log4j12 - all*.exclude group: 'org.slf4j', module: 'slf4j-log4j12' -} - test { useJUnitPlatform() } diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 2e6e589..41dfb87 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.4-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/installAndRun.sh b/installAndRun.sh index 28201e1..18e4437 100755 --- a/installAndRun.sh +++ b/installAndRun.sh @@ -17,7 +17,7 @@ if [[ justInstall -eq 1 && shouldRunInDocker -eq 1 ]]; then justInstall=0 fi -gradleVersion="7.3.3" +gradleVersion="7.4" if [[ justInstall -eq 0 ]]; then @@ -47,7 +47,7 @@ if [[ justInstall -eq 0 ]]; then sudo docker build -t "${dockerImage}" . echo -e "\nPushing docker image.. (the account password is required)..\n" (sudo docker login -u "${username}" && sudo docker push "${dockerImage}") || true - (sudo mkdir -p "$HOME"/tmp/config && sudo cp ./src/main/resources/application.properties "$HOME"/tmp/config) || true + (sudo mkdir -p "$HOME"/tmp/config && sudo cp ./src/main/resources/application.properties "$HOME"/tmp/config) || true # This also replaces an existing "application.properties". sudo docker run -d --mount type=bind,source="$HOME"/tmp/config,target=/mnt/config -p 1880:1880 "${dockerImage}" && echo "The docker container started running." # Run in "detached mode" (in the background). fi diff --git a/src/main/java/eu/openaire/urls_controller/controllers/UrlController.java b/src/main/java/eu/openaire/urls_controller/controllers/UrlController.java index 049ddd5..fce5c32 100644 --- a/src/main/java/eu/openaire/urls_controller/controllers/UrlController.java +++ b/src/main/java/eu/openaire/urls_controller/controllers/UrlController.java @@ -110,9 +110,12 @@ public class UrlController { try { jdbcTemplate.execute(createCurrentAssignmentsQuery); - } catch (Exception sqle) { + } catch (Exception e) { + String errorMsg = ImpalaConnector.handleQueryException("createCurrentAssignmentsQuery", createCurrentAssignmentsQuery, e); + String tmpErrMsg = dropCurrentAssignmentTable(); // The table may be partially created, e.g. in case of an "out of memory" error in the database-server, during the creation, resulting in an empty table (yes it has happened). + if ( tmpErrMsg != null ) + errorMsg += "\n" + tmpErrMsg; ImpalaConnector.databaseLock.unlock(); - String errorMsg = ImpalaConnector.handleQueryException("createCurrentAssignmentsQuery", createCurrentAssignmentsQuery, sqle); return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorMsg); } @@ -283,8 +286,8 @@ public class UrlController { payload.getLocation(), payload.getProvenance()}; jdbcTemplate.update(insertIntoPayloadBaseQuery, args, payloadArgTypes); - } catch (Exception sqle) { - logger.error("Problem when executing the \"insertIntoPayloadBaseQuery\": ", sqle); + } catch (Exception e) { + logger.error("Problem when executing the \"insertIntoPayloadBaseQuery\": " + e.getMessage()); failedCount.incrementAndGet(); } } @@ -310,8 +313,8 @@ public class UrlController { urlReport.getStatus().toString(), String.valueOf(error.getType()), error.getMessage()}; jdbcTemplate.update(insertIntoAttemptBaseQuery, args, attemptArgTypes); - } catch (Exception sqle) { - logger.error("Problem when executing the \"insertIntoAttemptBaseQuery\": " + sqle.getMessage()); + } catch (Exception e) { + logger.error("Problem when executing the \"insertIntoAttemptBaseQuery\": " + e.getMessage()); failedCount.incrementAndGet(); } } @@ -333,7 +336,7 @@ public class UrlController { } int failedQueries = failedCount.get(); - String failedQueriesMsg = failedQueries + " out of " + urlReports.size() + " failed to be processed!"; + String failedQueriesMsg = failedQueries + " out of " + (urlReports.size() *2) + " failed to be processed!"; logger.debug("Finished inserting the payloads and the attempts into the \"payload\" and \"attempt\" tables" + ((failedQueries > 0) ? (", although " + failedQueriesMsg) : ".") + " Going to merge the parquet files for those tables."); @@ -365,6 +368,17 @@ public class UrlController { } + private String dropCurrentAssignmentTable() { + String dropCurrentAssignmentsQuery = "DROP TABLE IF EXISTS " + databaseName + ".current_assignment PURGE"; + try { + jdbcTemplate.execute(dropCurrentAssignmentsQuery); + return null; + } catch (Exception e) { + return ImpalaConnector.handleQueryException("dropCurrentAssignmentsQuery", dropCurrentAssignmentsQuery, e); + } + } + + // The "batchExecute" does not work in this Impala-Database, so this is a "giant-query" solution. // Note: this causes an "Out of memory"-ERROR in the current version of the Impala JDBC driver. If a later version is provided, then this code should be tested. private static PreparedStatement constructLargeInsertQuery(Connection con, String baseInsertQuery, int dataSize, int numParamsPerRow) throws RuntimeException { @@ -393,15 +407,4 @@ public class UrlController { return preparedInsertStatement; } - - private String dropCurrentAssignmentTable() { - String dropCurrentAssignmentsQuery = "DROP TABLE " + databaseName + ".current_assignment PURGE"; - try { - jdbcTemplate.execute(dropCurrentAssignmentsQuery); - return null; - } catch (Exception e) { - return ImpalaConnector.handleQueryException("dropCurrentAssignmentsQuery", dropCurrentAssignmentsQuery, e); - } - } - }