diff --git a/build.gradle b/build.gradle index 52075eb..bfd0f9f 100644 --- a/build.gradle +++ b/build.gradle @@ -45,7 +45,7 @@ dependencies { // https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0' - implementation 'io.minio:minio:8.3.5' + implementation 'io.minio:minio:8.3.6' // https://mvnrepository.com/artifact/com.squareup.okhttp3/okhttp implementation group: 'com.squareup.okhttp3', name: 'okhttp', version: '4.9.3' @@ -69,12 +69,6 @@ tasks.withType(JavaExec) { jvmArgs = ['-Xms512m', '-Xmx8g'] } - -configurations { - // Eliminates slf4j-log4j12 - all*.exclude group: 'org.slf4j', module: 'slf4j-log4j12' -} - test { useJUnitPlatform() } diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 2e6e589..41dfb87 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.4-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/installAndRun.sh b/installAndRun.sh index 28201e1..18e4437 100755 --- a/installAndRun.sh +++ b/installAndRun.sh @@ -17,7 +17,7 @@ if [[ justInstall -eq 1 && shouldRunInDocker -eq 1 ]]; then justInstall=0 fi -gradleVersion="7.3.3" +gradleVersion="7.4" if [[ justInstall -eq 0 ]]; then @@ -47,7 +47,7 @@ if [[ justInstall -eq 0 ]]; then sudo docker build -t "${dockerImage}" . echo -e "\nPushing docker image.. (the account password is required)..\n" (sudo docker login -u "${username}" && sudo docker push "${dockerImage}") || true - (sudo mkdir -p "$HOME"/tmp/config && sudo cp ./src/main/resources/application.properties "$HOME"/tmp/config) || true + (sudo mkdir -p "$HOME"/tmp/config && sudo cp ./src/main/resources/application.properties "$HOME"/tmp/config) || true # This also replaces an existing "application.properties". sudo docker run -d --mount type=bind,source="$HOME"/tmp/config,target=/mnt/config -p 1880:1880 "${dockerImage}" && echo "The docker container started running." # Run in "detached mode" (in the background). fi diff --git a/src/main/java/eu/openaire/urls_controller/controllers/UrlController.java b/src/main/java/eu/openaire/urls_controller/controllers/UrlController.java index 049ddd5..fce5c32 100644 --- a/src/main/java/eu/openaire/urls_controller/controllers/UrlController.java +++ b/src/main/java/eu/openaire/urls_controller/controllers/UrlController.java @@ -110,9 +110,12 @@ public class UrlController { try { jdbcTemplate.execute(createCurrentAssignmentsQuery); - } catch (Exception sqle) { + } catch (Exception e) { + String errorMsg = ImpalaConnector.handleQueryException("createCurrentAssignmentsQuery", createCurrentAssignmentsQuery, e); + String tmpErrMsg = dropCurrentAssignmentTable(); // The table may be partially created, e.g. in case of an "out of memory" error in the database-server, during the creation, resulting in an empty table (yes it has happened). + if ( tmpErrMsg != null ) + errorMsg += "\n" + tmpErrMsg; ImpalaConnector.databaseLock.unlock(); - String errorMsg = ImpalaConnector.handleQueryException("createCurrentAssignmentsQuery", createCurrentAssignmentsQuery, sqle); return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorMsg); } @@ -283,8 +286,8 @@ public class UrlController { payload.getLocation(), payload.getProvenance()}; jdbcTemplate.update(insertIntoPayloadBaseQuery, args, payloadArgTypes); - } catch (Exception sqle) { - logger.error("Problem when executing the \"insertIntoPayloadBaseQuery\": ", sqle); + } catch (Exception e) { + logger.error("Problem when executing the \"insertIntoPayloadBaseQuery\": " + e.getMessage()); failedCount.incrementAndGet(); } } @@ -310,8 +313,8 @@ public class UrlController { urlReport.getStatus().toString(), String.valueOf(error.getType()), error.getMessage()}; jdbcTemplate.update(insertIntoAttemptBaseQuery, args, attemptArgTypes); - } catch (Exception sqle) { - logger.error("Problem when executing the \"insertIntoAttemptBaseQuery\": " + sqle.getMessage()); + } catch (Exception e) { + logger.error("Problem when executing the \"insertIntoAttemptBaseQuery\": " + e.getMessage()); failedCount.incrementAndGet(); } } @@ -333,7 +336,7 @@ public class UrlController { } int failedQueries = failedCount.get(); - String failedQueriesMsg = failedQueries + " out of " + urlReports.size() + " failed to be processed!"; + String failedQueriesMsg = failedQueries + " out of " + (urlReports.size() *2) + " failed to be processed!"; logger.debug("Finished inserting the payloads and the attempts into the \"payload\" and \"attempt\" tables" + ((failedQueries > 0) ? (", although " + failedQueriesMsg) : ".") + " Going to merge the parquet files for those tables."); @@ -365,6 +368,17 @@ public class UrlController { } + private String dropCurrentAssignmentTable() { + String dropCurrentAssignmentsQuery = "DROP TABLE IF EXISTS " + databaseName + ".current_assignment PURGE"; + try { + jdbcTemplate.execute(dropCurrentAssignmentsQuery); + return null; + } catch (Exception e) { + return ImpalaConnector.handleQueryException("dropCurrentAssignmentsQuery", dropCurrentAssignmentsQuery, e); + } + } + + // The "batchExecute" does not work in this Impala-Database, so this is a "giant-query" solution. // Note: this causes an "Out of memory"-ERROR in the current version of the Impala JDBC driver. If a later version is provided, then this code should be tested. private static PreparedStatement constructLargeInsertQuery(Connection con, String baseInsertQuery, int dataSize, int numParamsPerRow) throws RuntimeException { @@ -393,15 +407,4 @@ public class UrlController { return preparedInsertStatement; } - - private String dropCurrentAssignmentTable() { - String dropCurrentAssignmentsQuery = "DROP TABLE " + databaseName + ".current_assignment PURGE"; - try { - jdbcTemplate.execute(dropCurrentAssignmentsQuery); - return null; - } catch (Exception e) { - return ImpalaConnector.handleQueryException("dropCurrentAssignmentsQuery", dropCurrentAssignmentsQuery, e); - } - } - }