- In case of an error when creating the "current_assignment" table (e.g out of memory in the backend database server), check for partial-creation and drop it. Also, in any case, before we drop this table, now check if it exists firsts (in general it should always exist, unless the creation results in an error and the table was not created at all).

- Fix an error-message.
- Update dependencies.
- Code cleanup.

build.gradle

@@ -45,7 +45,7 @@ dependencies {
     // https://mvnrepository.com/artifact/org.apache.commons/commons-lang3
     implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0'
 
-    implementation 'io.minio:minio:8.3.5'
+    implementation 'io.minio:minio:8.3.6'
 
     // https://mvnrepository.com/artifact/com.squareup.okhttp3/okhttp
     implementation group: 'com.squareup.okhttp3', name: 'okhttp', version: '4.9.3'
@@ -69,12 +69,6 @@ tasks.withType(JavaExec) {
     jvmArgs = ['-Xms512m', '-Xmx8g']
 }
 
-
-configurations {
-    // Eliminates slf4j-log4j12
-    all*.exclude group: 'org.slf4j', module: 'slf4j-log4j12'
-}
-
 test {
     useJUnitPlatform()
 }

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.4-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

installAndRun.sh

@@ -17,7 +17,7 @@ if [[ justInstall -eq 1 && shouldRunInDocker -eq 1 ]]; then
   justInstall=0
 fi
 
-gradleVersion="7.3.3"
+gradleVersion="7.4"
 
 if [[ justInstall -eq 0 ]]; then
 
@@ -47,7 +47,7 @@ if [[ justInstall -eq 0 ]]; then
     sudo docker build -t "${dockerImage}" .
     echo -e "\nPushing docker image.. (the account password is required)..\n"
     (sudo docker login -u "${username}" && sudo docker push "${dockerImage}") || true
-    (sudo mkdir -p "$HOME"/tmp/config && sudo cp ./src/main/resources/application.properties "$HOME"/tmp/config) || true
+    (sudo mkdir -p "$HOME"/tmp/config && sudo cp ./src/main/resources/application.properties "$HOME"/tmp/config) || true  # This also replaces an existing "application.properties".
     sudo docker run -d --mount type=bind,source="$HOME"/tmp/config,target=/mnt/config -p 1880:1880 "${dockerImage}" && echo "The docker container started running."
     # Run in "detached mode" (in the background).
   fi

src/main/java/eu/openaire/urls_controller/controllers/UrlController.java

@@ -110,9 +110,12 @@ public class UrlController {
 
         try {
             jdbcTemplate.execute(createCurrentAssignmentsQuery);
-        } catch (Exception sqle) {
+        } catch (Exception e) {
+            String errorMsg = ImpalaConnector.handleQueryException("createCurrentAssignmentsQuery", createCurrentAssignmentsQuery, e);
+            String tmpErrMsg = dropCurrentAssignmentTable();    // The table may be partially created, e.g. in case of an "out of memory" error in the database-server, during the creation, resulting in an empty table (yes it has happened).
+            if ( tmpErrMsg != null )
+                errorMsg += "\n" + tmpErrMsg;
             ImpalaConnector.databaseLock.unlock();
-            String errorMsg = ImpalaConnector.handleQueryException("createCurrentAssignmentsQuery", createCurrentAssignmentsQuery, sqle);
             return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorMsg);
         }
 
@@ -283,8 +286,8 @@ public class UrlController {
                             payload.getLocation(), payload.getProvenance()};
 
                     jdbcTemplate.update(insertIntoPayloadBaseQuery, args, payloadArgTypes);
-                } catch (Exception sqle) {
-                    logger.error("Problem when executing the \"insertIntoPayloadBaseQuery\": ", sqle);
+                } catch (Exception e) {
+                    logger.error("Problem when executing the \"insertIntoPayloadBaseQuery\": " + e.getMessage());
                     failedCount.incrementAndGet();
                 }
             }
@@ -310,8 +313,8 @@ public class UrlController {
                             urlReport.getStatus().toString(), String.valueOf(error.getType()), error.getMessage()};
 
                     jdbcTemplate.update(insertIntoAttemptBaseQuery, args, attemptArgTypes);
-                } catch (Exception sqle) {
-                    logger.error("Problem when executing the \"insertIntoAttemptBaseQuery\": " + sqle.getMessage());
+                } catch (Exception e) {
+                    logger.error("Problem when executing the \"insertIntoAttemptBaseQuery\": " + e.getMessage());
                     failedCount.incrementAndGet();
                 }
             }
@@ -333,7 +336,7 @@ public class UrlController {
         }
 
         int failedQueries = failedCount.get();
-        String failedQueriesMsg = failedQueries + " out of " + urlReports.size() + " failed to be processed!";
+        String failedQueriesMsg = failedQueries + " out of " + (urlReports.size() *2) + " failed to be processed!";
         logger.debug("Finished inserting the payloads and the attempts into the \"payload\" and \"attempt\" tables" + ((failedQueries > 0) ? (", although " + failedQueriesMsg) : ".")
                 + " Going to merge the parquet files for those tables.");
 
@@ -365,6 +368,17 @@ public class UrlController {
     }
 
 
+    private String dropCurrentAssignmentTable() {
+        String dropCurrentAssignmentsQuery = "DROP TABLE IF EXISTS " + databaseName + ".current_assignment PURGE";
+        try {
+            jdbcTemplate.execute(dropCurrentAssignmentsQuery);
+            return null;
+        } catch (Exception e) {
+            return ImpalaConnector.handleQueryException("dropCurrentAssignmentsQuery", dropCurrentAssignmentsQuery, e);
+        }
+    }
+
+
     // The "batchExecute" does not work in this Impala-Database, so this is a "giant-query" solution.
     // Note: this causes an "Out of memory"-ERROR in the current version of the Impala JDBC driver. If a later version is provided, then this code should be tested.
     private static PreparedStatement constructLargeInsertQuery(Connection con, String baseInsertQuery, int dataSize, int numParamsPerRow) throws RuntimeException {
@@ -393,15 +407,4 @@ public class UrlController {
         return preparedInsertStatement;
     }
 
-
-    private String dropCurrentAssignmentTable() {
-        String dropCurrentAssignmentsQuery = "DROP TABLE " + databaseName + ".current_assignment PURGE";
-        try {
-            jdbcTemplate.execute(dropCurrentAssignmentsQuery);
-            return null;
-        } catch (Exception e) {
-            return ImpalaConnector.handleQueryException("dropCurrentAssignmentsQuery", dropCurrentAssignmentsQuery, e);
-        }
-    }
-
 }