Attempt to recover from serious failures, by individualizing the error-handling for each of the "table-merging" queries.

src/main/java/eu/openaire/urls_controller/util/FileUtils.java

@@ -86,7 +86,7 @@ public class FileUtils {
         if ( (tableName == null) || tableName.isEmpty() ) {
             errorMsg = "No tableName was given. Do not know the tableName for which we should merger the underlying files for!";
             logger.error(errorMsg);
-            return errorMsg;
+            return errorMsg;    // Return the error-msg to indicate that something went wrong and pass it down to the Worker.
         }
 
         // Make sure the following are empty strings (in case another method call this one in the future with a null-value).
@@ -98,15 +98,83 @@ public class FileUtils {
         else
             parameter = " '" + parameter + "'";  // This will be a "string-check", thus the single-quotes.
 
+        // Create a temp-table as a copy of the initial table.
         try {
             jdbcTemplate.execute("CREATE TABLE " + ImpalaConnector.databaseName + "." + tableName + "_tmp stored as parquet AS SELECT * FROM " + ImpalaConnector.databaseName + "." + tableName + " " + whereClause + parameter);
+        } catch (Exception e) {
+            errorMsg = "Problem when copying the contents of \"" + tableName + "\" table to a newly created \"" + tableName + "_tmp\" table, when merging the parquet-files!\n";
+            logger.error(errorMsg, e);
+            try {   // Make sure we delete the possibly half-created temp-table.
+                jdbcTemplate.execute("DROP TABLE IF EXISTS" + ImpalaConnector.databaseName + "." + tableName + "_tmp PURGE");
+                // We cannot move on with merging, but no harm happened, since the "table_tmp" name is still reserved for future use (after it was dropped immediately)..
+            } catch (Exception e1) {
+                logger.error("Failed to drop the \"" + tableName + "_tmp\" table!", e1);
+                // TODO - Should we shutdown the service? It is highly unlikely that anyone will observe this error live (and act immediately)..!
+            }
+            return errorMsg;    // We return only the initial error to the Worker, which is easily distinguished indie the "merge-queries".
+        }
+
+        // Drop the initial table.
+        try {
             jdbcTemplate.execute("DROP TABLE " + ImpalaConnector.databaseName + "." + tableName + " PURGE");
+        } catch (Exception e) {
+            errorMsg = "Problem when dropping the initial \"" + tableName + "\" table, when merging the parquet-files!\n";
+            logger.error(errorMsg, e);
+            // The original table could not be dropped, so the temp-table cannot be renamed to the original..!
+            try {   // Make sure we delete the already created temp-table, in order to be able to use it in the future. The merging has failed nevertheless.
+                jdbcTemplate.execute("DROP TABLE" + ImpalaConnector.databaseName + "." + tableName + "_tmp PURGE");
+            } catch (Exception e1) {
+                logger.error((errorMsg += "Failed to drop the \"" + tableName + "_tmp\" table!"), e1);  // Add this error to the original, both are very important.
+            }
+            // Here, the original table is created.
+            return errorMsg;
+            // TODO - Should we shutdown the service? It is highly unlikely that anyone will observe this error live (and act immediately)..!
+        }
+
+        // Rename the temp-table to have the initial-table's name.
+        try {
             jdbcTemplate.execute("ALTER TABLE " + ImpalaConnector.databaseName + "." + tableName + "_tmp RENAME TO " + ImpalaConnector.databaseName + "." + tableName);
+        } catch (Exception e) {
+            errorMsg = "Problem in renaming the \"" + tableName + "_tmp\" table to \"" + tableName + "\", when merging the parquet-files!\n";
+            logger.error(errorMsg, e);
+            // At this point we only have a "temp-table", the original is already deleted..
+            // Try to create the original, as a copy of the temp-table. If that succeeds, then try to delete the temp-table.
+            try {
+                jdbcTemplate.execute("CREATE TABLE " + ImpalaConnector.databaseName + "." + tableName + " stored as parquet AS SELECT * FROM " + ImpalaConnector.databaseName + "." + tableName + "_tmp");
+            } catch (Exception e1) {
+                errorMsg = "Problem when copying the contents of \"" + tableName + "_tmp\" table to a newly created \"" + tableName + "\" table, when merging the parquet-files!\n";
+                logger.error(errorMsg, e1);
+                // If the original table was not created, then we have to intervene manually, if it was created but without any data, then we can safely move on handling other assignments and workerReports, but the data will be lost! So this workerReport failed to be handled.
+                try {   // The below query normally returns a list, as it takes a "regex-pattern" as an input. BUT, we give just the table name, without wildcards. So the result is either the tableName itself or none (not any other table).
+                    jdbcTemplate.queryForObject("SHOW TABLES IN " + ImpalaConnector.databaseName + " LIKE '" + tableName + "'", List.class);
+                } catch (EmptyResultDataAccessException erdae) {
+                    // The table does not exist, so it was not even half-created by the previous query.
+                    // Not having the original table anymore is a serious error. A manual action is needed!
+                    logger.error((errorMsg += "The original table \"" + tableName + "\" must be created manually! Serious problems may appear otherwise!"));
+                    // TODO - Should we shutdown the service? It is highly unlikely that anyone will observe this error live (and act immediately)..!
+                }
+                // Here, the original-table exists in the DB, BUT without any data inside! This worker Report failed to handled! (some of its data could not be loaded to the database, and all previous data was lost).
+                return errorMsg;
+            }
+
+            // The creation of the original table was successful. Try to delete the temp-table.
+            try {
+                jdbcTemplate.execute("DROP TABLE " + ImpalaConnector.databaseName + "." + tableName + "_tmp PURGE");
+            } catch (Exception e2) {
+                logger.error((errorMsg += "Problem when dropping the \"" + tableName + "_tmp\" table, when merging the parquet-files!\n"), e2);
+                // Manual deletion should be performed!
+                return errorMsg;    // Return both errors here, as the second is so important that if it did not happen then we could move on with this workerReport.
+                // TODO - Should we shutdown the service? It is highly unlikely that anyone will observe this error live (and act immediately)..!
+            }
+            // Here the original table exists and the temp-table is deleted. We eventually have the same state as if the "ALTER TABLE" succeeded.
+        }
+
+        // Gather information to be used for queries-optimization.
+        try {
             jdbcTemplate.execute("COMPUTE STATS " + ImpalaConnector.databaseName + "." + tableName);
         } catch (Exception e) {
-            errorMsg = "Problem when executing the \"clone-drop-rename\" queries!\n";
-            logger.error(errorMsg, e);
-            return errorMsg;
+            logger.error("Problem when gathering information from table \"" + tableName + "\" to be used for queries-optimization.", e);
+            // In this case the error is not so important to the whole operation.. It's only that the performance of this specific table will be less optimal, only temporarily, unless every "COMPUTE STATS" query fails for future workerReports too.
         }
 
         return null;    // No errorMsg, everything is fine.