antonis.lempesis
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
1
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

Catch Spark-sql errors and show a log with the statement that failed.

This commit is contained in:
Lampros Smyrnaios 2024-05-29 12:10:11 +03:00
parent 888637773c
commit a644a6f4fe
1 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
dhp-common/src/main/java/eu/dnetlib/dhp/oozie/RunSQLSparkJob.java
View File

@ -65,7 +65,13 @@ public class RunSQLSparkJob {
for (String statement : sql.split(";\\s*/\\*\\s*EOS\\s*\\*/\\s*")) { for (String statement : sql.split(";\\s*/\\*\\s*EOS\\s*\\*/\\s*")) {
log.info("executing: {}", statement); log.info("executing: {}", statement);
long startTime = System.currentTimeMillis(); long startTime = System.currentTimeMillis();
spark.sql(statement).show(); try {
spark.sql(statement).show();
} catch (Exception e) {
log.error("Error executing statement: {}", statement, e);
System.err.println("Error executing statement: " + statement + "\n" + e);
throw e;
}
log log
.info( .info(
"executed in {}", "executed in {}",