From b19e4f8ae8ed2bc5421b8f8ebcf20a649599c3b7 Mon Sep 17 00:00:00 2001 From: Giambattista Bloisi Date: Sat, 9 Mar 2024 19:42:58 +0100 Subject: [PATCH] simple test DAG --- airflow/dags/spark_pi.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/airflow/dags/spark_pi.py b/airflow/dags/spark_pi.py index 32a101c..d6b5e35 100644 --- a/airflow/dags/spark_pi.py +++ b/airflow/dags/spark_pi.py @@ -68,10 +68,11 @@ submit = SparkKubernetesOperator( application_file="example_spark_kubernetes_operator_pi.yaml", kubernetes_conn_id="kubernetes_default", do_xcom_push=True, - dag=dag, - delete_on_termination=False + dag=dag ) +submit.delete_on_termination = False + sensor = SparkKubernetesSensor( task_id='spark_pi_monitor', namespace='lot1-spark-jobs',