From ba99672349c8b829b155b1c50871e900aa477eaf Mon Sep 17 00:00:00 2001 From: Giambattista Bloisi Date: Sat, 9 Mar 2024 18:15:21 +0100 Subject: [PATCH] simple test DAG --- airflow/dags/spark_pi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/dags/spark_pi.py b/airflow/dags/spark_pi.py index e66d5c8..6cca3d5 100644 --- a/airflow/dags/spark_pi.py +++ b/airflow/dags/spark_pi.py @@ -64,7 +64,7 @@ dag = DAG( submit = SparkKubernetesOperator( task_id='spark_pi_submit', - namespace='spark-jobs', + namespace='lot1-spark-jobs', application_file="example_spark_kubernetes_operator_pi.yaml", kubernetes_conn_id="kubernetes_default", do_xcom_push=True, @@ -73,7 +73,7 @@ submit = SparkKubernetesOperator( sensor = SparkKubernetesSensor( task_id='spark_pi_monitor', - namespace='spark-jobs', + namespace='lot1-spark-jobs', application_name="{{ task_instance.xcom_pull(task_ids='spark_pi_submit')['metadata']['name'] }}", kubernetes_conn_id="kubernetes_default", dag=dag,