diff --git a/airflow/dags/spark_pi.py b/airflow/dags/spark_pi.py index d6b5e35..7858522 100644 --- a/airflow/dags/spark_pi.py +++ b/airflow/dags/spark_pi.py @@ -68,6 +68,8 @@ submit = SparkKubernetesOperator( application_file="example_spark_kubernetes_operator_pi.yaml", kubernetes_conn_id="kubernetes_default", do_xcom_push=True, + in_cluster=True, + delete_on_termination=True, dag=dag )