giambattista.bloisi
/
lot1-kickoff
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

simple test DAG

This commit is contained in:
Giambattista Bloisi 2024-03-09 18:15:21 +01:00
parent 0a62276c42
commit ba99672349
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
airflow/dags/spark_pi.py
View File

@ -64,7 +64,7 @@ dag = DAG(
submit = SparkKubernetesOperator( submit = SparkKubernetesOperator(
task_id='spark_pi_submit', task_id='spark_pi_submit',
namespace='spark-jobs', namespace='lot1-spark-jobs',
application_file="example_spark_kubernetes_operator_pi.yaml", application_file="example_spark_kubernetes_operator_pi.yaml",
kubernetes_conn_id="kubernetes_default", kubernetes_conn_id="kubernetes_default",
do_xcom_push=True, do_xcom_push=True,
@ -73,7 +73,7 @@ submit = SparkKubernetesOperator(
sensor = SparkKubernetesSensor( sensor = SparkKubernetesSensor(
task_id='spark_pi_monitor', task_id='spark_pi_monitor',
namespace='spark-jobs', namespace='lot1-spark-jobs',
application_name="{{ task_instance.xcom_pull(task_ids='spark_pi_submit')['metadata']['name'] }}", application_name="{{ task_instance.xcom_pull(task_ids='spark_pi_submit')['metadata']['name'] }}",
kubernetes_conn_id="kubernetes_default", kubernetes_conn_id="kubernetes_default",
dag=dag, dag=dag,