From c8ffe36fbc54da89edc276ae9d60f9d6ba664aa9 Mon Sep 17 00:00:00 2001
From: Giambattista Bloisi <giambattista.bloisi@openaire.eu>
Date: Sat, 9 Mar 2024 23:48:10 +0100
Subject: [PATCH] simple test DAG

---
 airflow/dags/spark_pi.py | 41 +---------------------------------------
 1 file changed, 1 insertion(+), 40 deletions(-)

diff --git a/airflow/dags/spark_pi.py b/airflow/dags/spark_pi.py
index 8d77715..e5581a1 100644
--- a/airflow/dags/spark_pi.py
+++ b/airflow/dags/spark_pi.py
@@ -65,46 +65,7 @@ dag = DAG(
 submit = SparkKubernetesOperator(
     task_id='spark_pi_submit',
     namespace='lot1-spark-jobs',
-    template_spec="""apiVersion: "sparkoperator.k8s.io/v1beta2"
-kind: SparkApplication
-metadata:
-  name: spark-pi
-  namespace: lot1-spark-jobs
-spec:
-  type: Scala
-  mode: clusterß
-  image: "apache/spark:v3.1.3"
-  imagePullPolicy: Always
-  mainClass: org.apache.spark.examples.SparkPi
-  mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.12-3.1.3.jar"
-  sparkVersion: "3.1.3"
-  restartPolicy:
-    type: Never
-  volumes:
-    - name: "test-volume"
-      hostPath:
-        path: "/tmp"
-        type: Directory
-  driver:
-    cores: 1
-    coreLimit: "1200m"
-    memory: "512m"
-    labels:
-      version: 3.1.3
-    serviceAccount: spark
-    volumeMounts:
-      - name: "test-volume"
-        mountPath: "/tmp"
-  executor:
-    cores: 1
-    instances: 1
-    memory: "512m"
-    labels:
-      version: 3.1.3
-    volumeMounts:
-      - name: "test-volume"
-        mountPath: "/tmp"
-""",
+    application_file="/example_spark_kubernetes_operator_pi.yaml",
     kubernetes_conn_id="kubernetes_default",
     do_xcom_push=True,
     in_cluster=True,