From 7a8a2e6285591bc042d9f0cdad5cbcd1f920a75f Mon Sep 17 00:00:00 2001
From: Giambattista Bloisi <giambattista.bloisi@openaire.eu>
Date: Mon, 22 Apr 2024 10:20:20 +0200
Subject: [PATCH] - Update spark-operator version and image to support s3 -
 Change default code repo for Dag airflow to this repository, branch airflow,
 folder "airflow/dags"

---
 envs/local/airflow.yaml    |   2 +-
 modules/airflow/airflow.tf | 190 ++++++++++++++++++++-----------------
 2 files changed, 106 insertions(+), 86 deletions(-)

diff --git a/envs/local/airflow.yaml b/envs/local/airflow.yaml
index a284fa6..aa83a13 100644
--- a/envs/local/airflow.yaml
+++ b/envs/local/airflow.yaml
@@ -31,7 +31,7 @@ dags:
     enabled: true
   gitSync:
     enabled: true
-    repo: "https://code-repo.d4science.org/giambattista.bloisi/lot1-kickoff.git"
+    repo: "https://code-repo.d4science.org/D-Net/code-infrasturcutre-lab.git"
     branch: "airflow"
     subPath: "airflow/dags"
 
diff --git a/modules/airflow/airflow.tf b/modules/airflow/airflow.tf
index 41c98ea..e1ba509 100644
--- a/modules/airflow/airflow.tf
+++ b/modules/airflow/airflow.tf
@@ -5,91 +5,96 @@ resource "kubernetes_namespace" "spark_jobs_namespace" {
 }
 
 
-resource "kubernetes_service_account_v1" "spark_sa" {
-  metadata {
-    name      = "spark"
-    namespace = "${var.namespace_prefix}spark-jobs"
-  }
-}
-
-resource "kubernetes_role" "airflow_spark_role" {
-  metadata {
-    name      = "airflow-spark-role"
-    namespace = "${var.namespace_prefix}spark-jobs"
-  }
-
-  rule {
-    api_groups = ["sparkoperator.k8s.io"]
-    resources  = ["sparkapplications", "sparkapplications/status",
-        "scheduledsparkapplications", "scheduledsparkapplications/status"]
-    verbs      = ["*"]
-  }
-
-  rule {
-    api_groups = [""]
-    resources  = ["pods/log"]
-    verbs      = ["*"]
-  }
-}
-
-resource "kubernetes_role_binding_v1" "airflow_spark_role_binding" {
-  metadata {
-    name      = "airflow-spark-role-binding"
-    namespace = "${var.namespace_prefix}spark-jobs"
-  }
-
-  subject {
-    kind      = "ServiceAccount"
-    name      = "airflow-worker"
-    namespace = "${var.namespace_prefix}airflow"
-  }
-
-  role_ref {
-    api_group = "rbac.authorization.k8s.io"
-    kind      = "Role"
-    name      = "airflow-spark-role"
-  }
-}
-
+# resource "kubernetes_service_account_v1" "spark_sa" {
+#   depends_on = [kubernetes_namespace.spark_jobs_namespace]
+#   metadata {
+#     name      = "spark"
+#     namespace = "${var.namespace_prefix}spark-jobs"
+#   }
+# }
+#
+# resource "kubernetes_role" "airflow_spark_role" {
+#   depends_on = [kubernetes_namespace.spark_jobs_namespace]
+#   metadata {
+#     name      = "airflow-spark-role"
+#     namespace = "${var.namespace_prefix}spark-jobs"
+#   }
+#
+#   rule {
+#     api_groups = ["sparkoperator.k8s.io"]
+#     resources  = ["sparkapplications", "sparkapplications/status",
+#         "scheduledsparkapplications", "scheduledsparkapplications/status"]
+#     verbs      = ["*"]
+#   }
+#
+#   rule {
+#     api_groups = [""]
+#     resources  = ["pods/log"]
+#     verbs      = ["*"]
+#   }
+# }
+#
+# resource "kubernetes_role_binding_v1" "airflow_spark_role_binding" {
+#   depends_on = [kubernetes_namespace.spark_jobs_namespace]
+#   metadata {
+#     name      = "airflow-spark-role-binding"
+#     namespace = "${var.namespace_prefix}spark-jobs"
+#   }
+#
+#   subject {
+#     kind      = "ServiceAccount"
+#     name      = "airflow-worker"
+#     namespace = "${var.namespace_prefix}airflow"
+#   }
+#
+#   role_ref {
+#     api_group = "rbac.authorization.k8s.io"
+#     kind      = "Role"
+#     name      = "airflow-spark-role"
+#   }
+# }
+#
 resource "kubernetes_role_binding_v1" "airflow_spark_role_binding2" {
-  metadata {
-    name      = "airflow-spark-role-binding2"
-    namespace = "${var.namespace_prefix}spark-jobs"
-  }
+ depends_on = [kubernetes_namespace.spark_jobs_namespace]
+ metadata {
+   name      = "airflow-spark-role-binding2"
+   namespace = "${var.namespace_prefix}spark-jobs"
+ }
 
-  subject {
-    kind      = "ServiceAccount"
-    name      = "airflow-worker"
-    namespace = "${var.namespace_prefix}airflow"
-  }
+ subject {
+   kind      = "ServiceAccount"
+   name      = "airflow-worker"
+   namespace = "${var.namespace_prefix}airflow"
+ }
 
-  role_ref {
-    api_group = "rbac.authorization.k8s.io"
-    kind      = "Role"
-    name      = "spark-role"
-  }
+ role_ref {
+   api_group = "rbac.authorization.k8s.io"
+   kind      = "Role"
+   name      = "spark-role"
+ }
 }
-
-
-resource "kubernetes_role_binding_v1" "spark_role_binding" {
-  metadata {
-    name      = "spark-role-binding"
-    namespace = "${var.namespace_prefix}spark-jobs"
-  }
-
-  subject {
-    kind      = "ServiceAccount"
-    name      = "spark"
-    namespace = "${var.namespace_prefix}spark-jobs"
-  }
-
-  role_ref {
-    api_group = "rbac.authorization.k8s.io"
-    kind      = "Role"
-    name      = "spark-role"
-  }
-}
-
+#
+#
+# resource "kubernetes_role_binding_v1" "spark_role_binding" {
+#   depends_on = [kubernetes_namespace.spark_jobs_namespace]
+#   metadata {
+#     name      = "spark-role-binding"
+#     namespace = "${var.namespace_prefix}spark-jobs"
+#   }
+#
+#   subject {
+#     kind      = "ServiceAccount"
+#     name      = "spark"
+#     namespace = "${var.namespace_prefix}spark-jobs"
+#   }
+#
+#   role_ref {
+#     api_group = "rbac.authorization.k8s.io"
+#     kind      = "Role"
+#     name      = "spark-role"
+#   }
+# }
+#
 
 resource "helm_release" "gcp_spark_operator" {
   depends_on        = [kubernetes_namespace.spark_jobs_namespace]
@@ -99,15 +104,30 @@ resource "helm_release" "gcp_spark_operator" {
   create_namespace  = "true"
   namespace         = "${var.namespace_prefix}gcp-spark-operator"
   dependency_update = "true"
-  version           = "1.1.27"
+  version           = "1.2.7"
 
   set {
-    name  = "sparkJobNamespace"
-    value = "${var.namespace_prefix}spark-jobs"
+    name  = "image.repository"
+    value = "gbloisi/spark-operator"
   }
 
   set {
-    name  = "enableWebhook"
+    name  = "image.tag"
+    value = "v1beta2-1.4.3-3.5.1"
+  }
+
+  set {
+    name  = "sparkJobNamespaces"
+    value = "{${var.namespace_prefix}spark-jobs}"
+  }
+
+  set {
+    name  = "serviceAccounts.spark.name"
+    value = "spark"
+  }
+
+  set {
+    name  = "webhook.enabled"
     value = "true"
   }