- Update spark-operator version and image to support s3

- Change default code repo for Dag airflow to this repository, branch airflow, folder "airflow/dags"

envs/local/airflow.yaml

@@ -31,7 +31,7 @@ dags:
     enabled: true
   gitSync:
     enabled: true
-    repo: "https://code-repo.d4science.org/giambattista.bloisi/lot1-kickoff.git"
+    repo: "https://code-repo.d4science.org/D-Net/code-infrasturcutre-lab.git"
     branch: "airflow"
     subPath: "airflow/dags"
 

modules/airflow/airflow.tf

@@ -5,91 +5,96 @@ resource "kubernetes_namespace" "spark_jobs_namespace" {
 }
 
 
-resource "kubernetes_service_account_v1" "spark_sa" {
+# resource "kubernetes_service_account_v1" "spark_sa" {
-  metadata {
+#   depends_on = [kubernetes_namespace.spark_jobs_namespace]
-    name      = "spark"
+#   metadata {
-    namespace = "${var.namespace_prefix}spark-jobs"
+#     name      = "spark"
-  }
+#     namespace = "${var.namespace_prefix}spark-jobs"
-}
+#   }
-
+# }
-resource "kubernetes_role" "airflow_spark_role" {
+#
-  metadata {
+# resource "kubernetes_role" "airflow_spark_role" {
-    name      = "airflow-spark-role"
+#   depends_on = [kubernetes_namespace.spark_jobs_namespace]
-    namespace = "${var.namespace_prefix}spark-jobs"
+#   metadata {
-  }
+#     name      = "airflow-spark-role"
-
+#     namespace = "${var.namespace_prefix}spark-jobs"
-  rule {
+#   }
-    api_groups = ["sparkoperator.k8s.io"]
+#
-    resources  = ["sparkapplications", "sparkapplications/status",
+#   rule {
-        "scheduledsparkapplications", "scheduledsparkapplications/status"]
+#     api_groups = ["sparkoperator.k8s.io"]
-    verbs      = ["*"]
+#     resources  = ["sparkapplications", "sparkapplications/status",
-  }
+#         "scheduledsparkapplications", "scheduledsparkapplications/status"]
-
+#     verbs      = ["*"]
-  rule {
+#   }
-    api_groups = [""]
+#
-    resources  = ["pods/log"]
+#   rule {
-    verbs      = ["*"]
+#     api_groups = [""]
-  }
+#     resources  = ["pods/log"]
-}
+#     verbs      = ["*"]
-
+#   }
-resource "kubernetes_role_binding_v1" "airflow_spark_role_binding" {
+# }
-  metadata {
+#
-    name      = "airflow-spark-role-binding"
+# resource "kubernetes_role_binding_v1" "airflow_spark_role_binding" {
-    namespace = "${var.namespace_prefix}spark-jobs"
+#   depends_on = [kubernetes_namespace.spark_jobs_namespace]
-  }
+#   metadata {
-
+#     name      = "airflow-spark-role-binding"
-  subject {
+#     namespace = "${var.namespace_prefix}spark-jobs"
-    kind      = "ServiceAccount"
+#   }
-    name      = "airflow-worker"
+#
-    namespace = "${var.namespace_prefix}airflow"
+#   subject {
-  }
+#     kind      = "ServiceAccount"
-
+#     name      = "airflow-worker"
-  role_ref {
+#     namespace = "${var.namespace_prefix}airflow"
-    api_group = "rbac.authorization.k8s.io"
+#   }
-    kind      = "Role"
+#
-    name      = "airflow-spark-role"
+#   role_ref {
-  }
+#     api_group = "rbac.authorization.k8s.io"
-}
+#     kind      = "Role"
-
+#     name      = "airflow-spark-role"
+#   }
+# }
+#
 resource "kubernetes_role_binding_v1" "airflow_spark_role_binding2" {
-  metadata {
+ depends_on = [kubernetes_namespace.spark_jobs_namespace]
-    name      = "airflow-spark-role-binding2"
+ metadata {
-    namespace = "${var.namespace_prefix}spark-jobs"
+   name      = "airflow-spark-role-binding2"
-  }
+   namespace = "${var.namespace_prefix}spark-jobs"
+ }
 
-  subject {
+ subject {
-    kind      = "ServiceAccount"
+   kind      = "ServiceAccount"
-    name      = "airflow-worker"
+   name      = "airflow-worker"
-    namespace = "${var.namespace_prefix}airflow"
+   namespace = "${var.namespace_prefix}airflow"
-  }
+ }
 
-  role_ref {
+ role_ref {
-    api_group = "rbac.authorization.k8s.io"
+   api_group = "rbac.authorization.k8s.io"
-    kind      = "Role"
+   kind      = "Role"
-    name      = "spark-role"
+   name      = "spark-role"
-  }
+ }
 }
-
+#
-
+#
-resource "kubernetes_role_binding_v1" "spark_role_binding" {
+# resource "kubernetes_role_binding_v1" "spark_role_binding" {
-  metadata {
+#   depends_on = [kubernetes_namespace.spark_jobs_namespace]
-    name      = "spark-role-binding"
+#   metadata {
-    namespace = "${var.namespace_prefix}spark-jobs"
+#     name      = "spark-role-binding"
-  }
+#     namespace = "${var.namespace_prefix}spark-jobs"
-
+#   }
-  subject {
+#
-    kind      = "ServiceAccount"
+#   subject {
-    name      = "spark"
+#     kind      = "ServiceAccount"
-    namespace = "${var.namespace_prefix}spark-jobs"
+#     name      = "spark"
-  }
+#     namespace = "${var.namespace_prefix}spark-jobs"
-
+#   }
-  role_ref {
+#
-    api_group = "rbac.authorization.k8s.io"
+#   role_ref {
-    kind      = "Role"
+#     api_group = "rbac.authorization.k8s.io"
-    name      = "spark-role"
+#     kind      = "Role"
-  }
+#     name      = "spark-role"
-}
+#   }
-
+# }
+#
 
 resource "helm_release" "gcp_spark_operator" {
   depends_on        = [kubernetes_namespace.spark_jobs_namespace]
@@ -99,15 +104,30 @@ resource "helm_release" "gcp_spark_operator" {
   create_namespace  = "true"
   namespace         = "${var.namespace_prefix}gcp-spark-operator"
   dependency_update = "true"
-  version           = "1.1.27"
+  version           = "1.2.7"
 
   set {
-    name  = "sparkJobNamespace"
+    name  = "image.repository"
-    value = "${var.namespace_prefix}spark-jobs"
+    value = "gbloisi/spark-operator"
   }
 
   set {
-    name  = "enableWebhook"
+    name  = "image.tag"
+    value = "v1beta2-1.4.3-3.5.1"
+  }
+
+  set {
+    name  = "sparkJobNamespaces"
+    value = "{${var.namespace_prefix}spark-jobs}"
+  }
+
+  set {
+    name  = "serviceAccounts.spark.name"
+    value = "spark"
+  }
+
+  set {
+    name  = "webhook.enabled"
     value = "true"
   }