D-Net
/
code-infrastructure-lab
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

- Update spark-operator version and image to support s3 

- Change default code repo for Dag airflow to this repository, branch airflow, folder "airflow/dags"
This commit is contained in:
Giambattista Bloisi 2024-04-22 10:20:20 +02:00
parent 9631d0245b
commit 7a8a2e6285
2 changed files with 106 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
envs/local/airflow.yaml
View File

@ -31,7 +31,7 @@ dags:
enabled: true
gitSync:
enabled: true
repo: "https://code-repo.d4science.org/giambattista.bloisi/lot1-kickoff.git"
repo: "https://code-repo.d4science.org/D-Net/code-infrasturcutre-lab.git"
branch: "airflow"
subPath: "airflow/dags"

190
modules/airflow/airflow.tf
View File

@ -5,91 +5,96 @@ resource "kubernetes_namespace" "spark_jobs_namespace" {
}
resource "kubernetes_service_account_v1" "spark_sa" {
metadata {
name = "spark"
namespace = "${var.namespace_prefix}spark-jobs"
}
}
resource "kubernetes_role" "airflow_spark_role" {
metadata {
name = "airflow-spark-role"
namespace = "${var.namespace_prefix}spark-jobs"
}
rule {
api_groups = ["sparkoperator.k8s.io"]
resources = ["sparkapplications", "sparkapplications/status",
"scheduledsparkapplications", "scheduledsparkapplications/status"]
verbs = ["*"]
}
rule {
api_groups = [""]
resources = ["pods/log"]
verbs = ["*"]
}
}
resource "kubernetes_role_binding_v1" "airflow_spark_role_binding" {
metadata {
name = "airflow-spark-role-binding"
namespace = "${var.namespace_prefix}spark-jobs"
}
subject {
kind = "ServiceAccount"
name = "airflow-worker"
namespace = "${var.namespace_prefix}airflow"
}
role_ref {
api_group = "rbac.authorization.k8s.io"
kind = "Role"
name = "airflow-spark-role"
}
}
# resource "kubernetes_service_account_v1" "spark_sa" {
# depends_on = [kubernetes_namespace.spark_jobs_namespace]
# metadata {
# name = "spark"
# namespace = "${var.namespace_prefix}spark-jobs"
# }
# }
#
# resource "kubernetes_role" "airflow_spark_role" {
# depends_on = [kubernetes_namespace.spark_jobs_namespace]
# metadata {
# name = "airflow-spark-role"
# namespace = "${var.namespace_prefix}spark-jobs"
# }
#
# rule {
# api_groups = ["sparkoperator.k8s.io"]
# resources = ["sparkapplications", "sparkapplications/status",
# "scheduledsparkapplications", "scheduledsparkapplications/status"]
# verbs = ["*"]
# }
#
# rule {
# api_groups = [""]
# resources = ["pods/log"]
# verbs = ["*"]
# }
# }
#
# resource "kubernetes_role_binding_v1" "airflow_spark_role_binding" {
# depends_on = [kubernetes_namespace.spark_jobs_namespace]
# metadata {
# name = "airflow-spark-role-binding"
# namespace = "${var.namespace_prefix}spark-jobs"
# }
#
# subject {
# kind = "ServiceAccount"
# name = "airflow-worker"
# namespace = "${var.namespace_prefix}airflow"
# }
#
# role_ref {
# api_group = "rbac.authorization.k8s.io"
# kind = "Role"
# name = "airflow-spark-role"
# }
# }
#
resource "kubernetes_role_binding_v1" "airflow_spark_role_binding2" {
metadata {
name = "airflow-spark-role-binding2"
namespace = "${var.namespace_prefix}spark-jobs"
}
depends_on = [kubernetes_namespace.spark_jobs_namespace]
metadata {
name = "airflow-spark-role-binding2"
namespace = "${var.namespace_prefix}spark-jobs"
}
subject {
kind = "ServiceAccount"
name = "airflow-worker"
namespace = "${var.namespace_prefix}airflow"
}
subject {
kind = "ServiceAccount"
name = "airflow-worker"
namespace = "${var.namespace_prefix}airflow"
}
role_ref {
api_group = "rbac.authorization.k8s.io"
kind = "Role"
name = "spark-role"
}
role_ref {
api_group = "rbac.authorization.k8s.io"
kind = "Role"
name = "spark-role"
}
}
resource "kubernetes_role_binding_v1" "spark_role_binding" {
metadata {
name = "spark-role-binding"
namespace = "${var.namespace_prefix}spark-jobs"
}
subject {
kind = "ServiceAccount"
name = "spark"
namespace = "${var.namespace_prefix}spark-jobs"
}
role_ref {
api_group = "rbac.authorization.k8s.io"
kind = "Role"
name = "spark-role"
}
}
#
#
# resource "kubernetes_role_binding_v1" "spark_role_binding" {
# depends_on = [kubernetes_namespace.spark_jobs_namespace]
# metadata {
# name = "spark-role-binding"
# namespace = "${var.namespace_prefix}spark-jobs"
# }
#
# subject {
# kind = "ServiceAccount"
# name = "spark"
# namespace = "${var.namespace_prefix}spark-jobs"
# }
#
# role_ref {
# api_group = "rbac.authorization.k8s.io"
# kind = "Role"
# name = "spark-role"
# }
# }
#
resource "helm_release" "gcp_spark_operator" {
depends_on = [kubernetes_namespace.spark_jobs_namespace]
@ -99,15 +104,30 @@ resource "helm_release" "gcp_spark_operator" {
create_namespace = "true"
namespace = "${var.namespace_prefix}gcp-spark-operator"
dependency_update = "true"
version = "1.1.27"
version = "1.2.7"
set {
name = "sparkJobNamespace"
value = "${var.namespace_prefix}spark-jobs"
name = "image.repository"
value = "gbloisi/spark-operator"
}
set {
name = "enableWebhook"
name = "image.tag"
value = "v1beta2-1.4.3-3.5.1"
}
set {
name = "sparkJobNamespaces"
value = "{${var.namespace_prefix}spark-jobs}"
}
set {
name = "serviceAccounts.spark.name"
value = "spark"
}
set {
name = "webhook.enabled"
value = "true"
}