D-Net
/
code-infrastructure-lab
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

updated airflow roles

This commit is contained in:
Sandro La Bruzzo 2024-05-02 11:31:17 +02:00
parent 196ba0b54a
commit c9c80ad9b9
2 changed files with 42 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
envs/local/airflow.yaml
View File

@ -42,7 +42,7 @@ config:
logging:
remote_logging: "True"
logging_level: "INFO"
remote_base_log_folder: "s3://lot1-airflow/logs"
remote_base_log_folder: "s3://dnet-airflow/logs"
remote_log_conn_id: "s3_conn"
encrypt_s3_logs: "False"

82
modules/airflow/airflow.tf
View File

@ -13,47 +13,47 @@ resource "kubernetes_namespace" "spark_jobs_namespace" {
# }
# }
#
# resource "kubernetes_role" "airflow_spark_role" {
# depends_on = [kubernetes_namespace.spark_jobs_namespace]
# metadata {
# name = "airflow-spark-role"
# namespace = "${var.namespace_prefix}spark-jobs"
# }
#
# rule {
# api_groups = ["sparkoperator.k8s.io"]
# resources = ["sparkapplications", "sparkapplications/status",
# "scheduledsparkapplications", "scheduledsparkapplications/status"]
# verbs = ["*"]
# }
#
# rule {
# api_groups = [""]
# resources = ["pods/log"]
# verbs = ["*"]
# }
# }
#
# resource "kubernetes_role_binding_v1" "airflow_spark_role_binding" {
# depends_on = [kubernetes_namespace.spark_jobs_namespace]
# metadata {
# name = "airflow-spark-role-binding"
# namespace = "${var.namespace_prefix}spark-jobs"
# }
#
# subject {
# kind = "ServiceAccount"
# name = "airflow-worker"
# namespace = "${var.namespace_prefix}airflow"
# }
#
# role_ref {
# api_group = "rbac.authorization.k8s.io"
# kind = "Role"
# name = "airflow-spark-role"
# }
# }
#
resource "kubernetes_role" "airflow_spark_role" {
depends_on = [kubernetes_namespace.spark_jobs_namespace]
metadata {
name = "airflow-spark-role"
namespace = "${var.namespace_prefix}spark-jobs"
}
rule {
api_groups = ["sparkoperator.k8s.io"]
resources = ["sparkapplications", "sparkapplications/status",
"scheduledsparkapplications", "scheduledsparkapplications/status"]
verbs = ["*"]
}
rule {
api_groups = [""]
resources = ["pods/log"]
verbs = ["*"]
}
}
resource "kubernetes_role_binding_v1" "airflow_spark_role_binding" {
depends_on = [kubernetes_namespace.spark_jobs_namespace]
metadata {
name = "airflow-spark-role-binding"
namespace = "${var.namespace_prefix}spark-jobs"
}
subject {
kind = "ServiceAccount"
name = "airflow-worker"
namespace = "${var.namespace_prefix}airflow"
}
role_ref {
api_group = "rbac.authorization.k8s.io"
kind = "Role"
name = "airflow-spark-role"
}
}
resource "kubernetes_role_binding_v1" "airflow_spark_role_binding2" {
depends_on = [kubernetes_namespace.spark_jobs_namespace]
metadata {