D-Net
/
code-infrastructure-lab
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

updated airflow roles

This commit is contained in:
Sandro La Bruzzo 2024-05-02 11:31:17 +02:00
parent 196ba0b54a
commit c9c80ad9b9
2 changed files with 42 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
envs/local/airflow.yaml
View File

@ -42,7 +42,7 @@ config:
logging: logging:
remote_logging: "True" remote_logging: "True"
logging_level: "INFO" logging_level: "INFO"
remote_base_log_folder: "s3://lot1-airflow/logs" remote_base_log_folder: "s3://dnet-airflow/logs"
remote_log_conn_id: "s3_conn" remote_log_conn_id: "s3_conn"
encrypt_s3_logs: "False" encrypt_s3_logs: "False"

82
modules/airflow/airflow.tf
View File

@ -13,47 +13,47 @@ resource "kubernetes_namespace" "spark_jobs_namespace" {
# } # }
# } # }
# #
# resource "kubernetes_role" "airflow_spark_role" { resource "kubernetes_role" "airflow_spark_role" {
# depends_on = [kubernetes_namespace.spark_jobs_namespace] depends_on = [kubernetes_namespace.spark_jobs_namespace]
# metadata { metadata {
# name = "airflow-spark-role" name = "airflow-spark-role"
# namespace = "${var.namespace_prefix}spark-jobs" namespace = "${var.namespace_prefix}spark-jobs"
# } }
#
# rule { rule {
# api_groups = ["sparkoperator.k8s.io"] api_groups = ["sparkoperator.k8s.io"]
# resources = ["sparkapplications", "sparkapplications/status", resources = ["sparkapplications", "sparkapplications/status",
# "scheduledsparkapplications", "scheduledsparkapplications/status"] "scheduledsparkapplications", "scheduledsparkapplications/status"]
# verbs = ["*"] verbs = ["*"]
# } }
#
# rule { rule {
# api_groups = [""] api_groups = [""]
# resources = ["pods/log"] resources = ["pods/log"]
# verbs = ["*"] verbs = ["*"]
# } }
# } }
#
# resource "kubernetes_role_binding_v1" "airflow_spark_role_binding" { resource "kubernetes_role_binding_v1" "airflow_spark_role_binding" {
# depends_on = [kubernetes_namespace.spark_jobs_namespace] depends_on = [kubernetes_namespace.spark_jobs_namespace]
# metadata { metadata {
# name = "airflow-spark-role-binding" name = "airflow-spark-role-binding"
# namespace = "${var.namespace_prefix}spark-jobs" namespace = "${var.namespace_prefix}spark-jobs"
# } }
#
# subject { subject {
# kind = "ServiceAccount" kind = "ServiceAccount"
# name = "airflow-worker" name = "airflow-worker"
# namespace = "${var.namespace_prefix}airflow" namespace = "${var.namespace_prefix}airflow"
# } }
#
# role_ref { role_ref {
# api_group = "rbac.authorization.k8s.io" api_group = "rbac.authorization.k8s.io"
# kind = "Role" kind = "Role"
# name = "airflow-spark-role" name = "airflow-spark-role"
# } }
# } }
#
resource "kubernetes_role_binding_v1" "airflow_spark_role_binding2" { resource "kubernetes_role_binding_v1" "airflow_spark_role_binding2" {
depends_on = [kubernetes_namespace.spark_jobs_namespace] depends_on = [kubernetes_namespace.spark_jobs_namespace]
metadata { metadata {