diff --git a/clusters/local/kind-cluster-config.yaml b/clusters/local/kind-cluster-config.yaml index a0e810f..6128e70 100644 --- a/clusters/local/kind-cluster-config.yaml +++ b/clusters/local/kind-cluster-config.yaml @@ -19,7 +19,11 @@ nodes: - containerPort: 443 hostPort: 443 protocol: TCP - + extraMounts: + - hostPath: ./workflow/dnet + containerPath: /dnet-airflow + readOnly: true + propagation: None containerdConfigPatches: - |- diff --git a/envs/local/airflow.yaml b/envs/local/airflow.yaml index 6943482..0413b7d 100644 --- a/envs/local/airflow.yaml +++ b/envs/local/airflow.yaml @@ -25,15 +25,15 @@ secret: secretName: "s3-conn-secrets" secretKey: "AIRFLOW_CONN_S3_CONN" - +# - envName: "AIRFLOW_CONN_POSTGRES_CONN" +# secretName: "postgres-conn-secrets" +# secretKey: "AIRFLOW_CONN_POSTGRES_CONN" dags: persistence: enabled: true + existingClaim: "local-dnet-airflow" gitSync: - enabled: true - repo: "https://code-repo.d4science.org/D-Net/code-infrastructure-lab.git" - branch: "master" - subPath: "workflow/dnet" + enabled: false config: webserver: diff --git a/modules/airflow/airflow.tf b/modules/airflow/airflow.tf index f1f0d15..940f9a6 100644 --- a/modules/airflow/airflow.tf +++ b/modules/airflow/airflow.tf @@ -165,6 +165,53 @@ EOT type = "Opaque" } +resource "kubernetes_persistent_volume" "dags" { + metadata { + name = "local-dnet-airflow" + labels = { + type = "local" + } + } + spec { + node_affinity { + required { + node_selector_term { + match_expressions { + key = "kubernetes.io/hostname" + operator = "In" + values = ["dnet-data-platform-control-plane"] + } + } + } + } + storage_class_name = "standard" + capacity = { + storage = "1Gi" + } + access_modes = ["ReadWriteOnce"] + persistent_volume_source { + local { + path = "/dnet-airflow" + } + } + } +} + +resource "kubernetes_persistent_volume_claim" "dags" { + metadata { + name = "local-dnet-airflow" + namespace = "${var.namespace_prefix}airflow" + } + spec { + storage_class_name = "standard" + access_modes = ["ReadWriteOnce"] + resources { + requests = { + storage = "1Gi" + } + } + } +} resource "helm_release" "airflow" { depends_on = [kubernetes_secret.s3_conn_secrets]