Load Airflow dags from local folder

clusters/local/kind-cluster-config.yaml

@@ -19,7 +19,11 @@ nodes:
       - containerPort: 443
         hostPort: 443
         protocol: TCP
-
+    extraMounts:
+      - hostPath: ./workflow/dnet
+        containerPath: /dnet-airflow
+        readOnly: true
+        propagation: None
 
 containerdConfigPatches:
   - |-

envs/local/airflow.yaml

@@ -25,15 +25,15 @@ secret:
     secretName: "s3-conn-secrets"
     secretKey: "AIRFLOW_CONN_S3_CONN"
 
-
+#  - envName: "AIRFLOW_CONN_POSTGRES_CONN"
+#    secretName: "postgres-conn-secrets"
+#    secretKey: "AIRFLOW_CONN_POSTGRES_CONN"
 dags:
   persistence:
     enabled: true
+    existingClaim: "local-dnet-airflow"
   gitSync:
-    enabled: true
+    enabled: false
-    repo: "https://code-repo.d4science.org/D-Net/code-infrastructure-lab.git"
-    branch: "master"
-    subPath: "workflow/dnet"
 
 config:
   webserver:

modules/airflow/airflow.tf

@@ -165,6 +165,53 @@ EOT
   type = "Opaque"
 }
 
+resource "kubernetes_persistent_volume" "dags" {
+  metadata {
+    name      = "local-dnet-airflow"
+    labels = {
+      type = "local"
+    }
+  }
+  spec {
+    node_affinity {
+      required {
+        node_selector_term {
+          match_expressions {
+            key      = "kubernetes.io/hostname"
+            operator = "In"
+            values = ["dnet-data-platform-control-plane"]
+          }
+        }
+      }
+    }
+    storage_class_name = "standard"
+    capacity = {
+      storage = "1Gi"
+    }
+    access_modes = ["ReadWriteOnce"]
+    persistent_volume_source {
+      local {
+        path = "/dnet-airflow"
+      }
+    }
+  }
+}
+
+resource "kubernetes_persistent_volume_claim" "dags" {
+  metadata {
+    name      = "local-dnet-airflow"
+    namespace = "${var.namespace_prefix}airflow"
+  }
+  spec {
+    storage_class_name = "standard"
+    access_modes = ["ReadWriteOnce"]
+    resources {
+      requests = {
+        storage = "1Gi"
+      }
+    }
+  }
+}
 
 resource "helm_release" "airflow" {
   depends_on = [kubernetes_secret.s3_conn_secrets]