added git variable for airflow module

envs/local/airbyte.yaml

@@ -1,13 +0,0 @@
-
-webapp:
-  ingress:
-    enabled: true
-    className: "nginx"
-    annotations:
-      kubernetes.io/ingress.class: nginx
-    hosts:
-      - host: localhost
-        paths:
-          - path: /
-            pathType: ImplementationSpecific
-    tls: []

main.tf

@@ -6,17 +6,18 @@ module "minio" {
 }
 
 
-# module "airflow" {
+module "airflow" {
-#   source = "./modules/airflow"
+  source = "./modules/airflow"
-#   kube_context = var.kube_context
+  kube_context = var.kube_context
-#   admin_user = var.admin_user
+  admin_user = var.admin_user
-#   admin_password = var.admin_password
+  admin_password = var.admin_password
-#   namespace_prefix= var.namespace_prefix
+  namespace_prefix= var.namespace_prefix
-#   admin_hash = var.admin_hash
+  admin_hash = var.admin_hash
-#   env = var.env
+  env = var.env
-#   domain = var.domain
+  domain = var.domain
-#   s3_endpoint = var.s3_endpoint
+  s3_endpoint = var.s3_endpoint
-#   s3_key = var.s3_key
+  s3_key = var.s3_key
-#   s3_secret = var.s3_secret
+  s3_secret = var.s3_secret
+  branch_name= var.dag_branch_name
 
-# }
+}

modules/airflow/airflow.tf

@@ -104,17 +104,17 @@ resource "helm_release" "gcp_spark_operator" {
   create_namespace  = "true"
   namespace         = "${var.namespace_prefix}gcp-spark-operator"
   dependency_update = "true"
-  version           = "1.2.7"
+  version           = "2.0.2"
 
-  set {
+  # set {
-    name  = "image.repository"
+  #   name  = "image.repository"
-    value = "kubeflow/spark-operator"
+  #   value = "kubeflow/spark-operator"
-  }
+  # }
 
-  set {
+  # set {
-    name  = "image.tag"
+  #   name  = "image.tag"
-    value = "v1beta2-1.4.5-3.5.0"
+  #   value = "v1beta2-1.4.5-3.5.0"
-  }
+  # }
 
   set {
     name  = "sparkJobNamespaces"
@@ -180,7 +180,7 @@ resource "helm_release" "airflow" {
   repository        = "https://airflow.apache.org"
   namespace         = "${var.namespace_prefix}airflow"
   dependency_update = "true"
-  version           = "1.13.0"
+  version           = "1.15.0"
 
   values = [
     file("./envs/${var.env}/airflow.yaml")
@@ -211,15 +211,30 @@ resource "helm_release" "airflow" {
   }
 
   set {
-    name  = "images.airflow.repository"
+    name ="dags.gitSync.repo"
-    value = "gbloisi/airflow"
+    value = var.repo_url
   }
 
   set {
-    name  = "images.airflow.tag"
+    name ="dags.gitSync.branch"
-    value = "2.8.3rc1-python3.11"
+    value = var.branch_name
   }
 
+  set {
+    name ="dags.gitSync.subPath"
+    value = var.dag_path
+  }
+
+  # set {
+  #   name  = "images.airflow.repository"
+  #   value = "gbloisi/airflow"
+  # }
+
+  # set {
+  #   name  = "images.airflow.tag"
+  #   value = "2.8.3rc1-python3.11"
+  # }
+
   set {
     name  = "ingress.web.host"
     value = "airflow.${var.domain}"

modules/airflow/variables.tf

@@ -49,3 +49,18 @@ variable "admin_password" {
 variable "admin_hash" {
   type    = string
 }
+
+variable "repo_url" {
+  type    = string
+  default = "https://code-repo.d4science.org/D-Net/code-infrastructure-lab.git"
+}
+
+variable "branch_name" {
+  type    = string
+  default = "master"
+}
+
+variable "dag_path" {
+  type    = string
+  default = "workflow/dnet"
+}

variables.tf

@@ -54,3 +54,7 @@ variable "minio_buckets" {
   ]
 }
 
+variable "dag_branch_name" {
+  default = "master"
+}
+