From ed3422673fce936589aa673631857534c4ea39ce Mon Sep 17 00:00:00 2001
From: Sandro La Bruzzo <sandro.labruzzo@gmail.com>
Date: Wed, 16 Oct 2024 13:35:49 +0200
Subject: [PATCH] added git variable for airflow module

---
 envs/local/airbyte.yaml      | 13 -----------
 main.tf                      | 27 +++++++++++-----------
 modules/airflow/airflow.tf   | 43 ++++++++++++++++++++++++------------
 modules/airflow/variables.tf | 15 +++++++++++++
 variables.tf                 |  4 ++++
 5 files changed, 62 insertions(+), 40 deletions(-)
 delete mode 100644 envs/local/airbyte.yaml

diff --git a/envs/local/airbyte.yaml b/envs/local/airbyte.yaml
deleted file mode 100644
index aa79aa4..0000000
--- a/envs/local/airbyte.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-webapp:
-  ingress:
-    enabled: true
-    className: "nginx"
-    annotations:
-      kubernetes.io/ingress.class: nginx
-    hosts:
-      - host: localhost
-        paths:
-          - path: /
-            pathType: ImplementationSpecific
-    tls: []
\ No newline at end of file
diff --git a/main.tf b/main.tf
index 4c63dac..20a2ffd 100644
--- a/main.tf
+++ b/main.tf
@@ -6,17 +6,18 @@ module "minio" {
 }
 
 
-# module "airflow" {
-#   source = "./modules/airflow"
-#   kube_context = var.kube_context
-#   admin_user = var.admin_user
-#   admin_password = var.admin_password
-#   namespace_prefix= var.namespace_prefix
-#   admin_hash = var.admin_hash
-#   env = var.env
-#   domain = var.domain
-#   s3_endpoint = var.s3_endpoint
-#   s3_key = var.s3_key
-#   s3_secret = var.s3_secret
+module "airflow" {
+  source = "./modules/airflow"
+  kube_context = var.kube_context
+  admin_user = var.admin_user
+  admin_password = var.admin_password
+  namespace_prefix= var.namespace_prefix
+  admin_hash = var.admin_hash
+  env = var.env
+  domain = var.domain
+  s3_endpoint = var.s3_endpoint
+  s3_key = var.s3_key
+  s3_secret = var.s3_secret
+  branch_name= var.dag_branch_name
 
-# }
+}
diff --git a/modules/airflow/airflow.tf b/modules/airflow/airflow.tf
index fada7c8..b5000d4 100644
--- a/modules/airflow/airflow.tf
+++ b/modules/airflow/airflow.tf
@@ -104,17 +104,17 @@ resource "helm_release" "gcp_spark_operator" {
   create_namespace  = "true"
   namespace         = "${var.namespace_prefix}gcp-spark-operator"
   dependency_update = "true"
-  version           = "1.2.7"
+  version           = "2.0.2"
 
-  set {
-    name  = "image.repository"
-    value = "kubeflow/spark-operator"
-  }
+  # set {
+  #   name  = "image.repository"
+  #   value = "kubeflow/spark-operator"
+  # }
 
-  set {
-    name  = "image.tag"
-    value = "v1beta2-1.4.5-3.5.0"
-  }
+  # set {
+  #   name  = "image.tag"
+  #   value = "v1beta2-1.4.5-3.5.0"
+  # }
 
   set {
     name  = "sparkJobNamespaces"
@@ -180,7 +180,7 @@ resource "helm_release" "airflow" {
   repository        = "https://airflow.apache.org"
   namespace         = "${var.namespace_prefix}airflow"
   dependency_update = "true"
-  version           = "1.13.0"
+  version           = "1.15.0"
 
   values = [
     file("./envs/${var.env}/airflow.yaml")
@@ -211,15 +211,30 @@ resource "helm_release" "airflow" {
   }
 
   set {
-    name  = "images.airflow.repository"
-    value = "gbloisi/airflow"
+    name ="dags.gitSync.repo"
+    value = var.repo_url
   }
 
   set {
-    name  = "images.airflow.tag"
-    value = "2.8.3rc1-python3.11"
+    name ="dags.gitSync.branch"
+    value = var.branch_name
   }
 
+  set {
+    name ="dags.gitSync.subPath"
+    value = var.dag_path
+  }
+
+  # set {
+  #   name  = "images.airflow.repository"
+  #   value = "gbloisi/airflow"
+  # }
+
+  # set {
+  #   name  = "images.airflow.tag"
+  #   value = "2.8.3rc1-python3.11"
+  # }
+
   set {
     name  = "ingress.web.host"
     value = "airflow.${var.domain}"
diff --git a/modules/airflow/variables.tf b/modules/airflow/variables.tf
index 8c93472..6b4b7ae 100644
--- a/modules/airflow/variables.tf
+++ b/modules/airflow/variables.tf
@@ -49,3 +49,18 @@ variable "admin_password" {
 variable "admin_hash" {
   type    = string
 }
+
+variable "repo_url" {
+  type    = string
+  default = "https://code-repo.d4science.org/D-Net/code-infrastructure-lab.git"
+}
+
+variable "branch_name" {
+  type    = string
+  default = "master"
+}
+
+variable "dag_path" {
+  type    = string
+  default = "workflow/dnet"
+}
diff --git a/variables.tf b/variables.tf
index 207e4df..f624ad3 100644
--- a/variables.tf
+++ b/variables.tf
@@ -54,3 +54,7 @@ variable "minio_buckets" {
   ]
 }
 
+variable "dag_branch_name" {
+  default = "master"
+}
+