diff --git a/envs/local/airbyte.yaml b/envs/local/airbyte.yaml deleted file mode 100644 index aa79aa4..0000000 --- a/envs/local/airbyte.yaml +++ /dev/null @@ -1,13 +0,0 @@ - -webapp: - ingress: - enabled: true - className: "nginx" - annotations: - kubernetes.io/ingress.class: nginx - hosts: - - host: localhost - paths: - - path: / - pathType: ImplementationSpecific - tls: [] \ No newline at end of file diff --git a/main.tf b/main.tf index 4c63dac..20a2ffd 100644 --- a/main.tf +++ b/main.tf @@ -6,17 +6,18 @@ module "minio" { } -# module "airflow" { -# source = "./modules/airflow" -# kube_context = var.kube_context -# admin_user = var.admin_user -# admin_password = var.admin_password -# namespace_prefix= var.namespace_prefix -# admin_hash = var.admin_hash -# env = var.env -# domain = var.domain -# s3_endpoint = var.s3_endpoint -# s3_key = var.s3_key -# s3_secret = var.s3_secret +module "airflow" { + source = "./modules/airflow" + kube_context = var.kube_context + admin_user = var.admin_user + admin_password = var.admin_password + namespace_prefix= var.namespace_prefix + admin_hash = var.admin_hash + env = var.env + domain = var.domain + s3_endpoint = var.s3_endpoint + s3_key = var.s3_key + s3_secret = var.s3_secret + branch_name= var.dag_branch_name -# } +} diff --git a/modules/airflow/airflow.tf b/modules/airflow/airflow.tf index fada7c8..b5000d4 100644 --- a/modules/airflow/airflow.tf +++ b/modules/airflow/airflow.tf @@ -104,17 +104,17 @@ resource "helm_release" "gcp_spark_operator" { create_namespace = "true" namespace = "${var.namespace_prefix}gcp-spark-operator" dependency_update = "true" - version = "1.2.7" + version = "2.0.2" - set { - name = "image.repository" - value = "kubeflow/spark-operator" - } + # set { + # name = "image.repository" + # value = "kubeflow/spark-operator" + # } - set { - name = "image.tag" - value = "v1beta2-1.4.5-3.5.0" - } + # set { + # name = "image.tag" + # value = "v1beta2-1.4.5-3.5.0" + # } set { name = "sparkJobNamespaces" @@ -180,7 +180,7 @@ resource "helm_release" "airflow" { repository = "https://airflow.apache.org" namespace = "${var.namespace_prefix}airflow" dependency_update = "true" - version = "1.13.0" + version = "1.15.0" values = [ file("./envs/${var.env}/airflow.yaml") @@ -211,15 +211,30 @@ resource "helm_release" "airflow" { } set { - name = "images.airflow.repository" - value = "gbloisi/airflow" + name ="dags.gitSync.repo" + value = var.repo_url } set { - name = "images.airflow.tag" - value = "2.8.3rc1-python3.11" + name ="dags.gitSync.branch" + value = var.branch_name } + set { + name ="dags.gitSync.subPath" + value = var.dag_path + } + + # set { + # name = "images.airflow.repository" + # value = "gbloisi/airflow" + # } + + # set { + # name = "images.airflow.tag" + # value = "2.8.3rc1-python3.11" + # } + set { name = "ingress.web.host" value = "airflow.${var.domain}" diff --git a/modules/airflow/variables.tf b/modules/airflow/variables.tf index 8c93472..6b4b7ae 100644 --- a/modules/airflow/variables.tf +++ b/modules/airflow/variables.tf @@ -49,3 +49,18 @@ variable "admin_password" { variable "admin_hash" { type = string } + +variable "repo_url" { + type = string + default = "https://code-repo.d4science.org/D-Net/code-infrastructure-lab.git" +} + +variable "branch_name" { + type = string + default = "master" +} + +variable "dag_path" { + type = string + default = "workflow/dnet" +} diff --git a/variables.tf b/variables.tf index 207e4df..f624ad3 100644 --- a/variables.tf +++ b/variables.tf @@ -54,3 +54,7 @@ variable "minio_buckets" { ] } +variable "dag_branch_name" { + default = "master" +} +