commit ab7d2651ce6afb007d3f4c2d29f05ab2ceb84628 Author: s.licata Date: Mon May 20 13:04:04 2024 +0200 init: repository upload diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0704078 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +.vscode +.terraform +*.terraform.* +*.tfstate* +.terraform.lock.hcl +*symlink_provider.tf +.DS_Store +locals/ +**/private_keys \ No newline at end of file diff --git a/00-organization/branch-01-networking.tf b/00-organization/branch-01-networking.tf new file mode 100644 index 0000000..b917129 --- /dev/null +++ b/00-organization/branch-01-networking.tf @@ -0,0 +1,71 @@ +##------------------------------------------------------------------------------- +## d4science - NETWORKING Folder +##------------------------------------------------------------------------------- +module "d4science-networking-folder" { + source = "../assets/modules-fabric/v26/folder" + parent = "organizations/${var.organization.id}" + name = "Networking" + folder_create = true + + iam = { + "roles/owner" = [ + module.d4science-networking-tfsa.iam_email, + "group:foundationreply@d4science.org" + ] + "roles/compute.xpnAdmin" = [module.d4science-networking-tfsa.iam_email] #to enable shared VPC + #to enable 01-networking create the hub + "roles/resourcemanager.projectCreator" = [module.d4science-networking-tfsa.iam_email] + } + + # iam_additive = { + ## "roles/resourcemanager.projectCreator" = [module.common-terraform-sa.iam_email] # required to create project within this folder + # "roles/resourcemanager.projectCreator" = [] # required to create project within this folder + # + # } + +} + + +##------------------------------------------------------------------------------- +## 01 - Networking - TF SA, impersonated to apply Terraform config +##------------------------------------------------------------------------------- +module "d4science-networking-tfsa" { + source = "../assets/modules-fabric/v26/iam-service-account" + + project_id = module.d4science-seed-project.project_id + name = "d4science-com-tfnet-sa" + prefix = var.prefix + + iam = { + "roles/iam.serviceAccountTokenCreator" = ["group:foundationreply@d4science.org"] + #Impersonate service accounts (create OAuth2 access tokens, sign blobs or JWTs, etc). + } + + iam_billing_roles = { + "${var.billing_account_id}" = ["roles/billing.user"] + } +} + + +# +##------------------------------------------------------------------------------- +## 01 - Networking - TF Bucket, store Terraform state +##------------------------------------------------------------------------------- +module "d4science-networking-tfbucket" { + source = "../assets/modules-fabric/v26/gcs" + + name = "d4science-com-ew8-foundation-tfnet-bkt" + project_id = module.d4science-seed-project.project_id + prefix = var.prefix + + versioning = true + iam = { + "roles/storage.objectAdmin" = [module.d4science-networking-tfsa.iam_email] + } + + location = "EUROPE-WEST8" + storage_class = "STANDARD" + + labels = var.labels +} + diff --git a/00-organization/branch-02-security.tf b/00-organization/branch-02-security.tf new file mode 100644 index 0000000..de7b09d --- /dev/null +++ b/00-organization/branch-02-security.tf @@ -0,0 +1,42 @@ +##------------------------------------------------------------------------------- +## 02 - Security - TF SA, impersonated to apply Terraform config +##------------------------------------------------------------------------------- +module "d4science-security-tfsa" { + source = "../assets/modules-fabric/v26/iam-service-account" + + project_id = module.d4science-seed-project.project_id + name = "d4science-com-tfsec-sa" + prefix = var.prefix + + iam = { + "roles/iam.serviceAccountTokenCreator" = ["group:foundationreply@d4science.org"] + #Impersonate service accounts (create OAuth2 access tokens, sign blobs or JWTs, etc). + } + + iam_organization_roles = { + "${var.organization.id}" = ["roles/resourcemanager.organizationAdmin"] + } +} + + +# +##------------------------------------------------------------------------------- +## 01 - Networking - TF Bucket, store Terraform state +##------------------------------------------------------------------------------- +module "d4science-security-tfbucket" { + source = "../assets/modules-fabric/v26/gcs" + + name = "d4science-com-ew8-foundation-tfsec-bkt" + project_id = module.d4science-seed-project.project_id + prefix = var.prefix + + versioning = true + iam = { + "roles/storage.objectAdmin" = [module.d4science-security-tfsa.iam_email] + } + + location = "EUROPE-WEST8" + storage_class = "STANDARD" + + labels = var.labels +} diff --git a/00-organization/branch-03-project-factory.tf b/00-organization/branch-03-project-factory.tf new file mode 100644 index 0000000..e9a5210 --- /dev/null +++ b/00-organization/branch-03-project-factory.tf @@ -0,0 +1,127 @@ +##------------------------------------------------------------------------------- +## d4science/Prod Folder +##------------------------------------------------------------------------------- +module "d4science-prod-folder" { + source = "../assets/modules-fabric/v26/folder" + parent = "organizations/${var.organization.id}" + name = "Prod" + folder_create = true + + #https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/tree/master/blueprints/factories/project-factory + iam = { + #Required for Project Factory + "roles/logging.admin" = [module.d4science-project-factory-prod-tfsa.iam_email] + "roles/owner" = [module.d4science-project-factory-prod-tfsa.iam_email] + "roles/resourcemanager.folderAdmin" = [module.d4science-project-factory-prod-tfsa.iam_email] + "roles/resourcemanager.projectCreator" = [module.d4science-project-factory-prod-tfsa.iam_email] + + #To enable Shared VPC Service projects + "roles/compute.xpnAdmin" = [module.d4science-networking-tfsa.iam_email] + } +} + +##------------------------------------------------------------------------------- +## d4science/Test Folder +##------------------------------------------------------------------------------- +module "d4science-test-folder" { + source = "../assets/modules-fabric/v26/folder" + parent = "organizations/${var.organization.id}" + name = "Test" + folder_create = true + + #https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/tree/master/blueprints/factories/project-factory + iam = { + #Required for Project Factory + "roles/logging.admin" = [module.d4science-project-factory-test-tfsa.iam_email] + "roles/owner" = [module.d4science-project-factory-test-tfsa.iam_email] + "roles/resourcemanager.folderAdmin" = [module.d4science-project-factory-test-tfsa.iam_email] + "roles/resourcemanager.projectCreator" = [module.d4science-project-factory-test-tfsa.iam_email] + + #To enable Shared VPC Service projects + "roles/compute.xpnAdmin" = [module.d4science-networking-tfsa.iam_email] + } +} + +##------------------------------------------------------------------------------- +## 03 - Project Factory - TF SA, impersonated to apply Terraform config +##------------------------------------------------------------------------------- + +module "d4science-project-factory-test-tfsa" { + source = "../assets/modules-fabric/v26/iam-service-account" + + project_id = module.d4science-seed-project.project_id + name = "d4science-test-tfprj-sa" + prefix = var.prefix + + iam = { + "roles/iam.serviceAccountTokenCreator" = ["group:foundationreply@d4science.org"] + #Impersonate service accounts (create OAuth2 access tokens, sign blobs or JWTs, etc). + } + + iam_billing_roles = { + "${var.billing_account_id}" = ["roles/billing.admin"] + } +} + +module "d4science-project-factory-prod-tfsa" { + source = "../assets/modules-fabric/v26/iam-service-account" + + project_id = module.d4science-seed-project.project_id + name = "d4science-prod-tfprj-sa" + prefix = var.prefix + + iam = { + "roles/iam.serviceAccountTokenCreator" = ["group:foundationreply@d4science.org"] + #Impersonate service accounts (create OAuth2 access tokens, sign blobs or JWTs, etc). + } + + iam_billing_roles = { + "${var.billing_account_id}" = ["roles/billing.admin"] + } +} + +############## +############## +############## + +##------------------------------------------------------------------------------- +## 03 - Project Factory - TF Bucket, store Terraform state +##------------------------------------------------------------------------------- + +module "d4science-project-factory-test-tfbucket" { + source = "../assets/modules-fabric/v26/gcs" + + name = "d4science-test-ew8-foundation-tfprj-bkt" + project_id = module.d4science-seed-project.project_id + prefix = var.prefix + + versioning = true + iam = { + "roles/storage.objectAdmin" = [module.d4science-project-factory-test-tfsa.iam_email] + } + + location = "EUROPE-WEST8" + storage_class = "STANDARD" + + labels = var.labels +} + + + +module "d4science-project-factory-prod-tfbucket" { + source = "../assets/modules-fabric/v26/gcs" + + name = "d4science-prod-ew8-foundation-tfprj-bkt" + project_id = module.d4science-seed-project.project_id + prefix = var.prefix + + versioning = true + iam = { + "roles/storage.objectAdmin" = [module.d4science-project-factory-prod-tfsa.iam_email] + } + + location = "EUROPE-WEST8" + storage_class = "STANDARD" + + labels = var.labels +} diff --git a/00-organization/monitoring.tf b/00-organization/monitoring.tf new file mode 100644 index 0000000..1396f94 --- /dev/null +++ b/00-organization/monitoring.tf @@ -0,0 +1,73 @@ +module "common-organization-monitoring-project" { + source = "../assets/modules-fabric/v26/project" + name = "d4science-com-monitoring-prj" + project_create = true + parent = "organizations/${var.organization.id}" + billing_account = var.billing_account_id + services = [ + "cloudresourcemanager.googleapis.com", #required for SA impersonification + "iam.googleapis.com", #required for IAM and SA impersonification + "secretmanager.googleapis.com", #required for secrets + "monitoring.googleapis.com" #required for monitoring + ] + + iam = { + "roles/owner" = [ + module.d4science-organization-tfsa.iam_email, + ] + "roles/resourcemanager.projectIamAdmin" = [ + module.d4science-project-factory-prod-tfsa.iam_email, + module.d4science-project-factory-test-tfsa.iam_email + ], + "roles/monitoring.notificationChannelViewer" = [ + module.d4science-project-factory-test-tfsa.iam_email, + module.d4science-project-factory-prod-tfsa.iam_email + ] + } + + group_iam = { + "foundationreply@d4science.org" = [ + "roles/owner" + ] + } +} + +module "common-test-organization-monitoring-bucket" { + source = "../assets/modules-fabric/v26/logging-bucket" + parent_type = "project" + parent = module.common-organization-monitoring-project.project_id + id = "d4science-test-monitoring-sink-bucket" + location = "europe-west8" + description = "Terraform-managed." +} + +module "common-prod-organization-monitoring-bucket" { + source = "../assets/modules-fabric/v26/logging-bucket" + parent_type = "project" + parent = module.common-organization-monitoring-project.project_id + id = "d4science-prod-monitoring-sink-bucket" + location = "europe-west4" + description = "Terraform-managed." +} + +module "common-organization-monitoring-sa" { + source = "../assets/modules-fabric/v26/iam-service-account" + + project_id = module.common-organization-monitoring-project.project_id + name = "d4science-com-monitoring-sa" + prefix = var.prefix + + iam_organization_roles = { + "${var.organization.id}" = ["roles/monitoring.viewer"] + } +} + +resource "google_monitoring_notification_channel" "budget_alerting" { + display_name = "Budget alerting" + type = "email" + project = module.common-organization-monitoring-project.project_id + labels = { + email_address = "budget@d4science.org" + } + user_labels = {} +} \ No newline at end of file diff --git a/00-organization/organization.tf b/00-organization/organization.tf new file mode 100644 index 0000000..2f8c2bb --- /dev/null +++ b/00-organization/organization.tf @@ -0,0 +1,19 @@ +##------------------------------------------------------------------------------- +## BCC_PAY - Organization +##------------------------------------------------------------------------------- +module "organization" { + source = "../assets/modules-fabric/v26/organization" + organization_id = "organizations/${var.organization.id}" + + custom_roles = { + # Allow gke policy creation. Assign this to gke sa + "d4science_org_glb_gkemanagefwrules_role" = [ + "compute.networks.updatePolicy", + "compute.firewalls.list", + "compute.firewalls.get", + "compute.firewalls.create", + "compute.firewalls.update", + "compute.firewalls.delete" + ] + } +} diff --git a/00-organization/outputs-providers.tf b/00-organization/outputs-providers.tf new file mode 100644 index 0000000..8fc4710 --- /dev/null +++ b/00-organization/outputs-providers.tf @@ -0,0 +1,38 @@ +# tfdoc:file:description Providers output files. + +locals { + providers = { + "00-organization" = { + bucket = module.d4science-organization-tfbucket.name + service_account = module.d4science-organization-tfsa.email + } + "01-networking" = { + bucket = module.d4science-networking-tfbucket.name + service_account = module.d4science-networking-tfsa.email + } + "02-security" = { + bucket = module.d4science-security-tfbucket.name + service_account = module.d4science-security-tfsa.email + } + "03-project-factory-test" = { + bucket = module.d4science-project-factory-test-tfbucket.name + service_account = module.d4science-project-factory-test-tfsa.email + } + "03-project-factory-prod" = { + bucket = module.d4science-project-factory-prod-tfbucket.name + service_account = module.d4science-project-factory-prod-tfsa.email + } + } +} + +resource "local_file" "other_providers" { + for_each = var.outputs_location == null ? {} : local.providers + + file_permission = "0644" + filename = "${path.module}/${var.outputs_location}/providers/${each.key}.providers.tf" + content = templatefile("${path.module}/../assets/providers.tpl", { + bucket = each.value.bucket + sa = each.value.service_account + prefix = try(each.value.prefix, null) + }) +} diff --git a/00-organization/outputs-tfvars.tf b/00-organization/outputs-tfvars.tf new file mode 100644 index 0000000..f3cc837 --- /dev/null +++ b/00-organization/outputs-tfvars.tf @@ -0,0 +1,63 @@ +# tfdoc:file:description Terraform tfvars output files. + +locals { + tfvars = { + + billing_account_id = var.billing_account_id + organization = var.organization + prefix = var.prefix + labels = var.labels + groups = var.groups + + # Global variables + + seed-project = { + project_id = module.d4science-seed-project.project_id + project_number = module.d4science-seed-project.number + } + + service_accounts = { + networking = module.d4science-networking-tfsa.iam_email + security = module.d4science-security-tfsa.iam_email + project_factory_test = module.d4science-project-factory-test-tfsa.iam_email + project_factory_prod = module.d4science-project-factory-prod-tfsa.iam_email + } + + monitoring = { + bucket-sink-test = module.common-test-organization-monitoring-bucket + bucket-sink-prod = module.common-prod-organization-monitoring-bucket + project_id = module.common-organization-monitoring-project.project_id + channels = { + "budget-alerting" = google_monitoring_notification_channel.budget_alerting.id + } + } + + # Folders + folders = { + networking = { + id = module.d4science-networking-folder.id + name = module.d4science-networking-folder.name + } + prod = { + id = module.d4science-prod-folder.id + name = module.d4science-prod-folder.name + } + test = { + id = module.d4science-test-folder.id + name = module.d4science-test-folder.name + } + } + } +} + +resource "local_file" "tfvars" { + for_each = var.outputs_location == null ? {} : { 1 = 1 } + file_permission = "0644" + filename = "${path.module}/${var.outputs_location}/tfvars/00-organization.auto.tfvars.json" + content = jsonencode(local.tfvars) +} + +output "tfvars" { + sensitive = true + value = local.tfvars +} diff --git a/00-organization/providers.tf b/00-organization/providers.tf new file mode 100644 index 0000000..34aeca5 --- /dev/null +++ b/00-organization/providers.tf @@ -0,0 +1,18 @@ +terraform { + backend "gcs" { + bucket = "d4science-com-ew8-foundation-tforg-bkt" + impersonate_service_account = "d4science-com-tforg-sa@d4science-com-automation-prj.iam.gserviceaccount.com" + } +} + +provider "google" { + impersonate_service_account = "d4science-com-tforg-sa@d4science-com-automation-prj.iam.gserviceaccount.com" +} + +provider "google-beta" { + impersonate_service_account = "d4science-com-tforg-sa@d4science-com-automation-prj.iam.gserviceaccount.com" +} + +provider "google" { + alias = "no-impersonate" +} \ No newline at end of file diff --git a/00-organization/terraform.tfvars b/00-organization/terraform.tfvars new file mode 100644 index 0000000..0e5dd18 --- /dev/null +++ b/00-organization/terraform.tfvars @@ -0,0 +1,7 @@ +#D4Science +billing_account_id = "018258-BCA804-E9D4C6" + +organization = { + domain = "d4sscience.org" + id = 392184451762 +} diff --git a/00-organization/tf-automation.tf b/00-organization/tf-automation.tf new file mode 100644 index 0000000..e8992e1 --- /dev/null +++ b/00-organization/tf-automation.tf @@ -0,0 +1,87 @@ + +module "d4science-seed-project" { + source = "../assets/modules-fabric/v26/project" + name = "d4science-com-automation-prj" + project_create = true + billing_account = var.billing_account_id + parent = "organizations/${var.organization.id}" + services = [ + "cloudresourcemanager.googleapis.com", #required for SA impersonification + "serviceusage.googleapis.com", #required for SA impersonification + "iam.googleapis.com", #required for IAM and SA impersonification + "cloudbilling.googleapis.com", #required for project creation + "accesscontextmanager.googleapis.com", #required for VPC SC + "logging.googleapis.com", #required for sink creation + "servicenetworking.googleapis.com", #required for Private Service Access + "pubsub.googleapis.com", #required for PubSub + "monitoring.googleapis.com", #required for Monitoring + "billingbudgets.googleapis.com", #required for Billing budgets + "orgpolicy.googleapis.com" #required for Organizational policies + ] + + iam = { + "roles/owner" = [ + module.d4science-organization-tfsa.iam_email, + ] + } + + group_iam = { + "foundationreply@d4science.org" = [ + "roles/owner" + ] + } + + labels = var.labels +} +##------------------------------------------------------------------------------- +## Terraform bucket for storing TFSTATE +##------------------------------------------------------------------------------- +module "d4science-organization-tfbucket" { + source = "../assets/modules-fabric/v26/gcs" + + name = "d4science-com-ew8-foundation-tforg-bkt" + project_id = module.d4science-seed-project.project_id + + prefix = var.prefix + + versioning = true + + iam = { + "roles/storage.objectAdmin" = [module.d4science-organization-tfsa.iam_email] + } + + # Maintain last 3 versions (1 live plus 2). + lifecycle_rules = { + limit-versions = { + action = { + type = "Delete" + } + condition = { + num_newer_versions = 3 + with_state = "ARCHIVED" + } + } + } + + location = "EUROPE-WEST8" + storage_class = "STANDARD" + + labels = var.labels +} + + +##------------------------------------------------------------------------------- +## TF SA, impersonated to apply Terraform config +##------------------------------------------------------------------------------- +module "d4science-organization-tfsa" { + source = "../assets/modules-fabric/v26/iam-service-account" + + project_id = module.d4science-seed-project.project_id + name = "d4science-com-tforg-sa" + prefix = var.prefix + + iam = { + "roles/iam.serviceAccountTokenCreator" = ["group:foundationreply@d4science.org"] + #Impersonate service accounts (create OAuth2 access tokens, sign blobs or JWTs, etc). + } +} diff --git a/00-organization/variables.tf b/00-organization/variables.tf new file mode 100644 index 0000000..2173669 --- /dev/null +++ b/00-organization/variables.tf @@ -0,0 +1,35 @@ +variable "billing_account_id" { + description = "D4Science Billing account id." + type = string +} + +variable "groups" { + description = "Group names to grant organization-level permissions." + type = map(string) + default = {} +} + +variable "organization" { + description = "Organization details." + type = object({ + domain = string + id = number + }) +} + +variable "prefix" { + description = "Prefix for terraform resources" + type = string + default = null +} +variable "outputs_location" { + description = "Assets path location relative to the module" + type = string + default = "../assets" +} + +variable "labels" { + description = "Labels to add to the resources" + type = map(string) + default = {} +} diff --git a/01-networking/data/subnets/prod/d4science-prod-ew4-vpc-con-sub.yaml b/01-networking/data/subnets/prod/d4science-prod-ew4-vpc-con-sub.yaml new file mode 100644 index 0000000..9923bbd --- /dev/null +++ b/01-networking/data/subnets/prod/d4science-prod-ew4-vpc-con-sub.yaml @@ -0,0 +1,9 @@ +name : "d4science-prod-ew4-vpc-con-sub" +description : "Subnet assigned for D4Science prod environment's vpc connector" +region : "europe-west4" +ip_cidr_range : "10.254.2.0/28" + +iam: # This section must be added after the creation of the service project to allow it to operate on the VPC + roles/compute.networkUser: + - serviceAccount:service-398661014261@gcp-sa-vpcaccess.iam.gserviceaccount.com + - serviceAccount:398661014261@cloudservices.gserviceaccount.com \ No newline at end of file diff --git a/01-networking/data/subnets/prod/d4science-prod-ew4-vre-sub.yaml b/01-networking/data/subnets/prod/d4science-prod-ew4-vre-sub.yaml new file mode 100644 index 0000000..a5ba31f --- /dev/null +++ b/01-networking/data/subnets/prod/d4science-prod-ew4-vre-sub.yaml @@ -0,0 +1,14 @@ +name : "d4science-prod-ew4-vre-sub" +description : "Subnet assigned for D4Science prod environment's vre GKE cluster" +region : "europe-west4" +ip_cidr_range : "10.254.0.0/25" + +secondary_ip_ranges: # map of secondary ip ranges + pods: 10.250.0.0/17 + services: 10.250.128.0/17 + +iam: # This section must be added after the creation of the service project to allow it to operate on the VPC + roles/compute.networkUser: + - serviceAccount:service-783244529218@container-engine-robot.iam.gserviceaccount.com + - serviceAccount:783244529218@cloudservices.gserviceaccount.com + diff --git a/01-networking/data/subnets/test/d4science-test-ew8-vpc-con-sub.yaml b/01-networking/data/subnets/test/d4science-test-ew8-vpc-con-sub.yaml new file mode 100644 index 0000000..cdc79dd --- /dev/null +++ b/01-networking/data/subnets/test/d4science-test-ew8-vpc-con-sub.yaml @@ -0,0 +1,9 @@ +name : "d4science-test-ew8-vpc-con-sub" +description : "Subnet assigned for D4Science test environment's vpc connector" +region : "europe-west8" +ip_cidr_range : "10.254.66.0/28" + +iam: # This section must be added after the creation of the service project to allow it to operate on the VPC + roles/compute.networkUser: + - serviceAccount:service-31481777243@gcp-sa-vpcaccess.iam.gserviceaccount.com + - serviceAccount:31481777243@cloudservices.gserviceaccount.com \ No newline at end of file diff --git a/01-networking/data/subnets/test/d4science-test-ew8-vre-sub.yaml b/01-networking/data/subnets/test/d4science-test-ew8-vre-sub.yaml new file mode 100644 index 0000000..01279ff --- /dev/null +++ b/01-networking/data/subnets/test/d4science-test-ew8-vre-sub.yaml @@ -0,0 +1,13 @@ +name : "d4science-test-ew8-vre-sub" +description : "Subnet assigned for D4Science test environment's vre GKE cluster" +region : "europe-west8" +ip_cidr_range : "10.254.64.0/25" + +secondary_ip_ranges: # map of secondary ip ranges + pods: 10.251.0.0/17 + services: 10.251.128.0/17 + +iam: # This section must be added after the creation of the service project to allow it to operate on the VPC + roles/compute.networkUser: + - serviceAccount:service-804925782180@container-engine-robot.iam.gserviceaccount.com + - serviceAccount:804925782180@cloudservices.gserviceaccount.com \ No newline at end of file diff --git a/01-networking/main.tf b/01-networking/main.tf new file mode 100644 index 0000000..baf71ab --- /dev/null +++ b/01-networking/main.tf @@ -0,0 +1,35 @@ +locals { + organization_vars = jsondecode(file("../assets/tfvars/00-organization.auto.tfvars.json")) +} +locals { + + billing_account_id = local.organization_vars.billing_account_id + organization = local.organization_vars.organization + prefix = local.organization_vars.prefix + labels = local.organization_vars.labels + groups = local.organization_vars.groups + + # Global variables + + seed-project = { + project_id = local.organization_vars.seed-project.project_id + project_number = local.organization_vars.seed-project.project_number + } + + service_accounts = { + networking = local.organization_vars.service_accounts.networking + security = local.organization_vars.service_accounts.security + project_factory_test = local.organization_vars.service_accounts.project_factory_test + project_factory_prod = local.organization_vars.service_accounts.project_factory_prod + + } + + # Folders + folders = { + networking = { + id = local.organization_vars.folders.networking.id + name = local.organization_vars.folders.networking.name + + } + } +} \ No newline at end of file diff --git a/01-networking/outputs-tfvars.tf b/01-networking/outputs-tfvars.tf new file mode 100644 index 0000000..4fb72b2 --- /dev/null +++ b/01-networking/outputs-tfvars.tf @@ -0,0 +1,48 @@ +# tfdoc:file:description Terraform tfvars output files. + +locals { + tfvars = { + spoke-test = { + project_id = module.d4science-networking-spoke-test-project.project_id + number = module.d4science-networking-spoke-test-project.number + network = module.d4science-networking-test-vpc.self_link + subnets = { + for k, v in module.d4science-networking-test-vpc.subnet_ips : + k => { + self_link = module.d4science-networking-test-vpc.subnets[k].self_link + ip = v, + secondary_ranges = module.d4science-networking-test-vpc.subnet_secondary_ranges[k] + } + } + nat-address = module.d4science-networking-spoke-test-addresses.external_addresses["nat-ew8-00-addr-00"].address + } + + spoke-prod = { + project_id = module.d4science-networking-spoke-prod-project.project_id + number = module.d4science-networking-spoke-prod-project.number + network = module.d4science-networking-prod-vpc.self_link + subnets = { + for k, v in module.d4science-networking-prod-vpc.subnet_ips : + k => { + self_link = module.d4science-networking-prod-vpc.subnets[k].self_link + ip = v, + secondary_ranges = module.d4science-networking-prod-vpc.subnet_secondary_ranges[k] + } + } + nat-address = module.d4science-networking-spoke-prod-addresses.external_addresses["nat-ew4-00-addr-00"].address + } + } +} + +resource "local_file" "tfvars" { + for_each = var.outputs_location == null ? {} : { 1 = 1 } + file_permission = "0644" + filename = "${path.module}/${var.outputs_location}/tfvars/01-networking.auto.tfvars.json" + content = jsonencode(local.tfvars) +} + + +output "tfvars" { + sensitive = true + value = local.tfvars +} diff --git a/01-networking/providers.tf b/01-networking/providers.tf new file mode 100644 index 0000000..8fdee56 --- /dev/null +++ b/01-networking/providers.tf @@ -0,0 +1,37 @@ +terraform { + backend "gcs" { + bucket = "d4science-com-ew8-foundation-tfnet-bkt" + impersonate_service_account = "d4science-com-tfnet-sa@d4science-com-automation-prj.iam.gserviceaccount.com" + } + + required_version = "~> 1.6.5" + + required_providers { + google = { + source = "hashicorp/google" + version = "4.84.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = "4.84.0" + } + local = { + source = "hashicorp/local" + version = "~> 2.4.0" + } + } +} + +provider "google" { + impersonate_service_account = "d4science-com-tfnet-sa@d4science-com-automation-prj.iam.gserviceaccount.com" +} + +provider "google-beta" { + impersonate_service_account = "d4science-com-tfnet-sa@d4science-com-automation-prj.iam.gserviceaccount.com" +} + +provider "google" { + alias = "no-impersonate" +} + +provider "local" {} \ No newline at end of file diff --git a/01-networking/spoke-prod.tf b/01-networking/spoke-prod.tf new file mode 100644 index 0000000..3ef3395 --- /dev/null +++ b/01-networking/spoke-prod.tf @@ -0,0 +1,117 @@ +#---------------------------------- +# Project +#---------------------------------- +module "d4science-networking-spoke-prod-project" { + source = "../assets/modules-fabric/v26/project" + + name = "d4science-prod-spoke-prj" + prefix = local.prefix + billing_account = local.billing_account_id + parent = local.folders.networking.id + labels = local.labels + + auto_create_network = false + project_create = true + + # Enable Shared VPC + shared_vpc_host_config = { + enabled = true + # First run the next line must be commented, after you execute step 3 you can uncomment it + # because you first need to create the service projects + service_projects = var.service_projects["prod"] + } + + services = [ + "vpcaccess.googleapis.com", # to enable VPC Access Connector + "container.googleapis.com", + "certificatemanager.googleapis.com", # required for certificates + "secretmanager.googleapis.com", + "servicenetworking.googleapis.com" + ] + + iam = { + "roles/owner" = [] + "roles/editor" = ["serviceAccount:${module.d4science-networking-spoke-prod-project.service_accounts.cloud_services}"] + "roles/compute.securityAdmin" = [local.service_accounts.security] #to create firewall rules + + #https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/tree/master/blueprints/factories/project-factory + #Required for Project Factory + "roles/browser" = [local.service_accounts.project_factory_prod] + "roles/compute.viewer" = [local.service_accounts.project_factory_prod] + "roles/dns.admin" = [local.service_accounts.project_factory_prod] + + # These must be done after creating the service project + "roles/container.hostServiceAgentUser" = ["serviceAccount:service-783244529218@container-engine-robot.iam.gserviceaccount.com"] + "organizations/${local.organization_vars.organization.id}/roles/d4science_org_glb_gkemanagefwrules_role" = ["serviceAccount:service-783244529218@container-engine-robot.iam.gserviceaccount.com"] + } +} + + +#---------------------------------- +# VPC prod with subnets +#---------------------------------- +module "d4science-networking-prod-vpc" { + source = "../assets/modules-fabric/v26/net-vpc" + project_id = module.d4science-networking-spoke-prod-project.project_id + name = "d4science-prod-vpc" + factories_config = { + subnets_folder = "${path.root}/data/subnets/prod" + } + psa_config = { + ranges = { filestore = var.psa_ranges["prod"]["filestore"] } + } +} + +#---------------------------------- +# Network Adresses (GLB and NAT) +#---------------------------------- + +module "d4science-networking-spoke-prod-addresses" { + source = "../assets/modules-fabric/v26/net-address" + project_id = module.d4science-networking-spoke-prod-project.project_id + external_addresses = { + nat-ew4-00-addr-00 = { region = "europe-west4" } + } +} + + +#---------------------------------- +# Cloud NAT (Spoke PROD) +#---------------------------------- + +module "d4science-networking-spoke-prod-cloudnat-ew4" { + source = "../assets/modules-fabric/v26/net-cloudnat" + project_id = module.d4science-networking-spoke-prod-project.project_id + region = "europe-west4" + name = "d4science-prod-ew4-01-nat" + addresses = [module.d4science-networking-spoke-prod-addresses.external_addresses["nat-ew4-00-addr-00"].self_link] + + router_network = module.d4science-networking-prod-vpc.self_link + config_source_subnets = "LIST_OF_SUBNETWORKS" + subnetworks = [ + { + self_link = module.d4science-networking-prod-vpc.subnet_self_links["europe-west4/d4science-prod-ew4-vre-sub"] + config_source_ranges = ["ALL_IP_RANGES"] + secondary_ranges = null + } + ] +} + +##---------------------------------- +## Private Google Access (PROD) +##---------------------------------- + +module "d4science-networking-spoke-prod-pga" { + source = "../assets/modules-custom/pga" + project_id = module.d4science-networking-spoke-prod-project.project_id + name = "d4science-prod-pga" + domains = { + artifact = true + } + config = { + private = true + } + networks = [ + module.d4science-networking-prod-vpc.self_link + ] +} \ No newline at end of file diff --git a/01-networking/spoke-test.tf b/01-networking/spoke-test.tf new file mode 100644 index 0000000..c67bf0b --- /dev/null +++ b/01-networking/spoke-test.tf @@ -0,0 +1,117 @@ +#---------------------------------- +# Project +#---------------------------------- +module "d4science-networking-spoke-test-project" { + source = "../assets/modules-fabric/v26/project" + + name = "d4science-test-spoke-prj" + prefix = local.prefix + billing_account = local.billing_account_id + parent = local.folders.networking.id + labels = local.labels + + auto_create_network = false + project_create = true + + # Enable Shared VPC + shared_vpc_host_config = { + enabled = true + # First run the next line must be commented, after you execute step 3 you can uncomment it + # because you first need to create the service projects + service_projects = var.service_projects["test"] + } + + services = [ + "vpcaccess.googleapis.com", # to enable VPC Access Connector + "container.googleapis.com", + "certificatemanager.googleapis.com", # required for certificates + "secretmanager.googleapis.com", + "servicenetworking.googleapis.com" + ] + + iam = { + "roles/owner" = [] + "roles/editor" = ["serviceAccount:${module.d4science-networking-spoke-test-project.service_accounts.cloud_services}"] + "roles/compute.securityAdmin" = [local.service_accounts.security] #to create firewall rules + + #https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/tree/master/blueprints/factories/project-factory + #Required for Project Factory + "roles/browser" = [local.service_accounts.project_factory_test] + "roles/compute.viewer" = [local.service_accounts.project_factory_test] + "roles/dns.admin" = [local.service_accounts.project_factory_test] + + # These must be done after creating the service project + "roles/container.hostServiceAgentUser" = ["serviceAccount:service-804925782180@container-engine-robot.iam.gserviceaccount.com"] + "organizations/${local.organization_vars.organization.id}/roles/d4science_org_glb_gkemanagefwrules_role" = ["serviceAccount:service-804925782180@container-engine-robot.iam.gserviceaccount.com"] + } +} + + +#---------------------------------- +# VPC test with subnets +#---------------------------------- +module "d4science-networking-test-vpc" { + source = "../assets/modules-fabric/v26/net-vpc" + project_id = module.d4science-networking-spoke-test-project.project_id + name = "d4science-test-vpc" + factories_config = { + subnets_folder = "${path.root}/data/subnets/test" + } + psa_config = { + ranges = { filestore = var.psa_ranges["test"]["filestore"] } + } +} + +#---------------------------------- +# Network Adresses (GLB and NAT) +#---------------------------------- + +module "d4science-networking-spoke-test-addresses" { + source = "../assets/modules-fabric/v26/net-address" + project_id = module.d4science-networking-spoke-test-project.project_id + external_addresses = { + nat-ew8-00-addr-00 = { region = "europe-west8" } + } +} + + +#---------------------------------- +# Cloud NAT (Spoke TEST) +#---------------------------------- + +module "d4science-networking-spoke-test-cloudnat-ew8" { + source = "../assets/modules-fabric/v26/net-cloudnat" + project_id = module.d4science-networking-spoke-test-project.project_id + region = "europe-west8" + name = "d4science-test-ew8-01-nat" + addresses = [module.d4science-networking-spoke-test-addresses.external_addresses["nat-ew8-00-addr-00"].self_link] + + router_network = module.d4science-networking-test-vpc.self_link + config_source_subnets = "LIST_OF_SUBNETWORKS" + subnetworks = [ + { + self_link = module.d4science-networking-test-vpc.subnet_self_links["europe-west8/d4science-test-ew8-vre-sub"] + config_source_ranges = ["ALL_IP_RANGES"] + secondary_ranges = null + } + ] +} + +##---------------------------------- +## Private Google Access (TEST) +##---------------------------------- + +module "d4science-networking-spoke-test-pga" { + source = "../assets/modules-custom/pga" + project_id = module.d4science-networking-spoke-test-project.project_id + name = "d4science-test-pga" + domains = { + artifact = true + } + config = { + private = true + } + networks = [ + module.d4science-networking-test-vpc.self_link + ] +} \ No newline at end of file diff --git a/01-networking/terraform.tfvars b/01-networking/terraform.tfvars new file mode 100644 index 0000000..575961e --- /dev/null +++ b/01-networking/terraform.tfvars @@ -0,0 +1,10 @@ +service_projects = { + networking : ["d4science-spoke-prod", "d4science-spoke-test"], + test : ["d4science-test-vre-prj", "d4science-test-script-prj"], + prod : ["d4science-prod-vre-prj", "d4science-prod-script-prj"] +} + +psa_ranges = { + "test" : { "filestore" : "10.254.65.0/24" }, + "prod" : { "filestore" : "10.254.1.0/24" }, +} \ No newline at end of file diff --git a/01-networking/variables.tf b/01-networking/variables.tf new file mode 100644 index 0000000..4a0a647 --- /dev/null +++ b/01-networking/variables.tf @@ -0,0 +1,24 @@ +variable "outputs_location" { + description = "Assets path location relative to the module" + type = string + default = "../assets" +} + +variable "service_projects" { + description = "A map of lists reporting projects to attach to a shared VPC of each environment" + type = map(list(string)) + default = { + networking : [], + test : [], + prod : [] + } +} + +variable "psa_ranges" { + description = "A map of maps of address ranges for private service access" + type = map(map(string)) + default = { + test : {} + prod : {} + } +} \ No newline at end of file diff --git a/02-security/firewall/cidrs.yaml b/02-security/firewall/cidrs.yaml new file mode 100644 index 0000000..3135032 --- /dev/null +++ b/02-security/firewall/cidrs.yaml @@ -0,0 +1,37 @@ +health-check: + - 35.191.0.0/16 + - 130.211.0.0/22 + - 108.170.220.0/23 + +vpc-nat-ranges: + - 107.178.230.64/26 + - 35.199.224.0/19 + +google-restricted-api: + - 199.36.153.4/30 + +google-private-api: + - 199.36.153.8/30 + +#---------------------------------- +# VRE +#---------------------------------- +# TEST +d4science-test-vre-gke-controlplane: + - 10.249.0.64/28 +d4science-test-vre-gke-pods: + - 10.251.0.0/17 +d4science-test-vre-gke-services: + - 10.251.128.0/17 +d4science-test-vre-gke-subnet: + - 10.254.64.0/25 + +# PROD +d4science-prod-vre-gke-controlplane: + - 10.249.0.0/28 +d4science-prod-vre-gke-pods: + - 10.250.0.0/17 +d4science-prod-vre-gke-services: + - 10.250.128.0/17 +d4science-prod-vre-gke-subnet: + - 10.254.0.0/25 \ No newline at end of file diff --git a/02-security/firewall/rules/org-policies/cloudfunctions.yaml b/02-security/firewall/rules/org-policies/cloudfunctions.yaml new file mode 100644 index 0000000..6cda6e6 --- /dev/null +++ b/02-security/firewall/rules/org-policies/cloudfunctions.yaml @@ -0,0 +1,7 @@ +# Cloud Functions ORG policies + +cloudfunctions.allowedIngressSettings: + rules: + - allow: + values: + - ALLOW_INTERNAL_ONLY \ No newline at end of file diff --git a/02-security/firewall/rules/org-policies/gcp.yaml b/02-security/firewall/rules/org-policies/gcp.yaml new file mode 100644 index 0000000..bbe8b20 --- /dev/null +++ b/02-security/firewall/rules/org-policies/gcp.yaml @@ -0,0 +1,18 @@ +# GCP ORG policies + +gcp.detailedAuditLoggingMode: + rules: + - enforce: true + +gcp.resourceLocations: + rules: + - allow: + values: + - in:eu-locations + +gcp.restrictTLSVersion: + rules: + - deny: + values: + - TLS_VERSION_1 + - TLS_VERSION_1_1 \ No newline at end of file diff --git a/02-security/firewall/rules/org-policies/iam.yaml b/02-security/firewall/rules/org-policies/iam.yaml new file mode 100644 index 0000000..78c2e5c --- /dev/null +++ b/02-security/firewall/rules/org-policies/iam.yaml @@ -0,0 +1,15 @@ +# IAM ORG policies + +iam.disableServiceAccountKeyCreation: + rules: + - enforce: true + +iam.disableServiceAccountKeyUpload: + rules: + - enforce: true + +iam.allowedPolicyMemberDomains: + rules: + - allow: + values: + - C01slynf8 \ No newline at end of file diff --git a/02-security/firewall/rules/org-policies/run.yaml b/02-security/firewall/rules/org-policies/run.yaml new file mode 100644 index 0000000..8f4a38c --- /dev/null +++ b/02-security/firewall/rules/org-policies/run.yaml @@ -0,0 +1,25 @@ +# Cloud Run ORG policies + +# Dopo svariati test, l'unica sintassi funzionante necessita di +# due regole condizionali, con condizione opposta +# Warning: al tempo del test (08/2023) la UI di Cloud Run risulta buggata +# Testare Ingress tramite deploy gcloud +run.allowedIngress: + rules: + - allow: + values: + - internal + - internal-and-cloud-load-balancing + condition: + description: "Condizione per *non* permettere Ingress di tipo \"ALL\" ai servizi Cloud Run" + expression: "!resource.matchTag(\"812186410362/cloudrun-allowIngress-tag\", \"allow-all\")" + title: "Deny Ingress \"ALL\"" + - allow: + values: + - all + - internal + - internal-and-cloud-load-balancing + condition: + description: "Condizione per permettere Ingress di tipo \"ALL\" ai servizi Cloud Run" + expression: "resource.matchTag(\"812186410362/cloudrun-allowIngress-tag\", \"allow-all\")" + title: "Allow Ingress \"ALL\"" \ No newline at end of file diff --git a/02-security/firewall/rules/org-policies/storage.yaml b/02-security/firewall/rules/org-policies/storage.yaml new file mode 100644 index 0000000..b8c935c --- /dev/null +++ b/02-security/firewall/rules/org-policies/storage.yaml @@ -0,0 +1,10 @@ +# Storage ORG policies + +storage.publicAccessPrevention: + rules: + - enforce: true + +storage.retentionPolicySeconds: + rules: + - allow: + all: true \ No newline at end of file diff --git a/02-security/firewall/rules/spoke-prod/default.yaml b/02-security/firewall/rules/spoke-prod/default.yaml new file mode 100644 index 0000000..b26f308 --- /dev/null +++ b/02-security/firewall/rules/spoke-prod/default.yaml @@ -0,0 +1,21 @@ +ingress: + deny-ingress-all: + deny: true + description: "Block ingress." + priority: 65000 + enable_logging: + include_metadata: true + rules: + - protocol: all + ports: [] + +egress: + deny-egress-all: + deny: true + description: "Block egress." + priority: 65001 + enable_logging: + include_metadata: true + rules: + - protocol: all + ports: [] \ No newline at end of file diff --git a/02-security/firewall/rules/spoke-prod/vre.yaml b/02-security/firewall/rules/spoke-prod/vre.yaml new file mode 100644 index 0000000..6565b23 --- /dev/null +++ b/02-security/firewall/rules/spoke-prod/vre.yaml @@ -0,0 +1,103 @@ +ingress: + d4science-prod-glb-vre-ig-alw-healthcheck-fwr: + deny: false + description: Allow Healthcheck. + priority: 1000 + source_ranges: + - health-check + rules: + - protocol: tcp + ports: + - 80 + - 443 + - 667 + - 8080 + d4science-prod-glb-ing-alw-gkecontrolplane-fwr: + deny: false + description: "Allow GKE master to reach VPC." + priority: 1000 + source_ranges: + - d4science-prod-vre-gke-controlplane + rules: + - protocol: all + ports: [] + d4science-prod-glb-vre-ing-serverless-to-vpc-connector: + deny: false + description: Allow serverless services to reach vpc connectors. + priority: 1000 + source_ranges: + - vpc-nat-ranges + rules: + - protocol: tcp + ports: + - 667 + - protocol: udp + ports: + - 665 + - 666 + - protocol: icmp + ports: [] + +egress: + d4science-prod-glb-vre-egr-alw-dns-fwr: + deny: false + description: Allow DNS Egress. + priority: 1000 + rules: + - protocol: tcp + ports: [53] + - protocol: udp + ports: [53] + + d4science-prod-glb-vre-egr-alw-private-fwr: + deny: false + description: Allow PGA. + priority: 1000 + destination_ranges: + - google-private-api + rules: + - protocol: tcp + ports: [443] + + # This rule can be enforced only after the SA is created inside the service project. When you first create the cluster, activate it without the options for enforcing the SA. + d4science-prod-glb-vre-egr-alw-cluster-fwr: + deny: false + description: "Allow cluster communication." + use_service_accounts: true + targets: + - d4science-prod-vre-nodepoolsa@d4science-prod-vre-prj.iam.gserviceaccount.com + destination_ranges: + - 0.0.0.0/0 + priority: 1000 + rules: + - protocol: all + ports: [] + + d4science-prod-glb-vre-egr-vpc-connector-to-gke: + deny: false + description: "Allow vpc serverless connector to gke control plane communication." + targets: + - vpc-connector-europe-west4-d4science-prod-cf-vpc-con + destination_ranges: + - 0.0.0.0/0 + priority: 1000 + rules: + - protocol: all + ports: [] + + d4science-prod-glb-vre-egr-vpc-connector-to-serverless: + deny: false + description: Allow vpc connectors to reach serverless services. + priority: 1000 + destination_ranges: + - vpc-nat-ranges + rules: + - protocol: tcp + ports: + - 667 + - protocol: udp + ports: + - 665 + - 666 + - protocol: icmp + ports: [] \ No newline at end of file diff --git a/02-security/firewall/rules/spoke-test/default.yaml b/02-security/firewall/rules/spoke-test/default.yaml new file mode 100644 index 0000000..b26f308 --- /dev/null +++ b/02-security/firewall/rules/spoke-test/default.yaml @@ -0,0 +1,21 @@ +ingress: + deny-ingress-all: + deny: true + description: "Block ingress." + priority: 65000 + enable_logging: + include_metadata: true + rules: + - protocol: all + ports: [] + +egress: + deny-egress-all: + deny: true + description: "Block egress." + priority: 65001 + enable_logging: + include_metadata: true + rules: + - protocol: all + ports: [] \ No newline at end of file diff --git a/02-security/firewall/rules/spoke-test/vre.yaml b/02-security/firewall/rules/spoke-test/vre.yaml new file mode 100644 index 0000000..c43bd1e --- /dev/null +++ b/02-security/firewall/rules/spoke-test/vre.yaml @@ -0,0 +1,103 @@ +ingress: + d4science-test-glb-vre-ig-alw-healthcheck-fwr: + deny: false + description: Allow Healthcheck. + priority: 1000 + source_ranges: + - health-check + rules: + - protocol: tcp + ports: + - 80 + - 443 + - 667 + - 8080 + d4science-test-glb-ing-alw-gkecontrolplane-fwr: + deny: false + description: "Allow GKE master to reach VPC." + priority: 1000 + source_ranges: + - d4science-test-vre-gke-controlplane + rules: + - protocol: all + ports: [] + d4science-test-glb-vre-ing-serverless-to-vpc-connector: + deny: false + description: Allow serverless services to reach vpc connectors. + priority: 1000 + source_ranges: + - vpc-nat-ranges + rules: + - protocol: tcp + ports: + - 667 + - protocol: udp + ports: + - 665 + - 666 + - protocol: icmp + ports: [] + +egress: + d4science-test-glb-vre-egr-alw-dns-fwr: + deny: false + description: Allow DNS Egress. + priority: 1000 + rules: + - protocol: tcp + ports: [53] + - protocol: udp + ports: [53] + + d4science-test-glb-vre-egr-alw-private-fwr: + deny: false + description: Allow PGA. + priority: 1000 + destination_ranges: + - google-private-api + rules: + - protocol: tcp + ports: [443] + + # This rule can be enforced only after the SA is created inside the service project. When you first create the cluster, activate it without the options for enforcing the SA. + d4science-test-glb-vre-egr-alw-cluster-fwr: + deny: false + description: "Allow cluster communication." + use_service_accounts: true + targets: + - d4science-test-vre-nodepoolsa@d4science-test-vre-prj.iam.gserviceaccount.com + destination_ranges: + - 0.0.0.0/0 + priority: 1000 + rules: + - protocol: all + ports: [] + + d4science-test-glb-vre-egr-vpc-connector-to-gke: + deny: false + description: "Allow vpc serverless connector to gke control plane communication." + targets: + - vpc-connector-europe-west8-d4science-test-cf-vpc-con + destination_ranges: + - 0.0.0.0/0 + priority: 1000 + rules: + - protocol: all + ports: [] + + d4science-test-glb-vre-egr-vpc-connector-to-serverless: + deny: false + description: Allow vpc connectors to reach serverless services. + priority: 1000 + destination_ranges: + - vpc-nat-ranges + rules: + - protocol: tcp + ports: + - 667 + - protocol: udp + ports: + - 665 + - 666 + - protocol: icmp + ports: [] \ No newline at end of file diff --git a/02-security/main.tf b/02-security/main.tf new file mode 100644 index 0000000..8da6cab --- /dev/null +++ b/02-security/main.tf @@ -0,0 +1,13 @@ +locals { + networking_vars = jsondecode(file("../assets/tfvars/01-networking.auto.tfvars.json")) +} +locals { + spoke-test = { + project_id = local.networking_vars.spoke-test.project_id + network = local.networking_vars.spoke-test.network + } + spoke-prod = { + project_id = local.networking_vars.spoke-prod.project_id + network = local.networking_vars.spoke-prod.network + } +} \ No newline at end of file diff --git a/02-security/org_policies.tf b/02-security/org_policies.tf new file mode 100644 index 0000000..9368c91 --- /dev/null +++ b/02-security/org_policies.tf @@ -0,0 +1,8 @@ +##------------------------------------------------------------------------------- +## 02-Security - Organization - ORG POLICIES +##------------------------------------------------------------------------------- +module "org" { + source = "../assets/modules-fabric/v26/organization" + organization_id = "organizations/${var.organization.id}" + org_policies_data_path = "firewall/rules/org-policies" +} \ No newline at end of file diff --git a/02-security/providers.tf b/02-security/providers.tf new file mode 100644 index 0000000..af21b30 --- /dev/null +++ b/02-security/providers.tf @@ -0,0 +1,37 @@ +terraform { + backend "gcs" { + bucket = "d4science-com-ew8-foundation-tfsec-bkt" + impersonate_service_account = "d4science-com-tfsec-sa@d4science-com-automation-prj.iam.gserviceaccount.com" + } + + required_version = "~> 1.6.5" + + required_providers { + google = { + source = "hashicorp/google" + version = "4.84.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = "4.84.0" + } + local = { + source = "hashicorp/local" + version = "~> 2.4.0" + } + } +} + +provider "google" { + impersonate_service_account = "d4science-com-tfsec-sa@d4science-com-automation-prj.iam.gserviceaccount.com" +} + +provider "google-beta" { + impersonate_service_account = "d4science-com-tfsec-sa@d4science-com-automation-prj.iam.gserviceaccount.com" +} + +provider "google" { + alias = "no-impersonate" +} + +provider "local" {} \ No newline at end of file diff --git a/02-security/spoke-prod-firewall.tf b/02-security/spoke-prod-firewall.tf new file mode 100644 index 0000000..378f92c --- /dev/null +++ b/02-security/spoke-prod-firewall.tf @@ -0,0 +1,17 @@ +#---------------------------------- +# Spoke PROD Firewall +#---------------------------------- +module "d4science-security-spoke-prod-firewall" { + source = "../assets/modules-fabric/v26/net-vpc-firewall" + project_id = local.spoke-prod.project_id + network = local.spoke-prod.network + + default_rules_config = { + disabled = true + } + + factories_config = { + rules_folder = "firewall/rules/spoke-prod" + cidr_tpl_file = "firewall/cidrs.yaml" + } +} \ No newline at end of file diff --git a/02-security/spoke-test-firewall.tf b/02-security/spoke-test-firewall.tf new file mode 100644 index 0000000..bb8488a --- /dev/null +++ b/02-security/spoke-test-firewall.tf @@ -0,0 +1,17 @@ +#---------------------------------- +# Spoke TEST Firewall +#---------------------------------- +module "d4science-security-spoke-test-firewall" { + source = "../assets/modules-fabric/v26/net-vpc-firewall" + project_id = local.spoke-test.project_id + network = local.spoke-test.network + + default_rules_config = { + disabled = true + } + + factories_config = { + rules_folder = "firewall/rules/spoke-test" + cidr_tpl_file = "firewall/cidrs.yaml" + } +} \ No newline at end of file diff --git a/02-security/terraform.tfvars b/02-security/terraform.tfvars new file mode 100644 index 0000000..a087a2a --- /dev/null +++ b/02-security/terraform.tfvars @@ -0,0 +1,4 @@ +organization = { + domain = "d4sscience.org" + id = 392184451762 +} \ No newline at end of file diff --git a/02-security/variables.tf b/02-security/variables.tf new file mode 100644 index 0000000..19e0467 --- /dev/null +++ b/02-security/variables.tf @@ -0,0 +1,7 @@ +variable "organization" { + description = "Organization details." + type = object({ + domain = string + id = number + }) +} \ No newline at end of file diff --git a/03-project-factory-prod/main.tf b/03-project-factory-prod/main.tf new file mode 100644 index 0000000..c9689d6 --- /dev/null +++ b/03-project-factory-prod/main.tf @@ -0,0 +1,45 @@ +locals { + organization_vars = jsondecode(file("../assets/tfvars/00-organization.auto.tfvars.json")) + networking_vars = jsondecode(file("../assets/tfvars/01-networking.auto.tfvars.json")) +} +locals { + + billing_account_id = local.organization_vars.billing_account_id + organization = local.organization_vars.organization + prefix = local.organization_vars.prefix + labels = local.organization_vars.labels + groups = local.organization_vars.groups + + # Global variables + + seed-project = { + project_id = local.organization_vars.seed-project.project_id + project_number = local.organization_vars.seed-project.project_number + } + + service_accounts = { + networking = local.organization_vars.service_accounts.networking + security = local.organization_vars.service_accounts.security + project_factory_prod = local.organization_vars.service_accounts.project_factory_prod + } + + networking = { + spoke-prod-project = local.networking_vars.spoke-prod + } + + monitoring = { + bucket-sink = local.organization_vars.monitoring.bucket-sink-prod + project_id = local.organization_vars.monitoring.project_id + channels = local.organization_vars.monitoring.channels + } + + # Folders + folders = { + prod = { + id = local.organization_vars.folders.prod.id + name = local.organization_vars.folders.prod.name + + } + } +} + diff --git a/03-project-factory-prod/providers.tf b/03-project-factory-prod/providers.tf new file mode 100644 index 0000000..404e264 --- /dev/null +++ b/03-project-factory-prod/providers.tf @@ -0,0 +1,37 @@ +terraform { + backend "gcs" { + bucket = "d4science-prod-ew8-foundation-tfprj-bkt" + impersonate_service_account = "d4science-prod-tfprj-sa@d4science-com-automation-prj.iam.gserviceaccount.com" + } + + required_version = "~> 1.6.5" + + required_providers { + google = { + source = "hashicorp/google" + version = "4.84.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = "4.84.0" + } + local = { + source = "hashicorp/local" + version = "~> 2.4.0" + } + } +} + +provider "google" { + impersonate_service_account = "d4science-prod-tfprj-sa@d4science-com-automation-prj.iam.gserviceaccount.com" +} + +provider "google-beta" { + impersonate_service_account = "d4science-prod-tfprj-sa@d4science-com-automation-prj.iam.gserviceaccount.com" +} + +provider "google" { + alias = "no-impersonate" +} + +provider "local" {} \ No newline at end of file diff --git a/03-project-factory-prod/scripts-project.tf b/03-project-factory-prod/scripts-project.tf new file mode 100644 index 0000000..e18e983 --- /dev/null +++ b/03-project-factory-prod/scripts-project.tf @@ -0,0 +1,477 @@ +module "d4science-prod-budget-script-project" { + source = "../assets/modules-fabric/v26/project" + + name = "d4science-prod-script-prj" + prefix = local.prefix + billing_account = local.billing_account_id + parent = local.folders.prod.id + labels = local.labels + + auto_create_network = false + project_create = true + + services = [ + "cloudresourcemanager.googleapis.com", #required for SA impersonification + "iam.googleapis.com", #required for IAM and SA impersonification + "cloudfunctions.googleapis.com", #required for Cloud Functions + "pubsub.googleapis.com", #required for PubSub + "billingbudgets.googleapis.com", #required for Billing alerting, + "run.googleapis.com", #required for Cloud Functions 2nd gen + "cloudbuild.googleapis.com", #required for Cloud Functions 2nd gen + "eventarc.googleapis.com", #required for Cloud Functions 2nd gen + "vpcaccess.googleapis.com", #required for Serverless VPC access + ] + + iam = { + "roles/owner" = [ + local.service_accounts.project_factory_prod, + ] + } + + group_iam = { + "foundationreply@d4science.org" = [ + "roles/owner" + ] + } +} + +module "d4science-prod-budget-alert-topic" { + source = "../assets/modules-fabric/v26/pubsub" + project_id = module.d4science-prod-budget-script-project.project_id + name = "d4science-prod-budget-alerting-topic" +} + +module "d4science-billing-budget-overall-year-critical" { + source = "../assets/modules-custom/billing-budget" + billing_account = local.billing_account_id + name = "Critical - Overall yearly budget" + amount = 442500 + currency_code = "EUR" + thresholds = { + current = [0.95, 0.97] + forecasted = [] + } + calendar_period = "YEAR" + resource_ancestors = ["organizations/${local.organization.id}"] + notification_channels = [local.monitoring.channels["budget-alerting"]] +} + +module "d4science-billing-budget-overall-year" { + source = "../assets/modules-custom/billing-budget" + billing_account = local.billing_account_id + name = "Overall yearly budget" + amount = 442500 + currency_code = "EUR" + thresholds = { + current = [0.5, 0.8, 0.9] + forecasted = [] + } + calendar_period = "YEAR" + resource_ancestors = ["organizations/${local.organization.id}"] + notification_channels = [local.monitoring.channels["budget-alerting"]] +} + +module "d4science-billing-budget-overall-month" { + source = "../assets/modules-custom/billing-budget" + billing_account = local.billing_account_id + name = "Overall monthly budget" + amount = 36875 + currency_code = "EUR" + thresholds = { + current = [0.5, 0.8, 1.0] + forecasted = [] + } + calendar_period = "MONTH" + resource_ancestors = ["organizations/${local.organization.id}"] + notification_channels = [local.monitoring.channels["budget-alerting"]] +} + +module "d4science-prod-billing-budget-blue-cloud-total-year" { + source = "../assets/modules-custom/billing-budget" + billing_account = local.billing_account_id + name = "Prod - GKE - BlueCloud cumulative yearly budget" + amount = 168000 + currency_code = "EUR" + thresholds = { + current = [0.5, 0.8, 1.0] + forecasted = [] + } + labels = { + "k8s-namespace" = "blue-cloud" + } + calendar_period = "YEAR" + resource_ancestors = [local.folders.prod.id] + notification_channels = [local.monitoring.channels["budget-alerting"]] + pubsub_topic = module.d4science-prod-budget-alert-topic.id # Attention: this cannot be done without temporarily deactivating the org policy iam.allowedPolicyMemberDomains +} + +module "d4science-prod-billing-budget-i-marine-total-year" { + source = "../assets/modules-custom/billing-budget" + billing_account = local.billing_account_id + name = "Prod - GKE - iMarine cumulative yearly budget" + amount = 40500 + currency_code = "EUR" + thresholds = { + current = [0.5, 0.8, 1.0] + forecasted = [] + } + labels = { + "k8s-namespace" = "i-marine" + } + calendar_period = "YEAR" + resource_ancestors = [local.folders.prod.id] + notification_channels = [local.monitoring.channels["budget-alerting"]] + pubsub_topic = module.d4science-prod-budget-alert-topic.id # Attention: this cannot be done without temporarily deactivating the org policy iam.allowedPolicyMemberDomains +} + +module "d4science-prod-billing-budget-so-big-data-total-year" { + source = "../assets/modules-custom/billing-budget" + billing_account = local.billing_account_id + name = "Prod - GKE - SoBigData cumulative yearly budget" + amount = 147000 + currency_code = "EUR" + thresholds = { + current = [0.5, 0.8, 1.0] + forecasted = [] + } + labels = { + "k8s-namespace" = "so-big-data" + } + calendar_period = "YEAR" + resource_ancestors = [local.folders.prod.id] + notification_channels = [local.monitoring.channels["budget-alerting"]] + pubsub_topic = module.d4science-prod-budget-alert-topic.id # Attention: this cannot be done without temporarily deactivating the org policy iam.allowedPolicyMemberDomains +} + +module "d4science-prod-billing-budget-open-community-total-year" { + source = "../assets/modules-custom/billing-budget" + billing_account = local.billing_account_id + name = "Prod - GKE - OpenCommunity cumulative yearly budget" + amount = 87000 + currency_code = "EUR" + thresholds = { + current = [0.5, 0.8, 1.0] + forecasted = [] + } + labels = { + "k8s-namespace" = "open-community" + } + calendar_period = "YEAR" + resource_ancestors = [local.folders.prod.id] + notification_channels = [local.monitoring.channels["budget-alerting"]] + pubsub_topic = module.d4science-prod-budget-alert-topic.id # Attention: this cannot be done without temporarily deactivating the org policy iam.allowedPolicyMemberDomains +} + +module "d4science-prod-billing-budget-blue-cloud-year" { + source = "../assets/modules-custom/billing-budget" + billing_account = local.billing_account_id + name = "Prod - GKE - BlueCloud namespace yearly budget" + amount = 117000 + currency_code = "EUR" + thresholds = { + current = [0.5, 0.8, 1.0] + forecasted = [] + } + labels = { + "k8s-label/d4science-namespace" = "blue-cloud" + } + calendar_period = "YEAR" + resource_ancestors = [local.folders.prod.id] + notification_channels = [local.monitoring.channels["budget-alerting"]] +} + +module "d4science-prod-billing-budget-i-marine-year" { + source = "../assets/modules-custom/billing-budget" + billing_account = local.billing_account_id + name = "Prod - GKE - iMarine namespace yearly budget" + amount = 22500 + currency_code = "EUR" + thresholds = { + current = [0.5, 0.8, 1.0] + forecasted = [] + } + labels = { + "k8s-label/d4science-namespace" = "i-marine" + } + calendar_period = "YEAR" + resource_ancestors = [local.folders.prod.id] + notification_channels = [local.monitoring.channels["budget-alerting"]] +} + +module "d4science-prod-billing-budget-so-big-data-year" { + source = "../assets/modules-custom/billing-budget" + billing_account = local.billing_account_id + name = "Prod - GKE - SoBigData namespace yearly budget" + amount = 117000 + currency_code = "EUR" + thresholds = { + current = [0.5, 0.8, 1.0] + forecasted = [] + } + labels = { + "k8s-label/d4science-namespace" = "so-big-data" + } + calendar_period = "YEAR" + resource_ancestors = [local.folders.prod.id] + notification_channels = [local.monitoring.channels["budget-alerting"]] +} + +module "d4science-prod-billing-budget-open-community-year" { + source = "../assets/modules-custom/billing-budget" + billing_account = local.billing_account_id + name = "Prod - GKE - OpenCommunity namespace yearly budget" + amount = 87000 + currency_code = "EUR" + thresholds = { + current = [0.5, 0.8, 1.0] + forecasted = [] + } + labels = { + "k8s-label/d4science-namespace" = "open-community" + } + calendar_period = "YEAR" + resource_ancestors = [local.folders.prod.id] + notification_channels = [local.monitoring.channels["budget-alerting"]] +} + +module "d4science-prod-billing-budget-blue-cloud-datathon-year" { + source = "../assets/modules-custom/billing-budget" + billing_account = local.billing_account_id + name = "Prod - GKE - BlueCloudDatathon namespace yearly budget" + amount = 51000 + currency_code = "EUR" + thresholds = { + current = [0.5, 0.8, 1.0] + forecasted = [] + } + labels = { + "k8s-label/d4science-namespace" = "blue-cloud-datathon" + } + calendar_period = "YEAR" + resource_ancestors = [local.folders.prod.id] + notification_channels = [local.monitoring.channels["budget-alerting"]] +} + +module "d4science-prod-billing-budget-i-marine-datathon-year" { + source = "../assets/modules-custom/billing-budget" + billing_account = local.billing_account_id + name = "Prod - GKE - iMarineDatathon namespace yearly budget" + amount = 18000 + currency_code = "EUR" + thresholds = { + current = [0.5, 0.8, 1.0] + forecasted = [] + } + labels = { + "k8s-label/d4science-namespace" = "i-marine-datathon" + } + calendar_period = "YEAR" + resource_ancestors = [local.folders.prod.id] + notification_channels = [local.monitoring.channels["budget-alerting"]] +} + +module "d4science-prod-billing-budget-so-big-data-datathon-year" { + source = "../assets/modules-custom/billing-budget" + billing_account = local.billing_account_id + name = "Prod - GKE - SoBigDataDatathon namespace yearly budget" + amount = 30000 + currency_code = "EUR" + thresholds = { + current = [0.5, 0.8, 1.0] + forecasted = [] + } + labels = { + "k8s-label/d4science-namespace" = "so-big-data-datathon" + } + calendar_period = "YEAR" + resource_ancestors = [local.folders.prod.id] + notification_channels = [local.monitoring.channels["budget-alerting"]] +} + +# module "d4science-prod-billing-budget-open-community-year" { +# source = "../assets/modules-custom/billing-budget" +# billing_account = local.billing_account_id +# name = "Prod - GKE - OpenCommunityDatathon namespace yearly budget" +# amount = 0 +# currency_code = "EUR" +# thresholds = { +# current = [0.5, 0.8, 1.0] +# forecasted = [] +# } +# labels = { +# "k8s-label/d4science-namespace" = "open-community-datathon" +# } +# calendar_period = "YEAR" +# resource_ancestors = [local.folders.prod.id] +# notification_channels = [local.monitoring.channels["budget-alerting"]] +# } + +module "d4science-prod-billing-budget-blue-cloud-month" { + source = "../assets/modules-custom/billing-budget" + billing_account = local.billing_account_id + name = "Prod - GKE - BlueCloud namespace monthly budget" + amount = 9750 + currency_code = "EUR" + thresholds = { + current = [0.5, 0.8, 1.0] + forecasted = [] + } + labels = { + "k8s-label/d4science-namespace" = "blue-cloud" + } + calendar_period = "MONTH" + resource_ancestors = [local.folders.prod.id] + notification_channels = [local.monitoring.channels["budget-alerting"]] +} + +module "d4science-prod-billing-budget-i-marine-month" { + source = "../assets/modules-custom/billing-budget" + billing_account = local.billing_account_id + name = "Prod - GKE - iMarine namespace monthly budget" + amount = 1875 + currency_code = "EUR" + thresholds = { + current = [0.5, 0.8, 1.0] + forecasted = [] + } + labels = { + "k8s-label/d4science-namespace" = "i-marine" + } + calendar_period = "MONTH" + resource_ancestors = [local.folders.prod.id] + notification_channels = [local.monitoring.channels["budget-alerting"]] +} + +module "d4science-prod-billing-budget-so-big-data-month" { + source = "../assets/modules-custom/billing-budget" + billing_account = local.billing_account_id + name = "Prod - GKE - SoBigData namespace monthly budget" + amount = 9750 + currency_code = "EUR" + thresholds = { + current = [0.5, 0.8, 1.0] + forecasted = [] + } + labels = { + "k8s-label/d4science-namespace" = "so-big-data" + } + calendar_period = "MONTH" + resource_ancestors = [local.folders.prod.id] + notification_channels = [local.monitoring.channels["budget-alerting"]] +} + +module "d4science-prod-billing-budget-open-community-month" { + source = "../assets/modules-custom/billing-budget" + billing_account = local.billing_account_id + name = "Prod - GKE - OpenCommunity namespace monthly budget" + amount = 7250 + currency_code = "EUR" + thresholds = { + current = [0.5, 0.8, 1.0] + forecasted = [] + } + labels = { + "k8s-label/d4science-namespace" = "open-community" + } + calendar_period = "MONTH" + resource_ancestors = [local.folders.prod.id] + notification_channels = [local.monitoring.channels["budget-alerting"]] +} + +module "d4science-prod-billing-budget-blue-cloud-datathon-month" { + source = "../assets/modules-custom/billing-budget" + billing_account = local.billing_account_id + name = "Prod - GKE - BlueCloudDatathon namespace monthly budget" + amount = 4250 + currency_code = "EUR" + thresholds = { + current = [0.5, 0.8, 1.0] + forecasted = [] + } + labels = { + "k8s-label/d4science-namespace" = "blue-cloud-datathon" + } + calendar_period = "MONTH" + resource_ancestors = [local.folders.prod.id] + notification_channels = [local.monitoring.channels["budget-alerting"]] +} + +module "d4science-prod-billing-budget-i-marine-datathon-month" { + source = "../assets/modules-custom/billing-budget" + billing_account = local.billing_account_id + name = "Prod - GKE - iMarineDatathon namespace monthly budget" + amount = 1500 + currency_code = "EUR" + thresholds = { + current = [0.5, 0.8, 1.0] + forecasted = [] + } + labels = { + "k8s-label/d4science-namespace" = "i-marine-datathon" + } + calendar_period = "MONTH" + resource_ancestors = [local.folders.prod.id] + notification_channels = [local.monitoring.channels["budget-alerting"]] +} + +module "d4science-prod-billing-budget-so-big-data-datathon-month" { + source = "../assets/modules-custom/billing-budget" + billing_account = local.billing_account_id + name = "Prod - GKE - SoBigDataDatathon namespace monthly budget" + amount = 2500 + currency_code = "EUR" + thresholds = { + current = [0.5, 0.8, 1.0] + forecasted = [] + } + labels = { + "k8s-label/d4science-namespace" = "so-big-data-datathon" + } + calendar_period = "MONTH" + resource_ancestors = [local.folders.prod.id] + notification_channels = [local.monitoring.channels["budget-alerting"]] +} + +# module "d4science-prod-billing-budget-open-community-datathon-month" { +# source = "../assets/modules-custom/billing-budget" +# billing_account = local.billing_account_id +# name = "Prod - GKE - OpenCommunityDatathon namespace monthly budget" +# amount = 0 +# currency_code = "EUR" +# thresholds = { +# current = [0.5, 0.8, 1.0] +# forecasted = [] +# } +# labels = { +# "k8s-label/d4science-namespace" = "open-community-datathon" +# } +# calendar_period = "MONTH" +# resource_ancestors = [local.folders.prod.id] +# notification_channels = [local.monitoring.channels["budget-alerting"]] +# } + +module "d4science-prod-budget-script-sa" { + source = "../assets/modules-fabric/v26/iam-service-account" + + project_id = module.d4science-prod-budget-script-project.project_id + name = "d4science-prod-budget-cf-sa" + + iam = { + "roles/iam.serviceAccountTokenCreator" = ["group:foundationreply@d4science.org"] + #Impersonate service accounts (create OAuth2 access tokens, sign blobs or JWTs, etc). + } + + iam_billing_roles = { + "${local.billing_account_id}" = ["roles/billing.viewer"] + } +} + +resource "google_vpc_access_connector" "d4science-prod-budget-script-vpc-connector" { + project = module.d4science-prod-budget-script-project.project_id + name = "d4science-prod-cf-vpc-con" + region = var.gke_region + subnet { + name = "d4science-prod-ew4-vpc-con-sub" + project_id = local.networking.spoke-prod-project.project_id + } +} \ No newline at end of file diff --git a/03-project-factory-prod/terraform.tfvars b/03-project-factory-prod/terraform.tfvars new file mode 100644 index 0000000..5ece30d --- /dev/null +++ b/03-project-factory-prod/terraform.tfvars @@ -0,0 +1,13 @@ +gke_region = "europe-west4" +gke_subnet_id = "europe-west4/d4science-prod-ew4-vre-sub" +gke_master_auth_networks = { + "cnr-d4science-1" = "146.48.28.0/22", + "cnr-d4science-2" = "146.48.122.0/23", + "cnr-d4science-3" = "145.90.225.224/27", + # "cnr-d4science-4" = "2001:610:450:80::/64", + "reply-vpn-1" = "91.218.224.5/32", + "reply-vpn-2" = "91.218.224.15/32", + "reply-vpn-3" = "91.218.226.5/32", + "reply-vpn-4" = "91.218.226.15/32", + "budget-cf-vpc-access" = "10.254.2.0/28" +} \ No newline at end of file diff --git a/03-project-factory-prod/variables.tf b/03-project-factory-prod/variables.tf new file mode 100644 index 0000000..5ab76e9 --- /dev/null +++ b/03-project-factory-prod/variables.tf @@ -0,0 +1,23 @@ +variable "outputs_location" { + description = "Assets path location relative to the module" + type = string + default = "../assets" +} + +variable "gke_subnet_id" { + description = "Id of the subnet where gke cluster must reside" + type = string + default = "" +} + +variable "gke_region" { + description = "Region where gke cluster is located" + type = string + default = "" +} + +variable "gke_master_auth_networks" { + description = "Networks which are allowed to access gke control plane" + type = map(string) + default = {} +} \ No newline at end of file diff --git a/03-project-factory-prod/vre-gke.tf b/03-project-factory-prod/vre-gke.tf new file mode 100644 index 0000000..c2045e7 --- /dev/null +++ b/03-project-factory-prod/vre-gke.tf @@ -0,0 +1,96 @@ +module "d4science-prod-vre-gke-cluster" { + source = "../assets/modules-fabric/v26/gke-cluster-standard" + project_id = module.d4science-prod-vre-project.project_id + name = "d4science-prod-vre-gke-cluster" + location = var.gke_region + vpc_config = { + network = local.networking.spoke-prod-project.network + subnetwork = local.networking.spoke-prod-project.subnets[var.gke_subnet_id].self_link + secondary_range_names = { + pods = "pods" + services = "services" + } + master_authorized_ranges = var.gke_master_auth_networks + master_ipv4_cidr_block = "10.249.0.0/28" + } + private_cluster_config = { + enable_private_endpoint = false + master_global_access = false + } + enable_features = { + dataplane_v2 = true + workload_identity = true + cost_management = true + } + enable_addons = { + gce_persistent_disk_csi_driver = true + gcp_filestore_csi_driver = true + horizontal_pod_autoscaling = true + http_load_balancing = true + } + logging_config = { + enable_workloads_logs = true + enable_api_server_logs = true + enable_scheduler_logs = true + enable_controller_manager_logs = true + } + monitoring_config = { + enable_api_server_metrics = true + enable_controller_manager_metrics = true + enable_scheduler_metrics = true + enable_daemonset_metrics = true + enable_deployment_metrics = true + enable_hpa_metrics = true + enable_pod_metrics = true + enable_statefulset_metrics = true + enable_storage_metrics = true + } + maintenance_config = { + daily_window_start_time = null + recurring_window = { + start_time = "2024-02-17T01:00:00Z" + end_time = "2024-02-17T05:00:00Z" + recurrence = "FREQ=WEEKLY;BYDAY=SA,SU,MO" + } + } + labels = { + environment = "prod" + } +} + +module "d4science-prod-vre-gke-nodepool" { + source = "../assets/modules-fabric/v26/gke-nodepool" + project_id = module.d4science-prod-vre-project.project_id + cluster_name = module.d4science-prod-vre-gke-cluster.name + location = var.gke_region + name = "d4science-prod-vre-gke-nodepool" + service_account = { + create = true + email = "d4science-prod-vre-nodepoolsa" + oauth_scopes = ["https://www.googleapis.com/auth/cloud-platform"] + } + node_config = { + machine_type = "custom-20-73728" + gcfs = true + } + nodepool_config = { + autoscaling = { + max_node_count = 30 + min_node_count = 1 + } + management = { + auto_repair = true + auto_upgrade = true + } + } +} + +module "d4science_prod_artifact_registry" { + source = "../assets/modules-fabric/v26/artifact-registry" + project_id = module.d4science-prod-vre-project.project_id + location = var.gke_region + name = "d4science-prod-images" + iam = { + "roles/artifactregistry.reader" = [module.d4science-prod-vre-gke-nodepool.service_account_iam_email] + } +} \ No newline at end of file diff --git a/03-project-factory-prod/vre-project.tf b/03-project-factory-prod/vre-project.tf new file mode 100644 index 0000000..7e3ae45 --- /dev/null +++ b/03-project-factory-prod/vre-project.tf @@ -0,0 +1,105 @@ +module "d4science-prod-vre-project" { + source = "../assets/modules-fabric/v26/project" + + name = "d4science-prod-vre-prj" + prefix = local.prefix + billing_account = local.billing_account_id + parent = local.folders.prod.id + labels = local.labels + + auto_create_network = false + project_create = true + + services = [ + "cloudresourcemanager.googleapis.com", #required for SA impersonification + "iam.googleapis.com", #required for IAM and SA impersonification + "container.googleapis.com", #required for GKE + "artifactregistry.googleapis.com", #required for Artifact registry + "pubsub.googleapis.com", #required for PubSub + "billingbudgets.googleapis.com", #required for Billing alerting + "certificatemanager.googleapis.com", #required for certificates + "secretmanager.googleapis.com", #required for secrets + "logging.googleapis.com", #required for logging + "file.googleapis.com", #required for Filestore + "servicenetworking.googleapis.com", + "containerfilesystem.googleapis.com" #required for image streaming in GKE + ] + + iam = { + "roles/owner" = [ + local.service_accounts.project_factory_prod, + ] + "roles/container.developer" = ["serviceAccount:${module.d4science-prod-budget-script-sa.email}"] + } + + group_iam = { + "foundationreply@d4science.org" = [ + "roles/owner" + ] + } + + logging_sinks = { + gke = { + destination = local.monitoring.bucket-sink.id + filter = "resource.type=\"k8s_cluster\"" + type = "logging" + unique_writer = true + } + } +} + +module "d4science-prod-vre-addresses" { + source = "../assets/modules-fabric/v26/net-address" + project_id = module.d4science-prod-vre-project.project_id + global_addresses = ["d4science-prod-vre-address-ext"] +} + +module "d4science-prod-vre-ssl-secret" { + source = "../assets/modules-fabric/v26/secret-manager" + project_id = module.d4science-prod-vre-project.project_id + secrets = { + # Must be uploaded manually + d4science-prod-vre-ssl-private-key = ["europe-west4"], + d4science-prod-vre-ssl-public-cert = ["europe-west4"] + } +} + +resource "google_compute_ssl_policy" "d4science-prod-vre-ssl-policy-1_2-modern" { + project = module.d4science-prod-vre-project.project_id + name = "d4science-prod-ssl-policy-1-2-modern" + profile = "MODERN" + min_tls_version = "TLS_1_2" +} + +resource "google_compute_security_policy" "d4science-prod-vre-armor-policy" { + project = module.d4science-prod-vre-project.project_id + name = "d4science-prod-vre-armor-policy" + adaptive_protection_config { + layer_7_ddos_defense_config { + enable = true + } + } + + rule { + action = "deny(403)" + description = "Block RU" + priority = "1000" + match { + expr { + expression = "origin.region_code == 'RU'" + } + } + } + + rule { + action = "allow" + priority = "2147483647" + match { + versioned_expr = "SRC_IPS_V1" + config { + src_ip_ranges = ["*"] + } + } + description = "default rule" + } +} \ No newline at end of file diff --git a/03-project-factory-test/main.tf b/03-project-factory-test/main.tf new file mode 100644 index 0000000..7b40eb9 --- /dev/null +++ b/03-project-factory-test/main.tf @@ -0,0 +1,44 @@ +locals { + organization_vars = jsondecode(file("../assets/tfvars/00-organization.auto.tfvars.json")) + networking_vars = jsondecode(file("../assets/tfvars/01-networking.auto.tfvars.json")) +} +locals { + + billing_account_id = local.organization_vars.billing_account_id + organization = local.organization_vars.organization + prefix = local.organization_vars.prefix + labels = local.organization_vars.labels + groups = local.organization_vars.groups + + # Global variables + + seed-project = { + project_id = local.organization_vars.seed-project.project_id + project_number = local.organization_vars.seed-project.project_number + } + + service_accounts = { + networking = local.organization_vars.service_accounts.networking + security = local.organization_vars.service_accounts.security + project_factory_test = local.organization_vars.service_accounts.project_factory_test + } + + networking = { + spoke-test-project = local.networking_vars.spoke-test + } + + monitoring = { + bucket-sink = local.organization_vars.monitoring.bucket-sink-test + project_id = local.organization_vars.monitoring.project_id + channels = local.organization_vars.monitoring.channels + } + + # Folders + folders = { + test = { + id = local.organization_vars.folders.test.id + name = local.organization_vars.folders.test.name + } + } +} + diff --git a/03-project-factory-test/providers.tf b/03-project-factory-test/providers.tf new file mode 100644 index 0000000..145605f --- /dev/null +++ b/03-project-factory-test/providers.tf @@ -0,0 +1,37 @@ +terraform { + backend "gcs" { + bucket = "d4science-test-ew8-foundation-tfprj-bkt" + impersonate_service_account = "d4science-test-tfprj-sa@d4science-com-automation-prj.iam.gserviceaccount.com" + } + + required_version = "~> 1.6.5" + + required_providers { + google = { + source = "hashicorp/google" + version = "4.84.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = "4.84.0" + } + local = { + source = "hashicorp/local" + version = "~> 2.4.0" + } + } +} + +provider "google" { + impersonate_service_account = "d4science-test-tfprj-sa@d4science-com-automation-prj.iam.gserviceaccount.com" +} + +provider "google-beta" { + impersonate_service_account = "d4science-test-tfprj-sa@d4science-com-automation-prj.iam.gserviceaccount.com" +} + +provider "google" { + alias = "no-impersonate" +} + +provider "local" {} \ No newline at end of file diff --git a/03-project-factory-test/scripts-project.tf b/03-project-factory-test/scripts-project.tf new file mode 100644 index 0000000..8a147e8 --- /dev/null +++ b/03-project-factory-test/scripts-project.tf @@ -0,0 +1,177 @@ +module "d4science-test-budget-script-project" { + source = "../assets/modules-fabric/v26/project" + + name = "d4science-test-script-prj" + prefix = local.prefix + billing_account = local.billing_account_id + parent = local.folders.test.id + labels = local.labels + + auto_create_network = false + project_create = true + + services = [ + "cloudresourcemanager.googleapis.com", #required for SA impersonification + "iam.googleapis.com", #required for IAM and SA impersonification + "cloudfunctions.googleapis.com", #required for Cloud Functions + "pubsub.googleapis.com", #required for PubSub + "billingbudgets.googleapis.com", #required for Billing alerting, + "run.googleapis.com", #required for Cloud Functions 2nd gen + "cloudbuild.googleapis.com", #required for Cloud Functions 2nd gen + "eventarc.googleapis.com", #required for Cloud Functions 2nd gen + "vpcaccess.googleapis.com", #required for Serverless VPC access + ] + + iam = { + "roles/owner" = [ + local.service_accounts.project_factory_test, + ] + } + + group_iam = { + "foundationreply@d4science.org" = [ + "roles/owner" + ] + } +} + +module "d4science-test-budget-alert-topic" { + source = "../assets/modules-fabric/v26/pubsub" + project_id = module.d4science-test-budget-script-project.project_id + name = "d4science-test-budget-alerting-topic" +} + +# module "d4science-test-billing-budget-overall" { +# source = "../assets/modules-custom/billing-budget" +# billing_account = local.billing_account_id +# name = "Test - Overall budget" +# amount = 1000 +# currency_code = "EUR" +# thresholds = { +# current = [0.5, 0.75, 1.0] +# forecasted = [] +# } + +# projects = [ +# "projects/${local.networking.spoke-test-project.number}", +# "projects/${module.d4science-test-budget-script-project.number}", +# "projects/${module.d4science-test-vre-project.number}" +# ] +# notification_channels = [local.monitoring.channels["budget-alerting"]] +# } + +# module "d4science-test-billing-budget-jupyter-hub" { +# source = "../assets/modules-custom/billing-budget" +# billing_account = local.billing_account_id +# name = "Test - GKE - JupyterHub namespace budget" +# amount = 100 +# currency_code = "EUR" +# thresholds = { +# current = [0.5, 0.75, 1.0] +# forecasted = [] +# } +# labels = { +# "k8s-namespace" = "jupyter-hub" +# } +# resource_ancestors = [local.folders.test.id] +# notification_channels = [local.monitoring.channels["budget-alerting"]] +# pubsub_topic = module.d4science-test-budget-alert-topic.id # Attention: this cannot be done without temporarily deactivating the org policy iam.allowedPolicyMemberDomains +# } + +# module "d4science-test-billing-budget-blue-cloud" { +# source = "../assets/modules-custom/billing-budget" +# billing_account = local.billing_account_id +# name = "Test - GKE - BlueCloud namespace budget" +# amount = 100 +# currency_code = "EUR" +# thresholds = { +# current = [0.5, 0.75, 1.0] +# forecasted = [] +# } +# labels = { +# "k8s-namespace" = "blue-cloud" +# } +# resource_ancestors = [local.folders.test.id] +# notification_channels = [local.monitoring.channels["budget-alerting"]] +# pubsub_topic = module.d4science-test-budget-alert-topic.id # Attention: this cannot be done without temporarily deactivating the org policy iam.allowedPolicyMemberDomains +# } + +# module "d4science-test-billing-budget-i-marine" { +# source = "../assets/modules-custom/billing-budget" +# billing_account = local.billing_account_id +# name = "Test - GKE - iMarine namespace budget" +# amount = 100 +# currency_code = "EUR" +# thresholds = { +# current = [0.5, 0.75, 1.0] +# forecasted = [] +# } +# labels = { +# "k8s-namespace" = "i-marine" +# } +# resource_ancestors = [local.folders.test.id] +# notification_channels = [local.monitoring.channels["budget-alerting"]] +# pubsub_topic = module.d4science-test-budget-alert-topic.id # Attention: this cannot be done without temporarily deactivating the org policy iam.allowedPolicyMemberDomains +# } + +# module "d4science-test-billing-budget-so-big-data" { +# source = "../assets/modules-custom/billing-budget" +# billing_account = local.billing_account_id +# name = "Test - GKE - SoBigData namespace budget" +# amount = 100 +# currency_code = "EUR" +# thresholds = { +# current = [0.5, 0.75, 1.0] +# forecasted = [] +# } +# labels = { +# "k8s-namespace" = "so-big-data" +# } +# resource_ancestors = [local.folders.test.id] +# notification_channels = [local.monitoring.channels["budget-alerting"]] +# pubsub_topic = module.d4science-test-budget-alert-topic.id # Attention: this cannot be done without temporarily deactivating the org policy iam.allowedPolicyMemberDomains +# } + +# module "d4science-test-billing-budget-open-community" { +# source = "../assets/modules-custom/billing-budget" +# billing_account = local.billing_account_id +# name = "Test - GKE - OpenCommunity namespace budget" +# amount = 100 +# currency_code = "EUR" +# thresholds = { +# current = [0.5, 0.75, 1.0] +# forecasted = [] +# } +# labels = { +# "k8s-namespace" = "open-community" +# } +# resource_ancestors = [local.folders.test.id] +# notification_channels = [local.monitoring.channels["budget-alerting"]] +# pubsub_topic = module.d4science-test-budget-alert-topic.id # Attention: this cannot be done without temporarily deactivating the org policy iam.allowedPolicyMemberDomains +# } + +module "d4science-test-budget-script-sa" { + source = "../assets/modules-fabric/v26/iam-service-account" + + project_id = module.d4science-test-budget-script-project.project_id + name = "d4science-test-budget-cf-sa" + + iam = { + "roles/iam.serviceAccountTokenCreator" = ["group:foundationreply@d4science.org"] + #Impersonate service accounts (create OAuth2 access tokens, sign blobs or JWTs, etc). + } + + iam_billing_roles = { + "${local.billing_account_id}" = ["roles/billing.viewer"] + } +} + +resource "google_vpc_access_connector" "d4science-test-budget-script-vpc-connector" { + project = module.d4science-test-budget-script-project.project_id + name = "d4science-test-cf-vpc-con" + region = var.gke_region + subnet { + name = "d4science-test-ew8-vpc-con-sub" + project_id = local.networking.spoke-test-project.project_id + } +} \ No newline at end of file diff --git a/03-project-factory-test/terraform.tfvars b/03-project-factory-test/terraform.tfvars new file mode 100644 index 0000000..abc65f1 --- /dev/null +++ b/03-project-factory-test/terraform.tfvars @@ -0,0 +1,13 @@ +gke_region = "europe-west8" +gke_subnet_id = "europe-west8/d4science-test-ew8-vre-sub" +gke_master_auth_networks = { + "cnr-d4science-1" = "146.48.28.0/22", + "cnr-d4science-2" = "146.48.122.0/23", + "cnr-d4science-3" = "145.90.225.224/27", + # "cnr-d4science-4" = "2001:610:450:80::/64", + "reply-vpn-1" = "91.218.224.5/32", + "reply-vpn-2" = "91.218.224.15/32", + "reply-vpn-3" = "91.218.226.5/32", + "reply-vpn-4" = "91.218.226.15/32", + "budget-cf-vpc-access" = "10.254.66.0/28" +} \ No newline at end of file diff --git a/03-project-factory-test/variables.tf b/03-project-factory-test/variables.tf new file mode 100644 index 0000000..5ab76e9 --- /dev/null +++ b/03-project-factory-test/variables.tf @@ -0,0 +1,23 @@ +variable "outputs_location" { + description = "Assets path location relative to the module" + type = string + default = "../assets" +} + +variable "gke_subnet_id" { + description = "Id of the subnet where gke cluster must reside" + type = string + default = "" +} + +variable "gke_region" { + description = "Region where gke cluster is located" + type = string + default = "" +} + +variable "gke_master_auth_networks" { + description = "Networks which are allowed to access gke control plane" + type = map(string) + default = {} +} \ No newline at end of file diff --git a/03-project-factory-test/vre-gke.tf b/03-project-factory-test/vre-gke.tf new file mode 100644 index 0000000..9ddb031 --- /dev/null +++ b/03-project-factory-test/vre-gke.tf @@ -0,0 +1,88 @@ +module "d4science-test-vre-gke-cluster" { + source = "../assets/modules-fabric/v26/gke-cluster-standard" + project_id = module.d4science-test-vre-project.project_id + name = "d4science-test-vre-gke-cluster" + location = var.gke_region + vpc_config = { + network = local.networking.spoke-test-project.network + subnetwork = local.networking.spoke-test-project.subnets[var.gke_subnet_id].self_link + secondary_range_names = { + pods = "pods" + services = "services" + } + master_authorized_ranges = var.gke_master_auth_networks + master_ipv4_cidr_block = "10.249.0.64/28" + } + private_cluster_config = { + enable_private_endpoint = false + master_global_access = false + } + enable_features = { + dataplane_v2 = true + workload_identity = true + cost_management = true + } + enable_addons = { + gce_persistent_disk_csi_driver = true + gcp_filestore_csi_driver = true + horizontal_pod_autoscaling = true + http_load_balancing = true + } + logging_config = { + enable_workloads_logs = true + enable_api_server_logs = true + enable_scheduler_logs = true + enable_controller_manager_logs = true + } + monitoring_config = { + enable_api_server_metrics = true + enable_controller_manager_metrics = true + enable_scheduler_metrics = true + enable_daemonset_metrics = true + enable_deployment_metrics = true + enable_hpa_metrics = true + enable_pod_metrics = true + enable_statefulset_metrics = true + enable_storage_metrics = true + } + labels = { + environment = "test" + } +} + +module "d4science-test-vre-gke-nodepool" { + source = "../assets/modules-fabric/v26/gke-nodepool" + project_id = module.d4science-test-vre-project.project_id + cluster_name = module.d4science-test-vre-gke-cluster.name + location = var.gke_region + name = "d4science-test-vre-gke-nodepool" + service_account = { + create = true + email = "d4science-test-vre-nodepoolsa" + oauth_scopes = ["https://www.googleapis.com/auth/cloud-platform"] + } + node_config = { + machine_type = "e2-medium" + gcfs = true + } + nodepool_config = { + autoscaling = { + max_node_count = 2 + min_node_count = 0 + } + management = { + auto_repair = true + auto_upgrade = true + } + } +} + +module "d4science_test_artifact_registry" { + source = "../assets/modules-fabric/v26/artifact-registry" + project_id = module.d4science-test-vre-project.project_id + location = var.gke_region + name = "d4science-test-images" + iam = { + "roles/artifactregistry.reader" = [module.d4science-test-vre-gke-nodepool.service_account_iam_email] + } +} \ No newline at end of file diff --git a/03-project-factory-test/vre-project.tf b/03-project-factory-test/vre-project.tf new file mode 100644 index 0000000..45f5912 --- /dev/null +++ b/03-project-factory-test/vre-project.tf @@ -0,0 +1,105 @@ +module "d4science-test-vre-project" { + source = "../assets/modules-fabric/v26/project" + + name = "d4science-test-vre-prj" + prefix = local.prefix + billing_account = local.billing_account_id + parent = local.folders.test.id + labels = local.labels + + auto_create_network = false + project_create = true + + services = [ + "cloudresourcemanager.googleapis.com", #required for SA impersonification + "iam.googleapis.com", #required for IAM and SA impersonification + "container.googleapis.com", #required for GKE + "artifactregistry.googleapis.com", #required for Artifact registry + "pubsub.googleapis.com", #required for PubSub + "billingbudgets.googleapis.com", #required for Billing alerting + "certificatemanager.googleapis.com", #required for certificates + "secretmanager.googleapis.com", #required for secrets + "logging.googleapis.com", #required for logging + "file.googleapis.com", #required for Filestore + "servicenetworking.googleapis.com", + "containerfilesystem.googleapis.com" #required for image streaming in GKE + ] + + iam = { + "roles/owner" = [ + local.service_accounts.project_factory_test, + ] + "roles/container.developer" = ["serviceAccount:${module.d4science-test-budget-script-sa.email}"] + } + + group_iam = { + "foundationreply@d4science.org" = [ + "roles/owner" + ] + } + + logging_sinks = { + gke = { + destination = local.monitoring.bucket-sink.id + filter = "resource.type=\"k8s_cluster\"" + type = "logging" + unique_writer = true + } + } +} + +module "d4science-test-vre-addresses" { + source = "../assets/modules-fabric/v26/net-address" + project_id = module.d4science-test-vre-project.project_id + global_addresses = ["d4science-test-vre-address-ext"] +} + +module "d4science-test-vre-ssl-secret" { + source = "../assets/modules-fabric/v26/secret-manager" + project_id = module.d4science-test-vre-project.project_id + secrets = { + # Must be uploaded manually + d4science-test-vre-ssl-private-key = ["europe-west8"], + d4science-test-vre-ssl-public-cert = ["europe-west8"] + } +} + +resource "google_compute_ssl_policy" "d4science-test-vre-ssl-policy-1_2-modern" { + project = module.d4science-test-vre-project.project_id + name = "d4science-test-ssl-policy-1-2-modern" + profile = "MODERN" + min_tls_version = "TLS_1_2" +} + +resource "google_compute_security_policy" "d4science-test-vre-armor-policy" { + project = module.d4science-test-vre-project.project_id + name = "d4science-test-vre-armor-policy" + adaptive_protection_config { + layer_7_ddos_defense_config { + enable = true + } + } + + rule { + action = "deny(403)" + description = "Block RU" + priority = "1000" + match { + expr { + expression = "origin.region_code == 'RU'" + } + } + } + + rule { + action = "allow" + priority = "2147483647" + match { + versioned_expr = "SRC_IPS_V1" + config { + src_ip_ranges = ["*"] + } + } + description = "default rule" + } +} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..2adec1f --- /dev/null +++ b/README.md @@ -0,0 +1,76 @@ +gcloud auth application-default login + +## D4Science Foundation +This repository contains all the folder structure for D4Science foundation to support VRE applications. + +### PRJ Details +- Billing account: `018258-BCA804-E9D4C6` +- Tenant: + - `d4science` + +## Folder structure +``` +. +├── 00-organization +├── 01-networking +│   └── data +│   └── subnets +├── 02-security +│   └── firewall +│   └── rules +│   ├── 00-spoke-test +│   ├── 01-spoke-prod +├── 03-monitoring +│   └── data +├── 04-project-factory-test +├── 04-project-factory-prod +└── assets + ├── providers + └── tfvars +``` + +## Naming Convention +The naming convention for this foundation project is the following one: +`---prj` + +- tenant: `d4science` +- env: + - test + - prod +- name: project name + +### Resources +The naming convention to use for the resources of this project is: + +`----` + +#### Resource name examples +- Service Account: `d4science-dev-foundation-tfnet-sa` +- VPC: `d4science-dev-glb-spoke-vpc` (location glb = global) +- Subnet: `d4science-dev-ew8-01-sub` (unique id = 01 used as incremental counter) +- Bucket: `d4science-org-ew8-foundation-tforg-bkt` +- peering: `d4science-prod-hubspoke-peer` + +### Firewall Rule +The naming convention to use for the firewall rules of this project is: + +`--glb-----` + +#### FW rule example +- FW rule: `d4science-test-glb-paytas-ing-alw-gke-fwr` + +### General considerations +- Avoid fields with "-" +- unique id can be: + - numbers (e.g. 01) + - letters (e.g. abc) + - both (e.g. abc01) + +### SHORT NAMES +#### Resources +- SA: service account +- BKT: bucket +- PRJ: project +- SUB: subnet +- VPC: vpc +- FWR: Firewall \ No newline at end of file diff --git a/assets/modules-custom/billing-budget/README.md b/assets/modules-custom/billing-budget/README.md new file mode 100644 index 0000000..72fe574 --- /dev/null +++ b/assets/modules-custom/billing-budget/README.md @@ -0,0 +1,89 @@ +# Google Cloud Billing Budget Module + +This module allows creating a Cloud Billing budget for a set of services and projects. + +To create billing budgets you need one of the following IAM roles on the target billing account: + +* Billing Account Administrator +* Billing Account Costs Manager + +## Examples + +### Simple email notification + +Send a notification to an email when a set of projects reach $100 of spend. + +```hcl +module "budget" { + source = "./fabric/modules/billing-budget" + billing_account = var.billing_account_id + name = "$100 budget" + amount = 100 + thresholds = { + current = [0.5, 0.75, 1.0] + forecasted = [1.0] + } + projects = [ + "projects/123456789000", + "projects/123456789111" + ] + email_recipients = { + project_id = "my-project" + emails = ["user@example.com"] + } +} +# tftest modules=1 resources=2 inventory=email.yaml +``` + +### Pubsub notification + +Send a notification to a PubSub topic the total spend of a billing account reaches the previous month's spend. + + +```hcl +module "budget" { + source = "./fabric/modules/billing-budget" + billing_account = var.billing_account_id + name = "previous period budget" + amount = 0 + thresholds = { + current = [1.0] + forecasted = [] + } + pubsub_topic = module.pubsub.id +} + +module "pubsub" { + source = "./fabric/modules/pubsub" + project_id = var.project_id + name = "budget-topic" +} + +# tftest modules=2 resources=2 inventory=pubsub.yaml +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [billing_account](variables.tf#L23) | Billing account id. | string | ✓ | | +| [name](variables.tf#L50) | Budget name. | string | ✓ | | +| [thresholds](variables.tf#L85) | Thresholds percentages at which alerts are sent. Must be a value between 0 and 1. | object({…}) | ✓ | | +| [amount](variables.tf#L17) | Amount in the billing account's currency for the budget. Use 0 to set budget to 100% of last period's spend. | number | | 0 | +| [credit_treatment](variables.tf#L28) | How credits should be treated when determining spend for threshold calculations. Only INCLUDE_ALL_CREDITS or EXCLUDE_ALL_CREDITS are supported. | string | | "INCLUDE_ALL_CREDITS" | +| [email_recipients](variables.tf#L41) | Emails where budget notifications will be sent. Setting this will create a notification channel for each email in the specified project. | object({…}) | | null | +| [notification_channels](variables.tf#L55) | Monitoring notification channels where to send updates. | list(string) | | null | +| [notify_default_recipients](variables.tf#L61) | Notify Billing Account Administrators and Billing Account Users IAM roles for the target account. | bool | | false | +| [projects](variables.tf#L67) | List of projects of the form projects/{project_number}, specifying that usage from only this set of projects should be included in the budget. Set to null to include all projects linked to the billing account. | list(string) | | null | +| [pubsub_topic](variables.tf#L73) | The ID of the Cloud Pub/Sub topic where budget related messages will be published. | string | | null | +| [services](variables.tf#L79) | List of services of the form services/{service_id}, specifying that usage from only this set of services should be included in the budget. Set to null to include usage for all services. | list(string) | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [budget](outputs.tf#L17) | Budget resource. | | +| [id](outputs.tf#L22) | Fully qualified budget id. | | + + diff --git a/assets/modules-custom/billing-budget/main.tf b/assets/modules-custom/billing-budget/main.tf new file mode 100644 index 0000000..8ab87ff --- /dev/null +++ b/assets/modules-custom/billing-budget/main.tf @@ -0,0 +1,99 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + spend_basis = { + current = "CURRENT_SPEND" + forecasted = "FORECASTED_SPEND" + } + threshold_pairs = flatten([ + for type, values in var.thresholds : [ + for value in values : { + spend_basis = local.spend_basis[type] + threshold_percent = value + } + ] + ]) + + notification_channels = concat( + [for channel in google_monitoring_notification_channel.email_channels : channel.id], + coalesce(var.notification_channels, []) + ) +} + +resource "google_monitoring_notification_channel" "email_channels" { + for_each = toset(try(var.email_recipients.emails, [])) + display_name = "${var.name} budget email notification (${each.value})" + type = "email" + project = var.email_recipients.project_id + labels = { + email_address = each.value + } + user_labels = {} +} + + +resource "google_billing_budget" "budget" { + billing_account = var.billing_account + display_name = var.name + + budget_filter { + projects = var.projects + resource_ancestors = var.resource_ancestors + credit_types_treatment = var.credit_treatment + services = var.services + labels = var.labels + calendar_period = var.calendar_period + } + + dynamic "amount" { + for_each = var.amount == 0 ? [1] : [] + content { + last_period_amount = true + } + } + + dynamic "amount" { + for_each = var.amount != 0 ? [1] : [] + content { + dynamic "specified_amount" { + for_each = var.amount != 0 ? [1] : [] + content { + units = var.amount + currency_code = var.currency_code + } + } + } + } + + dynamic "threshold_rules" { + for_each = local.threshold_pairs + iterator = threshold + content { + threshold_percent = threshold.value.threshold_percent + spend_basis = threshold.value.spend_basis + } + } + + all_updates_rule { + monitoring_notification_channels = local.notification_channels + pubsub_topic = var.pubsub_topic + # disable_default_iam_recipients can only be set if + # monitoring_notification_channels is nonempty + disable_default_iam_recipients = try(length(var.notification_channels), 0) > 0 && !var.notify_default_recipients + schema_version = "1.0" + } +} diff --git a/assets/modules-custom/billing-budget/outputs.tf b/assets/modules-custom/billing-budget/outputs.tf new file mode 100644 index 0000000..530f857 --- /dev/null +++ b/assets/modules-custom/billing-budget/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "budget" { + description = "Budget resource." + value = google_billing_budget.budget +} + +output "id" { + description = "Fully qualified budget id." + value = google_billing_budget.budget.id +} diff --git a/assets/modules-custom/billing-budget/variables.tf b/assets/modules-custom/billing-budget/variables.tf new file mode 100644 index 0000000..e2b4f6c --- /dev/null +++ b/assets/modules-custom/billing-budget/variables.tf @@ -0,0 +1,123 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "amount" { + description = "Amount in the billing account's currency for the budget. Use 0 to set budget to 100% of last period's spend." + type = number + default = 0 +} + +variable "currency_code" { + description = "The 3-letter currency code defined in ISO 4217." + type = string + default = "USD" +} + +variable "billing_account" { + description = "Billing account id." + type = string +} + +variable "credit_treatment" { + description = "How credits should be treated when determining spend for threshold calculations. Only INCLUDE_ALL_CREDITS or EXCLUDE_ALL_CREDITS are supported." + type = string + default = "INCLUDE_ALL_CREDITS" + validation { + condition = ( + var.credit_treatment == "INCLUDE_ALL_CREDITS" || + var.credit_treatment == "EXCLUDE_ALL_CREDITS" + ) + error_message = "Argument credit_treatment must be INCLUDE_ALL_CREDITS or EXCLUDE_ALL_CREDITS." + } +} + +variable "email_recipients" { + description = "Emails where budget notifications will be sent. Setting this will create a notification channel for each email in the specified project." + type = object({ + project_id = string + emails = list(string) + }) + default = null +} + +variable "name" { + description = "Budget name." + type = string +} + +variable "notification_channels" { + description = "Monitoring notification channels where to send updates." + type = list(string) + default = null +} + +variable "notify_default_recipients" { + description = "Notify Billing Account Administrators and Billing Account Users IAM roles for the target account." + type = bool + default = false +} + +variable "projects" { + description = "List of projects of the form projects/{project_number}, specifying that usage from only this set of projects should be included in the budget. Set to null to include all projects linked to the billing account." + type = list(string) + default = null +} + +variable "resource_ancestors" { + description = "List of organizations of the form organizations/{organization_id} and/or folders of the form folders/{folder_id}, specifying that usage from only this set of ancestors should be included in the budget. Set to null to include all organizations and folders linked to the billing account." + type = list(string) + default = null +} + +variable "pubsub_topic" { + description = "The ID of the Cloud Pub/Sub topic where budget related messages will be published." + type = string + default = null +} + +variable "services" { + description = "List of services of the form services/{service_id}, specifying that usage from only this set of services should be included in the budget. Set to null to include usage for all services." + type = list(string) + default = null +} + +variable "labels" { + description = "A single label and value pair specifying that usage from only this set of labeled resources should be included in the budget. Set to null to include all labels." + type = map(string) + default = null +} + +variable "thresholds" { + description = "Thresholds percentages at which alerts are sent. Must be a value between 0 and 1." + type = object({ + current = list(number) + forecasted = list(number) + }) + validation { + condition = length(var.thresholds.current) > 0 || length(var.thresholds.forecasted) > 0 + error_message = "Must specify at least one budget threshold." + } +} + +variable "calendar_period" { + description = "The calendar period the budget spans. It can be either MONTH, QUARTER or YEAR." + type = string + default = null + validation { + condition = var.calendar_period == null || (var.calendar_period == null ? true : contains(["MONTH", "QUARTER", "YEAR"], var.calendar_period)) + error_message = "Allowed values for calendar_period are \"MONTH\", \"QUARTER\", or \"YEAR\"." + } +} diff --git a/assets/modules-custom/billing-budget/versions.tf b/assets/modules-custom/billing-budget/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-custom/billing-budget/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-custom/pga/main.tf b/assets/modules-custom/pga/main.tf new file mode 100644 index 0000000..757a7ed --- /dev/null +++ b/assets/modules-custom/pga/main.tf @@ -0,0 +1,107 @@ +#------------------------------------------------------------------------------- +# DNS Zones for PRIVATE GOOGLE ACCESS +#------------------------------------------------------------------------------- +/* !!!!! README !!!!! + * We use restricted.googleapis.com and additional domains linked to restricted APIS. + * Private Google Apis (private.googleapis.com) has been left here for future use, if needed. + */ +locals { + private_ips = ["199.36.153.8", "199.36.153.9", "199.36.153.10", "199.36.153.11"] + restricted_ips = ["199.36.153.4", "199.36.153.5", "199.36.153.6", "199.36.153.7"] + ips = var.config.private ? local.private_ips : local.restricted_ips + name = var.config.private ? "private" : "restricted" + + fixed_domains = { + accounts = ["accounts.google.com"] + appengine = ["appengine.google.com"] + appspot = ["*.appspot.com"] + artifact = ["*.gcr.io", "*.pkg.dev"] + functions = ["*.cloudfunctions.net"] + proxy = ["*.cloudproxy.app"] + run = ["*.run.app"] + composer = ["*.composer.cloud.google.com", "*.composer.googleusercontent.com"] + datafusion = ["*.datafusion.cloud.google.com", "*.datafusion.googleusercontent.com"] + dataproc = ["*.dataproc.cloud.google.com", "dataproc.cloud.google.com", "*.dataproc.googleusercontent.com", "dataproc.googleusercontent.com"] + download = ["dl.google.com"] + ad = ["*.googleadapis.com"] + gstatic = ["*.gstatic.com"] + lts = ["*.ltsapis.goog"] + notebooks = ["*.notebooks.cloud.google.com", "*.notebooks.googleusercontent.com"] + packages = ["packages.cloud.google.com"] + pki = ["*.pki.goog"] + source = ["source.developers.google.com"] + } + + enabled_fixed_domain = merge([ + for k, v in var.domains : { + for i, d in local.fixed_domains[k] : + "${k}-${i}" => d + } if v + ]...) + + enabled_custom_domain = merge([ + for k, v in var.custom_domains : { + for i, d in v : + "${k}-${i}" => d + } + ]...) + + enabled_domain = merge(local.enabled_fixed_domain, local.enabled_custom_domain) + +} + +#------------------------------------- +# PRIVATE.GOOGLEAPIS.COM +#------------------------------------- +# DNS zone for PGA: *.googleapis.com. +module "googleapis_zone" { + source = "../../modules-fabric/v26/dns" + project_id = var.project_id + name = "${var.name}-pga-${local.name}-apis" + zone_config = { + domain = "googleapis.com." + private = { + client_networks = var.networks + } + } + recordsets = { + "A ${local.name}.googleapis.com." = { ttl = 300, records = local.ips } + "CNAME *.googleapis.com." = { ttl = 300, records = ["${local.name}.googleapis.com."] } + } +} + +# DNS zone for others APIs +# All others api (different from *.googleapis.com) +module "pga_apis_zone" { + for_each = local.enabled_domain + source = "../../modules-fabric/v26/dns" + project_id = var.project_id + name = "${var.name}-pga-${local.name}-api-${each.key}" + zone_config = { + domain = "${trimprefix(each.value, "*.")}." + private = { + client_networks = var.networks + } + } + recordsets = { + # We are adding wildcard even if not needed, as per documentation. + "A ${trimprefix(each.value, "*.")}." = { ttl = 300, records = local.ips } + "CNAME *.${trimprefix(each.value, "*.")}." = { ttl = 300, records = ["${trimprefix(each.value, "*.")}."] } + } +} + +resource "google_service_networking_peered_dns_domain" "apis-dns-peering" { + for_each = can(var.config.value["service_networking_network"]) ? local.enabled_domain : {} + project = var.project_id + network = var.config.service_networking_network + name = "${replace(trimprefix(each.value, "*."), ".", "-")}-domain" + dns_suffix = "*.${trimprefix(each.value, "*.")}." +} + +resource "google_service_networking_peered_dns_domain" "googleapis-dns-peering" { + count = can(var.config.value["service_networking_network"]) ? 1 : 0 + project = var.project_id + name = "googleapis-domain" + network = var.config.service_networking_network + dns_suffix = "googleapis." +} diff --git a/assets/modules-custom/pga/outputs.tf b/assets/modules-custom/pga/outputs.tf new file mode 100644 index 0000000..e69de29 diff --git a/assets/modules-custom/pga/variables.tf b/assets/modules-custom/pga/variables.tf new file mode 100644 index 0000000..29e5133 --- /dev/null +++ b/assets/modules-custom/pga/variables.tf @@ -0,0 +1,59 @@ +variable "domains" { + description = "Declare which domains to redirect to PGA" + type = object({ + accounts = optional(bool, false) + appengine = optional(bool, false) + appspot = optional(bool, false) + artifact = optional(bool, false) + functions = optional(bool, false) + proxy = optional(bool, false) + run = optional(bool, false) + composer = optional(bool, false) + datafusion = optional(bool, false) + dataproc = optional(bool, false) + download = optional(bool, false) + ad = optional(bool, false) + gstatic = optional(bool, false) + lts = optional(bool, false) + notebooks = optional(bool, false) + packages = optional(bool, false) + pki = optional(bool, false) + source = optional(bool, false) + }) + default = {} +} + +variable "custom_domains" { + type = map(list(string)) + description = "Optional map {NAME}=>[{domain1},{domain2},...]" + default = {} +} + +variable "config" { + type = object({ + private = optional(bool, false) + restricted = optional(bool, false) + service_networking_network = optional(string) + }) + description = "(optional) describe your variable" + + validation { + condition = var.config.private != var.config.restricted + error_message = "One and only one between private and restricted must be set to true" + } +} + +variable "project_id" { + type = string + description = "DNS zone project id" +} + +variable "name" { + type = string + description = "Name for the created resources" +} + +variable "networks" { + type = list(string) + description = "List of networks self-links for DNS zone" +} diff --git a/assets/modules-fabric/v26/README.md b/assets/modules-fabric/v26/README.md new file mode 100644 index 0000000..72e6fb4 --- /dev/null +++ b/assets/modules-fabric/v26/README.md @@ -0,0 +1,109 @@ +# Terraform modules suite for Google Cloud + +The modules collected in this folder are designed as a suite: they are meant to be composed together, and are designed to be forked and modified where use of third party code and sources is not allowed. + +Modules try to stay close to the low level provider resources they encapsulate, and they all share a similar interface that combines management of one resource or set or resources, and the corresponding IAM bindings. + +Authoritative IAM bindings are primarily used (e.g. `google_storage_bucket_iam_binding` for GCS buckets) so that each module is authoritative for specific roles on the resources it manages, and can neutralize or reconcile IAM changes made elsewhere. + +Specific modules also offer support for non-authoritative bindings (e.g. `google_storage_bucket_iam_member` for service accounts), to allow granular permission management on resources that they don't manage directly. + +These modules are not necessarily backward compatible. Changes breaking compatibility in modules are marked by major releases (but not all major releases contain breaking changes). Please be mindful when upgrading Fabric modules in existing Terraform setups, and always try to use versioned references in module sources so you can easily revert back to a previous version. Since the introduction of the `moved` block in Terraform we try to use it whenever possible to make updates non-breaking, but that does not cover all changes we might need to make. + +These modules are used in the examples included in this repository. If you are using any of those examples in your own Terraform configuration, make sure that you are using the same version for all the modules, and switch module sources to GitHub format using references. The recommended approach to working with Fabric modules is the following: + +- Fork the repository and own the fork. This will allow you to: + - Evolve the existing modules. + - Create your own modules. + - Sync from the upstream repository to get all the updates. + +- Use GitHub sources with refs to reference the modules. See an example below: + + ```terraform + module "project" { + source = "../assets/modules-fabric/v26/project" + name = "my-project" + billing_account = "123456-123456-123456" + parent = "organizations/123456" + } + ``` + +## Foundational modules + +- [billing budget](./billing-budget) +- [Cloud Identity group](./cloud-identity-group/) +- [folder](./folder) +- [service accounts](./iam-service-account) +- [logging bucket](./logging-bucket) +- [organization](./organization) +- [project](./project) +- [projects-data-source](./projects-data-source) + +## Networking modules + +- [Address reservation](./net-address) +- [Cloud Endpoints](./endpoints) +- [DNS](./dns) +- [DNS Response Policy](./dns-response-policy/) +- [Firewall policy](./net-firewall-policy) +- [External Application Load Balancer](./net-lb-app-ext/) +- [External Passthrough Network Load Balancer](./net-lb-ext) +- [Internal Application Load Balancer](./net-lb-app-int) +- [Internal Passthrough Network Load Balancer](./net-lb-int) +- [Internal Proxy Network Load Balancer](./net-lb-proxy-int) +- [Internal ] +- [NAT](./net-cloudnat) +- [Service Directory](./service-directory) +- [VPC](./net-vpc) +- [VPC firewall](./net-vpc-firewall) +- [VPN dynamic](./net-vpn-dynamic) +- [VPC peering](./net-vpc-peering) +- [VPN HA](./net-vpn-ha) +- [VPN static](./net-vpn-static) + +## Compute/Container + +- [VM/VM group](./compute-vm) +- [MIG](./compute-mig) +- [COS container](./cloud-config-container/cos-generic-metadata/) (coredns/mysql/nva/onprem/squid) +- [GKE autopilot cluster](./gke-cluster-autopilot) +- [GKE standard cluster](./gke-cluster-standard) +- [GKE hub](./gke-hub) +- [GKE nodepool](./gke-nodepool) +- [GCVE private cloud](./gcve-private-cloud) + +## Data + +- [AlloyDB instance](./alloydb-instance) +- [BigQuery dataset](./bigquery-dataset) +- [Bigtable instance](./bigtable-instance) +- [Dataplex](./dataplex) +- [Dataplex DataScan](./dataplex-datascan/) +- [Cloud SQL instance](./cloudsql-instance) +- [Data Catalog Policy Tag](./data-catalog-policy-tag) +- [Datafusion](./datafusion) +- [Dataproc](./dataproc) +- [GCS](./gcs) +- [Pub/Sub](./pubsub) + +## Development + +- [API Gateway](./api-gateway) +- [Apigee](./apigee) +- [Artifact Registry](./artifact-registry) +- [Container Registry](./container-registry) +- [Cloud Source Repository](./source-repository) + +## Security + +- [Binauthz](./binauthz/) +- [KMS](./kms) +- [SecretManager](./secret-manager) +- [VPC Service Control](./vpc-sc) +- [Secure Web Proxy](./net-swp) + +## Serverless + +- [Cloud Functions v1](./cloud-function-v1) +- [Cloud Functions v2](./cloud-function-v2) +- [Cloud Run](./cloud-run) diff --git a/assets/modules-fabric/v26/__docs/20230816-iam-refactor.md b/assets/modules-fabric/v26/__docs/20230816-iam-refactor.md new file mode 100644 index 0000000..4691665 --- /dev/null +++ b/assets/modules-fabric/v26/__docs/20230816-iam-refactor.md @@ -0,0 +1,344 @@ +# Refactor IAM interface + +**authors:** [Ludo](https://github.com/ludoo), [Julio](https://github.com/juliocc) +**last modified:** August 17, 2023 + +## Status + +Implemented in [#1595](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1595). +Authoritative bindings type changed as per [#1622](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/issues/1622). + +## Context + +The IAM interface in our modules has evolved organically to progressively support more functionality, resulting in a large variable surface, lack of support for some key features like conditions, and some fragility for specific use cases. + +We currently support, with uneven coverage across modules: + +- authoritative `iam` in `ROLE => [PRINCIPALS]` format +- authoritative `group_iam` in `GROUP => [ROLES]` format +- legacy additive `iam_additive` in `ROLE => [PRINCIPALS]` format which breaks for dynamic values +- legacy additive `iam_additive_members` in `PRINCIPAL => [ROLES]` format which breaks for dynamic values +- new additive `iam_members` in `KEY => {role: ROLE, member: MEMBER, condition: CONDITION}` format which works with dynamic values and supports conditions +- policy authoritative `iam_policy` +- specific support for third party resource bindings in the service account module + +## Proposal + +### Authoritative bindings + +These tend to work well in practice, and the current `iam` and `group_iam` variables are simple to use with good coverage across modules. + +The only small use case that they do not cover is IAM conditions, which are easy to implement but would render the interface more verbose for the majority of cases where conditions are not needed. + +The **proposal** for authoritative bindings is to + +- leave the current interface in place (`iam` and `group_iam`) +- expand coverage so that all modules who have iam resources expose both +- add a new `iam_bindings` variable to support authoritative IAM with conditions + +The new `iam_bindings` variable will look like this: + +```hcl +variable "iam_bindings" { + description = "Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary." + type = map(object({ + members = list(string) + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} +} +``` + +This variable will not be internally merged in modules with `iam` or `group_iam`. + +### Additive bindings + +Additive bindings have evolved to mimick authoritative ones, but the result is an interface which is bloated (no one uses `iam_additive_members`), and hard to understand and use without triggering dynamic errors. Coverage is also spotty and uneven across modules, and the interface needs to support aliasing of project service accounts in the project module to work around dynamic errors. + +The `iam_additive` variable is used in a special patterns in data blueprints, to allow code to not mess up existing IAM bindings in an external project on destroy. This pattern only works in a limited set of cases, where principals are passed in via static variables or refer to "magic" static outputs in our modules. This is a simple example of the pattern: + +```hcl +locals { + iam = { + "roles/viewer" = [ + module.sa.iam_email, + var.group.admins + ] + } +} +module "project" { + iam = ( + var.project_create == null ? {} : local.iam + ) + iam_additive = ( + var.project_create != null ? {} : local.iam + ) +} +``` + +The **proposal** for authoritative bindings is to + +- remove `iam_additive` and `iam_additive_members` from the interface +- add a new `iam_bindings_additive` variable + +Once new variables are in place, migrate existing blueprints to using `iam_bindings_additive` using one of the two available patterns: + +- the flat verbose one where bindings are declared in the module call +- the more complex one that moves roles out to `locals` and uses them in `for` loops + +The new variable will closely follow the type of the authoritative `iam_bindings` variable described above: + +```hcl +variable "iam_bindings_additive" { + description = "Additive IAM bindings with support for conditions, in {KEY => { role = ROLE, members = [], condition = {}}} format." + type = map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) +} +``` + +### IAM policy + +The **proposal** is to remove the IAM policy variable and resources, as its coverage is very uneven and we never used it in practice. This will also simplify data access log management, which is currently split between its own variable/resource and the IAM policy ones. + +## Decision + +The proposal above summarizes the state of discussions between the authors, and implementation will be tested. + +## Consequences + +### FAST + +IAM implementation in the bootstrap stage and matching multitenant bootstrap has radically changed, with the addition of a new [`organization-iam.tf`](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/blob/master/fast/stages/0-bootstrap/organization-iam.tf) file which contains IAM binding definitions in an abstracted format, that is then converted to the specific formats required by the `iam`, `iam_bindings` and `iam_bindings_additive` variables. + +This brings several advantages over the previous handling of IAM: + +- authoritative and additive bindings are now grouped by principal in an easy to read and change format that serves as its own documentation +- support for IAM conditions has removed the need for standalone resources and made the intent behind those more explicit +- some subtle bugs on the intersection of user-specified bindings and internally-specified ones have been addressed + +### Blueprints + +A few data blueprints that leverage `iam_additive` have been refactored to use the new variable. This is most notable in data blueprints, where extra files have been added to the more complex examples like data foundations, to abstract IAM bindings in a way similar to what is described above for FAST. + +## Implementation + +The following sections provide a template for IAM-related variables and resources to ensure a consistent implementation of IAM across the repository. Use these code snippets to add IAM support to your module. + +### Top-level module IAM + +Use this template if your module manages a single instance of a given resource (e.g. a KMS keyring). + +```terraform +# variables.tf + +variable "iam" { + description = "IAM bindings in {ROLE => [MEMBERS]} format. Mutually exclusive with the access_* variables used for basic roles." + type = map(list(string)) + default = {} + nullable = false +} + +variable "iam_bindings" { + description = "Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary." + type = map(object({ + members = list(string) + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + default = {} + nullable = false +} + +variable "iam_bindings_additive" { + description = "Keyring individual additive IAM bindings. Keys are arbitrary." + type = map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + default = {} + nullable = false +} +``` + +```terraform +# iam.tf + +resource "google_RESOURCE_TYPE_iam_binding" "authoritative" { + for_each = var.iam + role = each.key + members = each.value + // add extra attributes (e.g. resource id) +} + +resource "google_RESOURCE_TYPE_iam_binding" "bindings" { + for_each = var.iam_bindings + role = each.value.role + members = each.value.members + // add extra attributes (e.g. resource id) + + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +resource "google_RESOURCE_TYPE_iam_member" "bindings" { + for_each = var.iam_bindings_additive + role = each.value.role + member = each.value.member + // add extra attributes (e.g. resource id) + + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} +``` + +### Sub-resources IAM + +Use this template if your module manages multiple instances of a resource (e.g. keys in KMS keyring). + +```terraform +# variables.tf +variable "sub_resources" { + type = map(object({ + # sub-resource configuration here + + iam = optional(map(list(string)), {}) + iam_bindings = optional(map(object({ + members = list(string) + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })), {}) + iam_bindings_additive = optional(map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })), {}) + })) + default = {} + nullable = false +} +``` + +```terraform +# iam.tf +locals { + SUB_RESOURCE_iam = flatten([ + for k, v in var.SUB_RESOURCEs : [ + for role, members in v.iam : { + key = k + role = role + members = members + } + ] + ]) + SUB_RESOURCE_iam_bindings = merge([ + for k, v in var.SUB_RESOURCEs : { + for binding_key, data in v.iam_bindings : + binding_key => { + SUB_RESOURCE = k + role = data.role + members = data.members + condition = data.condition + } + } + ]...) + SUB_RESOURCE_iam_bindings_additive = merge([ + for k, v in var.subresources : { + for binding_key, data in v.iam_bindings_additive : + binding_key => { + SUB_RESOURCE = k + role = data.role + member = data.member + condition = data.condition + } + } + ]...) +} +``` + +```terraform +# iam.tf + +resource "google_SUB_RESOURCE_iam_binding" "authoritative" { + for_each = { + for binding in local.SUB_RESOURCE_iam : + "${binding.key}.${binding.role}" => binding + } + role = each.value.role + members = each.value.members + // add extra attributes (e.g. sub resource id) +} + +resource "google_SUB_RESOURCE_iam_binding" "bindings" { + for_each = local.SUB_RESOURCE_iam_bindings + role = each.value.role + members = each.value.members + // add extra attributes (e.g. sub resource id) + + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +resource "google_SUB_RESOURCE_iam_member" "members" { + for_each = local.SUB_RESOURCE_iam_bindings_additive + role = each.value.role + member = each.value.member + // add extra attributes (e.g. sub resource id) + + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +``` diff --git a/assets/modules-fabric/v26/__docs/README.md b/assets/modules-fabric/v26/__docs/README.md new file mode 100644 index 0000000..da5c918 --- /dev/null +++ b/assets/modules-fabric/v26/__docs/README.md @@ -0,0 +1,3 @@ +# Fabric modules architectural documents + +This folder contains assorted bits of documentation used to log current architectural choices, or past decisions. Format is inspired by [Michael Nygard's decision record template](https://github.com/joelparkerhenderson/architecture-decision-record/blob/main/templates/decision-record-template-by-michael-nygard/index.md). diff --git a/assets/modules-fabric/v26/__experimental/net-dns-policy-address/README.md b/assets/modules-fabric/v26/__experimental/net-dns-policy-address/README.md new file mode 100644 index 0000000..7044ef2 --- /dev/null +++ b/assets/modules-fabric/v26/__experimental/net-dns-policy-address/README.md @@ -0,0 +1,35 @@ +# Google Cloud DNS Inbound Policy Addresses + +This module allows discovering the addresses reserved in subnets when [DNS Inbound Policies](https://cloud.google.com/dns/docs/policies) are configured. + +Since it's currently impossible to fetch those addresses using a GCP data source (see [this issue](https://github.com/hashicorp/terraform-provider-google/issues/3753) for more details), the workaround used here is to derive the authorization token from the Google provider, and do a direct HTTP call to the Compute API. + +## Examples + +```hcl +module "dns-policy-addresses" { + source = "./fabric/modules/__experimental/net-dns-policy-addresses" + project_id = "myproject" + regions = ["europe-west1", "europe-west3"] +} +# tftest skip (uses data sources) +``` + +The output is a map with lists of addresses of type `DNS_RESOLVER` for each region specified in variables. + + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [project_id](variables.tf#L17) | Project id. | string | ✓ | | +| [regions](variables.tf#L22) | Regions to fetch addresses from. | list(string) | | ["europe-west1"] | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [addresses](outputs.tf#L24) | DNS inbound policy addresses per region. | | + + diff --git a/assets/modules-fabric/v26/__experimental/net-dns-policy-address/main.tf b/assets/modules-fabric/v26/__experimental/net-dns-policy-address/main.tf new file mode 100644 index 0000000..cb3144d --- /dev/null +++ b/assets/modules-fabric/v26/__experimental/net-dns-policy-address/main.tf @@ -0,0 +1,32 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + url = format( + "https://content-compute.googleapis.com/compute/v1/projects/%s", + var.project_id + ) +} + +data "google_client_config" "current" {} + +data "http" "addresses" { + for_each = toset(var.regions) + url = "${local.url}/regions/${each.key}/addresses?filter=purpose%20%3D%20%22DNS_RESOLVER%22" + request_headers = { + Authorization = "Bearer ${data.google_client_config.current.access_token}" + } +} diff --git a/assets/modules-fabric/v26/__experimental/net-dns-policy-address/outputs.tf b/assets/modules-fabric/v26/__experimental/net-dns-policy-address/outputs.tf new file mode 100644 index 0000000..d379f26 --- /dev/null +++ b/assets/modules-fabric/v26/__experimental/net-dns-policy-address/outputs.tf @@ -0,0 +1,31 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + region_addresses = { + for k, v in data.http.addresses : k => try(jsondecode(v.body), {}) + } +} + + +output "addresses" { + description = "DNS inbound policy addresses per region." + value = { + for k, v in local.region_addresses : k => [ + for i in try(v.items, []) : i.address + ] + } +} diff --git a/assets/modules-fabric/v26/__experimental/net-dns-policy-address/variables.tf b/assets/modules-fabric/v26/__experimental/net-dns-policy-address/variables.tf new file mode 100644 index 0000000..1b80d16 --- /dev/null +++ b/assets/modules-fabric/v26/__experimental/net-dns-policy-address/variables.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project id." + type = string +} + +variable "regions" { + description = "Regions to fetch addresses from." + nullable = false + type = list(string) + default = ["europe-west1"] +} diff --git a/assets/modules-fabric/v26/__experimental/net-neg/README.md b/assets/modules-fabric/v26/__experimental/net-neg/README.md new file mode 100644 index 0000000..cb271c5 --- /dev/null +++ b/assets/modules-fabric/v26/__experimental/net-neg/README.md @@ -0,0 +1,48 @@ +# Network Endpoint Group Module + +This modules allows creating zonal network endpoint groups. + +Note: this module will integrated into a general-purpose load balancing module in the future. + +## Example +```hcl +module "neg" { + source = "./fabric/modules/__experimental/net-neg/" + project_id = "myproject" + name = "myneg" + network = var.vpc.self_link + subnetwork = var.subnet.self_link + zone = "europe-west1-b" + endpoints = [ + for instance in module.vm.instances : + { + instance = instance.name + port = 80 + ip_address = instance.network_interface[0].network_ip + } + ] +} +# tftest skip +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [endpoints](variables.tf#L17) | List of (instance, port, address) of the NEG. | list(object({…})) | ✓ | | +| [name](variables.tf#L26) | NEG name. | string | ✓ | | +| [network](variables.tf#L31) | Name or self link of the VPC used for the NEG. Use the self link for Shared VPC. | string | ✓ | | +| [project_id](variables.tf#L36) | NEG project id. | string | ✓ | | +| [subnetwork](variables.tf#L41) | VPC subnetwork name or self link. | string | ✓ | | +| [zone](variables.tf#L46) | NEG zone. | string | ✓ | | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [id](outputs.tf#L17) | Network endpoint group ID. | | +| [self_lnk](outputs.tf#L22) | Network endpoint group self link. | | +| [size](outputs.tf#L27) | Size of the network endpoint group. | | + + diff --git a/assets/modules-fabric/v26/__experimental/net-neg/main.tf b/assets/modules-fabric/v26/__experimental/net-neg/main.tf new file mode 100644 index 0000000..773a75c --- /dev/null +++ b/assets/modules-fabric/v26/__experimental/net-neg/main.tf @@ -0,0 +1,33 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_compute_network_endpoint_group" "group" { + project = var.project_id + name = var.name + network = var.network + subnetwork = var.subnetwork + zone = var.zone +} + +resource "google_compute_network_endpoint" "endpoint" { + for_each = { for endpoint in var.endpoints : endpoint.instance => endpoint } + project = var.project_id + network_endpoint_group = google_compute_network_endpoint_group.group.name + instance = each.value.instance + port = each.value.port + ip_address = each.value.ip_address + zone = var.zone +} diff --git a/assets/modules-fabric/v26/__experimental/net-neg/outputs.tf b/assets/modules-fabric/v26/__experimental/net-neg/outputs.tf new file mode 100644 index 0000000..cb496f5 --- /dev/null +++ b/assets/modules-fabric/v26/__experimental/net-neg/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "id" { + description = "Network endpoint group ID." + value = google_compute_network_endpoint_group.group.name +} + +output "self_lnk" { + description = "Network endpoint group self link." + value = google_compute_network_endpoint_group.group.self_link +} + +output "size" { + description = "Size of the network endpoint group." + value = google_compute_network_endpoint_group.group.size +} diff --git a/assets/modules-fabric/v26/__experimental/net-neg/variables.tf b/assets/modules-fabric/v26/__experimental/net-neg/variables.tf new file mode 100644 index 0000000..b4eb42a --- /dev/null +++ b/assets/modules-fabric/v26/__experimental/net-neg/variables.tf @@ -0,0 +1,49 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "endpoints" { + description = "List of (instance, port, address) of the NEG." + type = list(object({ + instance = string + port = number + ip_address = string + })) +} + +variable "name" { + description = "NEG name." + type = string +} + +variable "network" { + description = "Name or self link of the VPC used for the NEG. Use the self link for Shared VPC." + type = string +} + +variable "project_id" { + description = "NEG project id." + type = string +} + +variable "subnetwork" { + description = "VPC subnetwork name or self link." + type = string +} + +variable "zone" { + description = "NEG zone." + type = string +} diff --git a/assets/modules-fabric/v26/__experimental/net-neg/versions.tf b/assets/modules-fabric/v26/__experimental/net-neg/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/__experimental/net-neg/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/alloydb-instance/README.md b/assets/modules-fabric/v26/alloydb-instance/README.md new file mode 100644 index 0000000..1e24131 --- /dev/null +++ b/assets/modules-fabric/v26/alloydb-instance/README.md @@ -0,0 +1,88 @@ +# AlloyDB cluster and instance with read replicas + +This module manages the creation of AlloyDB cluster and configuration with/without automated backup policy, Primary node instance and Read Node Pools. + + +## Simple example + +This example shows how to create Alloydb cluster and instance with multiple read pools in GCP project. + +```hcl +module "alloydb" { + source = "./fabric/modules/alloydb-instance" + project_id = "myproject" + cluster_id = "alloydb-cluster-all" + location = "europe-west2" + labels = {} + display_name = "" + initial_user = { + user = "alloydb-cluster-full", + password = "alloydb-cluster-password" + } + network_self_link = "projects/myproject/global/networks/default" + + automated_backup_policy = null + + primary_instance_config = { + instance_id = "primary-instance-1", + instance_type = "PRIMARY", + machine_cpu_count = 2, + database_flags = {}, + display_name = "alloydb-primary-instance" + } + read_pool_instance = [ + { + instance_id = "read-instance-1", + display_name = "read-instance-1", + instance_type = "READ_POOL", + node_count = 1, + database_flags = {}, + machine_cpu_count = 1 + }, + { + instance_id = "read-instance-2", + display_name = "read-instance-2", + instance_type = "READ_POOL", + node_count = 1, + database_flags = {}, + machine_cpu_count = 1 + } + ] + +} + +# tftest modules=1 resources=7 +``` +## TODO +- [ ] Add IAM support +- [ ] support password in output + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [cluster_id](variables.tf#L35) | The ID of the alloydb cluster. | string | ✓ | | +| [network_self_link](variables.tf#L83) | Network ID where the AlloyDb cluster will be deployed. | string | ✓ | | +| [primary_instance_config](variables.tf#L88) | Primary cluster configuration that supports read and write operations. | object({…}) | ✓ | | +| [project_id](variables.tf#L110) | The ID of the project in which to provision resources. | string | ✓ | | +| [automated_backup_policy](variables.tf#L17) | The automated backup policy for this cluster. | object({…}) | | null | +| [display_name](variables.tf#L44) | Human readable display name for the Alloy DB Cluster. | string | | null | +| [encryption_key_name](variables.tf#L50) | The fully-qualified resource name of the KMS key for cluster encryption. | string | | null | +| [initial_user](variables.tf#L56) | Alloy DB Cluster Initial User Credentials. | object({…}) | | null | +| [labels](variables.tf#L65) | User-defined labels for the alloydb cluster. | map(string) | | {} | +| [location](variables.tf#L71) | Location where AlloyDb cluster will be deployed. | string | | "europe-west2" | +| [network_name](variables.tf#L77) | The network name of the project in which to provision resources. | string | | "multiple-readpool" | +| [read_pool_instance](variables.tf#L115) | List of Read Pool Instances to be created. | list(object({…})) | | [] | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [cluster](outputs.tf#L17) | Cluster created. | ✓ | +| [cluster_id](outputs.tf#L23) | ID of the Alloy DB Cluster created. | | +| [primary_instance](outputs.tf#L28) | Primary instance created. | | +| [primary_instance_id](outputs.tf#L33) | ID of the primary instance created. | | +| [read_pool_instance_ids](outputs.tf#L38) | IDs of the read instances created. | | + + diff --git a/assets/modules-fabric/v26/alloydb-instance/main.tf b/assets/modules-fabric/v26/alloydb-instance/main.tf new file mode 100644 index 0000000..1a01aed --- /dev/null +++ b/assets/modules-fabric/v26/alloydb-instance/main.tf @@ -0,0 +1,160 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + quantity_based_retention_count = ( + var.automated_backup_policy != null ? (var.automated_backup_policy.quantity_based_retention_count != null ? [var.automated_backup_policy.quantity_based_retention_count] : []) : [] + ) + read_pool_instance = ( + var.read_pool_instance != null ? + { for read_pool_instances in var.read_pool_instance : read_pool_instances.instance_id => read_pool_instances } : {} + ) + time_based_retention_count = ( + var.automated_backup_policy != null ? (var.automated_backup_policy.time_based_retention_count != null ? [var.automated_backup_policy.time_based_retention_count] : []) : [] + ) +} + +resource "google_alloydb_cluster" "default" { + cluster_id = var.cluster_id + location = var.location + network = var.network_self_link + display_name = var.display_name + project = var.project_id + labels = var.labels + + dynamic "automated_backup_policy" { + for_each = var.automated_backup_policy == null ? [] : [""] + content { + location = var.automated_backup_policy.location + backup_window = var.automated_backup_policy.backup_window + enabled = var.automated_backup_policy.enabled + labels = var.automated_backup_policy.labels + + + weekly_schedule { + days_of_week = automated_backup_policy.value.weekly_schedule.days_of_week + dynamic "start_times" { + for_each = { for i, time in automated_backup_policy.value.weekly_schedule.start_times : i => { + hours = tonumber(split(":", time)[0]) + minutes = tonumber(split(":", time)[1]) + seconds = tonumber(split(":", time)[2]) + nanos = tonumber(split(":", time)[3]) + } + } + content { + hours = start_times.value.hours + minutes = start_times.value.minutes + seconds = start_times.value.seconds + nanos = start_times.value.nanos + } + } + } + + dynamic "quantity_based_retention" { + for_each = local.quantity_based_retention_count + content { + count = quantity_based_retention.value + } + } + + dynamic "time_based_retention" { + for_each = local.time_based_retention_count + content { + retention_period = time_based_retention.value + } + } + + dynamic "encryption_config" { + for_each = automated_backup_policy.value.backup_encryption_key_name == null ? [] : ["encryption_config"] + content { + kms_key_name = automated_backup_policy.value.backup_encryption_key_name + } + } + + } + + } + + dynamic "initial_user" { + for_each = var.initial_user == null ? [] : ["initial_user"] + content { + user = var.initial_user.user + password = var.initial_user.password + } + } + + dynamic "encryption_config" { + for_each = var.encryption_key_name == null ? [] : ["encryption_config"] + content { + kms_key_name = var.encryption_key_name + } + } +} + +resource "google_alloydb_instance" "primary" { + cluster = google_alloydb_cluster.default.name + instance_id = var.primary_instance_config.instance_id + instance_type = "PRIMARY" + display_name = var.primary_instance_config.display_name + database_flags = var.primary_instance_config.database_flags + labels = var.primary_instance_config.labels + annotations = var.primary_instance_config.annotations + gce_zone = var.primary_instance_config.availability_type == "ZONAL" ? var.primary_instance_config.gce_zone : null + availability_type = var.primary_instance_config.availability_type + + machine_config { + cpu_count = var.primary_instance_config.machine_cpu_count + } + +} + +resource "google_alloydb_instance" "read_pool" { + for_each = local.read_pool_instance + cluster = google_alloydb_cluster.default.name + instance_id = each.key + instance_type = "READ_POOL" + availability_type = each.value.availability_type + gce_zone = each.value.availability_type == "ZONAL" ? each.value.availability_type.gce_zone : null + + read_pool_config { + node_count = each.value.node_count + } + + database_flags = each.value.database_flags + machine_config { + cpu_count = each.value.machine_cpu_count + } + + depends_on = [google_alloydb_instance.primary, google_compute_network.default, google_compute_global_address.private_ip_alloc, google_service_networking_connection.vpc_connection] +} + +resource "google_compute_network" "default" { + name = var.network_name +} + +resource "google_compute_global_address" "private_ip_alloc" { + name = "adb-all" + address_type = "INTERNAL" + purpose = "VPC_PEERING" + prefix_length = 16 + network = google_compute_network.default.id +} + +resource "google_service_networking_connection" "vpc_connection" { + network = google_compute_network.default.id + service = "servicenetworking.googleapis.com" + reserved_peering_ranges = [google_compute_global_address.private_ip_alloc.name] +} diff --git a/assets/modules-fabric/v26/alloydb-instance/outputs.tf b/assets/modules-fabric/v26/alloydb-instance/outputs.tf new file mode 100644 index 0000000..990278a --- /dev/null +++ b/assets/modules-fabric/v26/alloydb-instance/outputs.tf @@ -0,0 +1,43 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cluster" { + description = "Cluster created." + value = resource.google_alloydb_cluster.default.display_name + sensitive = true +} + +output "cluster_id" { + description = "ID of the Alloy DB Cluster created." + value = google_alloydb_cluster.default.cluster_id +} + +output "primary_instance" { + description = "Primary instance created." + value = resource.google_alloydb_instance.primary.display_name +} + +output "primary_instance_id" { + description = "ID of the primary instance created." + value = google_alloydb_instance.primary.instance_id +} + +output "read_pool_instance_ids" { + description = "IDs of the read instances created." + value = [ + for rd, details in google_alloydb_instance.read_pool : details.instance_id + ] +} diff --git a/assets/modules-fabric/v26/alloydb-instance/variables.tf b/assets/modules-fabric/v26/alloydb-instance/variables.tf new file mode 100644 index 0000000..97ed366 --- /dev/null +++ b/assets/modules-fabric/v26/alloydb-instance/variables.tf @@ -0,0 +1,127 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "automated_backup_policy" { + description = "The automated backup policy for this cluster." + type = object({ + location = optional(string) + backup_window = optional(string) + enabled = optional(bool) + weekly_schedule = optional(object({ + days_of_week = optional(list(string)) + start_times = list(string) + })), + quantity_based_retention_count = optional(number) + time_based_retention_count = optional(string) + labels = optional(map(string)) + backup_encryption_key_name = optional(string) + }) + default = null +} + +variable "cluster_id" { + description = "The ID of the alloydb cluster." + type = string + validation { + condition = can(regex("^[a-z0-9-]+$", var.cluster_id)) + error_message = "ERROR: Cluster ID must contain only Letters(lowercase), number, and hyphen." + } +} + +variable "display_name" { + description = "Human readable display name for the Alloy DB Cluster." + type = string + default = null +} + +variable "encryption_key_name" { + description = "The fully-qualified resource name of the KMS key for cluster encryption." + type = string + default = null +} + +variable "initial_user" { + description = "Alloy DB Cluster Initial User Credentials." + type = object({ + user = optional(string), + password = string + }) + default = null +} + +variable "labels" { + description = "User-defined labels for the alloydb cluster." + type = map(string) + default = {} +} + +variable "location" { + description = "Location where AlloyDb cluster will be deployed." + type = string + default = "europe-west2" +} + +variable "network_name" { + description = "The network name of the project in which to provision resources." + type = string + default = "multiple-readpool" +} + +variable "network_self_link" { + description = "Network ID where the AlloyDb cluster will be deployed." + type = string +} + +variable "primary_instance_config" { + description = "Primary cluster configuration that supports read and write operations." + type = object({ + instance_id = string, + display_name = optional(string), + database_flags = optional(map(string)) + labels = optional(map(string)) + annotations = optional(map(string)) + gce_zone = optional(string) + availability_type = optional(string) + machine_cpu_count = optional(number, 2), + }) + validation { + condition = can(regex("^(2|4|8|16|32|64)$", var.primary_instance_config.machine_cpu_count)) + error_message = "cpu count must be one of [2 4 8 16 32 64]." + } + validation { + condition = can(regex("^[a-z]([a-z0-9-]{0,61}[a-z0-9])?$", var.primary_instance_config.instance_id)) + error_message = "Primary Instance ID should satisfy the following pattern ^[a-z]([a-z0-9-]{0,61}[a-z0-9])?$." + } +} + +variable "project_id" { + description = "The ID of the project in which to provision resources." + type = string +} + +variable "read_pool_instance" { + description = "List of Read Pool Instances to be created." + type = list(object({ + instance_id = string + display_name = string + node_count = optional(number, 1) + database_flags = optional(map(string)) + availability_type = optional(string) + gce_zone = optional(string) + machine_cpu_count = optional(number, 2) + })) + default = [] +} diff --git a/assets/modules-fabric/v26/alloydb-instance/versions.tf b/assets/modules-fabric/v26/alloydb-instance/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/alloydb-instance/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/api-gateway/README.md b/assets/modules-fabric/v26/api-gateway/README.md new file mode 100644 index 0000000..d3c16d3 --- /dev/null +++ b/assets/modules-fabric/v26/api-gateway/README.md @@ -0,0 +1,89 @@ +# API Gateway +This module allows creating an API with its associated API config and API gateway. It also allows you grant IAM roles on the created resources. + +# Examples + +## Basic example +```hcl +module "gateway" { + source = "./fabric/modules/api-gateway" + project_id = "my-project" + api_id = "api" + region = "europe-west1" + spec = < + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [api_id](variables.tf#L17) | API identifier. | string | ✓ | | +| [project_id](variables.tf#L34) | Project identifier. | string | ✓ | | +| [region](variables.tf#L39) | Region. | string | ✓ | | +| [spec](variables.tf#L56) | String with the contents of the OpenAPI spec. | string | ✓ | | +| [iam](variables.tf#L22) | IAM bindings for the API in {ROLE => [MEMBERS]} format. | map(list(string)) | | null | +| [labels](variables.tf#L28) | Map of labels. | map(string) | | null | +| [service_account_create](variables.tf#L44) | Flag indicating whether a service account needs to be created. | bool | | false | +| [service_account_email](variables.tf#L50) | Service account for creating API configs. | string | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [api](outputs.tf#L17) | API. | | +| [api_config](outputs.tf#L28) | API configs. | | +| [api_config_id](outputs.tf#L39) | The identifiers of the API configs. | | +| [api_id](outputs.tf#L50) | API identifier. | | +| [default_hostname](outputs.tf#L61) | The default host names of the API gateway. | | +| [gateway](outputs.tf#L72) | API gateways. | | +| [gateway_id](outputs.tf#L83) | The identifiers of the API gateways. | | +| [service_account](outputs.tf#L94) | Service account resource. | | +| [service_account_email](outputs.tf#L99) | The service account for creating API configs. | | +| [service_account_iam_email](outputs.tf#L104) | The service account for creating API configs. | | + + diff --git a/assets/modules-fabric/v26/api-gateway/main.tf b/assets/modules-fabric/v26/api-gateway/main.tf new file mode 100644 index 0000000..5b6ce07 --- /dev/null +++ b/assets/modules-fabric/v26/api-gateway/main.tf @@ -0,0 +1,115 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + service_account_email = ( + var.service_account_create + ? ( + length(google_service_account.service_account) > 0 + ? google_service_account.service_account[0].email + : null + ) + : var.service_account_email + ) +} + +resource "google_api_gateway_api" "api" { + provider = google-beta + project = var.project_id + api_id = var.api_id + display_name = var.api_id + labels = var.labels +} + +resource "google_service_account" "service_account" { + count = var.service_account_create ? 1 : 0 + project = var.project_id + account_id = "sa-api-cfg-${google_api_gateway_api.api.api_id}" + display_name = "Service account to create API configs for ${google_api_gateway_api.api.api_id} API" +} + +resource "google_api_gateway_api_config" "api_config" { + provider = google-beta + project = google_api_gateway_api.api.project + api = google_api_gateway_api.api.api_id + api_config_id = "api-cfg-${google_api_gateway_api.api.api_id}-${md5(var.spec)}" + display_name = "api-cfg-${google_api_gateway_api.api.api_id}-${md5(var.spec)}" + openapi_documents { + document { + path = "spec.yaml" + contents = base64encode(var.spec) + } + } + dynamic "gateway_config" { + for_each = local.service_account_email == null ? [] : [""] + content { + backend_config { + google_service_account = local.service_account_email + } + } + } + lifecycle { + create_before_destroy = true + } +} + +resource "google_api_gateway_gateway" "gateway" { + provider = google-beta + project = google_api_gateway_api_config.api_config.project + api_config = google_api_gateway_api_config.api_config.id + gateway_id = "gw-${google_api_gateway_api.api.api_id}" + display_name = "gw-${google_api_gateway_api.api.api_id}" + region = var.region + lifecycle { + create_before_destroy = true + } +} + +resource "google_project_service" "service" { + project = google_api_gateway_gateway.gateway.project + service = google_api_gateway_api.api.managed_service + disable_on_destroy = true + disable_dependent_services = true +} + +resource "google_api_gateway_api_iam_binding" "api_iam_bindings" { + for_each = coalesce(var.iam, {}) + provider = google-beta + project = google_api_gateway_api.api.project + api = google_api_gateway_api.api.api_id + role = each.key + members = each.value +} + +resource "google_api_gateway_api_config_iam_binding" "api_config_iam_bindings" { + for_each = coalesce(var.iam, {}) + provider = google-beta + project = google_api_gateway_api_config.api_config.project + api = google_api_gateway_api.api.api_id + api_config = google_api_gateway_api_config.api_config.api_config_id + role = each.key + members = each.value +} + +resource "google_api_gateway_gateway_iam_binding" "gateway_iam_bindings" { + for_each = coalesce(var.iam, {}) + provider = google-beta + project = google_api_gateway_gateway.gateway.project + gateway = google_api_gateway_gateway.gateway.gateway_id + region = var.region + role = each.key + members = each.value +} diff --git a/assets/modules-fabric/v26/api-gateway/outputs.tf b/assets/modules-fabric/v26/api-gateway/outputs.tf new file mode 100644 index 0000000..863c382 --- /dev/null +++ b/assets/modules-fabric/v26/api-gateway/outputs.tf @@ -0,0 +1,107 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "api" { + description = "API." + value = google_api_gateway_api.api + depends_on = [ + google_project_service.service, + google_api_gateway_api_iam_binding.api_iam_bindings, + google_api_gateway_api_config_iam_binding.api_config_iam_bindings, + google_api_gateway_gateway_iam_binding.gateway_iam_bindings + ] +} + +output "api_config" { + description = "API configs." + value = google_api_gateway_api_config.api_config + depends_on = [ + google_project_service.service, + google_api_gateway_api_iam_binding.api_iam_bindings, + google_api_gateway_api_config_iam_binding.api_config_iam_bindings, + google_api_gateway_gateway_iam_binding.gateway_iam_bindings + ] +} + +output "api_config_id" { + description = "The identifiers of the API configs." + value = google_api_gateway_api_config.api_config.api_config_id + depends_on = [ + google_project_service.service, + google_api_gateway_api_iam_binding.api_iam_bindings, + google_api_gateway_api_config_iam_binding.api_config_iam_bindings, + google_api_gateway_gateway_iam_binding.gateway_iam_bindings + ] +} + +output "api_id" { + description = "API identifier." + value = google_api_gateway_api.api.api_id + depends_on = [ + google_project_service.service, + google_api_gateway_api_iam_binding.api_iam_bindings, + google_api_gateway_api_config_iam_binding.api_config_iam_bindings, + google_api_gateway_gateway_iam_binding.gateway_iam_bindings + ] +} + +output "default_hostname" { + description = "The default host names of the API gateway." + value = google_api_gateway_gateway.gateway.default_hostname + depends_on = [ + google_project_service.service, + google_api_gateway_api_iam_binding.api_iam_bindings, + google_api_gateway_api_config_iam_binding.api_config_iam_bindings, + google_api_gateway_gateway_iam_binding.gateway_iam_bindings + ] +} + +output "gateway" { + description = "API gateways." + value = google_api_gateway_gateway.gateway + depends_on = [ + google_project_service.service, + google_api_gateway_api_iam_binding.api_iam_bindings, + google_api_gateway_api_config_iam_binding.api_config_iam_bindings, + google_api_gateway_gateway_iam_binding.gateway_iam_bindings + ] +} + +output "gateway_id" { + description = "The identifiers of the API gateways." + value = google_api_gateway_gateway.gateway.gateway_id + depends_on = [ + google_project_service.service, + google_api_gateway_api_iam_binding.api_iam_bindings, + google_api_gateway_api_config_iam_binding.api_config_iam_bindings, + google_api_gateway_gateway_iam_binding.gateway_iam_bindings + ] +} + +output "service_account" { + description = "Service account resource." + value = try(google_service_account.service_account[0], null) +} + +output "service_account_email" { + description = "The service account for creating API configs." + value = local.service_account_email +} + +output "service_account_iam_email" { + description = "The service account for creating API configs." + value = local.service_account_email == null ? null : "serviceAccount:${local.service_account_email}" +} diff --git a/assets/modules-fabric/v26/api-gateway/variables.tf b/assets/modules-fabric/v26/api-gateway/variables.tf new file mode 100644 index 0000000..ef5bd41 --- /dev/null +++ b/assets/modules-fabric/v26/api-gateway/variables.tf @@ -0,0 +1,59 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "api_id" { + description = "API identifier." + type = string +} + +variable "iam" { + description = "IAM bindings for the API in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = null +} + +variable "labels" { + description = "Map of labels." + type = map(string) + default = null +} + +variable "project_id" { + description = "Project identifier." + type = string +} + +variable "region" { + description = "Region." + type = string +} + +variable "service_account_create" { + description = "Flag indicating whether a service account needs to be created." + type = bool + default = false +} + +variable "service_account_email" { + description = "Service account for creating API configs." + type = string + default = null +} + +variable "spec" { + description = "String with the contents of the OpenAPI spec." + type = string +} diff --git a/assets/modules-fabric/v26/api-gateway/versions.tf b/assets/modules-fabric/v26/api-gateway/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/api-gateway/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/apigee/README.md b/assets/modules-fabric/v26/apigee/README.md new file mode 100644 index 0000000..7692d6f --- /dev/null +++ b/assets/modules-fabric/v26/apigee/README.md @@ -0,0 +1,340 @@ +# Apigee + +This module simplifies the creation of a Apigee resources (organization, environment groups, environment group attachments, environments, instances and instance attachments). + +## Examples + + +- [Examples](#examples) + - [Minimal example (CLOUD)](#minimal-example-cloud) + - [Minimal example with existing organization (CLOUD)](#minimal-example-with-existing-organization-cloud) + - [Disable VPC Peering (CLOUD)](#disable-vpc-peering-cloud) + - [All resources (CLOUD)](#all-resources-cloud) + - [All resources (HYBRID control plane)](#all-resources-hybrid-control-plane) + - [New environment group](#new-environment-group) + - [New environment](#new-environment) + - [New instance (VPC Peering Provisioning Mode)](#new-instance-vpc-peering-provisioning-mode) + - [New instance (Non VPC Peering Provisioning Mode)](#new-instance-non-vpc-peering-provisioning-mode) + - [New endpoint attachment](#new-endpoint-attachment) + - [Apigee add-ons](#apigee-add-ons) +- [Variables](#variables) +- [Outputs](#outputs) + + +### Minimal example (CLOUD) + +This example shows how to create to create an Apigee organization and deploy instance in it. + +```hcl +module "apigee" { + source = "./fabric/modules/apigee" + project_id = var.project_id + organization = { + display_name = "Apigee" + billing_type = "PAYG" + analytics_region = "europe-west1" + authorized_network = var.vpc.id + runtime_type = "CLOUD" + } + envgroups = { + prod = ["prod.example.com"] + } + environments = { + apis-prod = { + display_name = "APIs prod" + description = "APIs Prod" + envgroups = ["prod"] + } + } + instances = { + europe-west1 = { + environments = ["apis-prod"] + runtime_ip_cidr_range = "10.32.0.0/22" + troubleshooting_ip_cidr_range = "10.64.0.0/28" + } + } +} +# tftest modules=1 resources=6 inventory=minimal-cloud.yaml +``` + +### Minimal example with existing organization (CLOUD) + +This example shows how to create to work with an existing organization in the project. Note that in this case we don't specify the IP ranges for the instance, so it requests and allocates an available /22 and /28 CIDR block from Service Networking to deploy the instance. + +```hcl +module "apigee" { + source = "./fabric/modules/apigee" + project_id = var.project_id + envgroups = { + prod = ["prod.example.com"] + } + environments = { + apis-prod = { + display_name = "APIs prod" + envgroups = ["prod"] + } + } + instances = { + europe-west1 = { + environments = ["apis-prod"] + } + } +} +# tftest modules=1 resources=5 inventory=minimal-cloud-no-org.yaml +``` + +### Disable VPC Peering (CLOUD) + +When a new Apigee organization is created, it is automatically peered to the authorized network. You can prevent this from happening by using the `disable_vpc_peering` key in the `organization` variable, as shown below: + + +```hcl +module "apigee" { + source = "./fabric/modules/apigee" + project_id = var.project_id + organization = { + display_name = "Apigee" + billing_type = "PAYG" + analytics_region = "europe-west1" + runtime_type = "CLOUD" + disable_vpc_peering = true + } + envgroups = { + prod = ["prod.example.com"] + } + environments = { + apis-prod = { + display_name = "APIs prod" + envgroups = ["prod"] + } + } + instances = { + europe-west1 = { + environments = ["apis-prod"] + } + } +} +# tftest modules=1 resources=6 inventory=no-peering.yaml +``` + + +### All resources (CLOUD) + +```hcl +module "apigee" { + source = "./fabric/modules/apigee" + project_id = "my-project" + organization = { + display_name = "My Organization" + description = "My Organization" + authorized_network = "my-vpc" + runtime_type = "CLOUD" + billing_type = "PAYG" + database_encryption_key = "123456789" + analytics_region = "europe-west1" + } + envgroups = { + test = ["test.example.com"] + prod = ["prod.example.com"] + } + environments = { + apis-test = { + display_name = "APIs test" + description = "APIs Test" + envgroups = ["test"] + } + apis-prod = { + display_name = "APIs prod" + description = "APIs prod" + envgroups = ["prod"] + iam = { + "roles/viewer" = ["group:devops@myorg.com"] + } + } + } + instances = { + europe-west1 = { + runtime_ip_cidr_range = "10.0.4.0/22" + troubleshooting_ip_cidr_range = "10.1.1.0.0/28" + environments = ["apis-test"] + } + europe-west3 = { + runtime_ip_cidr_range = "10.0.8.0/22" + troubleshooting_ip_cidr_range = "10.1.16.0/28" + environments = ["apis-prod"] + enable_nat = true + } + } + endpoint_attachments = { + endpoint-backend-1 = { + region = "europe-west1" + service_attachment = "projects/my-project-1/serviceAttachments/gkebackend1" + } + endpoint-backend-2 = { + region = "europe-west1" + service_attachment = "projects/my-project-2/serviceAttachments/gkebackend2" + } + } +} +# tftest modules=1 resources=15 +``` + +### All resources (HYBRID control plane) + +```hcl +module "apigee" { + source = "./fabric/modules/apigee" + project_id = "my-project" + organization = { + display_name = "My Organization" + description = "My Organization" + runtime_type = "HYBRID" + analytics_region = "europe-west1" + } + envgroups = { + test = ["test.example.com"] + prod = ["prod.example.com"] + } + environments = { + apis-test = { + display_name = "APIs test" + description = "APIs Test" + envgroups = ["test"] + } + apis-prod = { + display_name = "APIs prod" + description = "APIs prod" + envgroups = ["prod"] + iam = { + "roles/viewer" = ["group:devops@myorg.com"] + } + } + } +} +# tftest modules=1 resources=8 +``` + +### New environment group + +```hcl +module "apigee" { + source = "./fabric/modules/apigee" + project_id = "my-project" + envgroups = { + test = ["test.example.com"] + } +} +# tftest modules=1 resources=1 +``` + +### New environment + +```hcl +module "apigee" { + source = "./fabric/modules/apigee" + project_id = "my-project" + environments = { + apis-test = { + display_name = "APIs test" + description = "APIs Test" + } + } +} +# tftest modules=1 resources=1 +``` + +### New instance (VPC Peering Provisioning Mode) + +```hcl +module "apigee" { + source = "./fabric/modules/apigee" + project_id = "my-project" + instances = { + europe-west1 = { + runtime_ip_cidr_range = "10.0.4.0/22" + troubleshooting_ip_cidr_range = "10.1.1.0/28" + } + } +} +# tftest modules=1 resources=1 +``` + +### New instance (Non VPC Peering Provisioning Mode) + +```hcl +module "apigee" { + source = "./fabric/modules/apigee" + project_id = "my-project" + organization = { + display_name = "My Organization" + description = "My Organization" + runtime_type = "CLOUD" + billing_type = "Pay-as-you-go" + database_encryption_key = "123456789" + analytics_region = "europe-west1" + disable_vpc_peering = true + } + instances = { + europe-west1 = {} + } +} +# tftest modules=1 resources=2 +``` + +### New endpoint attachment + +Endpoint attachments allow to implement [Apigee southbound network patterns](https://cloud.google.com/apigee/docs/api-platform/architecture/southbound-networking-patterns-endpoints#create-the-psc-attachments). + +```hcl +module "apigee" { + source = "./fabric/modules/apigee" + project_id = "my-project" + endpoint_attachments = { + endpoint-backend-1 = { + region = "europe-west1" + service_attachment = "projects/my-project-1/serviceAttachments/gkebackend1" + } + } +} +# tftest modules=1 resources=1 +``` + +### Apigee add-ons + +```hcl +module "apigee" { + source = "./fabric/modules/apigee" + project_id = "my-project" + addons_config = { + monetization = true + } +} +# tftest modules=1 resources=1 +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [project_id](variables.tf#L117) | Project ID. | string | ✓ | | +| [addons_config](variables.tf#L17) | Addons configuration. | object({…}) | | null | +| [endpoint_attachments](variables.tf#L29) | Endpoint attachments. | map(object({…})) | | {} | +| [envgroups](variables.tf#L39) | Environment groups (NAME => [HOSTNAMES]). | map(list(string)) | | {} | +| [environments](variables.tf#L46) | Environments. | map(object({…})) | | {} | +| [instances](variables.tf#L64) | Instances ([REGION] => [INSTANCE]). | map(object({…})) | | {} | +| [organization](variables.tf#L89) | Apigee organization. If set to null the organization must already exist. | object({…}) | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [endpoint_attachment_hosts](outputs.tf#L17) | Endpoint hosts. | | +| [envgroups](outputs.tf#L22) | Environment groups. | | +| [environments](outputs.tf#L27) | Environment. | | +| [instances](outputs.tf#L32) | Instances. | | +| [nat_ips](outputs.tf#L37) | NAT IP addresses used in instances. | | +| [org_id](outputs.tf#L45) | Organization ID. | | +| [org_name](outputs.tf#L50) | Organization name. | | +| [organization](outputs.tf#L55) | Organization. | | +| [service_attachments](outputs.tf#L60) | Service attachments. | | + diff --git a/assets/modules-fabric/v26/apigee/main.tf b/assets/modules-fabric/v26/apigee/main.tf new file mode 100644 index 0000000..0b61f30 --- /dev/null +++ b/assets/modules-fabric/v26/apigee/main.tf @@ -0,0 +1,155 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + org_id = try(google_apigee_organization.organization[0].id, "organizations/${var.project_id}") + org_name = try(google_apigee_organization.organization[0].name, var.project_id) +} + +resource "google_apigee_organization" "organization" { + count = var.organization == null ? 0 : 1 + analytics_region = var.organization.analytics_region + project_id = var.project_id + authorized_network = var.organization.authorized_network + billing_type = var.organization.billing_type + runtime_type = var.organization.runtime_type + runtime_database_encryption_key_name = var.organization.database_encryption_key + retention = var.organization.retention + disable_vpc_peering = var.organization.disable_vpc_peering +} + +resource "google_apigee_envgroup" "envgroups" { + for_each = var.envgroups + name = each.key + hostnames = each.value + org_id = local.org_id +} + +resource "google_apigee_environment" "environments" { + for_each = var.environments + name = each.key + display_name = each.value.display_name + description = each.value.description + deployment_type = each.value.deployment_type + api_proxy_type = each.value.api_proxy_type + dynamic "node_config" { + for_each = try(each.value.node_config, null) != null ? [""] : [] + content { + min_node_count = each.value.node_config.min_node_count + max_node_count = each.value.node_config.max_node_count + } + } + org_id = local.org_id + lifecycle { + ignore_changes = [ + node_config["current_aggregate_node_count"] + ] + } +} + +resource "google_apigee_envgroup_attachment" "envgroup_attachments" { + for_each = merge(concat([for k1, v1 in var.environments : { + for v2 in coalesce(v1.envgroups, []) : "${k1}-${v2}" => { + environment = k1 + envgroup = v2 + } + }])...) + envgroup_id = try(google_apigee_envgroup.envgroups[each.value.envgroup].id, each.value.envgroup) + environment = google_apigee_environment.environments[each.value.environment].name +} + +resource "google_apigee_environment_iam_binding" "binding" { + for_each = merge(concat([for k1, v1 in var.environments : { + for k2, v2 in coalesce(v1.iam, {}) : "${k1}-${k2}" => { + environment = "${k1}" + role = k2 + members = v2 + } + }])...) + org_id = local.org_id + env_id = google_apigee_environment.environments[each.value.environment].name + role = each.value.role + members = each.value.members +} + +resource "google_apigee_instance" "instances" { + for_each = var.instances + name = coalesce(each.value.name, "instance-${each.key}") + display_name = each.value.display_name + description = each.value.description + location = each.key + org_id = local.org_id + ip_range = ( + compact([each.value.runtime_ip_cidr_range, each.value.troubleshooting_ip_cidr_range]) == [] + ? null + : join(",", compact([each.value.runtime_ip_cidr_range, each.value.troubleshooting_ip_cidr_range])) + ) + disk_encryption_key_name = each.value.disk_encryption_key + consumer_accept_list = each.value.consumer_accept_list +} + +resource "google_apigee_nat_address" "apigee_nat" { + for_each = { + for k, v in var.instances : + k => google_apigee_instance.instances[k].id + if v.enable_nat + } + name = each.key + instance_id = each.value +} + +resource "google_apigee_instance_attachment" "instance_attachments" { + for_each = merge(concat([for k1, v1 in var.instances : { + for v2 in coalesce(v1.environments, []) : + "${k1}-${v2}" => { + instance = k1 + environment = v2 + } + }])...) + instance_id = google_apigee_instance.instances[each.value.instance].id + environment = try(google_apigee_environment.environments[each.value.environment].name, + "${local.org_id}/environments/${each.value.environment}") +} + +resource "google_apigee_endpoint_attachment" "endpoint_attachments" { + for_each = var.endpoint_attachments + org_id = local.org_id + endpoint_attachment_id = each.key + location = each.value.region + service_attachment = each.value.service_attachment +} + +resource "google_apigee_addons_config" "addons_config" { + for_each = toset(var.addons_config == null ? [] : [""]) + org = local.org_name + addons_config { + advanced_api_ops_config { + enabled = var.addons_config.advanced_api_ops + } + api_security_config { + enabled = var.addons_config.api_security + } + connectors_platform_config { + enabled = var.addons_config.connectors_platform + } + integration_config { + enabled = var.addons_config.integration + } + monetization_config { + enabled = var.addons_config.monetization + } + } +} diff --git a/assets/modules-fabric/v26/apigee/outputs.tf b/assets/modules-fabric/v26/apigee/outputs.tf new file mode 100644 index 0000000..eb3ab2c --- /dev/null +++ b/assets/modules-fabric/v26/apigee/outputs.tf @@ -0,0 +1,63 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "endpoint_attachment_hosts" { + description = "Endpoint hosts." + value = { for k, v in google_apigee_endpoint_attachment.endpoint_attachments : k => v.host } +} + +output "envgroups" { + description = "Environment groups." + value = try(google_apigee_envgroup.envgroups, null) +} + +output "environments" { + description = "Environment." + value = try(google_apigee_environment.environments, null) +} + +output "instances" { + description = "Instances." + value = try(google_apigee_instance.instances, null) +} + +output "nat_ips" { + description = "NAT IP addresses used in instances." + value = { + for k, v in google_apigee_nat_address.apigee_nat : + k => v.ip_address + } +} + +output "org_id" { + description = "Organization ID." + value = local.org_id +} + +output "org_name" { + description = "Organization name." + value = try(google_apigee_organization.organization[0].name, var.project_id) +} + +output "organization" { + description = "Organization." + value = try(google_apigee_organization.organization[0], null) +} + +output "service_attachments" { + description = "Service attachments." + value = { for k, v in google_apigee_instance.instances : k => v.service_attachment } +} diff --git a/assets/modules-fabric/v26/apigee/variables.tf b/assets/modules-fabric/v26/apigee/variables.tf new file mode 100644 index 0000000..7854950 --- /dev/null +++ b/assets/modules-fabric/v26/apigee/variables.tf @@ -0,0 +1,120 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "addons_config" { + description = "Addons configuration." + type = object({ + advanced_api_ops = optional(bool, false) + api_security = optional(bool, false) + connectors_platform = optional(bool, false) + integration = optional(bool, false) + monetization = optional(bool, false) + }) + default = null +} + +variable "endpoint_attachments" { + description = "Endpoint attachments." + type = map(object({ + region = string + service_attachment = string + })) + default = {} + nullable = false +} + +variable "envgroups" { + description = "Environment groups (NAME => [HOSTNAMES])." + type = map(list(string)) + default = {} + nullable = false +} + +variable "environments" { + description = "Environments." + type = map(object({ + display_name = optional(string) + description = optional(string, "Terraform-managed") + deployment_type = optional(string) + api_proxy_type = optional(string) + node_config = optional(object({ + min_node_count = optional(number) + max_node_count = optional(number) + })) + iam = optional(map(list(string))) + envgroups = optional(list(string)) + })) + default = {} + nullable = false +} + +variable "instances" { + description = "Instances ([REGION] => [INSTANCE])." + type = map(object({ + name = optional(string) + display_name = optional(string) + description = optional(string, "Terraform-managed") + runtime_ip_cidr_range = optional(string) + troubleshooting_ip_cidr_range = optional(string) + disk_encryption_key = optional(string) + consumer_accept_list = optional(list(string)) + enable_nat = optional(bool, false) + environments = optional(list(string)) + })) + validation { + condition = alltrue([ + for k, v in var.instances : + # has troubleshooting_ip => has runtime_ip + v.runtime_ip_cidr_range != null || v.troubleshooting_ip_cidr_range == null + ]) + error_message = "Using a troubleshooting range requires specifying a runtime range too." + } + default = {} + nullable = false +} + +variable "organization" { + description = "Apigee organization. If set to null the organization must already exist." + type = object({ + display_name = optional(string) + description = optional(string, "Terraform-managed") + authorized_network = optional(string) + runtime_type = optional(string, "CLOUD") + billing_type = optional(string) + database_encryption_key = optional(string) + analytics_region = optional(string, "europe-west1") + retention = optional(string) + disable_vpc_peering = optional(bool, false) + }) + validation { + condition = var.organization == null || ( + try(var.organization.runtime_type, null) == "CLOUD" || !try(var.organization.disable_vpc_peering, false) + ) + error_message = "Disabling the VPC peering can only be done in organization using the CLOUD runtime." + } + validation { + condition = var.organization == null || ( + try(var.organization.authorized_network, null) == null || !try(var.organization.disable_vpc_peering, false) + ) + error_message = "Disabling the VPC peering is mutually exclusive with authorized_network." + } + default = null +} + +variable "project_id" { + description = "Project ID." + type = string +} diff --git a/assets/modules-fabric/v26/apigee/versions.tf b/assets/modules-fabric/v26/apigee/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/apigee/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/artifact-registry/README.md b/assets/modules-fabric/v26/artifact-registry/README.md new file mode 100644 index 0000000..14233a0 --- /dev/null +++ b/assets/modules-fabric/v26/artifact-registry/README.md @@ -0,0 +1,125 @@ +# Google Cloud Artifact Registry Module + +This module simplifies the creation of repositories using Google Cloud Artifact Registry. + + +- [Standard Repository](#standard-repository) +- [Remote and Virtual Repositories](#remote-and-virtual-repositories) +- [Additional Docker and Maven Options](#additional-docker-and-maven-options) +- [Variables](#variables) +- [Outputs](#outputs) + + +## Standard Repository + +```hcl +module "docker_artifact_registry" { + source = "./fabric/modules/artifact-registry" + project_id = "myproject" + location = "europe-west1" + name = "myregistry" + iam = { + "roles/artifactregistry.admin" = ["group:cicd@example.com"] + } +} +# tftest modules=1 resources=2 +``` + +## Remote and Virtual Repositories + +```hcl + +module "registry-local" { + source = "./fabric/modules/artifact-registry" + project_id = var.project_id + location = "europe-west1" + name = "local" + format = { python = {} } +} + +module "registry-remote" { + source = "./fabric/modules/artifact-registry" + project_id = var.project_id + location = "europe-west1" + name = "remote" + format = { python = {} } + mode = { remote = true } +} + +module "registry-virtual" { + source = "./fabric/modules/artifact-registry" + project_id = var.project_id + location = "europe-west1" + name = "virtual" + format = { python = {} } + mode = { + virtual = { + remote = { + repository = module.registry-remote.id + priority = 1 + } + local = { + repository = module.registry-local.id + priority = 10 + } + } + } +} + +# tftest modules=3 resources=3 inventory=remote-virtual.yaml +``` + +## Additional Docker and Maven Options + +```hcl + +module "registry-docker" { + source = "./fabric/modules/artifact-registry" + project_id = var.project_id + location = "europe-west1" + name = "docker" + format = { + docker = { + immutable_tags = true + } + } +} + +module "registry-maven" { + source = "./fabric/modules/artifact-registry" + project_id = var.project_id + location = "europe-west1" + name = "maven" + format = { + maven = { + allow_snapshot_overwrites = true + version_policy = "RELEASE" + } + } +} + +# tftest modules=2 resources=2 +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [location](variables.tf#L68) | Registry location. Use `gcloud beta artifacts locations list' to get valid values. | string | ✓ | | +| [name](variables.tf#L93) | Registry name. | string | ✓ | | +| [project_id](variables.tf#L98) | Registry project id. | string | ✓ | | +| [description](variables.tf#L17) | An optional description for the repository. | string | | "Terraform-managed registry" | +| [encryption_key](variables.tf#L23) | The KMS key name to use for encryption at rest. | string | | null | +| [format](variables.tf#L29) | Repository format. | object({…}) | | { docker = {} } | +| [iam](variables.tf#L56) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [labels](variables.tf#L62) | Labels to be attached to the registry. | map(string) | | {} | +| [mode](variables.tf#L73) | Repository mode. | object({…}) | | { standard = true } | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [id](outputs.tf#L17) | Fully qualified repository id. | | +| [image_path](outputs.tf#L22) | Repository path for images. | | +| [name](outputs.tf#L32) | Repository name. | | + diff --git a/assets/modules-fabric/v26/artifact-registry/main.tf b/assets/modules-fabric/v26/artifact-registry/main.tf new file mode 100644 index 0000000..5b23a19 --- /dev/null +++ b/assets/modules-fabric/v26/artifact-registry/main.tf @@ -0,0 +1,115 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + format_string = one([for k, v in var.format : k if v != null]) + mode_string = one([for k, v in var.mode : k if v != null && v != false]) +} + +resource "google_artifact_registry_repository" "registry" { + project = var.project_id + location = var.location + description = var.description + format = upper(local.format_string) + labels = var.labels + repository_id = var.name + mode = "${upper(local.mode_string)}_REPOSITORY" + kms_key_name = var.encryption_key + + dynamic "docker_config" { + # TODO: open a bug on the provider for this permadiff + for_each = ( + local.format_string == "docker" && try(var.format.docker.immutable_tags, null) == true + ? [""] + : [] + ) + content { + immutable_tags = var.format.docker.immutable_tags + } + } + + dynamic "maven_config" { + for_each = local.format_string == "maven" ? [""] : [] + content { + allow_snapshot_overwrites = var.format.maven.allow_snapshot_overwrites + version_policy = var.format.maven.version_policy + } + } + + dynamic "remote_repository_config" { + for_each = local.mode_string == "remote" ? [""] : [] + content { + dynamic "docker_repository" { + for_each = local.format_string == "docker" ? [""] : [] + content { + public_repository = "DOCKER_HUB" + } + } + dynamic "maven_repository" { + for_each = local.format_string == "maven" ? [""] : [] + content { + public_repository = "MAVEN_CENTRAL" + } + } + dynamic "npm_repository" { + for_each = local.format_string == "npm" ? [""] : [] + content { + public_repository = "NPMJS" + } + } + dynamic "python_repository" { + for_each = local.format_string == "python" ? [""] : [] + content { + public_repository = "PYPI" + } + } + } + } + + dynamic "virtual_repository_config" { + for_each = local.mode_string == "virtual" ? [""] : [] + content { + dynamic "upstream_policies" { + for_each = var.mode.virtual + content { + id = upstream_policies.key + repository = upstream_policies.value.repository + priority = upstream_policies.value.priority + } + } + } + } + + lifecycle { + precondition { + condition = local.mode_string != "remote" || contains( + ["docker", "maven", "npm", "python"], local.format_string + ) + error_message = "Invalid format for remote repository." + } + } + +} + +resource "google_artifact_registry_repository_iam_binding" "bindings" { + provider = google-beta + for_each = var.iam + project = var.project_id + location = google_artifact_registry_repository.registry.location + repository = google_artifact_registry_repository.registry.name + role = each.key + members = each.value +} diff --git a/assets/modules-fabric/v26/artifact-registry/outputs.tf b/assets/modules-fabric/v26/artifact-registry/outputs.tf new file mode 100644 index 0000000..bffd0fb --- /dev/null +++ b/assets/modules-fabric/v26/artifact-registry/outputs.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "id" { + description = "Fully qualified repository id." + value = google_artifact_registry_repository.registry.id +} + +output "image_path" { + description = "Repository path for images." + value = join("/", [ + "${var.location}-docker.pkg.dev", + var.project_id, + var.name + ]) + depends_on = [google_artifact_registry_repository.registry] +} + +output "name" { + description = "Repository name." + value = google_artifact_registry_repository.registry.name +} diff --git a/assets/modules-fabric/v26/artifact-registry/variables.tf b/assets/modules-fabric/v26/artifact-registry/variables.tf new file mode 100644 index 0000000..b49c9a5 --- /dev/null +++ b/assets/modules-fabric/v26/artifact-registry/variables.tf @@ -0,0 +1,101 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "description" { + description = "An optional description for the repository." + type = string + default = "Terraform-managed registry" +} + +variable "encryption_key" { + description = "The KMS key name to use for encryption at rest." + type = string + default = null +} + +variable "format" { + description = "Repository format." + type = object({ + apt = optional(object({})) + docker = optional(object({ + immutable_tags = optional(bool) + })) + kfp = optional(object({})) + go = optional(object({})) + maven = optional(object({ + allow_snapshot_overwrites = optional(bool) + version_policy = optional(string) + })) + npm = optional(object({})) + python = optional(object({})) + yum = optional(object({})) + }) + nullable = false + default = { docker = {} } + validation { + condition = ( + length([for k, v in var.format : k if v != null]) == 1 + ) + error_message = "Multiple or zero formats are not supported." + } +} + +variable "iam" { + description = "IAM bindings in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} +} + +variable "labels" { + description = "Labels to be attached to the registry." + type = map(string) + default = {} +} + +variable "location" { + description = "Registry location. Use `gcloud beta artifacts locations list' to get valid values." + type = string +} + +variable "mode" { + description = "Repository mode." + type = object({ + standard = optional(bool) + remote = optional(bool) + virtual = optional(map(object({ + repository = string + priority = number + }))) + }) + nullable = false + default = { standard = true } + validation { + condition = ( + length([for k, v in var.mode : k if v != null && v != false]) == 1 + ) + error_message = "Multiple or zero modes are not supported." + } +} + +variable "name" { + description = "Registry name." + type = string +} + +variable "project_id" { + description = "Registry project id." + type = string +} diff --git a/assets/modules-fabric/v26/artifact-registry/versions.tf b/assets/modules-fabric/v26/artifact-registry/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/artifact-registry/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/bigquery-dataset/README.md b/assets/modules-fabric/v26/bigquery-dataset/README.md new file mode 100644 index 0000000..48bc50a --- /dev/null +++ b/assets/modules-fabric/v26/bigquery-dataset/README.md @@ -0,0 +1,312 @@ +# Google Cloud Bigquery Module + +This module allows managing a single BigQuery dataset, including access configuration, tables and views. + +## TODO + +- [ ] check for dynamic values in tables and views +- [ ] add support for external tables + +## Examples + +### Simple dataset with access configuration + +Access configuration defaults to using the separate `google_bigquery_dataset_access` resource, so as to leave the default dataset access rules untouched. + +You can choose to manage the `google_bigquery_dataset` access rules instead via the `dataset_access` variable, but be sure to always have at least one `OWNER` access and to avoid duplicating accesses, or `terraform apply` will fail. + +The access variables are split into `access` and `access_identities` variables, so that dynamic values can be passed in for identities (eg a service account email generated by a different module or resource). + +```hcl +module "bigquery-dataset" { + source = "./fabric/modules/bigquery-dataset" + project_id = "my-project" + id = "my-dataset" + access = { + reader-group = { role = "READER", type = "group" } + owner = { role = "OWNER", type = "user" } + project_owners = { role = "OWNER", type = "special_group" } + view_1 = { role = "READER", type = "view" } + } + access_identities = { + reader-group = "playground-test@ludomagno.net" + owner = "ludo@ludomagno.net" + project_owners = "projectOwners" + view_1 = "my-project|my-dataset|my-table" + } +} +# tftest modules=1 resources=5 inventory=simple.yaml +``` + +### IAM roles + +Access configuration can also be specified via IAM instead of basic roles via the `iam` variable. When using IAM, basic roles cannot be used via the `access` family variables. + +```hcl +module "bigquery-dataset" { + source = "./fabric/modules/bigquery-dataset" + project_id = "my-project" + id = "my-dataset" + iam = { + "roles/bigquery.dataOwner" = ["user:user1@example.org"] + } +} +# tftest modules=1 resources=2 inventory=iam.yaml +``` + +### Authorized Views, Datasets, and Routines + +You can specify authorized [views](https://cloud.google.com/bigquery/docs/authorized-views), [datasets](https://cloud.google.com/bigquery/docs/authorized-datasets?hl=en), and [routines](https://cloud.google.com/bigquery/docs/authorized-routines) via the `authorized_views`, `authorized_datasets` and `authorized_routines` variables, respectively. + +```hcl +// Create private BigQuery dataset that will not be publicly accessible, except via the authorized BigQuery resources +module "bigquery-dataset-private" { + source = "./fabric/modules/bigquery-dataset" + project_id = "private_project" + id = "private_dataset" + authorized_views = [ + { + project_id = "auth_view_project" + dataset_id = "auth_view_dataset" + table_id = "auth_view" + } + ] + authorized_datasets = [ + { + project_id = "auth_dataset_project" + dataset_id = "auth_dataset" + } + ] + authorized_routines = [ + { + project_id = "auth_routine_project" + dataset_id = "auth_routine_dataset" + routine_id = "auth_routine" + } + ] +} + +// Create authorized view in a public dataset +module "bigquery-authorized-views-dataset-public" { + source = "./fabric/modules/bigquery-dataset" + project_id = "auth_view_project" + id = "auth_view_dataset" + views = { + auth_view = { + friendly_name = "Public" + labels = {} + query = "SELECT * FROM `private_project.private_dataset.private_table`" + use_legacy_sql = false + deletion_protection = true + } + } +} + +// Create public authorized dataset +module "bigquery-authorized-dataset-public" { + source = "./fabric/modules/bigquery-dataset" + project_id = "auth_dataset_project" + id = "auth_dataset" +} + +// Create public authorized routine +module "bigquery-authorized-authorized-routine-dataset-public" { + source = "./fabric/modules/bigquery-dataset" + project_id = "auth_routine_project" + id = "auth_routine_dataset" +} + +resource "google_bigquery_routine" "public-routine" { + dataset_id = module.bigquery-authorized-authorized-routine-dataset-public.dataset_id + routine_id = "auth_routine" + routine_type = "TABLE_VALUED_FUNCTION" + language = "SQL" + definition_body = <<-EOS + SELECT 1 + value AS value + EOS + arguments { + name = "value" + argument_kind = "FIXED_TYPE" + data_type = jsonencode({ "typeKind" = "INT64" }) + } + return_table_type = jsonencode({ "columns" = [ + { "name" = "value", "type" = { "typeKind" = "INT64" } }, + ] }) +} +# tftest modules=4 resources=9 inventory=authorized_resources.yaml +``` + +Authorized views can be specified both using the standard `access` options and the `authorized_views` blocks. The example configuration below uses both blocks, and will create a dataset with three authorized views `view_id_1`, `view_id_2`, and `view_id_3`. + +```hcl +module "bigquery-dataset" { + source = "./fabric/modules/bigquery-dataset" + project_id = "my-project" + id = "my-dataset" + authorized_views = [ + { + project_id = "view_project" + dataset_id = "view_dataset" + table_id = "view_id_1" + }, + { + project_id = "view_project" + dataset_id = "view_dataset" + table_id = "view_id_2" + } + ] + access = { + view_2 = { role = "READER", type = "view" } + view_3 = { role = "READER", type = "view" } + } + access_identities = { + view_2 = "view_project|view_dataset|view_id_2" + view_3 = "view_project|view_dataset|view_id_3" + } +} +# tftest modules=1 resources=4 inventory=authorized_resources_views.yaml +``` + +### Dataset options + +Dataset options are set via the `options` variable. all options must be specified, but a `null` value can be set to options that need to use defaults. + +```hcl +module "bigquery-dataset" { + source = "./fabric/modules/bigquery-dataset" + project_id = "my-project" + id = "my-dataset" + options = { + default_table_expiration_ms = 3600000 + default_partition_expiration_ms = null + delete_contents_on_destroy = false + max_time_travel_hours = 168 + } +} +# tftest modules=1 resources=1 inventory=options.yaml +``` + +### Tables and views + +Tables are created via the `tables` variable, or the `view` variable for views. Support for external tables will be added in a future release. + +```hcl +locals { + countries_schema = jsonencode([ + { name = "country", type = "STRING" }, + { name = "population", type = "INT64" }, + ]) +} + +module "bigquery-dataset" { + source = "./fabric/modules/bigquery-dataset" + project_id = "my-project" + id = "my_dataset" + tables = { + countries = { + friendly_name = "Countries" + schema = local.countries_schema + deletion_protection = true + } + } +} +# tftest modules=1 resources=2 inventory=tables.yaml +``` + +If partitioning is needed, populate the `partitioning` variable using either the `time` or `range` attribute. + +```hcl +locals { + countries_schema = jsonencode([ + { name = "country", type = "STRING" }, + { name = "population", type = "INT64" }, + ]) +} + +module "bigquery-dataset" { + source = "./fabric/modules/bigquery-dataset" + project_id = "my-project" + id = "my-dataset" + tables = { + table_a = { + deletion_protection = true + friendly_name = "Table a" + schema = local.countries_schema + partitioning = { + time = { type = "DAY", expiration_ms = null } + } + } + } +} +# tftest modules=1 resources=2 inventory=partitioning.yaml +``` + +To create views use the `view` variable. If you're querying a table created by the same module `terraform apply` will initially fail and eventually succeed once the underlying table has been created. You can probably also use the module's output in the view's query to create a dependency on the table. + +```hcl +locals { + countries_schema = jsonencode([ + { name = "country", type = "STRING" }, + { name = "population", type = "INT64" }, + ]) +} + +module "bigquery-dataset" { + source = "./fabric/modules/bigquery-dataset" + project_id = "my-project" + id = "my_dataset" + tables = { + countries = { + friendly_name = "Countries" + schema = local.countries_schema + deletion_protection = true + } + } + views = { + population = { + friendly_name = "Population" + query = "SELECT SUM(population) FROM my_dataset.countries" + use_legacy_sql = false + deletion_protection = true + } + } +} + +# tftest modules=1 resources=3 inventory=views.yaml +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [id](variables.tf#L98) | Dataset id. | string | ✓ | | +| [project_id](variables.tf#L128) | Id of the project where datasets will be created. | string | ✓ | | +| [access](variables.tf#L17) | Map of access rules with role and identity type. Keys are arbitrary and must match those in the `access_identities` variable, types are `domain`, `group`, `special_group`, `user`, `view`. | map(object({…})) | | {} | +| [access_identities](variables.tf#L33) | Map of access identities used for basic access roles. View identities have the format 'project_id\|dataset_id\|table_id'. | map(string) | | {} | +| [authorized_datasets](variables.tf#L39) | An array of datasets to be authorized on the dataset. | list(object({…})) | | [] | +| [authorized_routines](variables.tf#L48) | An array of authorized routine to be authorized on the dataset. | list(object({…})) | | [] | +| [authorized_views](variables.tf#L58) | An array of views to be authorized on the dataset. | list(object({…})) | | [] | +| [dataset_access](variables.tf#L68) | Set access in the dataset resource instead of using separate resources. | bool | | false | +| [description](variables.tf#L74) | Optional description. | string | | "Terraform managed." | +| [encryption_key](variables.tf#L80) | Self link of the KMS key that will be used to protect destination table. | string | | null | +| [friendly_name](variables.tf#L86) | Dataset friendly name. | string | | null | +| [iam](variables.tf#L92) | IAM bindings in {ROLE => [MEMBERS]} format. Mutually exclusive with the access_* variables used for basic roles. | map(list(string)) | | {} | +| [labels](variables.tf#L103) | Dataset labels. | map(string) | | {} | +| [location](variables.tf#L109) | Dataset location. | string | | "EU" | +| [options](variables.tf#L115) | Dataset options. | object({…}) | | {} | +| [tables](variables.tf#L133) | Table definitions. Options and partitioning default to null. Partitioning can only use `range` or `time`, set the unused one to null. | map(object({…})) | | {} | +| [views](variables.tf#L162) | View definitions. | map(object({…})) | | {} | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [dataset](outputs.tf#L17) | Dataset resource. | | +| [dataset_id](outputs.tf#L22) | Dataset id. | | +| [id](outputs.tf#L36) | Fully qualified dataset id. | | +| [self_link](outputs.tf#L50) | Dataset self link. | | +| [table_ids](outputs.tf#L64) | Map of fully qualified table ids keyed by table ids. | | +| [tables](outputs.tf#L69) | Table resources. | | +| [view_ids](outputs.tf#L74) | Map of fully qualified view ids keyed by view ids. | | +| [views](outputs.tf#L79) | View resources. | | + diff --git a/assets/modules-fabric/v26/bigquery-dataset/main.tf b/assets/modules-fabric/v26/bigquery-dataset/main.tf new file mode 100644 index 0000000..fafd75f --- /dev/null +++ b/assets/modules-fabric/v26/bigquery-dataset/main.tf @@ -0,0 +1,268 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + access_domain = { for k, v in var.access : k => v if v.type == "domain" } + access_group = { for k, v in var.access : k => v if v.type == "group" } + access_special = { for k, v in var.access : k => v if v.type == "special_group" } + access_user = { for k, v in var.access : k => v if v.type == "user" } + access_view = { for k, v in var.access : k => v if v.type == "view" } + + identities_view = { + for k, v in local.access_view : k => try( + zipmap( + ["project_id", "dataset_id", "table_id"], + split("|", var.access_identities[k]) + ), + { project_id = null, dataset_id = null, table_id = null } + ) + } + + authorized_views = merge( + { for access_key, view in local.identities_view : "${view["project_id"]}_${view["dataset_id"]}_${view["table_id"]}" => view }, + { for view in var.authorized_views : "${view["project_id"]}_${view["dataset_id"]}_${view["table_id"]}" => view }) + authorized_datasets = { for dataset in var.authorized_datasets : "${dataset["project_id"]}_${dataset["dataset_id"]}" => dataset } + authorized_routines = { for routine in var.authorized_routines : "${routine["project_id"]}_${routine["dataset_id"]}_${routine["routine_id"]}" => routine } + +} + +resource "google_bigquery_dataset" "default" { + project = var.project_id + dataset_id = var.id + friendly_name = var.friendly_name + description = var.description + labels = var.labels + location = var.location + + delete_contents_on_destroy = var.options.delete_contents_on_destroy + default_collation = var.options.default_collation + default_table_expiration_ms = var.options.default_table_expiration_ms + default_partition_expiration_ms = var.options.default_partition_expiration_ms + is_case_insensitive = var.options.is_case_insensitive + max_time_travel_hours = var.options.max_time_travel_hours + dynamic "access" { + for_each = var.dataset_access ? local.access_domain : {} + content { + role = access.value.role + domain = try(var.access_identities[access.key]) + } + } + + dynamic "access" { + for_each = var.dataset_access ? local.access_group : {} + content { + role = access.value.role + group_by_email = try(var.access_identities[access.key]) + } + } + + dynamic "access" { + for_each = var.dataset_access ? local.access_special : {} + content { + role = access.value.role + special_group = try(var.access_identities[access.key]) + } + } + + dynamic "access" { + for_each = var.dataset_access ? local.access_user : {} + content { + role = access.value.role + user_by_email = try(var.access_identities[access.key]) + } + } + + dynamic "access" { + for_each = var.dataset_access ? local.authorized_views : {} + content { + view { + project_id = each.value.project_id + dataset_id = each.value.dataset_id + table_id = each.value.table_id + } + } + } + + dynamic "access" { + for_each = var.dataset_access ? local.authorized_datasets : {} + content { + dataset { + dataset { + project_id = each.value.project_id + dataset_id = each.value.dataset_id + } + target_types = ["VIEWS"] + } + } + } + + dynamic "access" { + for_each = var.dataset_access ? local.authorized_routines : {} + content { + routine { + project_id = each.value.project_id + dataset_id = each.value.dataset_id + routine_id = each.value.routine_id + } + } + } + + dynamic "default_encryption_configuration" { + for_each = var.encryption_key == null ? [] : [""] + content { + kms_key_name = var.encryption_key + } + } +} + +resource "google_bigquery_dataset_access" "domain" { + for_each = var.dataset_access ? {} : local.access_domain + provider = google-beta + project = var.project_id + dataset_id = google_bigquery_dataset.default.dataset_id + role = each.value.role + domain = try(var.access_identities[each.key]) +} + +resource "google_bigquery_dataset_access" "group_by_email" { + for_each = var.dataset_access ? {} : local.access_group + provider = google-beta + project = var.project_id + dataset_id = google_bigquery_dataset.default.dataset_id + role = each.value.role + group_by_email = try(var.access_identities[each.key]) +} + +resource "google_bigquery_dataset_access" "special_group" { + for_each = var.dataset_access ? {} : local.access_special + provider = google-beta + project = var.project_id + dataset_id = google_bigquery_dataset.default.dataset_id + role = each.value.role + special_group = try(var.access_identities[each.key]) +} + +resource "google_bigquery_dataset_access" "user_by_email" { + for_each = var.dataset_access ? {} : local.access_user + provider = google-beta + project = var.project_id + dataset_id = google_bigquery_dataset.default.dataset_id + role = each.value.role + user_by_email = try(var.access_identities[each.key]) +} + +resource "google_bigquery_dataset_access" "authorized_views" { + for_each = var.dataset_access ? {} : local.authorized_views + project = var.project_id + dataset_id = google_bigquery_dataset.default.dataset_id + view { + project_id = each.value.project_id + dataset_id = each.value.dataset_id + table_id = each.value.table_id + } +} + +resource "google_bigquery_dataset_access" "authorized_datasets" { + for_each = var.dataset_access ? {} : local.authorized_datasets + project = var.project_id + dataset_id = google_bigquery_dataset.default.dataset_id + dataset { + dataset { + project_id = each.value.project_id + dataset_id = each.value.dataset_id + } + target_types = ["VIEWS"] + } +} + +resource "google_bigquery_dataset_access" "authorized_routines" { + for_each = var.dataset_access ? {} : local.authorized_routines + project = var.project_id + dataset_id = google_bigquery_dataset.default.dataset_id + routine { + project_id = each.value.project_id + dataset_id = each.value.dataset_id + routine_id = each.value.routine_id + } +} + +resource "google_bigquery_dataset_iam_binding" "bindings" { + for_each = var.iam + project = var.project_id + dataset_id = google_bigquery_dataset.default.dataset_id + role = each.key + members = each.value +} + +resource "google_bigquery_table" "default" { + provider = google-beta + for_each = var.tables + project = var.project_id + dataset_id = google_bigquery_dataset.default.dataset_id + table_id = each.key + friendly_name = each.value.friendly_name + description = each.value.description + clustering = each.value.options.clustering + expiration_time = each.value.options.expiration_time + labels = each.value.labels + schema = each.value.schema + deletion_protection = each.value.deletion_protection + + dynamic "encryption_configuration" { + for_each = each.value.options.encryption_key != null ? [""] : [] + content { + kms_key_name = each.value.options.encryption_key + } + } + + dynamic "range_partitioning" { + for_each = try(each.value.partitioning.range, null) != null ? [""] : [] + content { + field = each.value.partitioning.field + range { + start = each.value.partitioning.range.start + end = each.value.partitioning.range.end + interval = each.value.partitioning.range.interval + } + } + } + + dynamic "time_partitioning" { + for_each = try(each.value.partitioning.time, null) != null ? [""] : [] + content { + expiration_ms = each.value.partitioning.time.expiration_ms + field = each.value.partitioning.field + type = each.value.partitioning.time.type + } + } +} + +resource "google_bigquery_table" "views" { + depends_on = [google_bigquery_table.default] + for_each = var.views + project = var.project_id + dataset_id = google_bigquery_dataset.default.dataset_id + table_id = each.key + friendly_name = each.value.friendly_name + description = each.value.description + labels = each.value.labels + deletion_protection = each.value.deletion_protection + + view { + query = each.value.query + use_legacy_sql = each.value.use_legacy_sql + } +} diff --git a/assets/modules-fabric/v26/bigquery-dataset/outputs.tf b/assets/modules-fabric/v26/bigquery-dataset/outputs.tf new file mode 100644 index 0000000..5c2ee46 --- /dev/null +++ b/assets/modules-fabric/v26/bigquery-dataset/outputs.tf @@ -0,0 +1,82 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "dataset" { + description = "Dataset resource." + value = google_bigquery_dataset.default +} + +output "dataset_id" { + description = "Dataset id." + value = google_bigquery_dataset.default.dataset_id + depends_on = [ + google_bigquery_dataset_access.authorized_datasets, + google_bigquery_dataset_access.authorized_routines, + google_bigquery_dataset_access.authorized_views, + google_bigquery_dataset_access.domain, + google_bigquery_dataset_access.group_by_email, + google_bigquery_dataset_access.special_group, + google_bigquery_dataset_access.user_by_email + ] +} + +output "id" { + description = "Fully qualified dataset id." + value = google_bigquery_dataset.default.id + depends_on = [ + google_bigquery_dataset_access.authorized_datasets, + google_bigquery_dataset_access.authorized_routines, + google_bigquery_dataset_access.authorized_views, + google_bigquery_dataset_access.domain, + google_bigquery_dataset_access.group_by_email, + google_bigquery_dataset_access.special_group, + google_bigquery_dataset_access.user_by_email + ] +} + +output "self_link" { + description = "Dataset self link." + value = google_bigquery_dataset.default.self_link + depends_on = [ + google_bigquery_dataset_access.authorized_datasets, + google_bigquery_dataset_access.authorized_routines, + google_bigquery_dataset_access.authorized_views, + google_bigquery_dataset_access.domain, + google_bigquery_dataset_access.group_by_email, + google_bigquery_dataset_access.special_group, + google_bigquery_dataset_access.user_by_email + ] +} + +output "table_ids" { + description = "Map of fully qualified table ids keyed by table ids." + value = { for k, v in google_bigquery_table.default : v.table_id => v.id } +} + +output "tables" { + description = "Table resources." + value = google_bigquery_table.default +} + +output "view_ids" { + description = "Map of fully qualified view ids keyed by view ids." + value = { for k, v in google_bigquery_table.views : v.table_id => v.id } +} + +output "views" { + description = "View resources." + value = google_bigquery_table.views +} diff --git a/assets/modules-fabric/v26/bigquery-dataset/variables.tf b/assets/modules-fabric/v26/bigquery-dataset/variables.tf new file mode 100644 index 0000000..cb13eff --- /dev/null +++ b/assets/modules-fabric/v26/bigquery-dataset/variables.tf @@ -0,0 +1,173 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "access" { + description = "Map of access rules with role and identity type. Keys are arbitrary and must match those in the `access_identities` variable, types are `domain`, `group`, `special_group`, `user`, `view`." + type = map(object({ + role = string + type = string + })) + default = {} + validation { + condition = can([ + for k, v in var.access : + index(["domain", "group", "special_group", "user", "view"], v.type) + ]) + error_message = "Access type must be one of 'domain', 'group', 'special_group', 'user', 'view'." + } +} + +variable "access_identities" { + description = "Map of access identities used for basic access roles. View identities have the format 'project_id|dataset_id|table_id'." + type = map(string) + default = {} +} + +variable "authorized_datasets" { + description = "An array of datasets to be authorized on the dataset." + type = list(object({ + dataset_id = string, + project_id = string, + })) + default = [] +} + +variable "authorized_routines" { + description = "An array of authorized routine to be authorized on the dataset." + type = list(object({ + project_id = string, + dataset_id = string, + routine_id = string + })) + default = [] +} + +variable "authorized_views" { + description = "An array of views to be authorized on the dataset." + type = list(object({ + dataset_id = string, + project_id = string, + table_id = string # this is the view id, but we keep table_id to stay consistent as the resource + })) + default = [] +} + +variable "dataset_access" { + description = "Set access in the dataset resource instead of using separate resources." + type = bool + default = false +} + +variable "description" { + description = "Optional description." + type = string + default = "Terraform managed." +} + +variable "encryption_key" { + description = "Self link of the KMS key that will be used to protect destination table." + type = string + default = null +} + +variable "friendly_name" { + description = "Dataset friendly name." + type = string + default = null +} + +variable "iam" { + description = "IAM bindings in {ROLE => [MEMBERS]} format. Mutually exclusive with the access_* variables used for basic roles." + type = map(list(string)) + default = {} +} + +variable "id" { + description = "Dataset id." + type = string +} + +variable "labels" { + description = "Dataset labels." + type = map(string) + default = {} +} + +variable "location" { + description = "Dataset location." + type = string + default = "EU" +} + +variable "options" { + description = "Dataset options." + type = object({ + default_collation = optional(string) + default_table_expiration_ms = optional(number) + default_partition_expiration_ms = optional(number) + delete_contents_on_destroy = optional(bool, false) + is_case_insensitive = optional(bool) + max_time_travel_hours = optional(number, 168) + }) + default = {} +} + +variable "project_id" { + description = "Id of the project where datasets will be created." + type = string +} + +variable "tables" { + description = "Table definitions. Options and partitioning default to null. Partitioning can only use `range` or `time`, set the unused one to null." + type = map(object({ + deletion_protection = optional(bool) + description = optional(string, "Terraform managed.") + friendly_name = optional(string) + labels = optional(map(string), {}) + schema = optional(string) + options = optional(object({ + clustering = optional(list(string)) + encryption_key = optional(string) + expiration_time = optional(number) + }), {}) + partitioning = optional(object({ + field = optional(string) + range = optional(object({ + end = number + interval = number + start = number + })) + time = optional(object({ + expiration_ms = number + type = string + })) + })) + })) + default = {} +} + +variable "views" { + description = "View definitions." + type = map(object({ + query = string + deletion_protection = optional(bool) + description = optional(string, "Terraform managed.") + friendly_name = optional(string) + labels = optional(map(string), {}) + use_legacy_sql = optional(bool) + })) + default = {} +} diff --git a/assets/modules-fabric/v26/bigquery-dataset/versions.tf b/assets/modules-fabric/v26/bigquery-dataset/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/bigquery-dataset/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/bigtable-instance/README.md b/assets/modules-fabric/v26/bigtable-instance/README.md new file mode 100644 index 0000000..afe1ec4 --- /dev/null +++ b/assets/modules-fabric/v26/bigtable-instance/README.md @@ -0,0 +1,253 @@ +# Google Cloud BigTable Module + +This module allows managing a single BigTable instance, including access configuration and tables. + +## TODO + +- [ ] support bigtable_app_profile +- [ ] support IAM for tables + +## Examples + +### Instance with access configuration + +```hcl + +module "bigtable-instance" { + source = "./fabric/modules/bigtable-instance" + project_id = "my-project" + name = "instance" + clusters = { + my-cluster = { + zone = "europe-west1-b" + } + } + tables = { + test1 = {}, + test2 = { + split_keys = ["a", "b", "c"] + } + } + iam = { + "roles/bigtable.user" = ["user:viewer@testdomain.com"] + } +} +# tftest modules=1 resources=4 inventory=simple.yaml +``` + +### Instance with tables and column families + +```hcl + +module "bigtable-instance" { + source = "./fabric/modules/bigtable-instance" + project_id = "my-project" + name = "instance" + clusters = { + my-cluster = { + zone = "europe-west1-b" + } + } + tables = { + test1 = {}, + test2 = { + split_keys = ["a", "b", "c"] + column_families = { + cf1 = {} + cf2 = {} + cf3 = {} + } + } + test3 = { + column_families = { + cf1 = {} + } + } + } +} +# tftest modules=1 resources=4 inventory=columns.yaml +``` + +### Instance with replication enabled + +```hcl + +module "bigtable-instance" { + source = "./fabric/modules/bigtable-instance" + project_id = "my-project" + name = "instance" + clusters = { + first-cluster = { + zone = "europe-west1-b" + } + second-cluster = { + zone = "europe-southwest1-a" + } + third-cluster = { + zone = "us-central1-b" + } + } +} +# tftest modules=1 resources=1 inventory=replication.yaml +``` + +### Instance with garbage collection policy + +```hcl + +module "bigtable-instance" { + source = "./fabric/modules/bigtable-instance" + project_id = "my-project" + name = "instance" + clusters = { + my-cluster = { + zone = "europe-west1-b" + } + } + tables = { + test1 = { + column_families = { + cf1 = { + gc_policy = { + deletion_policy = "ABANDON" + max_age = "18h" + } + } + cf2 = {} + } + } + } +} +# tftest modules=1 resources=3 inventory=gc.yaml +``` + +### Instance with default garbage collection policy + +The default garbage collection policy is applied to any column family that does +not specify a `gc_policy`. If a column family specifies a `gc_policy`, the +default garbage collection policy is ignored for that column family. + +```hcl + +module "bigtable-instance" { + source = "./fabric/modules/bigtable-instance" + project_id = "my-project" + name = "instance" + clusters = { + my-cluster = { + zone = "europe-west1-b" + } + } + default_gc_policy = { + deletion_policy = "ABANDON" + max_age = "18h" + max_version = 7 + } + tables = { + test1 = { + column_families = { + cf1 = {} + cf2 = {} + } + } + } +} +# tftest modules=1 resources=4 +``` + +### Instance with static number of nodes + +If you are not using autoscaling settings, you must set a specific number of nodes with the variable `num_nodes`. + +```hcl + +module "bigtable-instance" { + source = "./fabric/modules/bigtable-instance" + project_id = "my-project" + name = "instance" + clusters = { + my-cluster = { + zone = "europe-west1-b" + num_nodes = 5 + } + } +} +# tftest modules=1 resources=1 inventory=static.yaml +``` + +### Instance with autoscaling (based on CPU only) + +If you use autoscaling, you should not set the variable `num_nodes`. + +```hcl + +module "bigtable-instance" { + source = "./fabric/modules/bigtable-instance" + project_id = "my-project" + name = "instance" + clusters = { + my-cluster = { + zone = "europe-southwest1-b" + autoscaling = { + min_nodes = 3 + max_nodes = 7 + cpu_target = 70 + } + } + } + + +} +# tftest modules=1 resources=1 inventory=autoscaling1.yaml +``` + +### Instance with autoscaling (based on CPU and/or storage) + +```hcl + +module "bigtable-instance" { + source = "./fabric/modules/bigtable-instance" + project_id = "my-project" + name = "instance" + clusters = { + my-cluster = { + zone = "europe-southwest1-a" + storage_type = "SSD" + autoscaling = { + min_nodes = 3 + max_nodes = 7 + cpu_target = 70 + storage_target = 4096 + } + } + } +} +# tftest modules=1 resources=1 inventory=autoscaling2.yaml +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [clusters](variables.tf#L17) | Clusters to be created in the BigTable instance. Set more than one cluster to enable replication. If you set autoscaling, num_nodes will be ignored. | map(object({…})) | ✓ | | +| [name](variables.tf#L78) | The name of the Cloud Bigtable instance. | string | ✓ | | +| [project_id](variables.tf#L83) | Id of the project where datasets will be created. | string | ✓ | | +| [default_autoscaling](variables.tf#L33) | Default settings for autoscaling of clusters. This will be the default autoscaling for any cluster not specifying any autoscaling details. | object({…}) | | null | +| [default_gc_policy](variables.tf#L44) | Default garbage collection policy, to be applied to all column families and all tables. Can be override in the tables variable for specific column families. | object({…}) | | null | +| [deletion_protection](variables.tf#L56) | Whether or not to allow Terraform to destroy the instance. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the instance will fail. | | | true | +| [display_name](variables.tf#L61) | The human-readable display name of the Bigtable instance. | | | null | +| [iam](variables.tf#L66) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [instance_type](variables.tf#L72) | (deprecated) The instance type to create. One of 'DEVELOPMENT' or 'PRODUCTION'. | string | | null | +| [tables](variables.tf#L88) | Tables to be created in the BigTable instance. | map(object({…})) | | {} | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [id](outputs.tf#L17) | Fully qualified instance id. | | +| [instance](outputs.tf#L26) | BigTable instance. | | +| [table_ids](outputs.tf#L35) | Map of fully qualified table ids keyed by table name. | | +| [tables](outputs.tf#L40) | Table resources. | | + + diff --git a/assets/modules-fabric/v26/bigtable-instance/main.tf b/assets/modules-fabric/v26/bigtable-instance/main.tf new file mode 100644 index 0000000..3a00eab --- /dev/null +++ b/assets/modules-fabric/v26/bigtable-instance/main.tf @@ -0,0 +1,116 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + gc_pairs = flatten([ + for table, table_obj in var.tables : [ + for cf, cf_obj in table_obj.column_families : { + table = table + column_family = cf + gc_policy = cf_obj.gc_policy == null ? var.default_gc_policy : cf_obj.gc_policy + } + ] + ]) + + clusters_autoscaling = { + for cluster_id, cluster in var.clusters : cluster_id => { + zone = cluster.zone + storage_type = cluster.storage_type + num_nodes = cluster.autoscaling == null && var.default_autoscaling == null ? cluster.num_nodes : null + autoscaling = cluster.autoscaling == null ? var.default_autoscaling : cluster.autoscaling + } + } +} + +resource "google_bigtable_instance" "default" { + project = var.project_id + name = var.name + + instance_type = var.instance_type + display_name = var.display_name == null ? var.display_name : var.name + deletion_protection = var.deletion_protection + + dynamic "cluster" { + for_each = local.clusters_autoscaling + content { + cluster_id = cluster.key + zone = cluster.value.zone + storage_type = cluster.value.storage_type + num_nodes = cluster.value.num_nodes + + dynamic "autoscaling_config" { + for_each = cluster.value.autoscaling == null ? [] : [""] + content { + min_nodes = cluster.value.autoscaling.min_nodes + max_nodes = cluster.value.autoscaling.max_nodes + cpu_target = cluster.value.autoscaling.cpu_target + storage_target = cluster.value.autoscaling.storage_target + } + } + } + } +} + +resource "google_bigtable_instance_iam_binding" "default" { + for_each = var.iam + project = var.project_id + instance = google_bigtable_instance.default.name + role = each.key + members = each.value +} + +resource "google_bigtable_table" "default" { + for_each = var.tables + project = var.project_id + instance_name = google_bigtable_instance.default.name + name = each.key + split_keys = each.value.split_keys + + dynamic "column_family" { + for_each = each.value.column_families + + content { + family = column_family.key + } + } +} + +resource "google_bigtable_gc_policy" "default" { + for_each = { for k, v in local.gc_pairs : k => v if v.gc_policy != null } + + table = each.value.table + column_family = each.value.column_family + instance_name = google_bigtable_instance.default.name + project = var.project_id + + gc_rules = try(each.value.gc_policy.gc_rules, null) + mode = try(each.value.gc_policy.mode, null) + deletion_policy = try(each.value.gc_policy.deletion_policy, null) + + dynamic "max_age" { + for_each = try(each.value.gc_policy.max_age, null) != null ? [""] : [] + content { + duration = each.value.gc_policy.max_age + } + } + + dynamic "max_version" { + for_each = try(each.value.gc_policy.max_version, null) != null ? [""] : [] + content { + number = each.value.gc_policy.max_version + } + } +} diff --git a/assets/modules-fabric/v26/bigtable-instance/outputs.tf b/assets/modules-fabric/v26/bigtable-instance/outputs.tf new file mode 100644 index 0000000..a2fd264 --- /dev/null +++ b/assets/modules-fabric/v26/bigtable-instance/outputs.tf @@ -0,0 +1,46 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "id" { + description = "Fully qualified instance id." + value = google_bigtable_instance.default.id + depends_on = [ + google_bigtable_instance_iam_binding.default, + google_bigtable_table.default + ] +} + +output "instance" { + description = "BigTable instance." + value = google_bigtable_instance.default + depends_on = [ + google_bigtable_instance_iam_binding.default, + google_bigtable_table.default + ] +} + +output "table_ids" { + description = "Map of fully qualified table ids keyed by table name." + value = { for k, v in google_bigtable_table.default : v.name => v.id } +} + +output "tables" { + description = "Table resources." + value = google_bigtable_table.default +} + + + diff --git a/assets/modules-fabric/v26/bigtable-instance/variables.tf b/assets/modules-fabric/v26/bigtable-instance/variables.tf new file mode 100644 index 0000000..f7b75c1 --- /dev/null +++ b/assets/modules-fabric/v26/bigtable-instance/variables.tf @@ -0,0 +1,105 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "clusters" { + description = "Clusters to be created in the BigTable instance. Set more than one cluster to enable replication. If you set autoscaling, num_nodes will be ignored." + nullable = false + type = map(object({ + zone = optional(string) + storage_type = optional(string) + num_nodes = optional(number) + autoscaling = optional(object({ + min_nodes = number + max_nodes = number + cpu_target = number + storage_target = optional(number) + })) + })) +} + +variable "default_autoscaling" { + description = "Default settings for autoscaling of clusters. This will be the default autoscaling for any cluster not specifying any autoscaling details." + type = object({ + min_nodes = number + max_nodes = number + cpu_target = number + storage_target = optional(number) + }) + default = null +} + +variable "default_gc_policy" { + description = "Default garbage collection policy, to be applied to all column families and all tables. Can be override in the tables variable for specific column families." + type = object({ + deletion_policy = optional(string) + gc_rules = optional(string) + mode = optional(string) + max_age = optional(string) + max_version = optional(string) + }) + default = null +} + +variable "deletion_protection" { + description = "Whether or not to allow Terraform to destroy the instance. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the instance will fail." + default = true +} + +variable "display_name" { + description = "The human-readable display name of the Bigtable instance." + default = null +} + +variable "iam" { + description = "IAM bindings for topic in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} +} + +variable "instance_type" { + description = "(deprecated) The instance type to create. One of 'DEVELOPMENT' or 'PRODUCTION'." + type = string + default = null +} + +variable "name" { + description = "The name of the Cloud Bigtable instance." + type = string +} + +variable "project_id" { + description = "Id of the project where datasets will be created." + type = string +} + +variable "tables" { + description = "Tables to be created in the BigTable instance." + nullable = false + type = map(object({ + split_keys = optional(list(string), []) + column_families = optional(map(object( + { + gc_policy = optional(object({ + deletion_policy = optional(string) + gc_rules = optional(string) + mode = optional(string) + max_age = optional(string) + max_version = optional(string) + }), null) + })), {}) + })) + default = {} +} diff --git a/assets/modules-fabric/v26/bigtable-instance/versions.tf b/assets/modules-fabric/v26/bigtable-instance/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/bigtable-instance/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/billing-budget/README.md b/assets/modules-fabric/v26/billing-budget/README.md new file mode 100644 index 0000000..72fe574 --- /dev/null +++ b/assets/modules-fabric/v26/billing-budget/README.md @@ -0,0 +1,89 @@ +# Google Cloud Billing Budget Module + +This module allows creating a Cloud Billing budget for a set of services and projects. + +To create billing budgets you need one of the following IAM roles on the target billing account: + +* Billing Account Administrator +* Billing Account Costs Manager + +## Examples + +### Simple email notification + +Send a notification to an email when a set of projects reach $100 of spend. + +```hcl +module "budget" { + source = "./fabric/modules/billing-budget" + billing_account = var.billing_account_id + name = "$100 budget" + amount = 100 + thresholds = { + current = [0.5, 0.75, 1.0] + forecasted = [1.0] + } + projects = [ + "projects/123456789000", + "projects/123456789111" + ] + email_recipients = { + project_id = "my-project" + emails = ["user@example.com"] + } +} +# tftest modules=1 resources=2 inventory=email.yaml +``` + +### Pubsub notification + +Send a notification to a PubSub topic the total spend of a billing account reaches the previous month's spend. + + +```hcl +module "budget" { + source = "./fabric/modules/billing-budget" + billing_account = var.billing_account_id + name = "previous period budget" + amount = 0 + thresholds = { + current = [1.0] + forecasted = [] + } + pubsub_topic = module.pubsub.id +} + +module "pubsub" { + source = "./fabric/modules/pubsub" + project_id = var.project_id + name = "budget-topic" +} + +# tftest modules=2 resources=2 inventory=pubsub.yaml +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [billing_account](variables.tf#L23) | Billing account id. | string | ✓ | | +| [name](variables.tf#L50) | Budget name. | string | ✓ | | +| [thresholds](variables.tf#L85) | Thresholds percentages at which alerts are sent. Must be a value between 0 and 1. | object({…}) | ✓ | | +| [amount](variables.tf#L17) | Amount in the billing account's currency for the budget. Use 0 to set budget to 100% of last period's spend. | number | | 0 | +| [credit_treatment](variables.tf#L28) | How credits should be treated when determining spend for threshold calculations. Only INCLUDE_ALL_CREDITS or EXCLUDE_ALL_CREDITS are supported. | string | | "INCLUDE_ALL_CREDITS" | +| [email_recipients](variables.tf#L41) | Emails where budget notifications will be sent. Setting this will create a notification channel for each email in the specified project. | object({…}) | | null | +| [notification_channels](variables.tf#L55) | Monitoring notification channels where to send updates. | list(string) | | null | +| [notify_default_recipients](variables.tf#L61) | Notify Billing Account Administrators and Billing Account Users IAM roles for the target account. | bool | | false | +| [projects](variables.tf#L67) | List of projects of the form projects/{project_number}, specifying that usage from only this set of projects should be included in the budget. Set to null to include all projects linked to the billing account. | list(string) | | null | +| [pubsub_topic](variables.tf#L73) | The ID of the Cloud Pub/Sub topic where budget related messages will be published. | string | | null | +| [services](variables.tf#L79) | List of services of the form services/{service_id}, specifying that usage from only this set of services should be included in the budget. Set to null to include usage for all services. | list(string) | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [budget](outputs.tf#L17) | Budget resource. | | +| [id](outputs.tf#L22) | Fully qualified budget id. | | + + diff --git a/assets/modules-fabric/v26/billing-budget/main.tf b/assets/modules-fabric/v26/billing-budget/main.tf new file mode 100644 index 0000000..2c6838d --- /dev/null +++ b/assets/modules-fabric/v26/billing-budget/main.tf @@ -0,0 +1,95 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + spend_basis = { + current = "CURRENT_SPEND" + forecasted = "FORECASTED_SPEND" + } + threshold_pairs = flatten([ + for type, values in var.thresholds : [ + for value in values : { + spend_basis = local.spend_basis[type] + threshold_percent = value + } + ] + ]) + + notification_channels = concat( + [for channel in google_monitoring_notification_channel.email_channels : channel.id], + coalesce(var.notification_channels, []) + ) +} + +resource "google_monitoring_notification_channel" "email_channels" { + for_each = toset(try(var.email_recipients.emails, [])) + display_name = "${var.name} budget email notification (${each.value})" + type = "email" + project = var.email_recipients.project_id + labels = { + email_address = each.value + } + user_labels = {} +} + + +resource "google_billing_budget" "budget" { + billing_account = var.billing_account + display_name = var.name + + budget_filter { + projects = var.projects + credit_types_treatment = var.credit_treatment + services = var.services + } + + dynamic "amount" { + for_each = var.amount == 0 ? [1] : [] + content { + last_period_amount = true + } + } + + dynamic "amount" { + for_each = var.amount != 0 ? [1] : [] + content { + dynamic "specified_amount" { + for_each = var.amount != 0 ? [1] : [] + content { + units = var.amount + } + } + } + } + + dynamic "threshold_rules" { + for_each = local.threshold_pairs + iterator = threshold + content { + threshold_percent = threshold.value.threshold_percent + spend_basis = threshold.value.spend_basis + } + } + + all_updates_rule { + monitoring_notification_channels = local.notification_channels + pubsub_topic = var.pubsub_topic + # disable_default_iam_recipients can only be set if + # monitoring_notification_channels is nonempty + disable_default_iam_recipients = try(length(var.notification_channels), 0) > 0 && !var.notify_default_recipients + schema_version = "1.0" + } +} diff --git a/assets/modules-fabric/v26/billing-budget/outputs.tf b/assets/modules-fabric/v26/billing-budget/outputs.tf new file mode 100644 index 0000000..530f857 --- /dev/null +++ b/assets/modules-fabric/v26/billing-budget/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "budget" { + description = "Budget resource." + value = google_billing_budget.budget +} + +output "id" { + description = "Fully qualified budget id." + value = google_billing_budget.budget.id +} diff --git a/assets/modules-fabric/v26/billing-budget/variables.tf b/assets/modules-fabric/v26/billing-budget/variables.tf new file mode 100644 index 0000000..003f928 --- /dev/null +++ b/assets/modules-fabric/v26/billing-budget/variables.tf @@ -0,0 +1,95 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "amount" { + description = "Amount in the billing account's currency for the budget. Use 0 to set budget to 100% of last period's spend." + type = number + default = 0 +} + +variable "billing_account" { + description = "Billing account id." + type = string +} + +variable "credit_treatment" { + description = "How credits should be treated when determining spend for threshold calculations. Only INCLUDE_ALL_CREDITS or EXCLUDE_ALL_CREDITS are supported." + type = string + default = "INCLUDE_ALL_CREDITS" + validation { + condition = ( + var.credit_treatment == "INCLUDE_ALL_CREDITS" || + var.credit_treatment == "EXCLUDE_ALL_CREDITS" + ) + error_message = "Argument credit_treatment must be INCLUDE_ALL_CREDITS or EXCLUDE_ALL_CREDITS." + } +} + +variable "email_recipients" { + description = "Emails where budget notifications will be sent. Setting this will create a notification channel for each email in the specified project." + type = object({ + project_id = string + emails = list(string) + }) + default = null +} + +variable "name" { + description = "Budget name." + type = string +} + +variable "notification_channels" { + description = "Monitoring notification channels where to send updates." + type = list(string) + default = null +} + +variable "notify_default_recipients" { + description = "Notify Billing Account Administrators and Billing Account Users IAM roles for the target account." + type = bool + default = false +} + +variable "projects" { + description = "List of projects of the form projects/{project_number}, specifying that usage from only this set of projects should be included in the budget. Set to null to include all projects linked to the billing account." + type = list(string) + default = null +} + +variable "pubsub_topic" { + description = "The ID of the Cloud Pub/Sub topic where budget related messages will be published." + type = string + default = null +} + +variable "services" { + description = "List of services of the form services/{service_id}, specifying that usage from only this set of services should be included in the budget. Set to null to include usage for all services." + type = list(string) + default = null +} + +variable "thresholds" { + description = "Thresholds percentages at which alerts are sent. Must be a value between 0 and 1." + type = object({ + current = list(number) + forecasted = list(number) + }) + validation { + condition = length(var.thresholds.current) > 0 || length(var.thresholds.forecasted) > 0 + error_message = "Must specify at least one budget threshold." + } +} diff --git a/assets/modules-fabric/v26/billing-budget/versions.tf b/assets/modules-fabric/v26/billing-budget/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/billing-budget/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/binauthz/README.md b/assets/modules-fabric/v26/binauthz/README.md new file mode 100644 index 0000000..64ef2d0 --- /dev/null +++ b/assets/modules-fabric/v26/binauthz/README.md @@ -0,0 +1,79 @@ +# Google Cloud Artifact Registry Module + +This module simplifies the creation of a Binary Authorization policy, attestors and attestor IAM bindings. + +## Example + +### Binary Authorization + +```hcl +module "binauthz" { + source = "./fabric/modules/binauthz" + project_id = "my_project" + global_policy_evaluation_mode = "DISABLE" + default_admission_rule = { + evaluation_mode = "ALWAYS_DENY" + enforcement_mode = "ENFORCED_BLOCK_AND_AUDIT_LOG" + attestors = null + } + cluster_admission_rules = { + "europe-west1-c.cluster" = { + evaluation_mode = "REQUIRE_ATTESTATION" + enforcement_mode = "ENFORCED_BLOCK_AND_AUDIT_LOG" + attestors = ["test"] + } + } + attestors_config = { + "test" : { + note_reference = null + pgp_public_keys = [ + < + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [project_id](variables.tf#L68) | Project ID. | string | ✓ | | +| [admission_whitelist_patterns](variables.tf#L17) | An image name pattern to allowlist. | list(string) | | null | +| [attestors_config](variables.tf#L23) | Attestors configuration. | map(object({…})) | | null | +| [cluster_admission_rules](variables.tf#L38) | Admission rules. | map(object({…})) | | null | +| [default_admission_rule](variables.tf#L48) | Default admission rule. | object({…}) | | {…} | +| [global_policy_evaluation_mode](variables.tf#L62) | Global policy evaluation mode. | string | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [attestors](outputs.tf#L17) | Attestors. | | +| [id](outputs.tf#L25) | Fully qualified Binary Authorization policy ID. | | +| [notes](outputs.tf#L30) | Notes. | | + + diff --git a/assets/modules-fabric/v26/binauthz/main.tf b/assets/modules-fabric/v26/binauthz/main.tf new file mode 100644 index 0000000..2c1af46 --- /dev/null +++ b/assets/modules-fabric/v26/binauthz/main.tf @@ -0,0 +1,91 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_binary_authorization_policy" "policy" { + project = var.project_id + dynamic "admission_whitelist_patterns" { + for_each = toset(coalesce(var.admission_whitelist_patterns, [])) + content { + name_pattern = admission_whitelist_patterns.value + } + } + default_admission_rule { + evaluation_mode = var.default_admission_rule.evaluation_mode + enforcement_mode = var.default_admission_rule.enforcement_mode + require_attestations_by = [for attestor in coalesce(var.default_admission_rule.attestors, []) : google_binary_authorization_attestor.attestors[attestor].name] + } + dynamic "cluster_admission_rules" { + for_each = coalesce(var.cluster_admission_rules, {}) + content { + cluster = cluster_admission_rules.key + evaluation_mode = cluster_admission_rules.value.evaluation_mode + enforcement_mode = cluster_admission_rules.value.enforcement_mode + require_attestations_by = [for attestor in cluster_admission_rules.value.attestors : google_binary_authorization_attestor.attestors[attestor].name] + } + } +} + +resource "google_binary_authorization_attestor" "attestors" { + for_each = coalesce(var.attestors_config, {}) + name = each.key + project = var.project_id + attestation_authority_note { + note_reference = each.value.note_reference == null ? google_container_analysis_note.notes[each.key].name : each.value.note_reference + dynamic "public_keys" { + for_each = coalesce(each.value.pgp_public_keys, []) + content { + ascii_armored_pgp_public_key = public_keys.value + } + } + dynamic "public_keys" { + for_each = { + for pkix_public_key in coalesce(each.value.pkix_public_keys, []) : + "${pkix_public_key.public_key_pem}-${pkix_public_key.signature_algorithm}" => pkix_public_key + } + content { + id = public_keys.value.id + pkix_public_key { + public_key_pem = public_keys.value.public_key_pem + signature_algorithm = public_keys.value.signature_algorithm + } + } + } + } +} + +resource "google_binary_authorization_attestor_iam_binding" "bindings" { + for_each = merge(flatten([ + for name, attestor_config in var.attestors_config : { for role, members in coalesce(attestor_config.iam, {}) : "${name}-${role}" => { + name = name + role = role + members = members + } }])...) + project = google_binary_authorization_attestor.attestors[each.value.name].project + attestor = google_binary_authorization_attestor.attestors[each.value.name].name + role = each.value.role + members = each.value.members +} + +resource "google_container_analysis_note" "notes" { + for_each = toset([for name, attestor_config in var.attestors_config : name if attestor_config.note_reference == null]) + name = "${each.value}-note" + project = var.project_id + attestation_authority { + hint { + human_readable_name = "Attestor ${each.value} note" + } + } +} diff --git a/assets/modules-fabric/v26/binauthz/outputs.tf b/assets/modules-fabric/v26/binauthz/outputs.tf new file mode 100644 index 0000000..874f9ae --- /dev/null +++ b/assets/modules-fabric/v26/binauthz/outputs.tf @@ -0,0 +1,33 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "attestors" { + description = "Attestors." + value = google_binary_authorization_attestor.attestors + depends_on = [ + google_binary_authorization_attestor_iam_binding.bindings + ] +} + +output "id" { + description = "Fully qualified Binary Authorization policy ID." + value = google_binary_authorization_policy.policy.id +} + +output "notes" { + description = "Notes." + value = google_container_analysis_note.notes +} diff --git a/assets/modules-fabric/v26/binauthz/variables.tf b/assets/modules-fabric/v26/binauthz/variables.tf new file mode 100644 index 0000000..6d21083 --- /dev/null +++ b/assets/modules-fabric/v26/binauthz/variables.tf @@ -0,0 +1,71 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "admission_whitelist_patterns" { + description = "An image name pattern to allowlist." + type = list(string) + default = null +} + +variable "attestors_config" { + description = "Attestors configuration." + type = map(object({ + note_reference = string + iam = map(list(string)) + pgp_public_keys = list(string) + pkix_public_keys = list(object({ + id = string + public_key_pem = string + signature_algorithm = string + })) + })) + default = null +} + +variable "cluster_admission_rules" { + description = "Admission rules." + type = map(object({ + evaluation_mode = string + enforcement_mode = string + attestors = list(string) + })) + default = null +} + +variable "default_admission_rule" { + description = "Default admission rule." + type = object({ + evaluation_mode = string + enforcement_mode = string + attestors = list(string) + }) + default = { + evaluation_mode = "ALWAYS_ALLOW" + enforcement_mode = "ENFORCED_BLOCK_AND_AUDIT_LOG" + attestors = null + } +} + +variable "global_policy_evaluation_mode" { + description = "Global policy evaluation mode." + type = string + default = null +} + +variable "project_id" { + description = "Project ID." + type = string +} diff --git a/assets/modules-fabric/v26/binauthz/versions.tf b/assets/modules-fabric/v26/binauthz/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/binauthz/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/cloud-config-container/.gitignore b/assets/modules-fabric/v26/cloud-config-container/.gitignore new file mode 100644 index 0000000..bbd801c --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/.gitignore @@ -0,0 +1 @@ +**/test.tf diff --git a/assets/modules-fabric/v26/cloud-config-container/README.md b/assets/modules-fabric/v26/cloud-config-container/README.md new file mode 100644 index 0000000..2307a76 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/README.md @@ -0,0 +1,28 @@ +# Instance Configuration via `cloud-config` + +This set of modules creates specialized [cloud-config](https://cloud.google.com/container-optimized-os/docs/how-to/run-container-instance#starting_a_docker_container_via_cloud-config) configurations, which are designed for use with [Container Optimized OS](https://cloud.google.com/container-optimized-os/docs) (the onprem module is the only exception) but can also be used as a basis for other image types or cloud providers. + +These modules are designed for several use cases: + +- to quickly prototype specialized services (eg MySQL access or HTTP serving) for prototyping infrastructure +- to emulate production services for performance testing +- to easily add glue components for services like DNS (eg to work around inbound/outbound forwarding limitations) +- to implement cloud-native production deployments that leverage cloud-init for configuration management, without the need of a separate tool + +## Available modules + +- [CoreDNS](./coredns) +- [MySQL](./mysql) +- [Nginx](./nginx) +- [Squid forward proxy](./squid) +- On-prem in Docker (*needs fixing*) + +## Using the modules + +All modules are designed to be as lightweight as possible, so that specialized modules like [compute-vm](../compute-vm) can be leveraged to manage instances or instance templates, and to allow simple forking to create custom derivatives. + +To use the modules with instances or instance templates, simply set use their `cloud_config` output for the `user-data` metadata. When updating the metadata after a variable change remember to manually restart the instances that use a module's output, or the changes won't effect the running system. + +## TODO + +- [ ] convert all `xxx_config` variables to use file content instead of path diff --git a/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/Corefile b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/Corefile new file mode 100644 index 0000000..4ed56d7 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/Corefile @@ -0,0 +1,9 @@ +. { + hosts /etc/coredns/onprem.hosts onprem.example.org { + 127.0.0.1 localhost.example.org localhost + } + forward . /etc/resolv.conf + reload + log + errors +} diff --git a/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/README.md b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/README.md new file mode 100644 index 0000000..1f6308c --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/README.md @@ -0,0 +1,90 @@ +# Containerized on-premises infrastructure + +This module manages a `cloud-config` configuration that starts an emulated on-premises infrastructure running in Docker Compose on a single instance, and connects it via static or dynamic VPN to a Google Cloud VPN gateway. + +The emulated on-premises infrastructure is composed of: + +- a [Strongswan container](./docker-images/strongswan) managing the VPN tunnel to GCP +- an optional Bird container managing the BGP session +- a CoreDNS container servng local DNS and forwarding to GCP +- an Nginx container serving a simple static web page +- a [generic Linux container](./docker-images/toolbox) used as a jump host inside the on-premises network + +A complete scenario using this module is available in the networking blueprints. + +The module renders the generated cloud config in the `cloud_config` output, to be used in instances or instance templates via the `user-data` metadata. + +## Examples + +### Static VPN + +```hcl +module "cloud-vpn" { + source = "./fabric/modules/net-vpn-static" + project_id = "my-project" + region = "europe-west1" + network = "my-vpc" + name = "to-on-prem" + remote_ranges = ["192.168.192.0/24"] + tunnels = { + remote-0 = { + peer_ip = module.vm.external_ip + traffic_selectors = { local = ["0.0.0.0/0"], remote = null } + } + } +} + +module "on-prem" { + source = "./fabric/modules/cloud-config-container/onprem" + vpn_config = { + type = "static" + peer_ip = module.cloud-vpn.address + shared_secret = module.cloud-vpn.random_secret + } +} + +module "vm" { + source = "./fabric/modules/compute-vm" + project_id = "my-project" + zone = "europe-west8-b" + name = "cos-nginx-tls" + network_interfaces = [{ + nat = true + network = "default" + subnetwork = "gce" + }] + metadata = { + user-data = module.on-prem.cloud_config + google-logging-enabled = true + } + boot_disk = { + initialize_params = { + image = "projects/cos-cloud/global/images/family/cos-stable" + type = "pd-ssd" + size = 10 + } + } + tags = ["ssh"] +} +# tftest skip +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [vpn_config](variables.tf#L35) | VPN configuration, type must be one of 'dynamic' or 'static'. | object({…}) | ✓ | | +| [config_variables](variables.tf#L17) | Additional variables used to render the cloud-config and CoreDNS templates. | map(any) | | {} | +| [coredns_config](variables.tf#L23) | CoreDNS configuration path, if null default will be used. | string | | null | +| [local_ip_cidr_range](variables.tf#L29) | IP CIDR range used for the Docker onprem network. | string | | "192.168.192.0/24" | +| [vpn_dynamic_config](variables.tf#L46) | BGP configuration for dynamic VPN, ignored if VPN type is 'static'. | object({…}) | | {…} | +| [vpn_static_ranges](variables.tf#L70) | Remote CIDR ranges for static VPN, ignored if VPN type is 'dynamic'. | list(string) | | ["10.0.0.0/8"] | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [cloud_config](outputs.tf#L17) | Rendered cloud-config file to be passed as user-data instance metadata. | | + + diff --git a/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/cloud-config.yaml b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/cloud-config.yaml new file mode 100644 index 0000000..ba27f84 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/cloud-config.yaml @@ -0,0 +1,375 @@ +#cloud-config + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package_update: true +package_upgrade: true +package_reboot_if_required: true + +packages: + - apt-transport-https + - ca-certificates + - curl + - gnupg-agent + - software-properties-common + +write_files: + +# Docker daemon configuration +- path: /etc/docker/daemon.json + owner: root:root + permissions: '0644' + content: | + { + "log-driver": "json-file", + "log-opts": { + "max-size": "10m" + } + } + +# Docker compose systemd unit for onprem +- path: /etc/systemd/system/docker-onprem.service + permissions: 0644 + owner: root + content: | + [Install] + WantedBy=multi-user.target + [Unit] + Description=Start Docker Compose onprem infrastructure + After=network-online.target docker.socket + Wants=network-online.target docker.socket + [Service] + ExecStart=/bin/sh -c "cd /var/lib/docker-compose/onprem && /usr/local/bin/docker-compose up" + ExecStop=/bin/sh -c "cd /var/lib/docker-compose/onprem && /usr/local/bin/docker-compose down" + +# Docker compose configuration file for onprem +- path: /var/lib/docker-compose/onprem/docker-compose.yaml + permissions: 0644 + owner: root + content: | + version: "3" + services: + vpn: + image: gcr.io/pso-cft-fabric/strongswan:latest + networks: + onprem: + ipv4_address: ${local_addresses.vpn} + ports: + - "500:500/udp" + - "4500:4500/udp" + %{~ if vpn_config.type == "dynamic" ~} + - "179:179/tcp" + %{~ endif ~} + privileged: true + cap_add: + - NET_ADMIN + volumes: + - "/lib/modules:/lib/modules:ro" + - "/etc/localtime:/etc/localtime:ro" + - "/var/lib/docker-compose/onprem/ipsec/ipsec.conf:/etc/ipsec.conf:ro" + - "/var/lib/docker-compose/onprem/ipsec/ipsec.secrets:/etc/ipsec.secrets:ro" + %{~ if vpn_config.type == "dynamic" ~} + - "/var/lib/docker-compose/onprem/ipsec/vti.conf:/etc/strongswan.d/vti.conf:ro" + %{~ endif ~} + environment: + - LAN_NETWORKS=${ip_cidr_ranges.local} + %{~ if vpn_config.type == "dynamic" ~} + bird: + image: pierky/bird + network_mode: service:vpn + cap_add: + - NET_ADMIN + - NET_BROADCAST + - NET_RAW + privileged: true + volumes: + - "/var/lib/docker-compose/onprem/bird/bird.conf:/etc/bird/bird.conf:ro" + %{~ endif ~} + dns: + image: coredns/coredns + command: "-conf /etc/coredns/Corefile" + depends_on: + - "vpn" + %{~ if vpn_config.type == "dynamic" ~} + - "bird" + %{~ endif ~} + networks: + onprem: + ipv4_address: ${local_addresses.dns} + volumes: + - "/var/lib/docker-compose/onprem/coredns:/etc/coredns:ro" + routing_sidecar_dns: + image: alpine + network_mode: service:dns + command: | + /bin/sh -c "\ + ip route del default &&\ + ip route add default via ${local_addresses.vpn}" + privileged: true + web: + image: nginx:stable-alpine + depends_on: + - "vpn" + %{~ if vpn_config.type == "dynamic" ~} + - "bird" + %{~ endif ~} + - "dns" + dns: + - ${local_addresses.dns} + networks: + onprem: + ipv4_address: ${local_addresses.www} + volumes: + - "/var/lib/docker-compose/onprem/nginx:/usr/share/nginx/html:ro" + routing_sidecar_web: + image: alpine + network_mode: service:web + command: | + /bin/sh -c "\ + ip route del default &&\ + ip route add default via ${local_addresses.vpn}" + privileged: true + toolbox: + image: gcr.io/pso-cft-fabric/toolbox:latest + networks: + onprem: + ipv4_address: ${local_addresses.shell} + depends_on: + - "vpn" + - "dns" + - "web" + dns: + - ${local_addresses.dns} + routing_sidecar_toolbox: + image: alpine + network_mode: service:toolbox + command: | + /bin/sh -c "\ + ip route del default &&\ + ip route add default via ${local_addresses.vpn}" + privileged: true + networks: + onprem: + ipam: + driver: default + config: + - subnet: ${ip_cidr_ranges.local} + +# IPSEC tunnel secret +- path: /var/lib/docker-compose/onprem/ipsec/ipsec.secrets + owner: root:root + permissions: '0600' + content: | + ${vpn_config.peer_ip} : PSK "${vpn_config.shared_secret}" + ${vpn_config.peer_ip2} : PSK "${vpn_config.shared_secret2}" + +# IPSEC tunnel configuration +- path: /var/lib/docker-compose/onprem/ipsec/ipsec.conf + owner: root:root + permissions: '0644' + content: | + conn %default + ikelifetime=600m + keylife=180m + rekeymargin=3m + keyingtries=3 + keyexchange=ikev2 + mobike=no + ike=aes256gcm16-sha512-modp2048 + esp=aes256gcm16-sha512-modp8192 + authby=psk + + conn gcp + %{~ if vpn_config.type == "dynamic" ~} + leftupdown="/var/lib/strongswan/ipsec-vti.sh 0 ${vpn_dynamic_config.peer_bgp_address}/30 ${vpn_dynamic_config.local_bgp_address}/30" + %{~ endif ~} + left=%any + leftid=%any + %{~ if vpn_config.type == "dynamic" ~} + leftsubnet=0.0.0.0/0 + %{~ else ~} + leftsubnet=${ip_cidr_ranges.local} + %{~ endif ~} + leftauth=psk + right=${vpn_config.peer_ip_wildcard} + rightid=${vpn_config.peer_ip} + %{~ if vpn_config.type == "dynamic" ~} + rightsubnet=0.0.0.0/0 + %{~ else ~} + rightsubnet=${ip_cidr_ranges.remote} + %{~ endif ~} + rightauth=psk + type=tunnel + auto=start + dpdaction=restart + closeaction=restart + %{~ if vpn_config.type == "dynamic" ~} + mark=%unique + %{~ endif ~} + + conn gcp2 + %{~ if vpn_config.type == "dynamic" ~} + leftupdown="/var/lib/strongswan/ipsec-vti.sh 1 ${vpn_dynamic_config.peer_bgp_address2}/30 ${vpn_dynamic_config.local_bgp_address2}/30" + %{~ endif ~} + left=%any + leftid=%any + %{~ if vpn_config.type == "dynamic" ~} + leftsubnet=0.0.0.0/0 + %{~ else ~} + leftsubnet=${ip_cidr_ranges.local} + %{~ endif ~} + leftauth=psk + right=${vpn_config.peer_ip_wildcard2} + rightid=${vpn_config.peer_ip2} + %{~ if vpn_config.type == "dynamic" ~} + rightsubnet=0.0.0.0/0 + %{~ else ~} + rightsubnet=${ip_cidr_ranges.remote} + %{~ endif ~} + rightauth=psk + type=tunnel + auto=start + dpdaction=restart + closeaction=restart + %{~ if vpn_config.type == "dynamic" ~} + mark=%unique + %{~ endif ~} + + %{~ if vpn_config.type == "dynamic" ~} + +# Charon configuration +- path: /var/lib/docker-compose/onprem/ipsec/vti.conf + owner: root:root + permissions: '0644' + content: | + charon { + install_routes = no + } + +# Bird bgp routing configuration +- path: /var/lib/docker-compose/onprem/bird/bird.conf + owner: root:root + permissions: '0644' + content: | + router id ${vpn_dynamic_config.local_bgp_address}; + + # watch interface up/down events + protocol device { + scan time 10; + } + + # sync routes to kernel + protocol kernel { + learn; + merge paths on; # For ECMP + export filter { + # internal IP of the strongswan VM + krt_prefsrc = ${local_addresses.vpn}; + # sync all routes to kernel + accept; + }; + import all; # Required due to /32 on GCE VMs for the static route below + } + + # Configure a static route to make sure route exists + protocol static { + # network connected to eth0 + route ${ip_cidr_ranges.local} recursive ${local_addresses.gw}; + %{~ for range in netblocks ~} + # route ${range} via ${vpn_dynamic_config.peer_bgp_address}; + %{~ endfor ~} + } + # prefix lists for routing security + # allow any possible GCP Subnet + define GCP_VPC_A_PREFIXES = [ 10.0.0.0/8{8,29}, 172.16.0.0/12{12,29}, 192.168.0.0/16{16,29} ]; + define GCP_NETBLOCKS = [ ${join(", ", netblocks)} ]; + define LOCAL_PREFIXES = [ ${ip_cidr_ranges.local} ]; + + # filter received prefixes + filter gcp_vpc_a_in { + if (net ~ GCP_VPC_A_PREFIXES || net ~ GCP_NETBLOCKS) then accept; + else reject; + } + + # filter advertised prefixes + filter gcp_vpc_a_out { + if (net ~ LOCAL_PREFIXES) then accept; + else reject; + } + + template bgp gcp_vpc_a { + keepalive time 20; + hold time 60; + # Cloud Router uses GR during maintenance + graceful restart aware; + import filter gcp_vpc_a_in; + import limit 10 action warn; # restart | block | disable + export filter gcp_vpc_a_out; + export limit 10 action warn; # restart | block | disable + } + + protocol bgp gcp_vpc_a_tun1 from gcp_vpc_a { + local ${vpn_dynamic_config.local_bgp_address} as ${vpn_dynamic_config.local_bgp_asn}; + neighbor ${vpn_dynamic_config.peer_bgp_address} as ${vpn_dynamic_config.peer_bgp_asn}; + } + protocol bgp gcp_vpc_a_tun2 from gcp_vpc_a { + local ${vpn_dynamic_config.local_bgp_address2} as ${vpn_dynamic_config.local_bgp_asn2}; + neighbor ${vpn_dynamic_config.peer_bgp_address2} as ${vpn_dynamic_config.peer_bgp_asn2}; + } + + %{~ endif ~} + +# CoreDNS configuration +- path: /var/lib/docker-compose/onprem/coredns/Corefile + owner: root:root + permissions: '0644' + content: | + ${coredns_config} + +# CoreDNS onprem hosts file +- path: /var/lib/docker-compose/onprem/coredns/onprem.hosts + owner: root:root + permissions: '0644' + content: | + %{~ for name, address in local_addresses ~} + ${address} ${name}.onprem.example.org + %{~ endfor ~} + +# Minimal nginx index page +- path: /var/lib/docker-compose/onprem/nginx/index.html + owner: root:root + permissions: '0644' + content: | + + + + +

On Prem in a Box

+

onprem

+ + + +runcmd: +- [systemctl, daemon-reload] +- [ sh, -c, 'curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -' ] +- [ sh, -c, 'add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"' ] +- [ sh, -c, 'apt update' ] +- [ sh, -c, 'apt install -y docker-ce docker-ce-cli containerd.io' ] +- [ sh, -c, 'curl -L https://github.com/docker/compose/releases/download/$(curl -s https://api.github.com/repos/docker/compose/releases/latest | grep "tag_name" | cut -d \" -f4)/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose' ] +- [ sh, -c, 'chmod 755 /usr/local/bin/docker-compose' ] +- [systemctl, enable, docker.service] +- [systemctl, start, docker.service] +- [systemctl, enable, docker-onprem.service] +- [systemctl, start, docker-onprem.service] diff --git a/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/README.md b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/README.md new file mode 100644 index 0000000..e9342f7 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/README.md @@ -0,0 +1,3 @@ +# Supporting container images + +The images in this folder are used by the [`onprem` module](../). \ No newline at end of file diff --git a/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/strongswan/Dockerfile b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/strongswan/Dockerfile new file mode 100644 index 0000000..8bb6165 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/strongswan/Dockerfile @@ -0,0 +1,37 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM debian:bullseye-slim + +ENV STRONGSWAN_VERSION=5.9 + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y sudo iptables procps strongswan=${STRONGSWAN_VERSION}* \ + && rm -rf /var/lib/apt/lists/* + +COPY entrypoint.sh /entrypoint.sh +RUN chmod 0755 /entrypoint.sh + +COPY ipsec-vti.sh /var/lib/strongswan/ipsec-vti.sh +RUN chmod 0755 /var/lib/strongswan/ipsec-vti.sh + +RUN echo 'ipsec ALL=NOPASSWD:SETENV:/usr/sbin/ipsec,/sbin/ip,/sbin/sysctl' > /etc/sudoers.d/ipsec +RUN chmod 0440 /etc/sudoers.d/ipsec + +ENV VPN_DEVICE=eth0 +ENV LAN_NETWORKS=192.168.0.0/24 + +EXPOSE 500/udp 4500/udp + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/strongswan/README.md b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/strongswan/README.md new file mode 100644 index 0000000..cc6eca1 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/strongswan/README.md @@ -0,0 +1,44 @@ + +# StrongSwan docker container + +## Build + +```bash +gcloud builds submit . --config=cloudbuild.yaml +``` + +## Docker compose example + +```yaml +version: "3" +services: + vpn: + image: gcr.io/pso-cft-fabric/strongswan:latest + networks: + default: + ipv4_address: 192.168.0.2 + cap_add: + - NET_ADMIN + ports: + - "500:500/udp" + - "4500:4500/udp" + - "179:179/tcp" + privileged: true + volumes: + - "/lib/modules:/lib/modules:ro" + - "/etc/localtime:/etc/localtime:ro" + - "/var/lib/docker-compose/onprem/ipsec/ipsec.conf:/etc/ipsec.conf:ro" + - "/var/lib/docker-compose/onprem/ipsec/ipsec.secrets:/etc/ipsec.secrets:ro" + - "/var/lib/docker-compose/onprem/ipsec/vti.conf:/etc/strongswan.d/vti.conf:ro" + bird: + image: pierky/bird + network_mode: service:vpn + cap_add: + - NET_ADMIN + - NET_BROADCAST + - NET_RAW + privileged: true + volumes: + - "/var/lib/docker-compose/onprem/bird/bird.conf:/etc/bird/bird.conf:ro" + +``` diff --git a/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/strongswan/cloudbuild.yaml b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/strongswan/cloudbuild.yaml new file mode 100644 index 0000000..b451e79 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/strongswan/cloudbuild.yaml @@ -0,0 +1,29 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# In this directory, run the following command to build this builder. +# $ gcloud builds submit . --config=cloudbuild.yaml + +steps: + - name: "gcr.io/cloud-builders/docker" + args: + - build + - --tag=gcr.io/$PROJECT_ID/strongswan + - --tag=gcr.io/$PROJECT_ID/strongswan:latest + - . + +images: + - "gcr.io/$PROJECT_ID/strongswan:latest" + +timeout: 1200s diff --git a/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/strongswan/entrypoint.sh b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/strongswan/entrypoint.sh new file mode 100644 index 0000000..1d80c1b --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/strongswan/entrypoint.sh @@ -0,0 +1,35 @@ +#!/bin/sh -e + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Enable IP forwarding +sysctl -w net.ipv4.ip_forward=1 + +# Stop ipsec when terminating +_stop_ipsec() { + echo "Shutting down strongSwan/ipsec..." + ipsec stop +} +trap _stop_ipsec TERM + +# Making the container to work as a default gateway for LAN_NETWORKS +iptables -t nat -A POSTROUTING -s ${LAN_NETWORKS} -o ${VPN_DEVICE} -m policy --dir out --pol ipsec -j ACCEPT +iptables -t nat -A POSTROUTING -s ${LAN_NETWORKS} -o ${VPN_DEVICE} -j MASQUERADE + +# Start ipsec +echo "Starting up strongSwan/ipsec..." +ipsec start --nofork "$@" & +child=$! +wait "$child" diff --git a/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/strongswan/ipsec-vti.sh b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/strongswan/ipsec-vti.sh new file mode 100644 index 0000000..5bff8bf --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/strongswan/ipsec-vti.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# originally published at +# https://cloud.google.com/community/tutorials/using-cloud-vpn-with-strongswan + +set -o nounset +set -o errexit + +IP=$(which ip) + +PLUTO_MARK_OUT_ARR=(${PLUTO_MARK_OUT//// }) +PLUTO_MARK_IN_ARR=(${PLUTO_MARK_IN//// }) + +VTI_TUNNEL_ID=${1} +VTI_REMOTE=${2} +VTI_LOCAL=${3} + +LOCAL_IF="${PLUTO_INTERFACE}" +VTI_IF="vti${VTI_TUNNEL_ID}" +# GCP's MTU is 1460 +GCP_MTU="1460" +# ipsec overhead is 73 bytes, we need to compute new mtu. +VTI_MTU=$((GCP_MTU-73)) + +case "${PLUTO_VERB}" in + up-client) + sudo ${IP} link add ${VTI_IF} type vti local ${PLUTO_ME} remote ${PLUTO_PEER} okey ${PLUTO_MARK_OUT_ARR[0]} ikey ${PLUTO_MARK_IN_ARR[0]} + sudo ${IP} addr add ${VTI_LOCAL} remote ${VTI_REMOTE} dev "${VTI_IF}" + sudo ${IP} link set ${VTI_IF} up mtu ${VTI_MTU} + + # Disable IPSEC Policy + sudo /sbin/sysctl -w net.ipv4.conf.${VTI_IF}.disable_policy=1 + + # Enable loosy source validation, if possible. Otherwise disable validation. + sudo /sbin/sysctl -w net.ipv4.conf.${VTI_IF}.rp_filter=2 || sysctl -w net.ipv4.conf.${VTI_IF}.rp_filter=0 + + # If you would like to use VTI for policy-based you should take care of routing by yourselv, e.x. + if [[ "${PLUTO_PEER_CLIENT}" != "0.0.0.0/0" ]]; then + ${IP} r add "${PLUTO_PEER_CLIENT}" dev "${VTI_IF}" + fi + ;; + down-client) + sudo ${IP} tunnel del "${VTI_IF}" + ;; +esac + +# Enable IPv4 forwarding +sudo /sbin/sysctl -w net.ipv4.ip_forward=1 + +# Disable IPSEC Encryption on local net +sudo /sbin/sysctl -w net.ipv4.conf.${LOCAL_IF}.disable_xfrm=1 +sudo /sbin/sysctl -w net.ipv4.conf.${LOCAL_IF}.disable_policy=1 diff --git a/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/toolbox/Dockerfile b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/toolbox/Dockerfile new file mode 100644 index 0000000..dfc8f6e --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/toolbox/Dockerfile @@ -0,0 +1,30 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +FROM google/cloud-sdk:alpine + +COPY entrypoint.sh /entrypoint.sh +RUN chmod 0755 /entrypoint.sh + +RUN apk update && \ + apk add bash curl bind-tools busybox-extras netcat-openbsd && \ + rm /var/cache/apk/* + +RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl && \ + chmod 755 kubectl && mv kubectl /usr/local/bin/ + +CMD ["/bin/bash"] + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/toolbox/README.md b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/toolbox/README.md new file mode 100644 index 0000000..6daada8 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/toolbox/README.md @@ -0,0 +1,26 @@ + +# ToolBox docker container + +Lightweight container with some basic console tools used for testing and probing. + +## Build + +```bash +gcloud builds submit . --config=cloudbuild.yaml +``` + +## Docker compose + +```yaml +version: "3" +services: + vpn: + image: gcr.io/pso-cft-fabric/toolbox:latest + networks: + default: + ipv4_address: 192.168.0.5 + cap_add: + - NET_ADMIN + privileged: true + +``` diff --git a/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/toolbox/cloudbuild.yaml b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/toolbox/cloudbuild.yaml new file mode 100644 index 0000000..6da9ed8 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/toolbox/cloudbuild.yaml @@ -0,0 +1,29 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# In this directory, run the following command to build this builder. +# $ gcloud builds submit . --config=cloudbuild.yaml + +steps: + - name: "gcr.io/cloud-builders/docker" + args: + - build + - --tag=gcr.io/$PROJECT_ID/toolbox + - --tag=gcr.io/$PROJECT_ID/toolbox:latest + - . + +images: + - "gcr.io/$PROJECT_ID/toolbox:latest" + +timeout: 1200s diff --git a/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/toolbox/entrypoint.sh b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/toolbox/entrypoint.sh new file mode 100644 index 0000000..bee48ff --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/docker-images/toolbox/entrypoint.sh @@ -0,0 +1,18 @@ +#!/bin/sh -e + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "Entering sleep..." +trap : TERM INT; (while true; do sleep 1000; done) & wait diff --git a/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/main.tf b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/main.tf new file mode 100644 index 0000000..2e61659 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/main.tf @@ -0,0 +1,68 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cloud_config = templatefile( + "${path.module}/cloud-config.yaml", + merge(local.cloud_config_vars, var.config_variables) + ) + corefile = ( + var.coredns_config == null ? + "${path.module}/Corefile" + : var.coredns_config + ) + cloud_config_vars = { + coredns_config = indent(4, templatefile(local.corefile, var.config_variables)) + ip_cidr_ranges = { + local = var.local_ip_cidr_range + remote = join(",", concat( + var.vpn_static_ranges, local.netblocks + )) + } + local_addresses = { + gw = cidrhost(var.local_ip_cidr_range, 1) + vpn = cidrhost(var.local_ip_cidr_range, 2) + dns = cidrhost(var.local_ip_cidr_range, 3) + www = cidrhost(var.local_ip_cidr_range, 4) + shell = cidrhost(var.local_ip_cidr_range, 5) + vpn2 = cidrhost(var.local_ip_cidr_range, 6) + } + netblocks = local.netblocks + vpn_config = local.vpn_config + vpn_dynamic_config = var.vpn_dynamic_config + } + netblocks = concat( + data.google_netblock_ip_ranges.dns-forwarders.cidr_blocks_ipv4, + data.google_netblock_ip_ranges.private-googleapis.cidr_blocks_ipv4, + data.google_netblock_ip_ranges.restricted-googleapis.cidr_blocks_ipv4 + ) + vpn_config = merge(var.vpn_config, { + peer_ip_wildcard = "%${var.vpn_config.peer_ip}" + peer_ip_wildcard2 = "%${var.vpn_config.peer_ip2}" + }) +} + +data "google_netblock_ip_ranges" "dns-forwarders" { + range_type = "dns-forwarders" +} + +data "google_netblock_ip_ranges" "private-googleapis" { + range_type = "private-googleapis" +} + +data "google_netblock_ip_ranges" "restricted-googleapis" { + range_type = "restricted-googleapis" +} diff --git a/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/outputs.tf b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/outputs.tf new file mode 100644 index 0000000..7d8d416 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cloud_config" { + description = "Rendered cloud-config file to be passed as user-data instance metadata." + value = local.cloud_config +} diff --git a/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/static-vpn-gw-cloud-init.yaml b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/static-vpn-gw-cloud-init.yaml new file mode 100644 index 0000000..36be78b --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/static-vpn-gw-cloud-init.yaml @@ -0,0 +1,236 @@ +#cloud-config + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package_update: true +package_upgrade: true +package_reboot_if_required: true + +packages: + - apt-transport-https + - ca-certificates + - curl + - gnupg-agent + - software-properties-common + +write_files: + # Docker daemon configuration + - path: /etc/docker/daemon.json + owner: root:root + permissions: "0644" + content: | + { + "log-driver": "json-file", + "log-opts": { + "max-size": "10m" + } + } + + # Docker compose systemd unit for onprem + - path: /etc/systemd/system/docker-onprem.service + permissions: 0644 + owner: root + content: | + [Install] + WantedBy=multi-user.target + [Unit] + Description=Start Docker Compose onprem infrastructure + After=network-online.target docker.socket + Wants=network-online.target docker.socket + [Service] + ExecStart=/bin/sh -c "cd /var/lib/docker-compose/onprem && /usr/local/bin/docker-compose up" + ExecStop=/bin/sh -c "cd /var/lib/docker-compose/onprem && /usr/local/bin/docker-compose down" + + # Docker compose configuration file for onprem + - path: /var/lib/docker-compose/onprem/docker-compose.yaml + permissions: 0644 + owner: root + content: | + version: "3" + services: + vpn: + image: gcr.io/pso-cft-fabric/strongswan:latest + networks: + onprem: + ipv4_address: ${vpn_ip_address} + ports: + - "500:500/udp" + - "4500:4500/udp" + privileged: true + cap_add: + - NET_ADMIN + volumes: + - "/lib/modules:/lib/modules:ro" + - "/etc/localtime:/etc/localtime:ro" + - "/var/lib/docker-compose/onprem/ipsec/ipsec.conf:/etc/ipsec.conf:ro" + - "/var/lib/docker-compose/onprem/ipsec/ipsec.secrets:/etc/ipsec.secrets:ro" + environment: + - LAN_NETWORKS=${local_ip_cidr_range} + dns: + image: coredns/coredns + command: "-conf /etc/coredns/Corefile" + depends_on: + - "vpn" + networks: + onprem: + ipv4_address: ${dns_ip_address} + volumes: + - "/var/lib/docker-compose/onprem/coredns:/etc/coredns:ro" + routing_sidecar_dns: + image: alpine + network_mode: service:dns + command: | + /bin/sh -c "\ + ip route del default &&\ + ip route add default via ${vpn_ip_address}" + privileged: true + web: + image: nginx:stable-alpine + depends_on: + - "vpn" + - "dns" + dns: + - ${dns_ip_address} + networks: + onprem: + ipv4_address: ${web_ip_address} + volumes: + - "/var/lib/docker-compose/onprem/nginx:/usr/share/nginx/html:ro" + routing_sidecar_web: + image: alpine + network_mode: service:web + command: | + /bin/sh -c "\ + ip route del default &&\ + ip route add default via ${vpn_ip_address}" + privileged: true + toolbox: + image: gcr.io/pso-cft-fabric/toolbox:latest + networks: + onprem: + ipv4_address: ${toolbox_ip_address} + depends_on: + - "vpn" + - "dns" + - "web" + dns: + - ${dns_ip_address} + routing_sidecar_toolbox: + image: alpine + network_mode: service:toolbox + command: | + /bin/sh -c "\ + ip route del default &&\ + ip route add default via ${vpn_ip_address}" + privileged: true + networks: + onprem: + ipam: + driver: default + config: + - subnet: ${local_ip_cidr_range} + + # IPSEC tunnel secret + - path: /var/lib/docker-compose/onprem/ipsec/ipsec.secrets + owner: root:root + permissions: "0600" + content: | + : PSK "${shared_secret}" + + # IPSEC tunnel configuration + - path: /var/lib/docker-compose/onprem/ipsec/ipsec.conf + owner: root:root + permissions: "0644" + content: | + conn %default + ikelifetime=600m + keylife=180m + rekeymargin=3m + keyingtries=3 + keyexchange=ikev2 + mobike=no + ike=aes256gcm16-sha512-modp2048 + esp=aes256gcm16-sha512-modp8192 + authby=psk + + conn gcp + left=%any + leftid=%any + leftsubnet=${local_ip_cidr_range} + leftauth=psk + right=${peer_ip_wildcard} + rightid=${peer_ip} + rightsubnet=199.36.153.4/30,35.199.192.0/19,${remote_ip_cidr_ranges} + rightauth=psk + type=tunnel + auto=start + dpdaction=restart + closeaction=restart + + # CoreDNS configuration + - path: /var/lib/docker-compose/onprem/coredns/Corefile + owner: root:root + permissions: "0644" + content: | + ${coredns_config} + + # CoreDNS onprem hosts file + - path: /var/lib/docker-compose/onprem/coredns/onprem.hosts + owner: root:root + permissions: "0644" + content: | + ${vpn_ip_address} gw.${dns_domain} + ${dns_ip_address} ns.${dns_domain} + ${web_ip_address} www.${dns_domain} + ${toolbox_ip_address} toolbox.${dns_domain} + + # Minimal nginx index page + - path: /var/lib/docker-compose/onprem/nginx/index.html + owner: root:root + permissions: "0644" + content: | + + + + +

On Prem in a Box

+

${instance_name}

+ + + +runcmd: + - [systemctl, daemon-reload] + - [ + sh, + -c, + "curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -", + ] + - [ + sh, + -c, + 'add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"', + ] + - [sh, -c, "apt update"] + - [sh, -c, "apt install -y docker-ce docker-ce-cli containerd.io"] + - [ + sh, + -c, + 'curl -L https://github.com/docker/compose/releases/download/$(curl -s https://api.github.com/repos/docker/compose/releases/latest | grep "tag_name" | cut -d \" -f4)/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose', + ] + - [sh, -c, "chmod 755 /usr/local/bin/docker-compose"] + - [systemctl, enable, docker.service] + - [systemctl, start, docker.service] + - [systemctl, enable, docker-onprem.service] + - [systemctl, start, docker-onprem.service] diff --git a/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/variables.tf b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/variables.tf new file mode 100644 index 0000000..06eb276 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/variables.tf @@ -0,0 +1,74 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "config_variables" { + description = "Additional variables used to render the cloud-config and CoreDNS templates." + type = map(any) + default = {} +} + +variable "coredns_config" { + description = "CoreDNS configuration path, if null default will be used." + type = string + default = null +} + +variable "local_ip_cidr_range" { + description = "IP CIDR range used for the Docker onprem network." + type = string + default = "192.168.192.0/24" +} + +variable "vpn_config" { + description = "VPN configuration, type must be one of 'dynamic' or 'static'." + type = object({ + peer_ip = string + shared_secret = string + type = optional(string, "static") + peer_ip2 = optional(string) + shared_secret2 = optional(string) + }) +} + +variable "vpn_dynamic_config" { + description = "BGP configuration for dynamic VPN, ignored if VPN type is 'static'." + type = object({ + local_bgp_asn = number + local_bgp_address = string + peer_bgp_asn = number + peer_bgp_address = string + local_bgp_asn2 = number + local_bgp_address2 = string + peer_bgp_asn2 = number + peer_bgp_address2 = string + }) + default = { + local_bgp_asn = 64514 + local_bgp_address = "169.254.1.2" + peer_bgp_asn = 64513 + peer_bgp_address = "169.254.1.1" + local_bgp_asn2 = 64514 + local_bgp_address2 = "169.254.2.2" + peer_bgp_asn2 = 64520 + peer_bgp_address2 = "169.254.2.1" + } +} + +variable "vpn_static_ranges" { + description = "Remote CIDR ranges for static VPN, ignored if VPN type is 'dynamic'." + type = list(string) + default = ["10.0.0.0/8"] +} diff --git a/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/versions.tf b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/__need_fixing/onprem/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/cloud-config-container/coredns/Corefile b/assets/modules-fabric/v26/cloud-config-container/coredns/Corefile new file mode 100644 index 0000000..e5a7674 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/coredns/Corefile @@ -0,0 +1,6 @@ +. { + forward . /etc/resolv.conf + reload + log + errors +} \ No newline at end of file diff --git a/assets/modules-fabric/v26/cloud-config-container/coredns/Corefile-hosts b/assets/modules-fabric/v26/cloud-config-container/coredns/Corefile-hosts new file mode 100644 index 0000000..1baa581 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/coredns/Corefile-hosts @@ -0,0 +1,9 @@ +. { + hosts /etc/coredns/example.hosts example.org { + 127.0.0.1 localhost.example.org localhost + } + forward . /etc/resolv.conf + reload + log + errors +} diff --git a/assets/modules-fabric/v26/cloud-config-container/coredns/README.md b/assets/modules-fabric/v26/cloud-config-container/coredns/README.md new file mode 100644 index 0000000..c4f63a9 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/coredns/README.md @@ -0,0 +1,91 @@ +# Containerized CoreDNS on Container Optimized OS + +This module manages a `cloud-config` configuration that starts a containerized [CoreDNS](https://coredns.io/) service on Container Optimized OS, using the [official image](https://hub.docker.com/r/coredns/coredns/). + +The resulting `cloud-config` can be customized in a number of ways: + +- a custom CoreDNS configuration can be set using the `coredns_config` variable +- additional files (eg for hosts or zone files) can be passed in via the `files` variable +- a completely custom `cloud-config` can be passed in via the `cloud_config` variable, and additional template variables can be passed in via `config_variables` + +The default instance configuration inserts iptables rules to allow traffic on the DNS TCP and UDP ports, and the 8080 port for the optional HTTP health check that can be enabled via the CoreDNS [health plugin](https://coredns.io/plugins/health/). + +Logging and monitoring are enabled via the [Google Cloud Logging agent](https://cloud.google.com/container-optimized-os/docs/how-to/logging) configured for the instance via the `google-logging-enabled` metadata property, and the [Node Problem Detector](https://cloud.google.com/container-optimized-os/docs/how-to/monitoring) service is started by default on boot. + +The module renders the generated cloud config in the `cloud_config` output, to be used in instances or instance templates via the `user-data` metadata. + +For convenience during development or for simple use cases, the module can optionally manage a single instance via the `test_instance` variable. If the instance is not needed the `instance*tf` files can be safely removed. Refer to the [top-level README](../README.md) for more details on the included instance. + +## Examples + +### Default CoreDNS configuration + +This example will create a `cloud-config` that uses the module's defaults, creating a simple DNS forwarder. + +```hcl +module "cos-coredns" { + source = "./fabric/modules/cloud-config-container/coredns" +} + +module "vm" { + source = "./fabric/modules/compute-vm" + project_id = "my-project" + zone = "europe-west8-b" + name = "cos-coredns" + network_interfaces = [{ + network = "default" + subnetwork = "gce" + }] + metadata = { + user-data = module.cos-coredns.cloud_config + google-logging-enabled = true + } + boot_disk = { + initialize_params = { + image = "projects/cos-cloud/global/images/family/cos-stable" + type = "pd-ssd" + size = 10 + } + } + tags = ["dns", "ssh"] +} +# tftest modules=1 resources=1 +``` + +### Custom CoreDNS configuration + +This example will create a `cloud-config` using a custom CoreDNS configuration, that leverages the [CoreDNS hosts plugin]() to serve a single zone via an included `hosts` format file. + +```hcl +module "cos-coredns" { + source = "./fabric/modules/cloud-config-container/coredns" + coredns_config = "./fabric/modules/cloud-config-container/coredns/Corefile-hosts" + files = { + "/etc/coredns/example.hosts" = { + content = "127.0.0.2 foo.example.org foo" + owner = null + permissions = "0644" + } + } +} +# tftest modules=0 resources=0 +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [cloud_config](variables.tf#L17) | Cloud config template path. If null default will be used. | string | | null | +| [config_variables](variables.tf#L23) | Additional variables used to render the cloud-config and CoreDNS templates. | map(any) | | {} | +| [coredns_config](variables.tf#L29) | CoreDNS configuration path, if null default will be used. | string | | null | +| [file_defaults](variables.tf#L35) | Default owner and permissions for files. | object({…}) | | {…} | +| [files](variables.tf#L47) | Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null. | map(object({…})) | | {} | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [cloud_config](outputs.tf#L17) | Rendered cloud-config file to be passed as user-data instance metadata. | | + + diff --git a/assets/modules-fabric/v26/cloud-config-container/coredns/cloud-config.yaml b/assets/modules-fabric/v26/cloud-config-container/coredns/cloud-config.yaml new file mode 100644 index 0000000..9fe929e --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/coredns/cloud-config.yaml @@ -0,0 +1,81 @@ +#cloud-config + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# https://hub.docker.com/r/coredns/coredns/ +# https://coredns.io/manual/toc/#installation + +write_files: + - path: /var/lib/docker/daemon.json + permissions: 0644 + owner: root + content: | + { + "live-restore": true, + "storage-driver": "overlay2", + "log-opts": { + "max-size": "1024m" + } + } + + # disable systemd-resolved to free port 53 on the loopback interface + - path: /etc/systemd/resolved.conf + permissions: 0644 + owner: root + content: | + [Resolve] + LLMNR=no + DNSStubListener=no + + - path: /etc/coredns/Corefile + permissions: 0644 + owner: root + content: | + ${indent(6, corefile)} + + # coredns container service + - path: /etc/systemd/system/coredns.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Start CoreDNS container + After=gcr-online.target docker.socket + Wants=gcr-online.target docker.socket docker-events-collector.service + [Service] + ExecStart=/usr/bin/docker run --rm --name=coredns \ + --network host \ + -v /etc/coredns:/etc/coredns \ + coredns/coredns -conf /etc/coredns/Corefile + ExecStop=/usr/bin/docker stop coredns + + %{ for path, data in files } + - path: ${path} + owner: ${lookup(data, "owner", "root")} + permissions: ${lookup(data, "permissions", "0644")} + content: | + ${indent(4, data.content)} + %{ endfor } + +bootcmd: + - systemctl start node-problem-detector + +runcmd: + - iptables -I INPUT 1 -p tcp -m tcp --dport 8080 -m state --state NEW,ESTABLISHED -j ACCEPT + - iptables -I INPUT 1 -p tcp -m tcp --dport 53 -m state --state NEW,ESTABLISHED -j ACCEPT + - iptables -I INPUT 1 -p udp -m udp --dport 53 -m state --state NEW,ESTABLISHED -j ACCEPT + - systemctl daemon-reload + - systemctl restart systemd-resolved.service + - systemctl start coredns diff --git a/assets/modules-fabric/v26/cloud-config-container/coredns/main.tf b/assets/modules-fabric/v26/cloud-config-container/coredns/main.tf new file mode 100644 index 0000000..789168c --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/coredns/main.tf @@ -0,0 +1,41 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cloud_config = templatefile(local.template, merge(var.config_variables, { + corefile = templatefile(local.corefile, var.config_variables) + files = local.files + })) + corefile = ( + var.coredns_config == null ? "${path.module}/Corefile" : var.coredns_config + ) + files = { + for path, attrs in var.files : path => { + content = attrs.content, + owner = attrs.owner == null ? var.file_defaults.owner : attrs.owner, + permissions = ( + attrs.permissions == null + ? var.file_defaults.permissions + : attrs.permissions + ) + } + } + template = ( + var.cloud_config == null + ? "${path.module}/cloud-config.yaml" + : var.cloud_config + ) +} diff --git a/assets/modules-fabric/v26/cloud-config-container/coredns/outputs.tf b/assets/modules-fabric/v26/cloud-config-container/coredns/outputs.tf new file mode 100644 index 0000000..7d8d416 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/coredns/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cloud_config" { + description = "Rendered cloud-config file to be passed as user-data instance metadata." + value = local.cloud_config +} diff --git a/assets/modules-fabric/v26/cloud-config-container/coredns/variables.tf b/assets/modules-fabric/v26/cloud-config-container/coredns/variables.tf new file mode 100644 index 0000000..c323017 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/coredns/variables.tf @@ -0,0 +1,55 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "cloud_config" { + description = "Cloud config template path. If null default will be used." + type = string + default = null +} + +variable "config_variables" { + description = "Additional variables used to render the cloud-config and CoreDNS templates." + type = map(any) + default = {} +} + +variable "coredns_config" { + description = "CoreDNS configuration path, if null default will be used." + type = string + default = null +} + +variable "file_defaults" { + description = "Default owner and permissions for files." + type = object({ + owner = string + permissions = string + }) + default = { + owner = "root" + permissions = "0644" + } +} + +variable "files" { + description = "Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null." + type = map(object({ + content = string + owner = string + permissions = string + })) + default = {} +} diff --git a/assets/modules-fabric/v26/cloud-config-container/coredns/versions.tf b/assets/modules-fabric/v26/cloud-config-container/coredns/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/coredns/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/cloud-config-container/cos-generic-metadata/README.md b/assets/modules-fabric/v26/cloud-config-container/cos-generic-metadata/README.md new file mode 100644 index 0000000..8807398 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/cos-generic-metadata/README.md @@ -0,0 +1,80 @@ +# Generic cloud-init generator for Container Optimized OS + +This helper module manages a `cloud-config` configuration that can start a container on [Container Optimized OS](https://cloud.google.com/container-optimized-os/docs) (COS). Either a complete `cloud-config` template can be provided via the `cloud_config` variable with optional template variables via the `config_variables`, or a generic `cloud-config` can be generated based on typical parameters needed to start a container. + +The module renders the generated cloud config in the `cloud_config` output, which can be directly used in instances or instance templates via the `user-data` metadata attribute. + +## Examples + +### Default configuration + +This example will create a `cloud-config` that starts [Envoy Proxy](https://www.envoyproxy.io) and expose it on port 80. For a complete example, look at the sibling [`envoy-traffic-director`](../envoy-traffic-director/README.md) module that uses this module to start Envoy Proxy and connect it to [Traffic Director](https://cloud.google.com/traffic-director). + +```hcl +module "cos-envoy" { + source = "./fabric/modules/cloud-config-container/cos-generic-metadata" + container_image = "envoyproxy/envoy:v1.14.1" + container_name = "envoy" + container_args = "-c /etc/envoy/envoy.yaml --log-level info --allow-unknown-static-fields" + container_volumes = [ + { host = "/etc/envoy/envoy.yaml", container = "/etc/envoy/envoy.yaml" } + ] + docker_args = "--network host --pid host" + # file paths are mocked to run this example in tests + files = { + "/var/run/envoy/customize.sh" = { + content = file("/dev/null") # file("customize.sh") + owner = "root" + permissions = "0744" + } + "/etc/envoy/envoy.yaml" = { + content = file("/dev/null") # file("envoy.yaml") + owner = "root" + permissions = "0644" + } + } + run_commands = [ + "iptables -t nat -N ENVOY_IN_REDIRECT", + "iptables -t nat -A ENVOY_IN_REDIRECT -p tcp -j REDIRECT --to-port 15001", + "iptables -t nat -A PREROUTING -p tcp -m tcp --dport 80 -j ENVOY_IN_REDIRECT", + "iptables -t filter -A INPUT -p tcp -m tcp --dport 15001 -m state --state NEW,ESTABLISHED -j ACCEPT", + "/var/run/envoy/customize.sh", + "systemctl daemon-reload", + "systemctl start envoy", + ] + users = [{ + username = "envoy", + uid = 1337 + }] +} + +# tftest modules=0 resources=0 +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [container_image](variables.tf#L47) | Container image. | string | ✓ | | +| [authenticate_gcr](variables.tf#L17) | Setup docker to pull images from private GCR. Requires at least one user since the token is stored in the home of the first user defined. | bool | | false | +| [boot_commands](variables.tf#L23) | List of cloud-init `bootcmd`s. | list(string) | | [] | +| [cloud_config](variables.tf#L29) | Cloud config template path. If provided, takes precedence over all other arguments. | string | | null | +| [config_variables](variables.tf#L35) | Additional variables used to render the template passed via `cloud_config`. | map(any) | | {} | +| [container_args](variables.tf#L41) | Arguments for container. | string | | "" | +| [container_name](variables.tf#L52) | Name of the container to be run. | string | | "container" | +| [container_volumes](variables.tf#L58) | List of volumes. | list(object({…})) | | [] | +| [docker_args](variables.tf#L67) | Extra arguments to be passed for docker. | string | | null | +| [file_defaults](variables.tf#L73) | Default owner and permissions for files. | object({…}) | | {…} | +| [files](variables.tf#L85) | Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null. | map(object({…})) | | {} | +| [run_as_first_user](variables.tf#L95) | Run as the first user if users are specified. | bool | | true | +| [run_commands](variables.tf#L101) | List of cloud-init `runcmd`s. | list(string) | | [] | +| [users](variables.tf#L107) | List of usernames to be created. If provided, first user will be used to run the container. | list(object({…})) | | […] | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [cloud_config](outputs.tf#L17) | Rendered cloud-config file to be passed as user-data instance metadata. | | + + diff --git a/assets/modules-fabric/v26/cloud-config-container/cos-generic-metadata/cloud-config.yaml b/assets/modules-fabric/v26/cloud-config-container/cos-generic-metadata/cloud-config.yaml new file mode 100644 index 0000000..a8d1f22 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/cos-generic-metadata/cloud-config.yaml @@ -0,0 +1,83 @@ +#cloud-config + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +%{ if length(users) > 0 ~} +users: +%{ for user in users ~} + - name: ${user.username} + uid: ${user.uid} +%{ endfor ~} +%{ endif ~} + +write_files: + - path: /var/lib/docker/daemon.json + permissions: 0644 + owner: root + content: | + { + "live-restore": true, + "storage-driver": "overlay2", + "log-opts": { + "max-size": "1024m" + } + } + # ${container_name} container service + - path: /etc/systemd/system/${container_name}.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Start ${container_name} container + After=gcr-online.target docker.socket + Wants=gcr-online.target docker.socket docker-events-collector.service + [Service] + %{~ if authenticate_gcr && length(users) > 0 ~} + Environment="HOME=/home/${users[0].username}" + ExecStartPre=/usr/bin/docker-credential-gcr configure-docker + %{~ endif ~} + ExecStart=/usr/bin/docker run --rm --name=${container_name} \ + %{~ if length(users) > 0 && run_as_first_user ~} + --user=${users[0].uid} \ + %{~ endif ~} + %{~ if docker_args != null ~} + ${docker_args} \ + %{~ endif ~} + %{~ for volume in container_volumes ~} + -v ${volume.host}:${volume.container} \ + %{~ endfor ~} + ${container_image} ${container_args} + ExecStop=/usr/bin/docker stop ${container_name} +%{ for path, data in files ~} + - path: ${path} + owner: ${lookup(data, "owner", "root")} + permissions: ${lookup(data, "permissions", "0644")} + content: | + ${indent(6, data.content)} +%{ endfor ~} + +%{ if length(boot_commands) > 0 ~} +bootcmd: +%{ for command in boot_commands ~} + - ${command} +%{ endfor ~} +%{ endif ~} + +%{ if length(run_commands) > 0 ~} +runcmd: +%{ for command in run_commands ~} + - ${command} +%{ endfor ~} +%{ endif ~} diff --git a/assets/modules-fabric/v26/cloud-config-container/cos-generic-metadata/main.tf b/assets/modules-fabric/v26/cloud-config-container/cos-generic-metadata/main.tf new file mode 100644 index 0000000..eb807c5 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/cos-generic-metadata/main.tf @@ -0,0 +1,47 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cloud_config = templatefile(local.template, merge(var.config_variables, { + boot_commands = var.boot_commands + container_args = var.container_args + container_image = var.container_image + container_name = var.container_name + container_volumes = var.container_volumes + docker_args = var.docker_args + files = local.files + run_commands = var.run_commands + users = var.users + authenticate_gcr = var.authenticate_gcr + run_as_first_user = var.run_as_first_user + })) + files = { + for path, attrs in var.files : path => { + content = attrs.content, + owner = attrs.owner == null ? var.file_defaults.owner : attrs.owner, + permissions = ( + attrs.permissions == null + ? var.file_defaults.permissions + : attrs.permissions + ) + } + } + template = ( + var.cloud_config == null + ? "${path.module}/cloud-config.yaml" + : var.cloud_config + ) +} diff --git a/assets/modules-fabric/v26/cloud-config-container/cos-generic-metadata/outputs.tf b/assets/modules-fabric/v26/cloud-config-container/cos-generic-metadata/outputs.tf new file mode 100644 index 0000000..7d8d416 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/cos-generic-metadata/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cloud_config" { + description = "Rendered cloud-config file to be passed as user-data instance metadata." + value = local.cloud_config +} diff --git a/assets/modules-fabric/v26/cloud-config-container/cos-generic-metadata/variables.tf b/assets/modules-fabric/v26/cloud-config-container/cos-generic-metadata/variables.tf new file mode 100644 index 0000000..0225916 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/cos-generic-metadata/variables.tf @@ -0,0 +1,115 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "authenticate_gcr" { + description = "Setup docker to pull images from private GCR. Requires at least one user since the token is stored in the home of the first user defined." + type = bool + default = false +} + +variable "boot_commands" { + description = "List of cloud-init `bootcmd`s." + type = list(string) + default = [] +} + +variable "cloud_config" { + description = "Cloud config template path. If provided, takes precedence over all other arguments." + type = string + default = null +} + +variable "config_variables" { + description = "Additional variables used to render the template passed via `cloud_config`." + type = map(any) + default = {} +} + +variable "container_args" { + description = "Arguments for container." + type = string + default = "" +} + +variable "container_image" { + description = "Container image." + type = string +} + +variable "container_name" { + description = "Name of the container to be run." + type = string + default = "container" +} + +variable "container_volumes" { + description = "List of volumes." + type = list(object({ + host = string, + container = string + })) + default = [] +} + +variable "docker_args" { + description = "Extra arguments to be passed for docker." + type = string + default = null +} + +variable "file_defaults" { + description = "Default owner and permissions for files." + type = object({ + owner = string + permissions = string + }) + default = { + owner = "root" + permissions = "0644" + } +} + +variable "files" { + description = "Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null." + type = map(object({ + content = string + owner = string + permissions = string + })) + default = {} +} + +variable "run_as_first_user" { + description = "Run as the first user if users are specified." + type = bool + default = true +} + +variable "run_commands" { + description = "List of cloud-init `runcmd`s." + type = list(string) + default = [] +} + +variable "users" { + description = "List of usernames to be created. If provided, first user will be used to run the container." + type = list(object({ + username = string, + uid = number, + })) + default = [ + ] +} diff --git a/assets/modules-fabric/v26/cloud-config-container/cos-generic-metadata/versions.tf b/assets/modules-fabric/v26/cloud-config-container/cos-generic-metadata/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/cos-generic-metadata/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/cloud-config-container/envoy-traffic-director/README.md b/assets/modules-fabric/v26/cloud-config-container/envoy-traffic-director/README.md new file mode 100644 index 0000000..caa0ec5 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/envoy-traffic-director/README.md @@ -0,0 +1,54 @@ +# Containerized Envoy Proxy with Traffic Director on Container Optimized OS + +This module manages a `cloud-config` configuration that starts a containerized Envoy Proxy on Container Optimized OS connected to Traffic Director. The default configuration creates a reverse proxy exposed on the node's port 80. Traffic routing policies and management should be managed by other means via Traffic Director. + +The generated cloud config is rendered in the `cloud_config` output, and is meant to be used in instances or instance templates via the `user-data` metadata. + +This module depends on the [`cos-generic-metadata` module](../cos-generic-metadata) being in the parent folder. If you change its location be sure to adjust the `source` attribute in `main.tf`. + +## Examples + +### Default configuration + +```hcl +module "cos-envoy-td" { + source = "./fabric/modules/cloud-config-container/envoy-traffic-director" +} + +module "vm" { + source = "./fabric/modules/compute-vm" + project_id = "my-project" + zone = "europe-west8-b" + name = "cos-envoy-td" + network_interfaces = [{ + network = "default" + subnetwork = "gce" + }] + metadata = { + user-data = module.cos-envoy-td.cloud_config + google-logging-enabled = true + } + boot_disk = { + image = "projects/cos-cloud/global/images/family/cos-stable" + type = "pd-ssd" + size = 10 + } + tags = ["http-server", "ssh"] +} +# tftest modules=1 resources=1 +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [envoy_image](variables.tf#L17) | Envoy Proxy container image to use. | string | | "envoyproxy/envoy:v1.15.5" | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [cloud_config](outputs.tf#L17) | Rendered cloud-config file to be passed as user-data instance metadata. | | + + diff --git a/assets/modules-fabric/v26/cloud-config-container/envoy-traffic-director/files/customize.sh b/assets/modules-fabric/v26/cloud-config-container/envoy-traffic-director/files/customize.sh new file mode 100644 index 0000000..eb9ae82 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/envoy-traffic-director/files/customize.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ENVOY_NODE_ID=$(uuidgen) +ENVOY_ZONE=$(curl -s -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/zone | cut -f 4 -d '/') +CONFIG_PROJECT_NUMBER=$(curl -s -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/network-interfaces/0/network | cut -f 2 -d '/') +VPC_NETWORK_NAME=$(curl -s -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/network-interfaces/0/network | cut -f 4 -d '/') +sed -i "s/ENVOY_NODE_ID/${ENVOY_NODE_ID}/" /etc/envoy/envoy.yaml +sed -i "s/ENVOY_ZONE/${ENVOY_ZONE}/" /etc/envoy/envoy.yaml +sed -i "s/CONFIG_PROJECT_NUMBER/${CONFIG_PROJECT_NUMBER}/" /etc/envoy/envoy.yaml +sed -i "s/VPC_NETWORK_NAME/${VPC_NETWORK_NAME}/" /etc/envoy/envoy.yaml diff --git a/assets/modules-fabric/v26/cloud-config-container/envoy-traffic-director/files/envoy.yaml b/assets/modules-fabric/v26/cloud-config-container/envoy-traffic-director/files/envoy.yaml new file mode 100644 index 0000000..d9a1462 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/envoy-traffic-director/files/envoy.yaml @@ -0,0 +1,101 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +node: + # The id must be in the following format: projects//networks//nodes/ + id: "projects/CONFIG_PROJECT_NUMBER/networks/VPC_NETWORK_NAME/nodes/ENVOY_NODE_ID" + cluster: cluster # unused + locality: + zone: "ENVOY_ZONE" + metadata: + TRAFFICDIRECTOR_INTERCEPTION_PORT: "15001" + TRAFFICDIRECTOR_ENABLE_TRACING: "false" + TRAFFICDIRECTOR_ACCESS_LOG_PATH: "" + TRAFFICDIRECTOR_INBOUND_BACKEND_PORTS: "" + +dynamic_resources: + lds_config: + ads: {} + resource_api_version: V3 + cds_config: + ads: {} + resource_api_version: V3 + ads_config: + api_type: GRPC + transport_api_version: V3 + grpc_services: + - google_grpc: + target_uri: trafficdirector.googleapis.com:443 + stat_prefix: trafficdirector + channel_credentials: + ssl_credentials: + root_certs: + filename: /etc/ssl/certs/ca-certificates.crt + call_credentials: + google_compute_engine: {} + channel_args: + args: + grpc.http2.max_pings_without_data: + int_value: 0 + grpc.keepalive_time_ms: + int_value: 10000 + grpc.keepalive_timeout_ms: + int_value: 20000 + +cluster_manager: + load_stats_config: + api_type: GRPC + transport_api_version: V3 + grpc_services: + - google_grpc: + target_uri: trafficdirector.googleapis.com:443 + stat_prefix: trafficdirector + channel_credentials: + ssl_credentials: + root_certs: + filename: /etc/ssl/certs/ca-certificates.crt + call_credentials: + google_compute_engine: {} + channel_args: + args: + grpc.http2.max_pings_without_data: + int_value: 0 + grpc.keepalive_time_ms: + int_value: 10000 + grpc.keepalive_timeout_ms: + int_value: 20000 + +admin: + access_log_path: /dev/stdout + address: + socket_address: + address: 127.0.0.1 # Admin page is only accessible locally. + port_value: 15000 + +tracing: + http: + name: envoy.tracers.opencensus + typed_config: + "@type": type.googleapis.com/envoy.config.trace.v3.OpenCensusConfig + stackdriver_exporter_enabled: "false" + stackdriver_project_id: "" + +layered_runtime: + layers: + - name: rtds_layer + rtds_layer: + name: traffic_director_runtime + rtds_config: + ads: {} + resource_api_version: V3 diff --git a/assets/modules-fabric/v26/cloud-config-container/envoy-traffic-director/main.tf b/assets/modules-fabric/v26/cloud-config-container/envoy-traffic-director/main.tf new file mode 100644 index 0000000..a6da784 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/envoy-traffic-director/main.tf @@ -0,0 +1,63 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "cos-envoy-td" { + source = "../cos-generic-metadata" + + boot_commands = [ + "systemctl start node-problem-detector", + ] + + container_image = var.envoy_image + container_name = "envoy" + container_args = "-c /etc/envoy/envoy.yaml --log-level info --allow-unknown-static-fields" + + container_volumes = [ + { host = "/etc/envoy/envoy.yaml", container = "/etc/envoy/envoy.yaml" } + ] + + docker_args = "--network host --pid host" + + files = { + "/var/run/envoy/customize.sh" = { + content = file("${path.module}/files/customize.sh") + owner = "root" + permissions = "0744" + } + "/etc/envoy/envoy.yaml" = { + content = file("${path.module}/files/envoy.yaml") + owner = "root" + permissions = "0644" + } + } + + run_commands = [ + "iptables -t nat -N ENVOY_IN_REDIRECT", + "iptables -t nat -A ENVOY_IN_REDIRECT -p tcp -j REDIRECT --to-port 15001", + "iptables -t nat -A PREROUTING -p tcp -m tcp --dport 80 -j ENVOY_IN_REDIRECT", + "iptables -t filter -A INPUT -p tcp -m tcp --dport 15001 -m state --state NEW,ESTABLISHED -j ACCEPT", + "/var/run/envoy/customize.sh", + "systemctl daemon-reload", + "systemctl start envoy", + ] + + users = [ + { + username = "envoy", + uid = 1337 + } + ] +} diff --git a/assets/modules-fabric/v26/cloud-config-container/envoy-traffic-director/outputs.tf b/assets/modules-fabric/v26/cloud-config-container/envoy-traffic-director/outputs.tf new file mode 100644 index 0000000..4ce8d24 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/envoy-traffic-director/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cloud_config" { + description = "Rendered cloud-config file to be passed as user-data instance metadata." + value = module.cos-envoy-td.cloud_config +} diff --git a/assets/modules-fabric/v26/cloud-config-container/envoy-traffic-director/variables.tf b/assets/modules-fabric/v26/cloud-config-container/envoy-traffic-director/variables.tf new file mode 100644 index 0000000..82cdbbd --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/envoy-traffic-director/variables.tf @@ -0,0 +1,21 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "envoy_image" { + description = "Envoy Proxy container image to use." + type = string + default = "envoyproxy/envoy:v1.15.5" +} diff --git a/assets/modules-fabric/v26/cloud-config-container/envoy-traffic-director/versions.tf b/assets/modules-fabric/v26/cloud-config-container/envoy-traffic-director/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/envoy-traffic-director/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/cloud-config-container/mysql/.gitignore b/assets/modules-fabric/v26/cloud-config-container/mysql/.gitignore new file mode 100644 index 0000000..95ea22d --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/mysql/.gitignore @@ -0,0 +1,2 @@ +kms.tf +kms.tf.sample diff --git a/assets/modules-fabric/v26/cloud-config-container/mysql/README.md b/assets/modules-fabric/v26/cloud-config-container/mysql/README.md new file mode 100644 index 0000000..3104580 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/mysql/README.md @@ -0,0 +1,94 @@ +# Containerized MySQL on Container Optimized OS + +This module manages a `cloud-config` configuration that starts a containerized [MySQL](https://www.mysql.com/) service on Container Optimized OS, using the [official image](https://hub.docker.com/_/mysql). + +The resulting `cloud-config` can be customized in a number of ways: + +- a custom MySQL configuration can be set using the `mysql_config` variable +- the container image can be changed via the `image` variable +- a data disk can be specified via the `mysql_data_disk` variable, the configuration will optionally format and mount it for container use +- a KMS encrypted root password can be passed to the container image, and decrypted at runtime on the instance using the attributes in the `kms_config` variable +- a completely custom `cloud-config` can be passed in via the `cloud_config` variable, and additional template variables can be passed in via `config_variables` + +The default instance configuration inserts a sngle iptables rule to allow traffic on the default MySQL port. + +Logging and monitoring are enabled via the [Google Cloud Logging agent](https://cloud.google.com/container-optimized-os/docs/how-to/logging) configured for the instance via the `google-logging-enabled` metadata property, and the [Node Problem Detector](https://cloud.google.com/container-optimized-os/docs/how-to/monitoring) service started by default on boot. + +The module renders the generated cloud config in the `cloud_config` output, to be used in instances or instance templates via the `user-data` metadata. + +For convenience during development or for simple use cases, the module can optionally manage a single instance via the `test_instance` variable. Please note that an `f1-micro` instance is too small to run MySQL. If the instance is not needed the `instance*tf` files can be safely removed. Refer to the [top-level README](../README.md) for more details on the included instance. + +## Examples + +### Default MySQL configuration + +This example will create a `cloud-config` that uses the container's default configuration, and a plaintext password for the MySQL root user. + +```hcl +module "cos-mysql" { + source = "./fabric/modules/cloud-config-container/mysql" + mysql_password = "foo" +} + +module "vm" { + source = "./fabric/modules/compute-vm" + project_id = "my-project" + zone = "europe-west8-b" + name = "cos-mysql" + network_interfaces = [{ + network = "default" + subnetwork = "gce" + }] + metadata = { + user-data = module.cos-mysql.cloud_config + google-logging-enabled = true + } + boot_disk = { + image = "projects/cos-cloud/global/images/family/cos-stable" + type = "pd-ssd" + size = 10 + } + tags = ["mysql", "ssh"] +} +# tftest modules=1 resources=1 +``` + +### Custom MySQL configuration and KMS encrypted password + +This example will create a `cloud-config` that uses a custom MySQL configuration, and passes in an encrypted password and the KMS attributes required to decrypt it. Please note that the instance service account needs the `roles/cloudkms.cryptoKeyDecrypter` on the specified KMS key. + +```hcl +module "cos-mysql" { + source = "./fabric/modules/cloud-config-container/mysql" + mysql_config = "./my.cnf" + mysql_password = "CiQAsd7WY==" + kms_config = { + project_id = "my-project" + keyring = "test-cos" + location = "europe-west1" + key = "mysql" + } +} +# tftest modules=0 resources=0 +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [mysql_password](variables.tf#L58) | MySQL root password. If an encrypted password is set, use the kms_config variable to specify KMS configuration. | string | ✓ | | +| [cloud_config](variables.tf#L17) | Cloud config template path. If null default will be used. | string | | null | +| [config_variables](variables.tf#L23) | Additional variables used to render the cloud-config template. | map(any) | | {} | +| [image](variables.tf#L29) | MySQL container image. | string | | "mysql:5.7" | +| [kms_config](variables.tf#L35) | Optional KMS configuration to decrypt passed-in password. Leave null if a plaintext password is used. | object({…}) | | null | +| [mysql_config](variables.tf#L46) | MySQL configuration file content, if null container default will be used. | string | | null | +| [mysql_data_disk](variables.tf#L52) | MySQL data disk name in /dev/disk/by-id/ including the google- prefix. If null the boot disk will be used for data. | string | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [cloud_config](outputs.tf#L17) | Rendered cloud-config file to be passed as user-data instance metadata. | | + + diff --git a/assets/modules-fabric/v26/cloud-config-container/mysql/cloud-config.yaml b/assets/modules-fabric/v26/cloud-config-container/mysql/cloud-config.yaml new file mode 100644 index 0000000..07706ae --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/mysql/cloud-config.yaml @@ -0,0 +1,116 @@ +#cloud-config + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +users: + - name: mysql + uid: 2000 + +write_files: + - path: /var/lib/docker/daemon.json + permissions: 0644 + owner: root + content: | + { + "live-restore": true, + "storage-driver": "overlay2", + "log-opts": { + "max-size": "1024m" + } + } + - path: /run/mysql/secrets/mysql-passwd${kms_config == null ? "" : "-cipher"}.txt + permissions: 0600 + owner: root + content: | + ${password} + %{~ if kms_config != null ~} + - path: /run/mysql/passwd.sh + permissions: 0700 + owner: root + content: | + #!/bin/bash + base64 -d /run/mysql/secrets/mysql-passwd-cipher.txt | docker run \ + --rm -i -v /run/mysql/secrets:/data google/cloud-sdk:alpine \ + gcloud kms decrypt --ciphertext-file - \ + --plaintext-file /data/mysql-passwd.txt \ + --keyring ${kms_config.keyring} \ + --key ${kms_config.key} \ + --project ${kms_config.project_id} \ + --location ${kms_config.location} + %{~ endif ~} + %{~ if mysql_config != null ~} + - path: /run/mysql/etc/my.cnf + permissions: 0644 + owner: mysql + content: | + ${indent(6, mysql_config)} + %{~ endif ~} + %{~ if mysql_data_disk != null ~} + - path: /etc/systemd/system/mysql-data.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=MySQL data disk + ConditionPathExists=/dev/disk/by-id/${mysql_data_disk} + Before=mysql.service + [Service] + Type=oneshot + ExecStart=/bin/mkdir -p /run/mysql/data + ExecStart=/bin/bash -c \ + "/bin/lsblk -fn -o FSTYPE \ + /dev/disk/by-id/${mysql_data_disk} |grep ext4 \ + || mkfs.ext4 -m 0 -F -E lazy_itable_init=0,lazy_journal_init=0,discard \ + /dev/disk/by-id/${mysql_data_disk}" + ExecStart=/bin/bash -c \ + "mount |grep /run/mysql/data \ + || mount -t ext4 /dev/disk/by-id/${mysql_data_disk} /run/mysql/data" + ExecStart=/sbin/resize2fs /dev/disk/by-id/${mysql_data_disk} + RemainAfterExit=true + %{~ endif ~} + - path: /etc/systemd/system/mysql.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=MySQL service + After=%{~ if mysql_data_disk != null ~}mysql-data.service %{ endif ~}gcr-online.target docker.socket docker-events-collector.service + Wants=%{~ if mysql_data_disk != null ~}mysql-data.service %{ endif ~}gcr-online.target docker.socket + [Service] + %{~ if kms_config != null ~} + ExecStartPre=/run/mysql/passwd.sh + %{~ endif ~} + ExecStartPre=/bin/mkdir -p /run/mysql/data + ExecStartPre=/bin/chown -R 2000 /run/mysql/secrets /run/mysql/data + ExecStart=/usr/bin/docker run --rm --name=mysql \ + --user 2000:2000 \ + --network host \ + -e MYSQL_ROOT_PASSWORD_FILE=/etc/secrets/mysql-passwd.txt \ + -v /run/mysql/secrets:/etc/secrets \ + -v /run/mysql/data:/var/lib/mysql \ + %{~ if mysql_config != null ~} + -v /run/mysql/etc:/etc/mysql \ + %{~ endif ~} + ${image} \ + --ignore-db-dir=lost+found + ExecStop=/usr/bin/docker stop mysql + +bootcmd: + - systemctl start node-problem-detector + +runcmd: + - iptables -I INPUT 1 -p tcp -m tcp --dport 3306 -m state --state NEW,ESTABLISHED -j ACCEPT + - systemctl daemon-reload + - systemctl start mysql diff --git a/assets/modules-fabric/v26/cloud-config-container/mysql/main.tf b/assets/modules-fabric/v26/cloud-config-container/mysql/main.tf new file mode 100644 index 0000000..4e44c46 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/mysql/main.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cloud_config = templatefile(local.template, merge(var.config_variables, { + image = var.image + kms_config = var.kms_config + mysql_config = var.mysql_config + mysql_data_disk = var.mysql_data_disk + password = var.mysql_password + })) + template = ( + var.cloud_config == null + ? "${path.module}/cloud-config.yaml" + : var.cloud_config + ) +} diff --git a/assets/modules-fabric/v26/cloud-config-container/mysql/outputs.tf b/assets/modules-fabric/v26/cloud-config-container/mysql/outputs.tf new file mode 100644 index 0000000..7d8d416 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/mysql/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cloud_config" { + description = "Rendered cloud-config file to be passed as user-data instance metadata." + value = local.cloud_config +} diff --git a/assets/modules-fabric/v26/cloud-config-container/mysql/variables.tf b/assets/modules-fabric/v26/cloud-config-container/mysql/variables.tf new file mode 100644 index 0000000..52bb3db --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/mysql/variables.tf @@ -0,0 +1,61 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "cloud_config" { + description = "Cloud config template path. If null default will be used." + type = string + default = null +} + +variable "config_variables" { + description = "Additional variables used to render the cloud-config template." + type = map(any) + default = {} +} + +variable "image" { + description = "MySQL container image." + type = string + default = "mysql:5.7" +} + +variable "kms_config" { + description = "Optional KMS configuration to decrypt passed-in password. Leave null if a plaintext password is used." + type = object({ + project_id = string + keyring = string + location = string + key = string + }) + default = null +} + +variable "mysql_config" { + description = "MySQL configuration file content, if null container default will be used." + type = string + default = null +} + +variable "mysql_data_disk" { + description = "MySQL data disk name in /dev/disk/by-id/ including the google- prefix. If null the boot disk will be used for data." + type = string + default = null +} + +variable "mysql_password" { + description = "MySQL root password. If an encrypted password is set, use the kms_config variable to specify KMS configuration." + type = string +} diff --git a/assets/modules-fabric/v26/cloud-config-container/mysql/versions.tf b/assets/modules-fabric/v26/cloud-config-container/mysql/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/mysql/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/cloud-config-container/nginx-tls/README.md b/assets/modules-fabric/v26/cloud-config-container/nginx-tls/README.md new file mode 100644 index 0000000..15f2ffe --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/nginx-tls/README.md @@ -0,0 +1,54 @@ +# Containerized Nginx with self-signed TLS on Container Optimized OS + +This module manages a `cloud-config` configuration that starts a containerized Nginx with a self-signed TLS cert on Container Optimized OS. This can be useful if you need quickly a VM or instance group answering HTTPS for prototyping. + +The generated cloud config is rendered in the `cloud_config` output, and is meant to be used in instances or instance templates via the `user-data` metadata. + +## Example + +```hcl +module "cos-nginx-tls" { + source = "./fabric/modules/cloud-config-container/nginx-tls" +} + +module "vm-nginx-tls" { + source = "./fabric/modules/compute-vm" + project_id = "my-project" + zone = "europe-west8-b" + name = "cos-nginx-tls" + network_interfaces = [{ + network = "default" + subnetwork = "gce" + }] + metadata = { + user-data = module.cos-nginx-tls.cloud_config + google-logging-enabled = true + } + boot_disk = { + initialize_params = { + image = "projects/cos-cloud/global/images/family/cos-stable" + type = "pd-ssd" + size = 10 + } + } + tags = ["http-server", "https-server", "ssh"] +} +# tftest modules=1 resources=1 +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [files](variables.tf#L17) | Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null. | map(object({…})) | | {} | +| [hello](variables.tf#L28) | Behave like the nginx hello image by returning plain text informative responses. | bool | | true | +| [image](variables.tf#L35) | Nginx container image to use. | string | | "nginx:1.23.1" | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [cloud_config](outputs.tf#L17) | Rendered cloud-config file to be passed as user-data instance metadata. | | + + diff --git a/assets/modules-fabric/v26/cloud-config-container/nginx-tls/assets/cloud-config.yaml b/assets/modules-fabric/v26/cloud-config-container/nginx-tls/assets/cloud-config.yaml new file mode 100644 index 0000000..2b7ebe8 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/nginx-tls/assets/cloud-config.yaml @@ -0,0 +1,63 @@ +#cloud-config + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +users: + - name: nginx + uid: 2000 + +write_files: + - path: /var/lib/docker/daemon.json + permissions: "0644" + owner: root + content: | + { + "live-restore": true, + "storage-driver": "overlay2", + "log-opts": { + "max-size": "1024m" + } + } + # nginx container service + - path: /etc/systemd/system/nginx.service + permissions: "0644" + owner: root + content: | + [Unit] + Description=Start nginx container + After=gcr-online.target docker.socket + Wants=gcr-online.target docker.socket docker-events-collector.service + [Service] + Environment="HOME=/home/nginx" + ExecStart=/usr/bin/docker run --rm --name=nginx \ + --network host --pid host \ + -v /etc/nginx/conf.d:/etc/nginx/conf.d \ + -v /etc/ssl:/etc/ssl \ + ${image} + ExecStop=/usr/bin/docker stop nginx +%{ for k, v in files ~} + - path: ${k} + owner: ${v.owner} + permissions: "${v.permissions}" + content: | + ${indent(6, v.content)} +%{ endfor ~} + +runcmd: + - iptables -I INPUT 1 -p tcp -m tcp --dport 80 -m state --state NEW,ESTABLISHED -j ACCEPT + - iptables -I INPUT 1 -p tcp -m tcp --dport 443 -m state --state NEW,ESTABLISHED -j ACCEPT + - /var/run/nginx/customize.sh + - systemctl daemon-reload + - systemctl start nginx diff --git a/assets/modules-fabric/v26/cloud-config-container/nginx-tls/assets/customize.sh b/assets/modules-fabric/v26/cloud-config-container/nginx-tls/assets/customize.sh new file mode 100644 index 0000000..22b4006 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/nginx-tls/assets/customize.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FQDN=$(\ + curl -s -H "Metadata-Flavor: Google" \ + http://metadata/computeMetadata/v1/instance/hostname) +HOSTNAME=$(echo $FQDN | cut -d"." -f1) +openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 \ + -subj /CN=$HOSTNAME/ -addext "subjectAltName = DNS:$FQDN" \ + -keyout /etc/ssl/self-signed.key -out /etc/ssl/self-signed.crt +chgrp nginx /etc/ssl/self-signed.key -out /etc/ssl/self-signed.crt +sed -i "s/HOSTNAME/${HOSTNAME}/" /etc/nginx/conf.d/default.conf \ No newline at end of file diff --git a/assets/modules-fabric/v26/cloud-config-container/nginx-tls/assets/default.conf b/assets/modules-fabric/v26/cloud-config-container/nginx-tls/assets/default.conf new file mode 100644 index 0000000..2be98ff --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/nginx-tls/assets/default.conf @@ -0,0 +1,24 @@ +server { + listen 80; + listen 443 ssl; + server_name HOSTNAME; + ssl_certificate /etc/ssl/self-signed.crt; + ssl_certificate_key /etc/ssl/self-signed.key; + + location / { + {% if hello %} + default_type text/plain; + expires -1; + return 200 'Server address: $server_addr:$server_port\nServer name: $hostname\nDate: $time_local\nURI: $request_uri\nRequest ID: $request_id\n'; + {% else %} + root /usr/share/nginx/html; + index index.html index.htm; + {% endif %} + } + + error_page 500 502 503 504 /50x.html; + + location = /50x.html { + root /usr/share/nginx/html; + } +} \ No newline at end of file diff --git a/assets/modules-fabric/v26/cloud-config-container/nginx-tls/outputs.tf b/assets/modules-fabric/v26/cloud-config-container/nginx-tls/outputs.tf new file mode 100644 index 0000000..2acd83f --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/nginx-tls/outputs.tf @@ -0,0 +1,38 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cloud_config" { + description = "Rendered cloud-config file to be passed as user-data instance metadata." + value = templatefile("${path.module}/assets/cloud-config.yaml", { + files = merge( + { + "/var/run/nginx/customize.sh" = { + content = file("${path.module}/assets/customize.sh") + owner = "root" + permissions = "0744" + } + "/etc/nginx/conf.d/default.conf" = { + content = templatefile( + "${path.module}/assets/default.conf", { hello = var.hello } + ) + owner = "root" + permissions = "0644" + } + }, var.files + ) + image = var.image + }) +} diff --git a/assets/modules-fabric/v26/cloud-config-container/nginx-tls/variables.tf b/assets/modules-fabric/v26/cloud-config-container/nginx-tls/variables.tf new file mode 100644 index 0000000..f3cab58 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/nginx-tls/variables.tf @@ -0,0 +1,39 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "files" { + description = "Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null." + type = map(object({ + content = string + owner = optional(string, "root") + permissions = optional(string, "0644") + })) + default = {} + nullable = false +} + +variable "hello" { + description = "Behave like the nginx hello image by returning plain text informative responses." + type = bool + default = true + nullable = false +} + +variable "image" { + description = "Nginx container image to use." + type = string + default = "nginx:1.23.1" +} diff --git a/assets/modules-fabric/v26/cloud-config-container/nginx-tls/versions.tf b/assets/modules-fabric/v26/cloud-config-container/nginx-tls/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/nginx-tls/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/cloud-config-container/nginx/README.md b/assets/modules-fabric/v26/cloud-config-container/nginx/README.md new file mode 100644 index 0000000..dad3d55 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/nginx/README.md @@ -0,0 +1,76 @@ +# Containerized Nginx on Container Optimized OS + +This module manages a `cloud-config` configuration that starts a containerized [Nginx](https://nginx.org/en/) service on Container Optimized OS, using the [hello demo image](https://hub.docker.com/r/nginxdemos/hello/). + +The resulting `cloud-config` can be customized in a number of ways: + +- a custom Nginx configuration can be set in `/etc/nginx/conf.d` using the `nginx_config` variable +- additional files (eg for hosts or zone files) can be passed in via the `files` variable +- a completely custom `cloud-config` can be passed in via the `cloud_config` variable, and additional template variables can be passed in via `config_variables` + +The default instance configuration inserts iptables rules to allow traffic on port 80. + +Logging and monitoring are enabled via the [Google Cloud Logging agent](https://cloud.google.com/container-optimized-os/docs/how-to/logging) configured for the instance via the `google-logging-enabled` metadata property, and the [Node Problem Detector](https://cloud.google.com/container-optimized-os/docs/how-to/monitoring) service started by default on boot. + +The module renders the generated cloud config in the `cloud_config` output, to be used in instances or instance templates via the `user-data` metadata. + +For convenience during development or for simple use cases, the module can optionally manage a single instance via the `test_instance` variable. If the instance is not needed the `instance*tf` files can be safely removed. Refer to the [top-level README](../README.md) for more details on the included instance. + +## Examples + +### Default configuration + +This example will create a `cloud-config` that uses the module's defaults, creating a simple hello web server showing host name and request id. + +```hcl +module "cos-nginx" { + source = "./fabric/modules/cloud-config-container/nginx" +} + +module "vm-nginx-tls" { + source = "./fabric/modules/compute-vm" + project_id = "my-project" + zone = "europe-west8-b" + name = "cos-nginx" + network_interfaces = [{ + network = "default" + subnetwork = "gce" + }] + metadata = { + user-data = module.cos-nginx.cloud_config + google-logging-enabled = true + } + boot_disk = { + initialize_params = { + image = "projects/cos-cloud/global/images/family/cos-stable" + type = "pd-ssd" + size = 10 + } + } + tags = ["http-server", "ssh"] +} +# tftest modules=1 resources=1 +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [cloud_config](variables.tf#L17) | Cloud config template path. If null default will be used. | string | | null | +| [config_variables](variables.tf#L23) | Additional variables used to render the cloud-config and Nginx templates. | map(any) | | {} | +| [file_defaults](variables.tf#L29) | Default owner and permissions for files. | object({…}) | | {…} | +| [files](variables.tf#L41) | Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null. | map(object({…})) | | {} | +| [image](variables.tf#L51) | Nginx container image. | string | | "nginxdemos/hello:plain-text" | +| [nginx_config](variables.tf#L57) | Nginx configuration path, if null container default will be used. | string | | null | +| [runcmd_post](variables.tf#L63) | Extra commands to run after starting nginx. | list(string) | | [] | +| [runcmd_pre](variables.tf#L69) | Extra commands to run before starting nginx. | list(string) | | [] | +| [users](variables.tf#L75) | List of additional usernames to be created. | list(object({…})) | | […] | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [cloud_config](outputs.tf#L17) | Rendered cloud-config file to be passed as user-data instance metadata. | | + + diff --git a/assets/modules-fabric/v26/cloud-config-container/nginx/cloud-config.yaml b/assets/modules-fabric/v26/cloud-config-container/nginx/cloud-config.yaml new file mode 100644 index 0000000..f4d05bc --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/nginx/cloud-config.yaml @@ -0,0 +1,89 @@ +#cloud-config + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# https://hub.docker.com/r/nginx/nginx/ +# https://nginx.io/manual/toc/#installation + +users: + - name: nginx + uid: 2000 + %{ for user in users } + - name: ${user.username} + uid: ${user.uid} + %{ endfor } + +write_files: + - path: /var/lib/docker/daemon.json + permissions: 0644 + owner: root + content: | + { + "live-restore": true, + "storage-driver": "overlay2", + "log-opts": { + "max-size": "1024m" + } + } + + %{~ if nginx_config != null ~} + - path: /etc/nginx/conf.d/nginx.conf + permissions: 0644 + owner: root + content: | + ${indent(6, nginx_config)} + %{~ endif ~} + + # nginx container service + - path: /etc/systemd/system/nginx.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Start nginx container + After=gcr-online.target docker.socket + Wants=gcr-online.target docker.socket docker-events-collector.service + [Service] + Environment="HOME=/home/nginx" + ExecStartPre=/usr/bin/docker-credential-gcr configure-docker + ExecStart=/usr/bin/docker run --rm --name=nginx \ + --network host \ + %{~ if etc_mount ~} + -v /etc/nginx/conf.d:/etc/nginx/conf.d \ + %{~ endif ~} + ${image} + ExecStop=/usr/bin/docker stop nginx + + %{ for path, data in files } + - path: ${path} + owner: ${lookup(data, "owner", "root")} + permissions: ${lookup(data, "permissions", "0644")} + content: | + ${indent(6, data.content)} + %{ endfor } + +bootcmd: + - systemctl start node-problem-detector + +runcmd: +%{ for cmd in runcmd_pre ~} + - ${cmd} +%{ endfor ~} + - iptables -I INPUT 1 -p tcp -m tcp --dport 80 -m state --state NEW,ESTABLISHED -j ACCEPT + - systemctl daemon-reload + - systemctl start nginx +%{ for cmd in runcmd_post ~} + - ${cmd} +%{ endfor ~} diff --git a/assets/modules-fabric/v26/cloud-config-container/nginx/main.tf b/assets/modules-fabric/v26/cloud-config-container/nginx/main.tf new file mode 100644 index 0000000..39c5993 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/nginx/main.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cloud_config = templatefile(local.template, merge(var.config_variables, { + etc_mount = ( + var.nginx_config != null || length([ + for name in keys(var.files) : + name if substr(name, 0, 18) == "/etc/nginx/conf.d/" + ]) > 0 + ) + files = local.files + users = var.users + image = var.image + nginx_config = (var.nginx_config == null ? null : templatefile( + var.nginx_config, var.config_variables + )) + runcmd_pre = var.runcmd_pre + runcmd_post = var.runcmd_post + })) + files = { + for path, attrs in var.files : path => { + content = attrs.content, + owner = attrs.owner == null ? var.file_defaults.owner : attrs.owner, + permissions = ( + attrs.permissions == null + ? var.file_defaults.permissions + : attrs.permissions + ) + } + } + template = ( + var.cloud_config == null + ? "${path.module}/cloud-config.yaml" + : var.cloud_config + ) +} diff --git a/assets/modules-fabric/v26/cloud-config-container/nginx/outputs.tf b/assets/modules-fabric/v26/cloud-config-container/nginx/outputs.tf new file mode 100644 index 0000000..7d8d416 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/nginx/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cloud_config" { + description = "Rendered cloud-config file to be passed as user-data instance metadata." + value = local.cloud_config +} diff --git a/assets/modules-fabric/v26/cloud-config-container/nginx/variables.tf b/assets/modules-fabric/v26/cloud-config-container/nginx/variables.tf new file mode 100644 index 0000000..973baff --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/nginx/variables.tf @@ -0,0 +1,83 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "cloud_config" { + description = "Cloud config template path. If null default will be used." + type = string + default = null +} + +variable "config_variables" { + description = "Additional variables used to render the cloud-config and Nginx templates." + type = map(any) + default = {} +} + +variable "file_defaults" { + description = "Default owner and permissions for files." + type = object({ + owner = string + permissions = string + }) + default = { + owner = "root" + permissions = "0644" + } +} + +variable "files" { + description = "Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null." + type = map(object({ + content = string + owner = string + permissions = string + })) + default = {} +} + +variable "image" { + description = "Nginx container image." + type = string + default = "nginxdemos/hello:plain-text" +} + +variable "nginx_config" { + description = "Nginx configuration path, if null container default will be used." + type = string + default = null +} + +variable "runcmd_post" { + description = "Extra commands to run after starting nginx." + type = list(string) + default = [] +} + +variable "runcmd_pre" { + description = "Extra commands to run before starting nginx." + type = list(string) + default = [] +} + +variable "users" { + description = "List of additional usernames to be created." + type = list(object({ + username = string, + uid = number, + })) + default = [ + ] +} diff --git a/assets/modules-fabric/v26/cloud-config-container/nginx/versions.tf b/assets/modules-fabric/v26/cloud-config-container/nginx/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/nginx/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/cloud-config-container/simple-nva/README.md b/assets/modules-fabric/v26/cloud-config-container/simple-nva/README.md new file mode 100644 index 0000000..3fb279c --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/simple-nva/README.md @@ -0,0 +1,176 @@ +# Google Simple NVA Module + +The module allows you to create Network Virtual Appliances (NVAs) as a stub for future appliances deployments. + +This NVAs can be used to interconnect up to 8 VPCs. + +The NVAs run [Container-Optimized OS (COS)](https://cloud.google.com/container-optimized-os/docs). COS is a Linux-based OS designed for running containers. By default, it only allows SSH ingress connections. To see the exact host firewall configuration, run `sudo iptables -L -v`. More info available in the [official](https://cloud.google.com/container-optimized-os/docs/how-to/firewall) documentation. + +To configure the firewall, you can either +- use the [open_ports](variables.tf#L84) variable +- for a thiner grain control, pass a custom bash script at startup with iptables commands + +## Examples + +### Simple example + +```hcl +locals { + network_interfaces = [ + { + addresses = null + name = "dev" + nat = false + network = "dev_vpc_self_link" + routes = ["10.128.0.0/9"] + subnetwork = "dev_vpc_nva_subnet_self_link" + }, + { + addresses = null + name = "prod" + nat = false + network = "prod_vpc_self_link" + routes = ["10.0.0.0/9"] + subnetwork = "prod_vpc_nva_subnet_self_link" + } + ] +} + +module "cos-nva" { + source = "./fabric/modules/cloud-config-container/simple-nva" + enable_health_checks = true + network_interfaces = local.network_interfaces + # files = { + # "/var/lib/cloud/scripts/per-boot/firewall-rules.sh" = { + # content = file("./your_path/to/firewall-rules.sh") + # owner = "root" + # permissions = 0700 + # } + # } +} + +module "vm" { + source = "./fabric/modules/compute-vm" + project_id = "my-project" + zone = "europe-west8-b" + name = "cos-nva" + network_interfaces = local.network_interfaces + metadata = { + user-data = module.cos-nva.cloud_config + google-logging-enabled = true + } + boot_disk = { + initialize_params = { + image = "projects/cos-cloud/global/images/family/cos-stable" + type = "pd-ssd" + size = 10 + } + } + tags = ["nva", "ssh"] +} +# tftest modules=1 resources=1 +``` + +### Example with advanced routing capabilities (FRR) + +The sample code brings up [FRRouting](https://frrouting.org/) container. + +``` +# tftest-file id=frr_conf path=./frr.conf +# Example frr.conmf file + +log syslog informational +no ipv6 forwarding +router bgp 65001 + neighbor 10.128.0.2 remote-as 65002 +line vty +``` + +Following code assumes a file in the same folder named frr.conf exists. + +```hcl +locals { + network_interfaces = [ + { + addresses = null + name = "dev" + nat = false + network = "dev_vpc_self_link" + routes = ["10.128.0.0/9"] + subnetwork = "dev_vpc_nva_subnet_self_link" + enable_masquerading = true + non_masq_cidrs = ["10.0.0.0/8"] + }, + { + addresses = null + name = "prod" + nat = false + network = "prod_vpc_self_link" + routes = ["10.0.0.0/9"] + subnetwork = "prod_vpc_nva_subnet_self_link" + } + ] +} + +module "cos-nva" { + source = "./fabric/modules/cloud-config-container/simple-nva" + enable_health_checks = true + network_interfaces = local.network_interfaces + frr_config = { config_file = "./frr.conf", daemons_enabled = ["bgpd"] } + run_cmds = ["ls -l"] +} + +module "vm" { + source = "./fabric/modules/compute-vm" + project_id = "my-project" + zone = "europe-west8-b" + name = "cos-nva" + network_interfaces = local.network_interfaces + metadata = { + user-data = module.cos-nva.cloud_config + google-logging-enabled = true + } + boot_disk = { + image = "projects/cos-cloud/global/images/family/cos-stable" + type = "pd-ssd" + size = 10 + } + tags = ["nva", "ssh"] +} +# tftest modules=1 resources=1 files=frr_conf +``` + +The FRR container is managed as a systemd service. To interact with the service, use the standard systemd commands: `sudo systemctl {start|stop|restart} frr`. + +To interact with the FRR CLI run: + +```shell +# get the container ID +CONTAINER_ID =`sudo docker ps -a -q` +sudo docker exec -it $CONTAINER_ID vtysh +``` + +Check FRR running configuration with `show running-config` from vtysh. Please always refer to the official documentation for more information how to deal with vtysh and useful commands. + +Sample frr.conf file is based on the documentation available [here](https://docs.frrouting.org/en/latest/basic.html). It configures a BGP service with ASN 65001 on FRR container establishing a BGP session with a remote neighbor with IP address 10.128.0.2 and ASN 65002. Check BGP status for FRR with `show bgp summary` from vtysh. + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [network_interfaces](variables.tf#L75) | Network interfaces configuration. | list(object({…})) | ✓ | | +| [cloud_config](variables.tf#L17) | Cloud config template path. If null default will be used. | string | | null | +| [enable_health_checks](variables.tf#L23) | Configures routing to enable responses to health check probes. | bool | | false | +| [files](variables.tf#L29) | Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null. | map(object({…})) | | {} | +| [frr_config](variables.tf#L39) | FRR configuration for container running on the NVA. | object({…}) | | null | +| [open_ports](variables.tf#L84) | Optional firewall ports to open. | object({…}) | | {…} | +| [run_cmds](variables.tf#L96) | Optional cloud init run commands to execute. | list(string) | | [] | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [cloud_config](outputs.tf#L17) | Rendered cloud-config file to be passed as user-data instance metadata. | | + + diff --git a/assets/modules-fabric/v26/cloud-config-container/simple-nva/cloud-config.yaml b/assets/modules-fabric/v26/cloud-config-container/simple-nva/cloud-config.yaml new file mode 100644 index 0000000..328ace7 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/simple-nva/cloud-config.yaml @@ -0,0 +1,74 @@ +#cloud-config + +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +write_files: +%{ for path, data in files } + - path: ${path} + owner: ${lookup(data, "owner", "root")} + permissions: ${lookup(data, "permissions", "0644")} + content: | + ${indent(6, data.content)} +%{ endfor } + + - path: /etc/systemd/system/routing.service + permissions: 0644 + owner: root + content: | + [Install] + WantedBy=multi-user.target + [Unit] + Description=Start routing + After=network-online.target + Wants=network-online.target + [Service] + RemainAfterExit=true + ExecStart=/bin/sh -c "/var/run/nva/start-routing.sh" + - path: /var/run/nva/start-routing.sh + permissions: 0744 + owner: root + content: | + iptables --policy FORWARD ACCEPT +%{ for interface in network_interfaces ~} +%{ if enable_health_checks ~} + /var/run/nva/policy_based_routing.sh ${interface.name} &>/dev/null & +%{ endif ~} +%{ if interface.enable_masquerading ~} +%{ for cidr in interface.non_masq_cidrs ~} + iptables -t nat -A POSTROUTING -o ${interface.name} -d ${cidr} -j ACCEPT +%{ endfor ~} + iptables -t nat -A POSTROUTING -o ${interface.name} -j MASQUERADE +%{ endif ~} +%{ for route in interface.routes ~} + ip route add ${route} via `curl http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/${interface.number}/gateway -H "Metadata-Flavor:Google"` dev ${interface.name} +%{ endfor ~} +%{ endfor ~} +%{ for port in open_tcp_ports ~} + iptables -A INPUT -p tcp --dport ${port} -j ACCEPT +%{ endfor ~} +%{ for port in open_udp_ports ~} + iptables -A INPUT -p udp --dport ${port} -j ACCEPT +%{ endfor ~} + +bootcmd: + - systemctl start node-problem-detector + +runcmd: + - systemctl daemon-reload + - systemctl enable routing + - systemctl start routing +%{ for cmd in run_cmds ~} + - ${cmd} +%{ endfor ~} diff --git a/assets/modules-fabric/v26/cloud-config-container/simple-nva/files/frr/daemons b/assets/modules-fabric/v26/cloud-config-container/simple-nva/files/frr/daemons new file mode 100644 index 0000000..0a388df --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/simple-nva/files/frr/daemons @@ -0,0 +1,65 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +zebra=${zebra_enabled} +bgpd=${bgpd_enabled} +ospfd=${ospfd_enabled} +ospf6d=${ospf6d_enabled} +ripd=${ripd_enabled} +ripngd=${ripngd_enabled} +isisd=${isisd_enabled} +pimd=${pimd_enabled} +ldpd=${ldpd_enabled} +nhrpd=${nhrpd_enabled} +eigrpd=${eigrpd_enabled} +babeld=${babeld_enabled} +sharpd=${sharpd_enabled} +staticd=${staticd_enabled} +pbrd=${pbrd_enabled} +bfdd=${bfdd_enabled} +fabricd=${fabricd_enabled} + +# If this option is set the /etc/init.d/frr script automatically loads +# the config via "vtysh -b" when the servers are started. +# Check /etc/pam.d/frr if you intend to use "vtysh"! + +vtysh_enable=yes +zebra_options=" -A 127.0.0.1 -s 90000000" +bgpd_options=" -A 127.0.0.1" +ospfd_options=" --daemon -A 127.0.0.1" +ospf6d_options=" --daemon -A ::1" +ripd_options=" --daemon -A 127.0.0.1" +ripngd_options=" --daemon -A ::1" +isisd_options=" --daemon -A 127.0.0.1" +pimd_options=" --daemon -A 127.0.0.1" +ldpd_options=" --daemon -A 127.0.0.1" +nhrpd_options=" --daemon -A 127.0.0.1" +eigrpd_options=" --daemon -A 127.0.0.1" +babeld_options=" --daemon -A 127.0.0.1" +sharpd_options=" --daemon -A 127.0.0.1" +staticd_options=" --daemon -A 127.0.0.1" +pbrd_options=" --daemon -A 127.0.0.1" +bfdd_options=" --daemon -A 127.0.0.1" +fabricd_options=" --daemon -A 127.0.0.1" + +#MAX_FDS=1024 +# The list of daemons to watch is automatically generated by the init script. +#watchfrr_options="" + +# for debugging purposes, you can specify a "wrap" command to start instead +# of starting the daemon directly, e.g. to use valgrind on ospfd: +# ospfd_wrap="/usr/bin/valgrind" +# or you can use "all_wrap" for all daemons, e.g. to use perf record: +# all_wrap="/usr/bin/perf record --call-graph -" +# the normal daemon command is added to this at the end. diff --git a/assets/modules-fabric/v26/cloud-config-container/simple-nva/files/frr/frr.service b/assets/modules-fabric/v26/cloud-config-container/simple-nva/files/frr/frr.service new file mode 100644 index 0000000..a560602 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/simple-nva/files/frr/frr.service @@ -0,0 +1,27 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[Unit] +Description=Start FRR container +After=gcr-online.target docker.socket +Wants=gcr-online.target docker.socket docker-events-collector.service +[Service] +Environment="HOME=/home/frr" +ExecStart=/usr/bin/docker run --rm --name=frr \ +--privileged \ +--network host \ +-v /etc/frr:/etc/frr \ +frrouting/frr +ExecStop=/usr/bin/docker stop frr +ExecStopPost=/usr/bin/docker rm frr diff --git a/assets/modules-fabric/v26/cloud-config-container/simple-nva/files/frr/vtysh.conf b/assets/modules-fabric/v26/cloud-config-container/simple-nva/files/frr/vtysh.conf new file mode 100644 index 0000000..48e9580 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/simple-nva/files/frr/vtysh.conf @@ -0,0 +1,16 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a sample file used to remove warnings +# when users open the vtysh console. diff --git a/assets/modules-fabric/v26/cloud-config-container/simple-nva/files/ipprefix_by_netmask.sh b/assets/modules-fabric/v26/cloud-config-container/simple-nva/files/ipprefix_by_netmask.sh new file mode 100644 index 0000000..1694382 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/simple-nva/files/ipprefix_by_netmask.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# https://stackoverflow.com/questions/50413579/bash-convert-netmask-in-cidr-notation +c=0 x=0$(printf '%o' ${1//./ }) +while [ $x -gt 0 ]; do + let c+=$((x % 2)) 'x>>=1' +done +echo $c diff --git a/assets/modules-fabric/v26/cloud-config-container/simple-nva/files/policy_based_routing.sh b/assets/modules-fabric/v26/cloud-config-container/simple-nva/files/policy_based_routing.sh new file mode 100644 index 0000000..008aa0b --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/simple-nva/files/policy_based_routing.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +IF_NAME=$1 +IF_NUMBER=$(echo $IF_NAME | sed -e s/eth//) +IF_GW=$(curl http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/$IF_NUMBER/gateway -H "Metadata-Flavor: Google") +IF_IP=$(curl http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/$IF_NUMBER/ip -H "Metadata-Flavor: Google") +IF_NETMASK=$(curl http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/$IF_NUMBER/subnetmask -H "Metadata-Flavor: Google") +IF_IP_PREFIX=$(/var/run/nva/ipprefix_by_netmask.sh $IF_NETMASK) + +# Sleep while there's no load balancer IP route for this IF +while true +do + IPS_LB_STR=$(ip r show table local | grep "$IF_NAME proto 66" | cut -f 2 -d " " | tr -s '\n' ' ') + IPS_LB=($IPS_LB_STR) + for IP in "${IPS_LB[@]}" + do + # Configure hc routing table if not available for this network interface + grep -qxF "$((200 + $IF_NUMBER)) hc-$IF_NAME" /etc/iproute2/rt_tables || { + echo "$((200 + $IF_NUMBER)) hc-$IF_NAME" >>/etc/iproute2/rt_tables + ip route add $IF_GW src $IF_IP dev $IF_NAME table hc-$IF_NAME + ip route add default via $IF_GW dev $IF_NAME table hc-$IF_NAME + } + + # configure PBR route for LB + ip rule list | grep -qF "$IP" || ip rule add from $IP/32 table hc-$IF_NAME + done + + # remove previously configure PBR for old LB removed from network interface + # first get list of PBR on this network interface and retrieve LB IP addresses + PBR_LB_IPS_STR=$(ip rule list | grep "hc-$IF_NAME" | cut -f 2 -d " " | tr -s '\n' ' ') + PBR_LB_IPS=($PBR_LB_IPS_STR) + + # iterate over PBR LB IP addresses + for PBR_IP in "${PBR_LB_IPS[@]}" + do + # check if the PBR LB IP belongs to the current array of LB IPs attached to the + # network interface, if not delete the corresponding PBR rule + if [ -z "$IPS_LB" ] || ! echo ${IPS_LB[@]} | grep --quiet "$PBR_IP" ; then + ip rule del from $PBR_IP + fi + done + sleep 2 +done diff --git a/assets/modules-fabric/v26/cloud-config-container/simple-nva/main.tf b/assets/modules-fabric/v26/cloud-config-container/simple-nva/main.tf new file mode 100644 index 0000000..7fb0f16 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/simple-nva/main.tf @@ -0,0 +1,147 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + _files = merge( + { + "/var/run/nva/ipprefix_by_netmask.sh" = { + content = file("${path.module}/files/ipprefix_by_netmask.sh") + owner = "root" + permissions = "0744" + } + "/var/run/nva/policy_based_routing.sh" = { + content = file("${path.module}/files/policy_based_routing.sh") + owner = "root" + permissions = "0744" + } + }, { + for path, attrs in var.files : path => { + content = attrs.content, + owner = attrs.owner, + permissions = attrs.permissions + } + }, + try(var.frr_config != null, false) ? { + "/etc/frr/daemons" = { + content = templatefile("${path.module}/files/frr/daemons", local._frr_daemons_enabled) + owner = "root" + permissions = "0744" + } + "/etc/frr/frr.conf" = { + # content can either be the path to the config file or the config string + content = try(file(var.frr_config.config_file), var.frr_config.config_file) + owner = "root" + permissions = "0744" + } + "/etc/frr/vtysh.conf" = { + # content can either be the path to the config file or the config string + content = file("${path.module}/files/frr/vtysh.conf") + owner = "root" + permissions = "0644" + } + "/etc/profile.d/00-aliases.sh" = { + content = "alias vtysh='sudo docker exec -it frr sh -c vtysh'" + owner = "root" + permissions = "0644" + } + "/etc/systemd/system/frr.service" = { + content = file("${path.module}/files/frr/frr.service") + owner = "root" + permissions = "0644" + } + "/var/lib/docker/daemon.json" = { + content = < contains(var.frr_config.daemons_enabled, daemon) ? "yes" : "no" + }, {}) + + _network_interfaces = [ + for index, interface in var.network_interfaces : { + name = "eth${index}" + number = index + routes = interface.routes + enable_masquerading = interface.enable_masquerading != null ? interface.enable_masquerading : false + non_masq_cidrs = interface.non_masq_cidrs != null ? interface.non_masq_cidrs : [] + } + ] + + _run_cmds = ( + try(var.frr_config != null, false) + ? concat(["systemctl start frr"], var.run_cmds) + : var.run_cmds + ) + + _tcp_ports = concat(flatten(try( + [ + for daemon, ports in local._frr_daemons : contains(var.frr_config.daemons_enabled, daemon) ? ports.tcp : [] + ], [])), var.open_ports.tcp) + + _template = ( + var.cloud_config == null + ? "${path.module}/cloud-config.yaml" + : var.cloud_config + ) + + _udp_ports = concat(flatten(try( + [ + for daemon, ports in local._frr_daemons : contains(var.frr_config.daemons_enabled, daemon) ? ports.udp : [] + ], [])), var.open_ports.udp) + + cloud_config = templatefile(local._template, { + enable_health_checks = var.enable_health_checks + files = local._files + network_interfaces = local._network_interfaces + open_tcp_ports = local._tcp_ports + open_udp_ports = local._udp_ports + run_cmds = local._run_cmds + }) +} diff --git a/assets/modules-fabric/v26/cloud-config-container/simple-nva/outputs.tf b/assets/modules-fabric/v26/cloud-config-container/simple-nva/outputs.tf new file mode 100644 index 0000000..54942c1 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/simple-nva/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cloud_config" { + description = "Rendered cloud-config file to be passed as user-data instance metadata." + value = local.cloud_config +} diff --git a/assets/modules-fabric/v26/cloud-config-container/simple-nva/variables.tf b/assets/modules-fabric/v26/cloud-config-container/simple-nva/variables.tf new file mode 100644 index 0000000..20eecd0 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/simple-nva/variables.tf @@ -0,0 +1,100 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "cloud_config" { + description = "Cloud config template path. If null default will be used." + type = string + default = null +} + +variable "enable_health_checks" { + description = "Configures routing to enable responses to health check probes." + type = bool + default = false +} + +variable "files" { + description = "Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null." + type = map(object({ + content = string + owner = string + permissions = string + })) + default = {} +} + +variable "frr_config" { + description = "FRR configuration for container running on the NVA." + type = object({ + config_file = string + daemons_enabled = optional(list(string)) + }) + default = null + validation { + condition = try(alltrue([ + for daemon in var.frr_config.daemons_enabled : contains([ + "zebra", + "bgpd", + "ospfd", + "ospf6d", + "ripd", + "ripngd", + "isisd", + "pimd", + "ldpd", + "nhrpd", + "eigrpd", + "babeld", + "sharpd", + "staticd", + "pbrd", + "bfdd", + "fabricd" + ], daemon) + ]), true) + error_message = < + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [allow](variables.tf#L18) | List of domains Squid will allow connections to. | list(string) | | [] | +| [clients](variables.tf#L24) | List of CIDR ranges from which Squid will allow connections. | list(string) | | [] | +| [cloud_config](variables.tf#L30) | Cloud config template path. If null default will be used. | string | | null | +| [config_variables](variables.tf#L36) | Additional variables used to render the cloud-config and Squid templates. | map(any) | | {} | +| [default_action](variables.tf#L42) | Default action for domains not matching neither the allow or deny lists. | string | | "deny" | +| [deny](variables.tf#L52) | List of domains Squid will deny connections to. | list(string) | | [] | +| [file_defaults](variables.tf#L58) | Default owner and permissions for files. | object({…}) | | {…} | +| [files](variables.tf#L70) | Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null. | map(object({…})) | | {} | +| [squid_config](variables.tf#L80) | Squid configuration path, if null default will be used. | string | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [cloud_config](outputs.tf#L17) | Rendered cloud-config file to be passed as user-data instance metadata. | | + + diff --git a/assets/modules-fabric/v26/cloud-config-container/squid/cloud-config.yaml b/assets/modules-fabric/v26/cloud-config-container/squid/cloud-config.yaml new file mode 100644 index 0000000..5ba6e98 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/squid/cloud-config.yaml @@ -0,0 +1,92 @@ +#cloud-config + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +users: +- name: squid + uid: 2000 + +write_files: + - path: /var/lib/docker/daemon.json + permissions: 0644 + owner: root + content: | + { + "live-restore": true, + "storage-driver": "overlay2", + "log-opts": { + "max-size": "1024m" + } + } + + - path: /etc/squid/squid.conf + permissions: 0644 + owner: root + content: | + ${indent(6, squid_config)} + + - path: /etc/squid/allowlist.txt + permissions: 0644 + owner: root + content: | + ${indent(6, join("\n", allow))} + + - path: /etc/squid/denylist.txt + permissions: 0644 + owner: root + content: | + ${indent(6, join("\n", deny))} + + - path: /etc/squid/clients.txt + permissions: 0644 + owner: root + content: | + ${indent(6, join("\n", clients))} + + # squid container service + - path: /etc/systemd/system/squid.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Start squid container + After=gcr-online.target docker.socket + Wants=gcr-online.target docker.socket docker-events-collector.service + + [Service] + Environment="HOME=/home/squid" + ExecStartPre=/usr/bin/docker-credential-gcr configure-docker + ExecStart=/usr/bin/docker run --rm --name=squid \ + --network host \ + -v /etc/squid:/etc/squid \ + gcr.io/pso-cft-fabric/squid:20221116 + ExecStop=/usr/bin/docker stop squid + ExecStopPost=/usr/bin/docker rm squid + + %{ for path, data in files } + - path: ${path} + owner: ${lookup(data, "owner", "root")} + permissions: ${lookup(data, "permissions", "0644")} + content: | + ${indent(4, data.content)} + %{ endfor } + +bootcmd: + - systemctl start node-problem-detector + +runcmd: + - iptables -I INPUT 1 -p tcp -m tcp --dport 3128 -m state --state NEW,ESTABLISHED -j ACCEPT + - systemctl daemon-reload + - systemctl start squid diff --git a/assets/modules-fabric/v26/cloud-config-container/squid/docker/Dockerfile b/assets/modules-fabric/v26/cloud-config-container/squid/docker/Dockerfile new file mode 100644 index 0000000..2ae03a4 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/squid/docker/Dockerfile @@ -0,0 +1,38 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM debian:buster-slim + +ENV SQUID_VERSION=4.6 \ + SQUID_CACHE_DIR=/var/spool/squid \ + SQUID_LOG_DIR=/var/log/squid \ + SQUID_PID_DIR=/var/run/squid \ + SQUID_USER=proxy + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y squid=${SQUID_VERSION}* \ + && rm -rf /var/lib/apt/lists/* + +COPY entrypoint.sh /sbin/entrypoint.sh +RUN chmod 755 /sbin/entrypoint.sh + +# Create the PID file directory as root, as the non-privileged user squid is not +# allowed to write in /var/run. +RUN mkdir -p ${SQUID_PID_DIR} \ + && chown ${SQUID_USER}:${SQUID_USER} ${SQUID_PID_DIR} + +USER ${SQUID_USER} + +EXPOSE 3128/tcp +ENTRYPOINT ["/sbin/entrypoint.sh"] diff --git a/assets/modules-fabric/v26/cloud-config-container/squid/docker/cloudbuild.yaml b/assets/modules-fabric/v26/cloud-config-container/squid/docker/cloudbuild.yaml new file mode 100644 index 0000000..aca00b9 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/squid/docker/cloudbuild.yaml @@ -0,0 +1,30 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# In this directory, run the following command to build this builder. +# $ gcloud builds submit . + +steps: + - name: "gcr.io/cloud-builders/docker" + args: + - build + - --tag=gcr.io/$PROJECT_ID/squid:${_IMAGE_VERSION} + - --tag=gcr.io/$PROJECT_ID/squid:latest + - . + +substitutions: + _IMAGE_VERSION: "20221116" +images: + - "gcr.io/$PROJECT_ID/squid:${_IMAGE_VERSION}" + - "gcr.io/$PROJECT_ID/squid:latest" diff --git a/assets/modules-fabric/v26/cloud-config-container/squid/docker/entrypoint.sh b/assets/modules-fabric/v26/cloud-config-container/squid/docker/entrypoint.sh new file mode 100644 index 0000000..880eaf3 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/squid/docker/entrypoint.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This container is based off Sameer Naik's existing squid container. +# https://github.com/sameersbn/docker-squid + +set -e + +create_log_dir() { + mkdir -p ${SQUID_LOG_DIR} + chmod -R 755 ${SQUID_LOG_DIR} + chown -R ${SQUID_USER}:${SQUID_USER} ${SQUID_LOG_DIR} +} + +create_cache_dir() { + mkdir -p ${SQUID_CACHE_DIR} + chown -R ${SQUID_USER}:${SQUID_USER} ${SQUID_CACHE_DIR} +} + +create_log_dir +create_cache_dir + +# allow arguments to be passed to squid +if [[ ${1:0:1} = '-' ]]; then + EXTRA_ARGS="$@" + set -- +elif [[ ${1} == squid || ${1} == $(which squid) ]]; then + EXTRA_ARGS="${@:2}" + set -- +fi + +# default behaviour is to launch squid +if [[ -z ${1} ]]; then + if [[ ! -d ${SQUID_CACHE_DIR}/00 ]]; then + echo "Initializing cache..." + $(which squid) -N -f /etc/squid/squid.conf -z + fi + echo "Starting squid..." + exec $(which squid) -f /etc/squid/squid.conf -NYCd 1 ${EXTRA_ARGS} +else + exec "$@" +fi diff --git a/assets/modules-fabric/v26/cloud-config-container/squid/main.tf b/assets/modules-fabric/v26/cloud-config-container/squid/main.tf new file mode 100644 index 0000000..ad895c1 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/squid/main.tf @@ -0,0 +1,47 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cloud_config = templatefile(local.template, merge(local.config_variables, { + squid_config = templatefile(local.squid_config, local.config_variables) + files = local.files + })) + squid_config = ( + var.squid_config == null ? "${path.module}/squid.conf" : var.squid_config + ) + files = { + for path, attrs in var.files : path => { + content = attrs.content, + owner = attrs.owner == null ? var.file_defaults.owner : attrs.owner, + permissions = ( + attrs.permissions == null + ? var.file_defaults.permissions + : attrs.permissions + ) + } + } + template = ( + var.cloud_config == null + ? "${path.module}/cloud-config.yaml" + : var.cloud_config + ) + config_variables = merge(var.config_variables, { + allow = var.allow + deny = var.deny + clients = var.clients + default_action = var.default_action + }) +} diff --git a/assets/modules-fabric/v26/cloud-config-container/squid/outputs.tf b/assets/modules-fabric/v26/cloud-config-container/squid/outputs.tf new file mode 100644 index 0000000..7d8d416 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/squid/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cloud_config" { + description = "Rendered cloud-config file to be passed as user-data instance metadata." + value = local.cloud_config +} diff --git a/assets/modules-fabric/v26/cloud-config-container/squid/squid.conf b/assets/modules-fabric/v26/cloud-config-container/squid/squid.conf new file mode 100644 index 0000000..4800257 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/squid/squid.conf @@ -0,0 +1,52 @@ +# bind to port 3128 +http_port 0.0.0.0:3128 + +# only proxy, don't cache +cache deny all + +# redirect all logs to /dev/stdout +logfile_rotate 0 +cache_log stdio:/dev/stdout +access_log stdio:/dev/stdout +cache_store_log stdio:/dev/stdout + +pid_filename /var/run/squid/squid.pid + +acl ssl_ports port 443 +acl safe_ports port 80 +acl safe_ports port 443 +acl CONNECT method CONNECT +acl to_metadata dst 169.254.169.254 + +# read client CIDR ranges from clients.txt +acl clients src "/etc/squid/clients.txt" + +# read allowed domains from allowlist.txt +acl allowlist dstdomain "/etc/squid/allowlist.txt" + +# read denied domains from denylist.txt +acl denylist dstdomain "/etc/squid/denylist.txt" + +# deny access to anything other than ports 80 and 443 +http_access deny !safe_ports + +# deny CONNECT if connection is not using ssl +http_access deny CONNECT !ssl_ports + +# deny access to cachemgr +http_access deny manager + +# deny access to localhost through the proxy +http_access deny to_localhost + +# deny access to the local metadata server through the proxy +http_access deny to_metadata + +# deny connection from allowed clients to any denied domains +http_access deny clients denylist + +# allow connection from allowed clients only to the allowed domains +http_access allow clients allowlist + +# deny everything else +http_access ${default_action} all diff --git a/assets/modules-fabric/v26/cloud-config-container/squid/variables.tf b/assets/modules-fabric/v26/cloud-config-container/squid/variables.tf new file mode 100644 index 0000000..b770882 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/squid/variables.tf @@ -0,0 +1,84 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +variable "allow" { + description = "List of domains Squid will allow connections to." + type = list(string) + default = [] +} + +variable "clients" { + description = "List of CIDR ranges from which Squid will allow connections." + type = list(string) + default = [] +} + +variable "cloud_config" { + description = "Cloud config template path. If null default will be used." + type = string + default = null +} + +variable "config_variables" { + description = "Additional variables used to render the cloud-config and Squid templates." + type = map(any) + default = {} +} + +variable "default_action" { + description = "Default action for domains not matching neither the allow or deny lists." + type = string + default = "deny" + validation { + condition = var.default_action == "deny" || var.default_action == "allow" + error_message = "Default action must be allow or deny." + } +} + +variable "deny" { + description = "List of domains Squid will deny connections to." + type = list(string) + default = [] +} + +variable "file_defaults" { + description = "Default owner and permissions for files." + type = object({ + owner = string + permissions = string + }) + default = { + owner = "root" + permissions = "0644" + } +} + +variable "files" { + description = "Map of extra files to create on the instance, path as key. Owner and permissions will use defaults if null." + type = map(object({ + content = string + owner = string + permissions = string + })) + default = {} +} + +variable "squid_config" { + description = "Squid configuration path, if null default will be used." + type = string + default = null +} diff --git a/assets/modules-fabric/v26/cloud-config-container/squid/versions.tf b/assets/modules-fabric/v26/cloud-config-container/squid/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-config-container/squid/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/cloud-function-v1/README.md b/assets/modules-fabric/v26/cloud-function-v1/README.md new file mode 100644 index 0000000..5fc72ca --- /dev/null +++ b/assets/modules-fabric/v26/cloud-function-v1/README.md @@ -0,0 +1,236 @@ +# Cloud Function Module (V1) + +Cloud Function management, with support for IAM roles and optional bucket creation. + +The GCS object used for deployment uses a hash of the bundle zip contents in its name, which ensures change tracking and avoids recreating the function if the GCS object is deleted and needs recreating. + +## TODO + +- [ ] add support for `source_repository` + +## Examples + +### HTTP trigger + +This deploys a Cloud Function with an HTTP endpoint, using a pre-existing GCS bucket for deployment, setting the service account to the Cloud Function default one, and delegating access control to the containing project. + +```hcl +module "cf-http" { + source = "./fabric/modules/cloud-function-v1" + project_id = "my-project" + name = "test-cf-http" + bucket_name = "test-cf-bundles" + bundle_config = { + source_dir = "fabric/assets/" + output_path = "bundle.zip" + } +} +# tftest modules=1 resources=2 +``` + +### PubSub and non-HTTP triggers + +Other trigger types other than HTTP are configured via the `trigger_config` variable. This example shows a PubSub trigger. + +```hcl +module "cf-http" { + source = "./fabric/modules/cloud-function-v1" + project_id = "my-project" + name = "test-cf-http" + bucket_name = "test-cf-bundles" + bundle_config = { + source_dir = "fabric/assets/" + output_path = "bundle.zip" + } + trigger_config = { + event = "google.pubsub.topic.publish" + resource = "local.my-topic" + } +} +# tftest modules=1 resources=2 +``` + +### Controlling HTTP access + +To allow anonymous access to the function, grant the `roles/cloudfunctions.invoker` role to the special `allUsers` identifier. Use specific identities (service accounts, groups, etc.) instead of `allUsers` to only allow selective access. + +```hcl +module "cf-http" { + source = "./fabric/modules/cloud-function-v1" + project_id = "my-project" + name = "test-cf-http" + bucket_name = "test-cf-bundles" + bundle_config = { + source_dir = "fabric/assets/" + output_path = "bundle.zip" + } + iam = { + "roles/cloudfunctions.invoker" = ["allUsers"] + } +} +# tftest modules=1 resources=3 inventory=iam.yaml +``` + +### GCS bucket creation + +You can have the module auto-create the GCS bucket used for deployment via the `bucket_config` variable. Setting `bucket_config.location` to `null` will also use the function region for GCS. + +```hcl +module "cf-http" { + source = "./fabric/modules/cloud-function-v1" + project_id = "my-project" + name = "test-cf-http" + bucket_name = "test-cf-bundles" + bucket_config = { + lifecycle_delete_age_days = 1 + } + bundle_config = { + source_dir = "fabric/assets/" + } +} +# tftest modules=1 resources=3 +``` + +### Service account management + +To use a custom service account managed by the module, set `service_account_create` to `true` and leave `service_account` set to `null` value (default). + +```hcl +module "cf-http" { + source = "./fabric/modules/cloud-function-v1" + project_id = "my-project" + name = "test-cf-http" + bucket_name = "test-cf-bundles" + bundle_config = { + source_dir = "fabric/assets/" + output_path = "bundle.zip" + } + service_account_create = true +} +# tftest modules=1 resources=3 +``` + +To use an externally managed service account, pass its email in `service_account` and leave `service_account_create` to `false` (the default). + +```hcl +module "cf-http" { + source = "./fabric/modules/cloud-function-v1" + project_id = "my-project" + name = "test-cf-http" + bucket_name = "test-cf-bundles" + bundle_config = { + source_dir = "fabric/assets/" + output_path = "bundle.zip" + } + service_account = "non-existent@serice.account.email" +} +# tftest modules=1 resources=2 +``` + +### Custom bundle config + +In order to help prevent `archive_zip.output_md5` from changing cross platform (e.g. Cloud Build vs your local development environment), you'll have to make sure that the files included in the zip are always the same. + +```hcl +module "cf-http" { + source = "./fabric/modules/cloud-function-v1" + project_id = "my-project" + name = "test-cf-http" + bucket_name = "test-cf-bundles" + bundle_config = { + source_dir = "fabric/assets" + output_path = "bundle.zip" + excludes = ["__pycache__"] + } +} +# tftest modules=1 resources=2 +``` + +### Private Cloud Build Pool + +This deploys a Cloud Function with an HTTP endpoint, using a pre-existing GCS bucket for deployment using a pre existing private Cloud Build worker pool. + +```hcl +module "cf-http" { + source = "./fabric/modules/cloud-function-v1" + project_id = "my-project" + name = "test-cf-http" + bucket_name = "test-cf-bundles" + build_worker_pool = "projects/my-project/locations/europe-west1/workerPools/my_build_worker_pool" + bundle_config = { + source_dir = "fabric/assets" + output_path = "bundle.zip" + } +} +# tftest modules=1 resources=2 +``` + +### Multiple Cloud Functions within project + +When deploying multiple functions do not reuse `bundle_config.output_path` between instances as the result is undefined. Default `output_path` creates file in `/tmp` folder using project Id and function name to avoid name conflicts. + +```hcl +module "cf-http-one" { + source = "./fabric/modules/cloud-function-v1" + project_id = "my-project" + name = "test-cf-http-one" + bucket_name = "test-cf-bundles" + bundle_config = { + source_dir = "fabric/assets" + } +} + +module "cf-http-two" { + source = "./fabric/modules/cloud-function-v1" + project_id = "my-project" + name = "test-cf-http-two" + bucket_name = "test-cf-bundles" + bundle_config = { + source_dir = "fabric/assets" + } +} +# tftest modules=2 resources=4 inventory=multiple_functions.yaml +``` + + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [bucket_name](variables.tf#L26) | Name of the bucket that will be used for the function code. It will be created with prefix prepended if bucket_config is not null. | string | ✓ | | +| [bundle_config](variables.tf#L37) | Cloud function source folder and generated zip bundle paths. Output path defaults to '/tmp/bundle.zip' if null. | object({…}) | ✓ | | +| [name](variables.tf#L96) | Name used for cloud function and associated resources. | string | ✓ | | +| [project_id](variables.tf#L111) | Project id used for all resources. | string | ✓ | | +| [bucket_config](variables.tf#L17) | Enable and configure auto-created bucket. Set fields to null to use defaults. | object({…}) | | null | +| [build_worker_pool](variables.tf#L31) | Build worker pool, in projects//locations//workerPools/ format. | string | | null | +| [description](variables.tf#L46) | Optional description. | string | | "Terraform managed." | +| [environment_variables](variables.tf#L52) | Cloud function environment variables. | map(string) | | {} | +| [function_config](variables.tf#L58) | Cloud function configuration. Defaults to using main as entrypoint, 1 instance with 256MiB of memory, and 180 second timeout. | object({…}) | | {…} | +| [iam](variables.tf#L78) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [ingress_settings](variables.tf#L84) | Control traffic that reaches the cloud function. Allowed values are ALLOW_ALL, ALLOW_INTERNAL_AND_GCLB and ALLOW_INTERNAL_ONLY . | string | | null | +| [labels](variables.tf#L90) | Resource labels. | map(string) | | {} | +| [prefix](variables.tf#L101) | Optional prefix used for resource names. | string | | null | +| [region](variables.tf#L116) | Region used for all resources. | string | | "europe-west1" | +| [secrets](variables.tf#L122) | Secret Manager secrets. Key is the variable name or mountpoint, volume versions are in version:path format. | map(object({…})) | | {} | +| [service_account](variables.tf#L134) | Service account email. Unused if service account is auto-created. | string | | null | +| [service_account_create](variables.tf#L140) | Auto-create service account. | bool | | false | +| [trigger_config](variables.tf#L146) | Function trigger configuration. Leave null for HTTP trigger. | object({…}) | | null | +| [vpc_connector](variables.tf#L156) | VPC connector configuration. Set create to 'true' if a new connector needs to be created. | object({…}) | | null | +| [vpc_connector_config](variables.tf#L166) | VPC connector network configuration. Must be provided if new VPC connector is being created. | object({…}) | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [bucket](outputs.tf#L17) | Bucket resource (only if auto-created). | | +| [bucket_name](outputs.tf#L24) | Bucket name. | | +| [function](outputs.tf#L29) | Cloud function resources. | | +| [function_name](outputs.tf#L34) | Cloud function name. | | +| [id](outputs.tf#L39) | Fully qualified function id. | | +| [service_account](outputs.tf#L44) | Service account resource. | | +| [service_account_email](outputs.tf#L49) | Service account email. | | +| [service_account_iam_email](outputs.tf#L54) | Service account email. | | +| [vpc_connector](outputs.tf#L62) | VPC connector resource if created. | | + + diff --git a/assets/modules-fabric/v26/cloud-function-v1/main.tf b/assets/modules-fabric/v26/cloud-function-v1/main.tf new file mode 100644 index 0000000..e965b8a --- /dev/null +++ b/assets/modules-fabric/v26/cloud-function-v1/main.tf @@ -0,0 +1,181 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + bucket = ( + var.bucket_name != null + ? var.bucket_name + : ( + length(google_storage_bucket.bucket) > 0 + ? google_storage_bucket.bucket[0].name + : null + ) + ) + prefix = var.prefix == null ? "" : "${var.prefix}-" + service_account_email = ( + var.service_account_create + ? google_service_account.service_account[0].email + : var.service_account + ) + vpc_connector = ( + var.vpc_connector == null + ? null + : ( + try(var.vpc_connector.create, false) == false + ? var.vpc_connector.name + : google_vpc_access_connector.connector.0.id + ) + ) +} + +resource "google_vpc_access_connector" "connector" { + count = try(var.vpc_connector.create, false) == false ? 0 : 1 + project = var.project_id + name = var.vpc_connector.name + region = var.region + ip_cidr_range = var.vpc_connector_config.ip_cidr_range + network = var.vpc_connector_config.network +} + +resource "google_cloudfunctions_function" "function" { + project = var.project_id + region = var.region + name = "${local.prefix}${var.name}" + description = var.description + runtime = var.function_config.runtime + available_memory_mb = var.function_config.memory_mb + max_instances = var.function_config.instance_count + timeout = var.function_config.timeout_seconds + entry_point = var.function_config.entry_point + environment_variables = var.environment_variables + service_account_email = local.service_account_email + source_archive_bucket = local.bucket + source_archive_object = google_storage_bucket_object.bundle.name + labels = var.labels + trigger_http = var.trigger_config == null ? true : null + + ingress_settings = var.ingress_settings + build_worker_pool = var.build_worker_pool + + vpc_connector = local.vpc_connector + vpc_connector_egress_settings = try( + var.vpc_connector.egress_settings, null + ) + + dynamic "event_trigger" { + for_each = var.trigger_config == null ? [] : [""] + content { + event_type = var.trigger_config.event + resource = var.trigger_config.resource + dynamic "failure_policy" { + for_each = var.trigger_config.retry == null ? [] : [""] + content { + retry = var.trigger_config.retry + } + } + } + } + + dynamic "secret_environment_variables" { + for_each = { for k, v in var.secrets : k => v if !v.is_volume } + iterator = secret + content { + key = secret.key + project_id = secret.value.project_id + secret = secret.value.secret + version = try(secret.value.versions.0, "latest") + } + } + + dynamic "secret_volumes" { + for_each = { for k, v in var.secrets : k => v if v.is_volume } + iterator = secret + content { + mount_path = secret.key + project_id = secret.value.project_id + secret = secret.value.secret + dynamic "versions" { + for_each = secret.value.versions + iterator = version + content { + path = split(":", version)[1] + version = split(":", version)[0] + } + } + } + } +} + +resource "google_cloudfunctions_function_iam_binding" "default" { + for_each = var.iam + project = var.project_id + region = var.region + cloud_function = google_cloudfunctions_function.function.id + role = each.key + members = each.value +} + +resource "google_storage_bucket" "bucket" { + count = var.bucket_config == null ? 0 : 1 + project = var.project_id + name = "${local.prefix}${var.bucket_name}" + uniform_bucket_level_access = true + location = ( + var.bucket_config.location == null + ? var.region + : var.bucket_config.location + ) + labels = var.labels + + dynamic "lifecycle_rule" { + for_each = var.bucket_config.lifecycle_delete_age_days == null ? [] : [""] + content { + action { type = "Delete" } + condition { + age = var.bucket_config.lifecycle_delete_age_days + with_state = "ARCHIVED" + } + } + } + + dynamic "versioning" { + for_each = var.bucket_config.lifecycle_delete_age_days == null ? [] : [""] + content { + enabled = true + } + } +} + +resource "google_storage_bucket_object" "bundle" { + name = "bundle-${data.archive_file.bundle.output_md5}.zip" + bucket = local.bucket + source = data.archive_file.bundle.output_path +} + +data "archive_file" "bundle" { + type = "zip" + source_dir = var.bundle_config.source_dir + output_path = coalesce(var.bundle_config.output_path, "/tmp/bundle-${var.project_id}-${var.name}.zip") + output_file_mode = "0644" + excludes = var.bundle_config.excludes +} + +resource "google_service_account" "service_account" { + count = var.service_account_create ? 1 : 0 + project = var.project_id + account_id = "tf-cf-${var.name}" + display_name = "Terraform Cloud Function ${var.name}." +} diff --git a/assets/modules-fabric/v26/cloud-function-v1/outputs.tf b/assets/modules-fabric/v26/cloud-function-v1/outputs.tf new file mode 100644 index 0000000..9f2fd68 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-function-v1/outputs.tf @@ -0,0 +1,65 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "bucket" { + description = "Bucket resource (only if auto-created)." + value = try( + var.bucket_config == null ? null : google_storage_bucket.bucket.0, null + ) +} + +output "bucket_name" { + description = "Bucket name." + value = local.bucket +} + +output "function" { + description = "Cloud function resources." + value = google_cloudfunctions_function.function +} + +output "function_name" { + description = "Cloud function name." + value = google_cloudfunctions_function.function.name +} + +output "id" { + description = "Fully qualified function id." + value = google_cloudfunctions_function.function.id +} + +output "service_account" { + description = "Service account resource." + value = try(google_service_account.service_account[0], null) +} + +output "service_account_email" { + description = "Service account email." + value = local.service_account_email +} + +output "service_account_iam_email" { + description = "Service account email." + value = join("", [ + "serviceAccount:", + local.service_account_email == null ? "" : local.service_account_email + ]) +} + +output "vpc_connector" { + description = "VPC connector resource if created." + value = try(google_vpc_access_connector.connector.0.id, null) +} diff --git a/assets/modules-fabric/v26/cloud-function-v1/variables.tf b/assets/modules-fabric/v26/cloud-function-v1/variables.tf new file mode 100644 index 0000000..543bc01 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-function-v1/variables.tf @@ -0,0 +1,175 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "bucket_config" { + description = "Enable and configure auto-created bucket. Set fields to null to use defaults." + type = object({ + location = optional(string) + lifecycle_delete_age_days = optional(number) + }) + default = null +} + +variable "bucket_name" { + description = "Name of the bucket that will be used for the function code. It will be created with prefix prepended if bucket_config is not null." + type = string +} + +variable "build_worker_pool" { + description = "Build worker pool, in projects//locations//workerPools/ format." + type = string + default = null +} + +variable "bundle_config" { + description = "Cloud function source folder and generated zip bundle paths. Output path defaults to '/tmp/bundle.zip' if null." + type = object({ + source_dir = string + output_path = optional(string) + excludes = optional(list(string)) + }) +} + +variable "description" { + description = "Optional description." + type = string + default = "Terraform managed." +} + +variable "environment_variables" { + description = "Cloud function environment variables." + type = map(string) + default = {} +} + +variable "function_config" { + description = "Cloud function configuration. Defaults to using main as entrypoint, 1 instance with 256MiB of memory, and 180 second timeout." + type = object({ + entry_point = optional(string, "main") + instance_count = optional(number, 1) + memory_mb = optional(number, 256) # Memory in MB + cpu = optional(string, "0.166") + runtime = optional(string, "python310") + timeout_seconds = optional(number, 180) + }) + default = { + entry_point = "main" + instance_count = 1 + memory_mb = 256 + cpu = "0.166" + runtime = "python310" + timeout_seconds = 180 + } +} + +variable "iam" { + description = "IAM bindings for topic in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} +} + +variable "ingress_settings" { + description = "Control traffic that reaches the cloud function. Allowed values are ALLOW_ALL, ALLOW_INTERNAL_AND_GCLB and ALLOW_INTERNAL_ONLY ." + type = string + default = null +} + +variable "labels" { + description = "Resource labels." + type = map(string) + default = {} +} + +variable "name" { + description = "Name used for cloud function and associated resources." + type = string +} + +variable "prefix" { + description = "Optional prefix used for resource names." + type = string + default = null + validation { + condition = var.prefix != "" + error_message = "Prefix cannot be empty, please use null instead." + } +} + +variable "project_id" { + description = "Project id used for all resources." + type = string +} + +variable "region" { + description = "Region used for all resources." + type = string + default = "europe-west1" +} + +variable "secrets" { + description = "Secret Manager secrets. Key is the variable name or mountpoint, volume versions are in version:path format." + type = map(object({ + is_volume = bool + project_id = number + secret = string + versions = list(string) + })) + nullable = false + default = {} +} + +variable "service_account" { + description = "Service account email. Unused if service account is auto-created." + type = string + default = null +} + +variable "service_account_create" { + description = "Auto-create service account." + type = bool + default = false +} + +variable "trigger_config" { + description = "Function trigger configuration. Leave null for HTTP trigger." + type = object({ + event = string + resource = string + retry = optional(bool) + }) + default = null +} + +variable "vpc_connector" { + description = "VPC connector configuration. Set create to 'true' if a new connector needs to be created." + type = object({ + create = bool + name = string + egress_settings = string + }) + default = null +} + +variable "vpc_connector_config" { + description = "VPC connector network configuration. Must be provided if new VPC connector is being created." + type = object({ + ip_cidr_range = string + network = string + }) + default = null +} + + diff --git a/assets/modules-fabric/v26/cloud-function-v1/versions.tf b/assets/modules-fabric/v26/cloud-function-v1/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-function-v1/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/cloud-function-v2/README.md b/assets/modules-fabric/v26/cloud-function-v2/README.md new file mode 100644 index 0000000..fa56fa1 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-function-v2/README.md @@ -0,0 +1,269 @@ +# Cloud Function Module (v2) + +Cloud Function management, with support for IAM roles and optional bucket creation. + +The GCS object used for deployment uses a hash of the bundle zip contents in its name, which ensures change tracking and avoids recreating the function if the GCS object is deleted and needs recreating. + + +- [TODO](#todo) +- [Examples](#examples) + - [HTTP trigger](#http-trigger) + - [PubSub and non-HTTP triggers](#pubsub-and-non-http-triggers) + - [Controlling HTTP access](#controlling-http-access) + - [GCS bucket creation](#gcs-bucket-creation) + - [Service account management](#service-account-management) + - [Custom bundle config](#custom-bundle-config) + - [Private Cloud Build Pool](#private-cloud-build-pool) + - [Multiple Cloud Functions within project](#multiple-cloud-functions-within-project) +- [Variables](#variables) +- [Outputs](#outputs) + + +## TODO + +- [ ] add support for `source_repository` + +## Examples + +### HTTP trigger + +This deploys a Cloud Function with an HTTP endpoint, using a pre-existing GCS bucket for deployment, setting the service account to the Cloud Function default one, and delegating access control to the containing project. + +```hcl +module "cf-http" { + source = "./fabric/modules/cloud-function-v2" + project_id = "my-project" + name = "test-cf-http" + bucket_name = "test-cf-bundles" + bundle_config = { + source_dir = "fabric/assets/" + output_path = "bundle.zip" + } +} +# tftest modules=1 resources=2 +``` + +### PubSub and non-HTTP triggers + +Other trigger types other than HTTP are configured via the `trigger_config` variable. This example shows a PubSub trigger via [Eventarc](https://cloud.google.com/eventarc/docs): + +```hcl +module "trigger-service-account" { + source = "./fabric/modules/iam-service-account" + project_id = "my-project" + name = "sa-cloudfunction" + iam_project_roles = { + "my-project" = [ + "roles/run.invoker" + ] + } +} + +module "cf-http" { + source = "./fabric/modules/cloud-function-v2" + project_id = "my-project" + name = "test-cf-http" + bucket_name = "test-cf-bundles" + bundle_config = { + source_dir = "fabric/assets/" + output_path = "bundle.zip" + } + trigger_config = { + event_type = "google.cloud.pubsub.topic.v1.messagePublished" + pubsub_topic = "local.my-topic" + service_account_email = module.trigger-service-account.email + } +} +# tftest modules=2 resources=4 +``` + +Ensure that pubsub service identity (`service-[project number]@gcp-sa-pubsub.iam.gserviceaccount.com` has `roles/iam.serviceAccountTokenCreator` +as documented [here](https://cloud.google.com/eventarc/docs/roles-permissions#pubsub-topic). + +### Controlling HTTP access + +To allow anonymous access to the function, grant the `roles/run.invoker` role to the special `allUsers` identifier. Use specific identities (service accounts, groups, etc.) instead of `allUsers` to only allow selective access. The Cloud Run role needs to be used as explained in the [gcloud documentation](https://cloud.google.com/sdk/gcloud/reference/functions/add-invoker-policy-binding#DESCRIPTION). + +```hcl +module "cf-http" { + source = "./fabric/modules/cloud-function-v2" + project_id = "my-project" + name = "test-cf-http" + bucket_name = "test-cf-bundles" + bundle_config = { + source_dir = "fabric/assets/" + output_path = "bundle.zip" + } + iam = { + "roles/run.invoker" = ["allUsers"] + } +} +# tftest modules=1 resources=3 inventory=iam.yaml +``` + +### GCS bucket creation + +You can have the module auto-create the GCS bucket used for deployment via the `bucket_config` variable. Setting `bucket_config.location` to `null` will also use the function region for GCS. + +```hcl +module "cf-http" { + source = "./fabric/modules/cloud-function-v2" + project_id = "my-project" + name = "test-cf-http" + bucket_name = "test-cf-bundles" + bucket_config = { + lifecycle_delete_age_days = 1 + } + bundle_config = { + source_dir = "fabric/assets/" + } +} +# tftest modules=1 resources=3 +``` + +### Service account management + +To use a custom service account managed by the module, set `service_account_create` to `true` and leave `service_account` set to `null` value (default). + +```hcl +module "cf-http" { + source = "./fabric/modules/cloud-function-v2" + project_id = "my-project" + name = "test-cf-http" + bucket_name = "test-cf-bundles" + bundle_config = { + source_dir = "fabric/assets/" + output_path = "bundle.zip" + } + service_account_create = true +} +# tftest modules=1 resources=3 +``` + +To use an externally managed service account, pass its email in `service_account` and leave `service_account_create` to `false` (the default). + +```hcl +module "cf-http" { + source = "./fabric/modules/cloud-function-v2" + project_id = "my-project" + name = "test-cf-http" + bucket_name = "test-cf-bundles" + bundle_config = { + source_dir = "fabric/assets/" + output_path = "bundle.zip" + } + service_account = "non-existent@serice.account.email" +} +# tftest modules=1 resources=2 +``` + +### Custom bundle config + +In order to help prevent `archive_zip.output_md5` from changing cross platform (e.g. Cloud Build vs your local development environment), you'll have to make sure that the files included in the zip are always the same. + +```hcl +module "cf-http" { + source = "./fabric/modules/cloud-function-v2" + project_id = "my-project" + name = "test-cf-http" + bucket_name = "test-cf-bundles" + bundle_config = { + source_dir = "fabric/assets" + output_path = "bundle.zip" + excludes = ["__pycache__"] + } +} +# tftest modules=1 resources=2 +``` + +### Private Cloud Build Pool + +This deploys a Cloud Function with an HTTP endpoint, using a pre-existing GCS bucket for deployment using a pre existing private Cloud Build worker pool. + +```hcl +module "cf-http" { + source = "./fabric/modules/cloud-function-v2" + project_id = "my-project" + name = "test-cf-http" + bucket_name = "test-cf-bundles" + build_worker_pool = "projects/my-project/locations/europe-west1/workerPools/my_build_worker_pool" + bundle_config = { + source_dir = "fabric/assets" + output_path = "bundle.zip" + } +} +# tftest modules=1 resources=2 +``` + +### Multiple Cloud Functions within project + +When deploying multiple functions do not reuse `bundle_config.output_path` between instances as the result is undefined. Default `output_path` creates file in `/tmp` folder using project Id and function name to avoid name conflicts. + +```hcl +module "cf-http-one" { + source = "./fabric/modules/cloud-function-v2" + project_id = "my-project" + name = "test-cf-http-one" + bucket_name = "test-cf-bundles" + bundle_config = { + source_dir = "fabric/assets" + } +} + +module "cf-http-two" { + source = "./fabric/modules/cloud-function-v2" + project_id = "my-project" + name = "test-cf-http-two" + bucket_name = "test-cf-bundles" + bundle_config = { + source_dir = "fabric/assets" + } +} +# tftest modules=2 resources=4 inventory=multiple_functions.yaml +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [bucket_name](variables.tf#L26) | Name of the bucket that will be used for the function code. It will be created with prefix prepended if bucket_config is not null. | string | ✓ | | +| [bundle_config](variables.tf#L37) | Cloud function source folder and generated zip bundle paths. Output path defaults to '/tmp/bundle.zip' if null. | object({…}) | ✓ | | +| [name](variables.tf#L96) | Name used for cloud function and associated resources. | string | ✓ | | +| [project_id](variables.tf#L111) | Project id used for all resources. | string | ✓ | | +| [bucket_config](variables.tf#L17) | Enable and configure auto-created bucket. Set fields to null to use defaults. | object({…}) | | null | +| [build_worker_pool](variables.tf#L31) | Build worker pool, in projects//locations//workerPools/ format. | string | | null | +| [description](variables.tf#L46) | Optional description. | string | | "Terraform managed." | +| [environment_variables](variables.tf#L52) | Cloud function environment variables. | map(string) | | {} | +| [function_config](variables.tf#L58) | Cloud function configuration. Defaults to using main as entrypoint, 1 instance with 256MiB of memory, and 180 second timeout. | object({…}) | | {…} | +| [iam](variables.tf#L78) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [ingress_settings](variables.tf#L84) | Control traffic that reaches the cloud function. Allowed values are ALLOW_ALL, ALLOW_INTERNAL_AND_GCLB and ALLOW_INTERNAL_ONLY . | string | | null | +| [labels](variables.tf#L90) | Resource labels. | map(string) | | {} | +| [prefix](variables.tf#L101) | Optional prefix used for resource names. | string | | null | +| [region](variables.tf#L116) | Region used for all resources. | string | | "europe-west1" | +| [secrets](variables.tf#L122) | Secret Manager secrets. Key is the variable name or mountpoint, volume versions are in version:path format. | map(object({…})) | | {} | +| [service_account](variables.tf#L134) | Service account email. Unused if service account is auto-created. | string | | null | +| [service_account_create](variables.tf#L140) | Auto-create service account. | bool | | false | +| [trigger_config](variables.tf#L146) | Function trigger configuration. Leave null for HTTP trigger. | object({…}) | | null | +| [vpc_connector](variables.tf#L164) | VPC connector configuration. Set create to 'true' if a new connector needs to be created. | object({…}) | | null | +| [vpc_connector_config](variables.tf#L174) | VPC connector network configuration. Must be provided if new VPC connector is being created. | object({…}) | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [bucket](outputs.tf#L17) | Bucket resource (only if auto-created). | | +| [bucket_name](outputs.tf#L24) | Bucket name. | | +| [function](outputs.tf#L29) | Cloud function resources. | | +| [function_name](outputs.tf#L34) | Cloud function name. | | +| [id](outputs.tf#L39) | Fully qualified function id. | | +| [service_account](outputs.tf#L44) | Service account resource. | | +| [service_account_email](outputs.tf#L49) | Service account email. | | +| [service_account_iam_email](outputs.tf#L54) | Service account email. | | +| [trigger_service_account](outputs.tf#L62) | Service account resource. | | +| [trigger_service_account_email](outputs.tf#L67) | Service account email. | | +| [trigger_service_account_iam_email](outputs.tf#L72) | Service account email. | | +| [uri](outputs.tf#L80) | Cloud function service uri. | | +| [vpc_connector](outputs.tf#L85) | VPC connector resource if created. | | + + diff --git a/assets/modules-fabric/v26/cloud-function-v2/main.tf b/assets/modules-fabric/v26/cloud-function-v2/main.tf new file mode 100644 index 0000000..5b506f2 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-function-v2/main.tf @@ -0,0 +1,247 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + bucket = ( + var.bucket_name != null + ? var.bucket_name + : ( + length(google_storage_bucket.bucket) > 0 + ? google_storage_bucket.bucket[0].name + : null + ) + ) + prefix = var.prefix == null ? "" : "${var.prefix}-" + service_account_email = ( + var.service_account_create + ? google_service_account.service_account[0].email + : var.service_account + ) + trigger_sa_create = ( + try(var.trigger_config.service_account_create, false) == true + ) + trigger_sa_email = try( + google_service_account.trigger_service_account[0].email, null + ) + vpc_connector = ( + var.vpc_connector == null + ? null + : ( + try(var.vpc_connector.create, false) == false + ? var.vpc_connector.name + : google_vpc_access_connector.connector.0.id + ) + ) +} + +resource "google_vpc_access_connector" "connector" { + count = try(var.vpc_connector.create, false) == true ? 1 : 0 + project = var.project_id + name = var.vpc_connector.name + region = var.region + ip_cidr_range = var.vpc_connector_config.ip_cidr_range + network = var.vpc_connector_config.network +} + +resource "google_cloudfunctions2_function" "function" { + provider = google-beta + project = var.project_id + location = var.region + name = "${local.prefix}${var.name}" + description = var.description + build_config { + worker_pool = var.build_worker_pool + runtime = var.function_config.runtime + entry_point = var.function_config.entry_point + environment_variables = var.environment_variables + source { + storage_source { + bucket = local.bucket + object = google_storage_bucket_object.bundle.name + } + } + } + dynamic "event_trigger" { + for_each = var.trigger_config == null ? [] : [""] + content { + event_type = var.trigger_config.event_type + pubsub_topic = var.trigger_config.pubsub_topic + trigger_region = ( + var.trigger_config.region == null + ? var.region + : var.trigger_config.region + ) + dynamic "event_filters" { + for_each = var.trigger_config.event_filters + iterator = event_filter + content { + attribute = event_filter.value.attribute + value = event_filter.value.value + operator = event_filter.value.operator + } + } + service_account_email = local.trigger_sa_email + retry_policy = var.trigger_config.retry_policy + } + } + service_config { + max_instance_count = var.function_config.instance_count + min_instance_count = 0 + available_memory = "${var.function_config.memory_mb}M" + available_cpu = var.function_config.cpu + timeout_seconds = var.function_config.timeout_seconds + environment_variables = var.environment_variables + ingress_settings = var.ingress_settings + all_traffic_on_latest_revision = true + service_account_email = local.service_account_email + vpc_connector = local.vpc_connector + vpc_connector_egress_settings = try( + var.vpc_connector.egress_settings, null) + + dynamic "secret_environment_variables" { + for_each = { for k, v in var.secrets : k => v if !v.is_volume } + iterator = secret + content { + key = secret.key + project_id = secret.value.project_id + secret = secret.value.secret + version = try(secret.value.versions.0, "latest") + } + } + + dynamic "secret_volumes" { + for_each = { for k, v in var.secrets : k => v if v.is_volume } + iterator = secret + content { + mount_path = secret.key + project_id = secret.value.project_id + secret = secret.value.secret + dynamic "versions" { + for_each = secret.value.versions + iterator = version + content { + path = split(":", version)[1] + version = split(":", version)[0] + } + } + } + } + } + labels = var.labels +} + +resource "google_cloudfunctions2_function_iam_binding" "binding" { + for_each = { + for k, v in var.iam : k => v if k != "roles/run.invoker" + } + project = var.project_id + location = google_cloudfunctions2_function.function.location + cloud_function = google_cloudfunctions2_function.function.name + role = each.key + members = each.value +} + +resource "google_cloud_run_service_iam_binding" "invoker" { + # cloud run resources are needed for invoker role to the underlying service + count = ( + lookup(var.iam, "roles/run.invoker", null) != null + ) ? 1 : 0 + project = var.project_id + location = google_cloudfunctions2_function.function.location + service = google_cloudfunctions2_function.function.name + role = "roles/run.invoker" + members = distinct(compact(concat( + lookup(var.iam, "roles/run.invoker", []), + ( + !local.trigger_sa_create + ? [] + : ["serviceAccount:${local.trigger_sa_email}"] + ) + ))) +} + +resource "google_cloud_run_service_iam_member" "invoker" { + # if authoritative invoker role is not present and we create trigger sa + # use additive binding to grant it the role + count = ( + lookup(var.iam, "roles/run.invoker", null) == null && + local.trigger_sa_create + ) ? 1 : 0 + project = var.project_id + location = google_cloudfunctions2_function.function.location + service = google_cloudfunctions2_function.function.name + role = "roles/run.invoker" + member = "serviceAccount:${local.trigger_sa_email}" +} + +resource "google_storage_bucket" "bucket" { + count = var.bucket_config == null ? 0 : 1 + project = var.project_id + name = "${local.prefix}${var.bucket_name}" + uniform_bucket_level_access = true + location = ( + var.bucket_config.location == null + ? var.region + : var.bucket_config.location + ) + labels = var.labels + + dynamic "lifecycle_rule" { + for_each = var.bucket_config.lifecycle_delete_age_days == null ? [] : [""] + content { + action { type = "Delete" } + condition { + age = var.bucket_config.lifecycle_delete_age_days + with_state = "ARCHIVED" + } + } + } + + dynamic "versioning" { + for_each = var.bucket_config.lifecycle_delete_age_days == null ? [] : [""] + content { + enabled = true + } + } +} + +resource "google_storage_bucket_object" "bundle" { + name = "bundle-${data.archive_file.bundle.output_md5}.zip" + bucket = local.bucket + source = data.archive_file.bundle.output_path +} + +data "archive_file" "bundle" { + type = "zip" + source_dir = var.bundle_config.source_dir + output_path = coalesce(var.bundle_config.output_path, "/tmp/bundle-${var.project_id}-${var.name}.zip") + output_file_mode = "0644" + excludes = var.bundle_config.excludes +} + +resource "google_service_account" "service_account" { + count = var.service_account_create ? 1 : 0 + project = var.project_id + account_id = "tf-cf-${var.name}" + display_name = "Terraform Cloud Function ${var.name}." +} + +resource "google_service_account" "trigger_service_account" { + count = local.trigger_sa_create ? 1 : 0 + project = var.project_id + account_id = "tf-cf-trigger-${var.name}" + display_name = "Terraform trigger for Cloud Function ${var.name}." +} diff --git a/assets/modules-fabric/v26/cloud-function-v2/outputs.tf b/assets/modules-fabric/v26/cloud-function-v2/outputs.tf new file mode 100644 index 0000000..4e42a00 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-function-v2/outputs.tf @@ -0,0 +1,88 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "bucket" { + description = "Bucket resource (only if auto-created)." + value = try( + var.bucket_config == null ? null : google_storage_bucket.bucket.0, null + ) +} + +output "bucket_name" { + description = "Bucket name." + value = local.bucket +} + +output "function" { + description = "Cloud function resources." + value = google_cloudfunctions2_function.function +} + +output "function_name" { + description = "Cloud function name." + value = google_cloudfunctions2_function.function.name +} + +output "id" { + description = "Fully qualified function id." + value = google_cloudfunctions2_function.function.id +} + +output "service_account" { + description = "Service account resource." + value = try(google_service_account.service_account[0], null) +} + +output "service_account_email" { + description = "Service account email." + value = local.service_account_email +} + +output "service_account_iam_email" { + description = "Service account email." + value = join("", [ + "serviceAccount:", + local.service_account_email == null ? "" : local.service_account_email + ]) +} + +output "trigger_service_account" { + description = "Service account resource." + value = try(google_service_account.trigger_service_account[0], null) +} + +output "trigger_service_account_email" { + description = "Service account email." + value = local.trigger_sa_email +} + +output "trigger_service_account_iam_email" { + description = "Service account email." + value = join("", [ + "serviceAccount:", + local.trigger_sa_email == null ? "" : local.trigger_sa_email + ]) +} + +output "uri" { + description = "Cloud function service uri." + value = google_cloudfunctions2_function.function.service_config[0].uri +} + +output "vpc_connector" { + description = "VPC connector resource if created." + value = try(google_vpc_access_connector.connector.0.id, null) +} diff --git a/assets/modules-fabric/v26/cloud-function-v2/variables.tf b/assets/modules-fabric/v26/cloud-function-v2/variables.tf new file mode 100644 index 0000000..b8f12e9 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-function-v2/variables.tf @@ -0,0 +1,183 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "bucket_config" { + description = "Enable and configure auto-created bucket. Set fields to null to use defaults." + type = object({ + location = optional(string) + lifecycle_delete_age_days = optional(number) + }) + default = null +} + +variable "bucket_name" { + description = "Name of the bucket that will be used for the function code. It will be created with prefix prepended if bucket_config is not null." + type = string +} + +variable "build_worker_pool" { + description = "Build worker pool, in projects//locations//workerPools/ format." + type = string + default = null +} + +variable "bundle_config" { + description = "Cloud function source folder and generated zip bundle paths. Output path defaults to '/tmp/bundle.zip' if null." + type = object({ + source_dir = string + output_path = optional(string) + excludes = optional(list(string)) + }) +} + +variable "description" { + description = "Optional description." + type = string + default = "Terraform managed." +} + +variable "environment_variables" { + description = "Cloud function environment variables." + type = map(string) + default = {} +} + +variable "function_config" { + description = "Cloud function configuration. Defaults to using main as entrypoint, 1 instance with 256MiB of memory, and 180 second timeout." + type = object({ + entry_point = optional(string, "main") + instance_count = optional(number, 1) + memory_mb = optional(number, 256) # Memory in MB + cpu = optional(string, "0.166") + runtime = optional(string, "python310") + timeout_seconds = optional(number, 180) + }) + default = { + entry_point = "main" + instance_count = 1 + memory_mb = 256 + cpu = "0.166" + runtime = "python310" + timeout_seconds = 180 + } +} + +variable "iam" { + description = "IAM bindings for topic in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} +} + +variable "ingress_settings" { + description = "Control traffic that reaches the cloud function. Allowed values are ALLOW_ALL, ALLOW_INTERNAL_AND_GCLB and ALLOW_INTERNAL_ONLY ." + type = string + default = null +} + +variable "labels" { + description = "Resource labels." + type = map(string) + default = {} +} + +variable "name" { + description = "Name used for cloud function and associated resources." + type = string +} + +variable "prefix" { + description = "Optional prefix used for resource names." + type = string + default = null + validation { + condition = var.prefix != "" + error_message = "Prefix cannot be empty, please use null instead." + } +} + +variable "project_id" { + description = "Project id used for all resources." + type = string +} + +variable "region" { + description = "Region used for all resources." + type = string + default = "europe-west1" +} + +variable "secrets" { + description = "Secret Manager secrets. Key is the variable name or mountpoint, volume versions are in version:path format." + type = map(object({ + is_volume = bool + project_id = number + secret = string + versions = list(string) + })) + nullable = false + default = {} +} + +variable "service_account" { + description = "Service account email. Unused if service account is auto-created." + type = string + default = null +} + +variable "service_account_create" { + description = "Auto-create service account." + type = bool + default = false +} + +variable "trigger_config" { + description = "Function trigger configuration. Leave null for HTTP trigger." + type = object({ + event_type = string + pubsub_topic = optional(string) + region = optional(string) + event_filters = optional(list(object({ + attribute = string + value = string + operator = optional(string) + })), []) + service_account_email = optional(string) + service_account_create = optional(bool, false) + retry_policy = optional(string) + }) + default = null +} + +variable "vpc_connector" { + description = "VPC connector configuration. Set create to 'true' if a new connector needs to be created." + type = object({ + create = bool + name = string + egress_settings = string + }) + default = null +} + +variable "vpc_connector_config" { + description = "VPC connector network configuration. Must be provided if new VPC connector is being created." + type = object({ + ip_cidr_range = string + network = string + }) + default = null +} + + diff --git a/assets/modules-fabric/v26/cloud-function-v2/versions.tf b/assets/modules-fabric/v26/cloud-function-v2/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-function-v2/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/cloud-identity-group/README.md b/assets/modules-fabric/v26/cloud-identity-group/README.md new file mode 100644 index 0000000..eee01c4 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-identity-group/README.md @@ -0,0 +1,73 @@ +# Cloud Identity Group Module + +This module allows creating a Cloud Identity group and assigning members. + +## Usage +To use this module you must either run terraform as a user that has the Groups Admin role in Cloud Identity or [enable domain-wide delegation](https://developers.google.com/identity/protocols/oauth2/service-account#delegatingauthority) to the service account used by terraform. If you use a service account, you must also grant that service account the Groups Admin role in Cloud Identity. + +Please note that the underlying terraform resources only allow the creation of groups with members that are part of the organization. If you want to create memberships for identities outside your own organization, you have to manually allow members outside your organization in the Cloud Identity admin console. + +As of version 4.34 of the GCP Terraform provider one operation is not working: +- removing a group that has at least one OWNER managed by terraform ([bug](https://github.com/hashicorp/terraform-provider-google/issues/7617)) + +Until that bug is fixed, this module will only support the creation of MEMBER and MANAGER memberships. + +## Examples + +### Simple Group +```hcl +module "group" { + source = "./fabric/modules/cloud-identity-group" + customer_id = "customers/C01234567" + name = "mygroup@example.com" + display_name = "My group name" + description = "My group Description" + members = [ + "user1@example.com", + "user2@example.com", + "service-account@my-gcp-project.iam.gserviceaccount.com" + ] +} +# tftest modules=1 resources=4 inventory=members.yaml +``` + +### Group with managers +```hcl +module "group" { + source = "./fabric/modules/cloud-identity-group" + customer_id = "customers/C01234567" + name = "mygroup2@example.com" + display_name = "My group name 2" + description = "My group 2 Description" + members = [ + "user1@example.com", + "user2@example.com", + "service-account@my-gcp-project.iam.gserviceaccount.com" + ] + managers = [ + "user3@example.com" + ] +} +# tftest modules=1 resources=5 +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [customer_id](variables.tf#L17) | Directory customer ID in the form customers/C0xxxxxxx. | string | ✓ | | +| [display_name](variables.tf#L32) | Group display name. | string | ✓ | | +| [name](variables.tf#L49) | Group ID (usually an email). | string | ✓ | | +| [description](variables.tf#L26) | Group description. | string | | null | +| [managers](variables.tf#L37) | List of group managers. | list(string) | | [] | +| [members](variables.tf#L43) | List of group members. | list(string) | | [] | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [id](outputs.tf#L17) | Fully qualified group id. | | +| [name](outputs.tf#L22) | Group name. | | + + diff --git a/assets/modules-fabric/v26/cloud-identity-group/main.tf b/assets/modules-fabric/v26/cloud-identity-group/main.tf new file mode 100644 index 0000000..7e0455e --- /dev/null +++ b/assets/modules-fabric/v26/cloud-identity-group/main.tf @@ -0,0 +1,54 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_cloud_identity_group" "group" { + display_name = var.display_name + parent = var.customer_id + description = var.description + + group_key { + id = var.name + } + + labels = { + "cloudidentity.googleapis.com/groups.discussion_forum" = "" + } +} + +# resource "google_cloud_identity_group_membership" "owners" { +# group = google_cloud_identity_group.group.id +# for_each = toset(var.owners) +# preferred_member_key { id = each.key } +# roles { name = "OWNER" } +# roles { name = "MEMBER" } +# roles { name = "MANAGER" } +# } + +resource "google_cloud_identity_group_membership" "managers" { + group = google_cloud_identity_group.group.id + for_each = toset(var.managers) + preferred_member_key { id = each.key } + roles { name = "MEMBER" } + roles { name = "MANAGER" } +} + +resource "google_cloud_identity_group_membership" "members" { + group = google_cloud_identity_group.group.id + for_each = toset(var.members) + preferred_member_key { id = each.key } + roles { name = "MEMBER" } + depends_on = [google_cloud_identity_group_membership.managers] +} diff --git a/assets/modules-fabric/v26/cloud-identity-group/outputs.tf b/assets/modules-fabric/v26/cloud-identity-group/outputs.tf new file mode 100644 index 0000000..95c31de --- /dev/null +++ b/assets/modules-fabric/v26/cloud-identity-group/outputs.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "id" { + description = "Fully qualified group id." + value = google_cloud_identity_group.group.id +} + +output "name" { + description = "Group name." + value = var.name + depends_on = [ + google_cloud_identity_group.group + ] +} diff --git a/assets/modules-fabric/v26/cloud-identity-group/variables.tf b/assets/modules-fabric/v26/cloud-identity-group/variables.tf new file mode 100644 index 0000000..221bcb8 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-identity-group/variables.tf @@ -0,0 +1,58 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "customer_id" { + description = "Directory customer ID in the form customers/C0xxxxxxx." + type = string + validation { + condition = can(regex("^customers/C0[a-z0-9]{7}$", var.customer_id)) + error_message = "Customer ID must be in the form customers/C0xxxxxxx." + } +} + +variable "description" { + description = "Group description." + type = string + default = null +} + +variable "display_name" { + description = "Group display name." + type = string +} + +variable "managers" { + description = "List of group managers." + type = list(string) + default = [] +} + +variable "members" { + description = "List of group members." + type = list(string) + default = [] +} + +variable "name" { + description = "Group ID (usually an email)." + type = string +} + +# variable "owners" { +# description = "List of group owners." +# type = list(string) +# default = [] +# } diff --git a/assets/modules-fabric/v26/cloud-identity-group/versions.tf b/assets/modules-fabric/v26/cloud-identity-group/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-identity-group/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/cloud-run/README.md b/assets/modules-fabric/v26/cloud-run/README.md new file mode 100644 index 0000000..5090aba --- /dev/null +++ b/assets/modules-fabric/v26/cloud-run/README.md @@ -0,0 +1,373 @@ +# Cloud Run Module + +Cloud Run management, with support for IAM roles, revision annotations and optional Eventarc trigger creation. + +## Examples + + +- [Examples](#examples) + - [IAM and environment variables](#iam-and-environment-variables) + - [Mounting secrets as volumes](#mounting-secrets-as-volumes) + - [Revision annotations](#revision-annotations) + - [Second generation execution environment](#second-generation-execution-environment) + - [VPC Access Connector creation](#vpc-access-connector-creation) + - [Traffic split](#traffic-split) + - [Eventarc triggers](#eventarc-triggers) + - [PubSub](#pubsub) + - [Audit logs](#audit-logs) + - [Using custom service accounts for triggers](#using-custom-service-accounts-for-triggers) + - [Service account](#service-account) +- [Variables](#variables) +- [Outputs](#outputs) + + +### IAM and environment variables + +IAM bindings support the usual syntax. Container environment values can be declared as key-value strings or as references to Secret Manager secrets. Both can be combined as long as there's no duplication of keys: + +```hcl +module "cloud_run" { + source = "./fabric/modules/cloud-run" + project_id = "my-project" + name = "hello" + containers = { + hello = { + image = "us-docker.pkg.dev/cloudrun/container/hello" + env = { + VAR1 = "VALUE1" + VAR2 = "VALUE2" + } + env_from = { + SECRET1 = { + name = "credentials" + key = "1" + } + } + } + } + iam = { + "roles/run.invoker" = ["allUsers"] + } +} +# tftest modules=1 resources=2 inventory=simple.yaml +``` + +### Mounting secrets as volumes + +```hcl +module "cloud_run" { + source = "./fabric/modules/cloud-run" + project_id = var.project_id + name = "hello" + region = var.region + containers = { + hello = { + image = "us-docker.pkg.dev/cloudrun/container/hello" + volume_mounts = { + "credentials" = "/credentials" + } + } + } + volumes = { + credentials = { + name = "credentials" + secret_name = "credentials" + items = { + v1 = { path = "v1.txt" } + } + } + } +} +# tftest modules=1 resources=1 inventory=secrets.yaml +``` + +### Revision annotations + +Annotations can be specified via the `revision_annotations` variable: + +```hcl +module "cloud_run" { + source = "./fabric/modules/cloud-run" + project_id = var.project_id + name = "hello" + containers = { + hello = { + image = "us-docker.pkg.dev/cloudrun/container/hello" + } + } + revision_annotations = { + autoscaling = { + max_scale = 10 + min_scale = 1 + } + cloudsql_unstances = ["sql-0", "sql-1"] + vpcaccess_connector = "foo" + vpcaccess_egress = "all-traffic" + } +} +# tftest modules=1 resources=1 inventory=revision-annotations.yaml +``` + +### Second generation execution environment + +Second generation execution environment (gen2) can be enabled by setting the `gen2_execution_environment` variable to true: + +```hcl +module "cloud_run" { + source = "./fabric/modules/cloud-run" + project_id = var.project_id + name = "hello" + containers = { + hello = { + image = "us-docker.pkg.dev/cloudrun/container/hello" + } + } + gen2_execution_environment = true +} +# tftest modules=1 resources=1 inventory=gen2.yaml +``` + +### VPC Access Connector creation + +If creation of a [VPC Access Connector](https://cloud.google.com/vpc/docs/serverless-vpc-access) is required, use the `vpc_connector_create` variable which also support optional attributes for number of instances, machine type, and throughput (not shown here). The annotation to use the connector will be added automatically. + +```hcl +module "cloud_run" { + source = "./fabric/modules/cloud-run" + project_id = var.project_id + name = "hello" + containers = { + hello = { + image = "us-docker.pkg.dev/cloudrun/container/hello" + } + } + vpc_connector_create = { + ip_cidr_range = "10.10.10.0/24" + vpc_self_link = "projects/example/host/global/networks/host" + } +} +# tftest modules=1 resources=2 inventory=connector.yaml +``` + +Note that if you are using Shared VPC you need to specify a subnet: + +```hcl +module "cloud_run" { + source = "./fabric/modules/cloud-run" + project_id = var.project_id + name = "hello" + containers = { + hello = { + image = "us-docker.pkg.dev/cloudrun/container/hello" + } + } + vpc_connector_create = { + subnet = { + name = "subnet-vpc-access" + project_id = "host-project" + } + } +} +# tftest modules=1 resources=2 inventory=connector-shared.yaml +``` + +### Traffic split + +This deploys a Cloud Run service with traffic split between two revisions. + +```hcl +module "cloud_run" { + source = "./fabric/modules/cloud-run" + project_id = "my-project" + name = "hello" + revision_name = "green" + containers = { + hello = { + image = "us-docker.pkg.dev/cloudrun/container/hello" + } + } + traffic = { + blue = { percent = 25 } + green = { percent = 75 } + } +} +# tftest modules=1 resources=1 inventory=traffic.yaml +``` + +### Eventarc triggers + +#### PubSub + +This deploys a Cloud Run service that will be triggered when messages are published to Pub/Sub topics. + +```hcl +module "cloud_run" { + source = "./fabric/modules/cloud-run" + project_id = "my-project" + name = "hello" + containers = { + hello = { + image = "us-docker.pkg.dev/cloudrun/container/hello" + } + } + eventarc_triggers = { + pubsub = { + topic-1 = "topic1" + topic-2 = "topic2" + } + } +} +# tftest modules=1 resources=3 inventory=eventarc.yaml +``` + +#### Audit logs + +This deploys a Cloud Run service that will be triggered when specific log events are written to Google Cloud audit logs. + +```hcl +module "cloud_run" { + source = "./fabric/modules/cloud-run" + project_id = "my-project" + name = "hello" + containers = { + hello = { + image = "us-docker.pkg.dev/cloudrun/container/hello" + } + } + eventarc_triggers = { + audit_log = { + setiampolicy = { + method = "SetIamPolicy" + service = "cloudresourcemanager.googleapis.com" + } + } + } +} +# tftest modules=1 resources=2 inventory=audit-logs.yaml +``` + +#### Using custom service accounts for triggers + +By default `Compute default service account` is used to trigger Cloud Run. If you want to use custom Service Account you can either provide your own in `eventarc_triggers.service_account_email` or set `eventarc_triggers.service_account_create` to true and service account named `tf-cr-trigger-${var.name}` will be created with `roles/run.invoker` granted on this Cloud Run service. + +Example using provided service account: + +```hcl +module "cloud_run" { + source = "./fabric/modules/cloud-run" + project_id = "my-project" + name = "hello" + containers = { + hello = { + image = "us-docker.pkg.dev/cloudrun/container/hello" + } + } + eventarc_triggers = { + audit_log = { + setiampolicy = { + method = "SetIamPolicy" + service = "cloudresourcemanager.googleapis.com" + } + } + service_account_email = "cloud-run-trigger@my-project.iam.gserviceaccount.com" + } +} +# tftest modules=1 resources=2 inventory=trigger-service-account-external.yaml +``` + +Example using automatically created service account: + +```hcl +module "cloud_run" { + source = "./fabric/modules/cloud-run" + project_id = "my-project" + name = "hello" + containers = { + hello = { + image = "us-docker.pkg.dev/cloudrun/container/hello" + } + } + eventarc_triggers = { + pubsub = { + topic-1 = "topic1" + topic-2 = "topic2" + } + service_account_create = true + } +} +# tftest modules=1 resources=5 inventory=trigger-service-account.yaml +``` + +### Service account + +To use a custom service account managed by the module, set `service_account_create` to `true` and leave `service_account` set to `null` value (default). + +```hcl +module "cloud_run" { + source = "./fabric/modules/cloud-run" + project_id = "my-project" + name = "hello" + containers = { + hello = { + image = "us-docker.pkg.dev/cloudrun/container/hello" + } + } + service_account_create = true +} +# tftest modules=1 resources=2 inventory=service-account.yaml +``` + +To use an externally managed service account, pass its email in `service_account` and leave `service_account_create` to `false` (the default). + +```hcl +module "cloud_run" { + source = "./fabric/modules/cloud-run" + project_id = "my-project" + name = "hello" + containers = { + hello = { + image = "us-docker.pkg.dev/cloudrun/container/hello" + } + } + service_account = "cloud-run@my-project.iam.gserviceaccount.com" +} +# tftest modules=1 resources=1 inventory=service-account-external.yaml +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [name](variables.tf#L136) | Name used for cloud run service. | string | ✓ | | +| [project_id](variables.tf#L151) | Project id used for all resources. | string | ✓ | | +| [container_concurrency](variables.tf#L18) | Maximum allowed in-flight (concurrent) requests per container of the revision. | string | | null | +| [containers](variables.tf#L24) | Containers in arbitrary key => attributes format. | map(object({…})) | | {} | +| [eventarc_triggers](variables.tf#L91) | Event arc triggers for different sources. | object({…}) | | {} | +| [gen2_execution_environment](variables.tf#L105) | Use second generation execution environment. | bool | | false | +| [iam](variables.tf#L111) | IAM bindings for Cloud Run service in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [ingress_settings](variables.tf#L117) | Ingress settings. | string | | null | +| [labels](variables.tf#L130) | Resource labels. | map(string) | | {} | +| [prefix](variables.tf#L141) | Optional prefix used for resource names. | string | | null | +| [region](variables.tf#L156) | Region used for all resources. | string | | "europe-west1" | +| [revision_annotations](variables.tf#L162) | Configure revision template annotations. | object({…}) | | {} | +| [revision_name](variables.tf#L177) | Revision name. | string | | null | +| [service_account](variables.tf#L183) | Service account email. Unused if service account is auto-created. | string | | null | +| [service_account_create](variables.tf#L189) | Auto-create service account. | bool | | false | +| [startup_cpu_boost](variables.tf#L195) | Enable startup cpu boost. | bool | | false | +| [timeout_seconds](variables.tf#L201) | Maximum duration the instance is allowed for responding to a request. | number | | null | +| [traffic](variables.tf#L207) | Traffic steering configuration. If revision name is null the latest revision will be used. | map(object({…})) | | {} | +| [volumes](variables.tf#L218) | Named volumes in containers in name => attributes format. | map(object({…})) | | {} | +| [vpc_connector_create](variables.tf#L232) | Populate this to create a VPC connector. You can then refer to it in the template annotations. | object({…}) | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [id](outputs.tf#L18) | Fully qualified service id. | | +| [service](outputs.tf#L23) | Cloud Run service. | | +| [service_account](outputs.tf#L28) | Service account resource. | | +| [service_account_email](outputs.tf#L33) | Service account email. | | +| [service_account_iam_email](outputs.tf#L38) | Service account email. | | +| [service_name](outputs.tf#L46) | Cloud Run service name. | | +| [vpc_connector](outputs.tf#L52) | VPC connector resource if created. | | + diff --git a/assets/modules-fabric/v26/cloud-run/main.tf b/assets/modules-fabric/v26/cloud-run/main.tf new file mode 100644 index 0000000..46f6e3a --- /dev/null +++ b/assets/modules-fabric/v26/cloud-run/main.tf @@ -0,0 +1,394 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + _vpcaccess_annotation = ( + local.vpc_connector_create + ? { + "run.googleapis.com/vpc-access-connector" = google_vpc_access_connector.connector.0.id + } + : ( + var.revision_annotations.vpcaccess_connector == null + ? {} + : { + "run.googleapis.com/vpc-access-connector" = ( + var.revision_annotations.vpcaccess_connector + ) + } + ) + ) + annotations = merge( + var.ingress_settings == null ? {} : { + "run.googleapis.com/ingress" = var.ingress_settings + }, + ) + prefix = var.prefix == null ? "" : "${var.prefix}-" + revision_annotations = merge( + try(var.revision_annotations.autoscaling, null) == null ? {} : { + "autoscaling.knative.dev/maxScale" = ( + var.revision_annotations.autoscaling.max_scale + ) + }, + try(var.revision_annotations.autoscaling.min_scale, null) == null ? {} : { + "autoscaling.knative.dev/minScale" = ( + var.revision_annotations.autoscaling.min_scale + ) + }, + length(var.revision_annotations.cloudsql_instances) == 0 ? {} : { + "run.googleapis.com/cloudsql-instances" = ( + join(",", var.revision_annotations.cloudsql_instances) + ) + }, + local._vpcaccess_annotation, + var.revision_annotations.vpcaccess_egress == null ? {} : { + "run.googleapis.com/vpc-access-egress" = ( + var.revision_annotations.vpcaccess_egress + ) + }, + var.gen2_execution_environment ? { + "run.googleapis.com/execution-environment" = "gen2" + } : {}, + var.startup_cpu_boost ? { + "run.googleapis.com/startup-cpu-boost" = "true" + } : {}, + ) + revision_name = ( + try(var.revision_name, null) == null + ? null + : "${var.name}-${var.revision_name}" + ) + service_account_email = ( + var.service_account_create + ? ( + length(google_service_account.service_account) > 0 + ? google_service_account.service_account[0].email + : null + ) + : var.service_account + ) + trigger_sa_create = try( + var.eventarc_triggers.service_account_create, false + ) + trigger_sa_email = try( + google_service_account.trigger_service_account[0].email, null + ) + vpc_connector_create = var.vpc_connector_create != null +} + +resource "google_vpc_access_connector" "connector" { + count = local.vpc_connector_create ? 1 : 0 + project = var.project_id + name = ( + var.vpc_connector_create.name != null + ? var.vpc_connector_create.name + : var.name + ) + region = var.region + ip_cidr_range = var.vpc_connector_create.ip_cidr_range + network = var.vpc_connector_create.vpc_self_link + machine_type = var.vpc_connector_create.machine_type + max_instances = var.vpc_connector_create.instances.max + max_throughput = var.vpc_connector_create.throughput.max + min_instances = var.vpc_connector_create.instances.min + min_throughput = var.vpc_connector_create.throughput.min + subnet { + name = var.vpc_connector_create.subnet.name + project_id = var.vpc_connector_create.subnet.project_id + } +} + +resource "google_cloud_run_service" "service" { + provider = google-beta + project = var.project_id + location = var.region + name = "${local.prefix}${var.name}" + + template { + spec { + container_concurrency = var.container_concurrency + service_account_name = local.service_account_email + timeout_seconds = var.timeout_seconds + dynamic "containers" { + for_each = var.containers + content { + image = containers.value.image + args = containers.value.args + command = containers.value.command + dynamic "env" { + for_each = containers.value.env + content { + name = env.key + value = env.value + } + } + dynamic "env" { + for_each = containers.value.env_from_key + content { + name = env.key + value_from { + secret_key_ref { + key = env.value.key + name = env.value.name + } + } + } + } + dynamic "liveness_probe" { + for_each = containers.value.liveness_probe == null ? [] : [""] + content { + failure_threshold = containers.value.liveness_probe.failure_threshold + initial_delay_seconds = containers.value.liveness_probe.initial_delay_seconds + period_seconds = containers.value.liveness_probe.period_seconds + timeout_seconds = containers.value.liveness_probe.timeout_seconds + dynamic "grpc" { + for_each = ( + containers.value.liveness_probe.action.grpc == null ? [] : [""] + ) + content { + port = containers.value.liveness_probe.action.grpc.port + service = containers.value.liveness_probe.action.grpc.service + } + } + dynamic "http_get" { + for_each = ( + containers.value.liveness_probe.action.http_get == null ? [] : [""] + ) + content { + path = containers.value.liveness_probe.action.http_get.path + dynamic "http_headers" { + for_each = ( + containers.value.liveness_probe.action.http_get.http_headers + ) + content { + name = http_headers.key + value = http_headers.value + } + } + } + } + } + } + dynamic "ports" { + for_each = containers.value.ports + content { + container_port = ports.value.container_port + name = ports.value.name + protocol = ports.value.protocol + } + } + dynamic "resources" { + for_each = containers.value.resources == null ? [] : [""] + content { + limits = containers.value.resources.limits + requests = containers.value.resources.requests + } + } + dynamic "startup_probe" { + for_each = containers.value.startup_probe == null ? [] : [""] + content { + failure_threshold = containers.value.startup_probe.failure_threshold + initial_delay_seconds = containers.value.startup_probe.initial_delay_seconds + period_seconds = containers.value.startup_probe.period_seconds + timeout_seconds = containers.value.startup_probe.timeout_seconds + dynamic "grpc" { + for_each = ( + containers.value.startup_probe.action.grpc == null ? [] : [""] + ) + content { + port = containers.value.startup_probe.action.grpc.port + service = containers.value.startup_probe.action.grpc.service + } + } + dynamic "http_get" { + for_each = ( + containers.value.startup_probe.action.http_get == null ? [] : [""] + ) + content { + path = containers.value.startup_probe.action.http_get.path + dynamic "http_headers" { + for_each = ( + containers.value.startup_probe.action.http_get.http_headers + ) + content { + name = http_headers.key + value = http_headers.value + } + } + } + } + dynamic "tcp_socket" { + for_each = ( + containers.value.startup_probe.action.tcp_socket == null ? [] : [""] + ) + content { + port = containers.value.startup_probe.action.tcp_socket.port + } + } + } + } + dynamic "volume_mounts" { + for_each = containers.value.volume_mounts + content { + name = volume_mounts.key + mount_path = volume_mounts.value + } + } + } + } + dynamic "volumes" { + for_each = var.volumes + content { + name = volumes.key + secret { + secret_name = volumes.value.secret_name + default_mode = volumes.value.default_mode + dynamic "items" { + for_each = volumes.value.items + content { + key = items.key + path = items.value.path + mode = items.value.mode + } + } + } + } + } + } + metadata { + name = local.revision_name + annotations = local.revision_annotations + } + } + + metadata { + annotations = local.annotations + labels = var.labels + } + + dynamic "traffic" { + for_each = var.traffic + content { + percent = traffic.value.percent + latest_revision = traffic.value.latest == true + revision_name = ( + traffic.value.latest == true + ? null + : "${var.name}-${traffic.key}" + ) + tag = traffic.value.tag + } + } + + lifecycle { + ignore_changes = [ + metadata.0.annotations["run.googleapis.com/operation-id"], + template.0.metadata.0.labels["run.googleapis.com/startupProbeType"] + ] + } +} + +resource "google_cloud_run_service_iam_binding" "binding" { + for_each = var.iam + project = google_cloud_run_service.service.project + location = google_cloud_run_service.service.location + service = google_cloud_run_service.service.name + role = each.key + members = ( + each.key != "roles/run.invoker" || !local.trigger_sa_create + ? each.value + # if invoker role is present and we create trigger sa, add it as member + : concat( + each.value, ["serviceAccount:${local.trigger_sa_email}"] + ) + ) +} + +resource "google_cloud_run_service_iam_member" "default" { + # if authoritative invoker role is not present and we create trigger sa + # use additive binding to grant it the role + count = ( + lookup(var.iam, "roles/run.invoker", null) == null && + local.trigger_sa_create + ) ? 1 : 0 + project = google_cloud_run_service.service.project + location = google_cloud_run_service.service.location + service = google_cloud_run_service.service.name + role = "roles/run.invoker" + member = "serviceAccount:${local.trigger_sa_email}" +} + +resource "google_service_account" "service_account" { + count = var.service_account_create ? 1 : 0 + project = var.project_id + account_id = "tf-cr-${var.name}" + display_name = "Terraform Cloud Run ${var.name}." +} + +resource "google_eventarc_trigger" "audit_log_triggers" { + for_each = var.eventarc_triggers.audit_log + name = "${local.prefix}audit-log-${each.key}" + location = google_cloud_run_service.service.location + project = google_cloud_run_service.service.project + matching_criteria { + attribute = "type" + value = "google.cloud.audit.log.v1.written" + } + matching_criteria { + attribute = "serviceName" + value = each.value.service + } + matching_criteria { + attribute = "methodName" + value = each.value.method + } + destination { + cloud_run_service { + service = google_cloud_run_service.service.name + region = google_cloud_run_service.service.location + } + } + service_account = local.trigger_sa_email +} + +resource "google_eventarc_trigger" "pubsub_triggers" { + for_each = var.eventarc_triggers.pubsub + name = "${local.prefix}pubsub-${each.key}" + location = google_cloud_run_service.service.location + project = google_cloud_run_service.service.project + matching_criteria { + attribute = "type" + value = "google.cloud.pubsub.topic.v1.messagePublished" + } + transport { + pubsub { + topic = each.value + } + } + destination { + cloud_run_service { + service = google_cloud_run_service.service.name + region = google_cloud_run_service.service.location + } + } + service_account = local.trigger_sa_email +} + +resource "google_service_account" "trigger_service_account" { + count = local.trigger_sa_create ? 1 : 0 + project = var.project_id + account_id = "tf-cr-trigger-${var.name}" + display_name = "Terraform trigger for Cloud Run ${var.name}." +} diff --git a/assets/modules-fabric/v26/cloud-run/outputs.tf b/assets/modules-fabric/v26/cloud-run/outputs.tf new file mode 100644 index 0000000..2aec9f6 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-run/outputs.tf @@ -0,0 +1,55 @@ + +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "id" { + description = "Fully qualified service id." + value = google_cloud_run_service.service.id +} + +output "service" { + description = "Cloud Run service." + value = google_cloud_run_service.service +} + +output "service_account" { + description = "Service account resource." + value = try(google_service_account.service_account[0], null) +} + +output "service_account_email" { + description = "Service account email." + value = local.service_account_email +} + +output "service_account_iam_email" { + description = "Service account email." + value = join("", [ + "serviceAccount:", + local.service_account_email == null ? "" : local.service_account_email + ]) +} + +output "service_name" { + description = "Cloud Run service name." + value = google_cloud_run_service.service.name +} + + +output "vpc_connector" { + description = "VPC connector resource if created." + value = try(google_vpc_access_connector.connector.0.id, null) +} diff --git a/assets/modules-fabric/v26/cloud-run/variables.tf b/assets/modules-fabric/v26/cloud-run/variables.tf new file mode 100644 index 0000000..2b7fd3a --- /dev/null +++ b/assets/modules-fabric/v26/cloud-run/variables.tf @@ -0,0 +1,253 @@ + +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "container_concurrency" { + description = "Maximum allowed in-flight (concurrent) requests per container of the revision." + type = string + default = null +} + +variable "containers" { + description = "Containers in arbitrary key => attributes format." + type = map(object({ + image = string + args = optional(list(string)) + command = optional(list(string)) + env = optional(map(string), {}) + env_from_key = optional(map(object({ + key = string + name = string + })), {}) + liveness_probe = optional(object({ + action = object({ + grpc = optional(object({ + port = optional(number) + service = optional(string) + })) + http_get = optional(object({ + http_headers = optional(map(string), {}) + path = optional(string) + })) + }) + failure_threshold = optional(number) + initial_delay_seconds = optional(number) + period_seconds = optional(number) + timeout_seconds = optional(number) + })) + ports = optional(map(object({ + container_port = optional(number) + name = optional(string) + protocol = optional(string) + })), {}) + resources = optional(object({ + limits = optional(object({ + cpu = string + memory = string + })) + requests = optional(object({ + cpu = string + memory = string + })) + })) + startup_probe = optional(object({ + action = object({ + grpc = optional(object({ + port = optional(number) + service = optional(string) + })) + http_get = optional(object({ + http_headers = optional(map(string), {}) + path = optional(string) + })) + tcp_socket = optional(object({ + port = optional(number) + })) + }) + failure_threshold = optional(number) + initial_delay_seconds = optional(number) + period_seconds = optional(number) + timeout_seconds = optional(number) + })) + volume_mounts = optional(map(string), {}) + })) + default = {} + nullable = false +} + +variable "eventarc_triggers" { + description = "Event arc triggers for different sources." + type = object({ + audit_log = optional(map(object({ + method = string + service = string + })), {}) + pubsub = optional(map(string), {}) + service_account_email = optional(string) + service_account_create = optional(bool, false) + }) + default = {} +} + +variable "gen2_execution_environment" { + description = "Use second generation execution environment." + type = bool + default = false +} + +variable "iam" { + description = "IAM bindings for Cloud Run service in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} +} + +variable "ingress_settings" { + description = "Ingress settings." + type = string + default = null + validation { + condition = contains( + ["all", "internal", "internal-and-cloud-load-balancing"], + coalesce(var.ingress_settings, "all") + ) + error_message = "Ingress settings can be one of 'all', 'internal', 'internal-and-cloud-load-balancing'." + } +} + +variable "labels" { + description = "Resource labels." + type = map(string) + default = {} +} + +variable "name" { + description = "Name used for cloud run service." + type = string +} + +variable "prefix" { + description = "Optional prefix used for resource names." + type = string + default = null + validation { + condition = var.prefix != "" + error_message = "Prefix cannot be empty, please use null instead." + } +} + +variable "project_id" { + description = "Project id used for all resources." + type = string +} + +variable "region" { + description = "Region used for all resources." + type = string + default = "europe-west1" +} + +variable "revision_annotations" { + description = "Configure revision template annotations." + type = object({ + autoscaling = optional(object({ + max_scale = number + min_scale = number + })) + cloudsql_instances = optional(list(string), []) + vpcaccess_connector = optional(string) + vpcaccess_egress = optional(string) + }) + default = {} + nullable = false +} + +variable "revision_name" { + description = "Revision name." + type = string + default = null +} + +variable "service_account" { + description = "Service account email. Unused if service account is auto-created." + type = string + default = null +} + +variable "service_account_create" { + description = "Auto-create service account." + type = bool + default = false +} + +variable "startup_cpu_boost" { + description = "Enable startup cpu boost." + type = bool + default = false +} + +variable "timeout_seconds" { + description = "Maximum duration the instance is allowed for responding to a request." + type = number + default = null +} + +variable "traffic" { + description = "Traffic steering configuration. If revision name is null the latest revision will be used." + type = map(object({ + percent = number + latest = optional(bool) + tag = optional(string) + })) + default = {} + nullable = false +} + +variable "volumes" { + description = "Named volumes in containers in name => attributes format." + type = map(object({ + secret_name = string + default_mode = optional(string) + items = optional(map(object({ + path = string + mode = optional(string) + }))) + })) + default = {} + nullable = false +} + +variable "vpc_connector_create" { + description = "Populate this to create a VPC connector. You can then refer to it in the template annotations." + type = object({ + ip_cidr_range = optional(string) + vpc_self_link = optional(string) + machine_type = optional(string) + name = optional(string) + instances = optional(object({ + max = optional(number) + min = optional(number) + }), {}) + throughput = optional(object({ + max = optional(number) + min = optional(number) + }), {}) + subnet = optional(object({ + name = optional(string) + project_id = optional(string) + }), {}) + }) + default = null +} diff --git a/assets/modules-fabric/v26/cloud-run/versions.tf b/assets/modules-fabric/v26/cloud-run/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/cloud-run/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/cloudsql-instance/README.md b/assets/modules-fabric/v26/cloudsql-instance/README.md new file mode 100644 index 0000000..67b7675 --- /dev/null +++ b/assets/modules-fabric/v26/cloudsql-instance/README.md @@ -0,0 +1,234 @@ +# Cloud SQL instance with read replicas + +This module manages the creation of Cloud SQL instances with potential read replicas in other regions. It can also create an initial set of users and databases via the `users` and `databases` parameters. + +Note that this module assumes that some options are the same for both the primary instance and all the replicas (e.g. tier, disks, labels, flags, etc). + +*Warning:* if you use the `users` field, you terraform state will contain each user's password in plain text. + +## Simple example + +This example shows how to setup a project, VPC and a standalone Cloud SQL instance. + +```hcl +module "project" { + source = "./fabric/modules/project" + billing_account = var.billing_account_id + parent = var.organization_id + name = "my-db-project" + services = [ + "servicenetworking.googleapis.com" + ] +} + +module "vpc" { + source = "./fabric/modules/net-vpc" + project_id = module.project.project_id + name = "my-network" + psa_config = { + ranges = { cloud-sql = "10.60.0.0/16" } + } +} + +module "db" { + source = "./fabric/modules/cloudsql-instance" + project_id = module.project.project_id + network = module.vpc.self_link + name = "db" + region = "europe-west1" + database_version = "POSTGRES_13" + tier = "db-g1-small" +} +# tftest modules=3 resources=11 inventory=simple.yaml +``` + +## Cross-regional read replica + +```hcl +module "db" { + source = "./fabric/modules/cloudsql-instance" + project_id = var.project_id + network = var.vpc.self_link + prefix = "myprefix" + name = "db" + region = "europe-west1" + database_version = "POSTGRES_13" + tier = "db-g1-small" + + replicas = { + replica1 = { region = "europe-west3", encryption_key_name = null } + replica2 = { region = "us-central1", encryption_key_name = null } + } +} +# tftest modules=1 resources=3 inventory=replicas.yaml +``` + +## Custom flags, databases and users + +```hcl +module "db" { + source = "./fabric/modules/cloudsql-instance" + project_id = var.project_id + network = var.vpc.self_link + name = "db" + region = "europe-west1" + database_version = "MYSQL_8_0" + tier = "db-g1-small" + + flags = { + disconnect_on_expired_password = "on" + } + + databases = [ + "people", + "departments" + ] + + users = { + # generatea password for user1 + user1 = null + # assign a password to user2 + user2 = "mypassword" + } +} +# tftest modules=1 resources=6 inventory=custom.yaml +``` + +### CMEK encryption +```hcl + +module "project" { + source = "./fabric/modules/project" + billing_account = var.billing_account_id + parent = var.organization_id + name = "my-db-project" + services = [ + "servicenetworking.googleapis.com", + "sqladmin.googleapis.com", + ] +} + +module "kms" { + source = "./fabric/modules/kms" + project_id = module.project.project_id + keyring = { + name = "keyring" + location = var.region + } + keys = { + key-sql = { + iam = { + "roles/cloudkms.cryptoKeyEncrypterDecrypter" = [ + "serviceAccount:${module.project.service_accounts.robots.sqladmin}" + ] + } + } + } +} + +module "db" { + source = "./fabric/modules/cloudsql-instance" + project_id = module.project.project_id + encryption_key_name = module.kms.keys["key-sql"].id + network = var.vpc.self_link + name = "db" + region = var.region + database_version = "POSTGRES_13" + tier = "db-g1-small" +} + +# tftest modules=3 resources=10 +``` + +### Enable public IP + +Use `ipv_enabled` to create instances with a public IP. + +```hcl +module "db" { + source = "./fabric/modules/cloudsql-instance" + project_id = var.project_id + network = var.vpc.self_link + name = "db" + region = "europe-west1" + tier = "db-g1-small" + database_version = "MYSQL_8_0" + ipv4_enabled = true + replicas = { + replica1 = { region = "europe-west3", encryption_key_name = null } + } +} +# tftest modules=1 resources=2 inventory=public-ip.yaml +``` + +### Query Insights + +Provide `insights_config` (can be just empty `{}`) to enable [Query Insights](https://cloud.google.com/sql/docs/postgres/using-query-insights) + +```hcl +module "db" { + source = "./fabric/modules/cloudsql-instance" + project_id = var.project_id + network = var.vpc.self_link + name = "db" + region = "europe-west1" + database_version = "POSTGRES_13" + tier = "db-g1-small" + + insights_config = { + query_string_length = 2048 + } +} +# tftest modules=1 resources=1 inventory=insights.yaml +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [database_version](variables.tf#L71) | Database type and version to create. | string | ✓ | | +| [name](variables.tf#L141) | Name of primary instance. | string | ✓ | | +| [network](variables.tf#L146) | VPC self link where the instances will be deployed. Private Service Networking must be enabled and configured in this VPC. | string | ✓ | | +| [project_id](variables.tf#L167) | The ID of the project where this instances will be created. | string | ✓ | | +| [region](variables.tf#L172) | Region of the primary instance. | string | ✓ | | +| [tier](variables.tf#L198) | The machine type to use for the instances. | string | ✓ | | +| [activation_policy](variables.tf#L16) | This variable specifies when the instance should be active. Can be either ALWAYS, NEVER or ON_DEMAND. Default is ALWAYS. | string | | "ALWAYS" | +| [allocated_ip_ranges](variables.tf#L27) | (Optional)The name of the allocated ip range for the private ip CloudSQL instance. For example: \"google-managed-services-default\". If set, the instance ip will be created in the allocated range. The range name must comply with RFC 1035. Specifically, the name must be 1-63 characters long and match the regular expression a-z?. | object({…}) | | {} | +| [authorized_networks](variables.tf#L36) | Map of NAME=>CIDR_RANGE to allow to connect to the database(s). | map(string) | | null | +| [availability_type](variables.tf#L42) | Availability type for the primary replica. Either `ZONAL` or `REGIONAL`. | string | | "ZONAL" | +| [backup_configuration](variables.tf#L48) | Backup settings for primary instance. Will be automatically enabled if using MySQL with one or more replicas. | object({…}) | | {…} | +| [databases](variables.tf#L76) | Databases to create once the primary instance is created. | list(string) | | null | +| [deletion_protection](variables.tf#L82) | Allow terraform to delete instances. | bool | | false | +| [deletion_protection_enabled](variables.tf#L88) | Set Google's deletion protection attribute which applies across all surfaces (UI, API, & Terraform). | bool | | false | +| [disk_size](variables.tf#L94) | Disk size in GB. Set to null to enable autoresize. | number | | null | +| [disk_type](variables.tf#L100) | The type of data disk: `PD_SSD` or `PD_HDD`. | string | | "PD_SSD" | +| [encryption_key_name](variables.tf#L106) | The full path to the encryption key used for the CMEK disk encryption of the primary instance. | string | | null | +| [flags](variables.tf#L112) | Map FLAG_NAME=>VALUE for database-specific tuning. | map(string) | | null | +| [insights_config](variables.tf#L118) | Query Insights configuration. Defaults to null which disables Query Insights. | object({…}) | | null | +| [ipv4_enabled](variables.tf#L129) | Add a public IP address to database instance. | bool | | false | +| [labels](variables.tf#L135) | Labels to be attached to all instances. | map(string) | | null | +| [postgres_client_certificates](variables.tf#L151) | Map of cert keys connect to the application(s) using public IP. | list(string) | | null | +| [prefix](variables.tf#L157) | Optional prefix used to generate instance names. | string | | null | +| [replicas](variables.tf#L177) | Map of NAME=> {REGION, KMS_KEY} for additional read replicas. Set to null to disable replica creation. | map(object({…})) | | {} | +| [require_ssl](variables.tf#L186) | Enable SSL connections only. | bool | | null | +| [root_password](variables.tf#L192) | Root password of the Cloud SQL instance. Required for MS SQL Server. | string | | null | +| [users](variables.tf#L203) | Map of users to create in the primary instance (and replicated to other replicas) in the format USER=>PASSWORD. For MySQL, anything afterr the first `@` (if persent) will be used as the user's host. Set PASSWORD to null if you want to get an autogenerated password. | map(string) | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [connection_name](outputs.tf#L24) | Connection name of the primary instance. | | +| [connection_names](outputs.tf#L29) | Connection names of all instances. | | +| [id](outputs.tf#L37) | Fully qualified primary instance id. | | +| [ids](outputs.tf#L42) | Fully qualified ids of all instances. | | +| [instances](outputs.tf#L50) | Cloud SQL instance resources. | ✓ | +| [ip](outputs.tf#L56) | IP address of the primary instance. | | +| [ips](outputs.tf#L61) | IP addresses of all instances. | | +| [name](outputs.tf#L69) | Name of the primary instance. | | +| [names](outputs.tf#L74) | Names of all instances. | | +| [postgres_client_certificates](outputs.tf#L82) | The CA Certificate used to connect to the SQL Instance via SSL. | ✓ | +| [self_link](outputs.tf#L88) | Self link of the primary instance. | | +| [self_links](outputs.tf#L93) | Self links of all instances. | | +| [user_passwords](outputs.tf#L101) | Map of containing the password of all users created through terraform. | ✓ | + diff --git a/assets/modules-fabric/v26/cloudsql-instance/main.tf b/assets/modules-fabric/v26/cloudsql-instance/main.tf new file mode 100644 index 0000000..fd3d9ab --- /dev/null +++ b/assets/modules-fabric/v26/cloudsql-instance/main.tf @@ -0,0 +1,204 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + prefix = var.prefix == null ? "" : "${var.prefix}-" + is_mysql = can(regex("^MYSQL", var.database_version)) + has_replicas = try(length(var.replicas) > 0, false) + is_regional = var.availability_type == "REGIONAL" ? true : false + + // Enable backup if the user asks for it or if the user is deploying + // MySQL in HA configuration (regional or with specified replicas) + enable_backup = var.backup_configuration.enabled || (local.is_mysql && local.has_replicas) || (local.is_mysql && local.is_regional) + + users = { + for user, password in coalesce(var.users, {}) : + (user) => ( + local.is_mysql + ? { + name = split("@", user)[0] + host = try(split("@", user)[1], null) + password = try(random_password.passwords[user].result, password) + } + : { + name = user + host = null + password = try(random_password.passwords[user].result, password) + } + ) + } + +} + +resource "google_sql_database_instance" "primary" { + provider = google-beta + project = var.project_id + name = "${local.prefix}${var.name}" + region = var.region + database_version = var.database_version + encryption_key_name = var.encryption_key_name + root_password = var.root_password + + settings { + tier = var.tier + deletion_protection_enabled = var.deletion_protection_enabled + disk_autoresize = var.disk_size == null + disk_size = var.disk_size + disk_type = var.disk_type + availability_type = var.availability_type + user_labels = var.labels + activation_policy = var.activation_policy + + ip_configuration { + ipv4_enabled = var.ipv4_enabled + private_network = var.network + allocated_ip_range = var.allocated_ip_ranges.primary + require_ssl = var.require_ssl + dynamic "authorized_networks" { + for_each = var.authorized_networks != null ? var.authorized_networks : {} + iterator = network + content { + name = network.key + value = network.value + } + } + } + + dynamic "backup_configuration" { + for_each = local.enable_backup ? { 1 = 1 } : {} + content { + enabled = true + + // enable binary log if the user asks for it or we have replicas (default in regional), + // but only for MySQL + binary_log_enabled = ( + local.is_mysql + ? var.backup_configuration.binary_log_enabled || local.has_replicas || local.is_regional + : null + ) + start_time = var.backup_configuration.start_time + location = var.backup_configuration.location + point_in_time_recovery_enabled = var.backup_configuration.point_in_time_recovery_enabled + transaction_log_retention_days = var.backup_configuration.log_retention_days + backup_retention_settings { + retained_backups = var.backup_configuration.retention_count + retention_unit = "COUNT" + } + } + } + + dynamic "database_flags" { + for_each = var.flags != null ? var.flags : {} + iterator = flag + content { + name = flag.key + value = flag.value + } + } + + dynamic "insights_config" { + for_each = var.insights_config != null ? [1] : [] + content { + query_insights_enabled = true + query_string_length = var.insights_config.query_string_length + record_application_tags = var.insights_config.record_application_tags + record_client_address = var.insights_config.record_client_address + query_plans_per_minute = var.insights_config.query_plans_per_minute + } + } + } + deletion_protection = var.deletion_protection +} + +resource "google_sql_database_instance" "replicas" { + provider = google-beta + for_each = local.has_replicas ? var.replicas : {} + project = var.project_id + name = "${local.prefix}${each.key}" + region = each.value.region + database_version = var.database_version + encryption_key_name = each.value.encryption_key_name + master_instance_name = google_sql_database_instance.primary.name + + settings { + tier = var.tier + deletion_protection_enabled = var.deletion_protection_enabled + disk_autoresize = var.disk_size == null + disk_size = var.disk_size + disk_type = var.disk_type + # availability_type = var.availability_type + user_labels = var.labels + activation_policy = var.activation_policy + + ip_configuration { + ipv4_enabled = var.ipv4_enabled + private_network = var.network + allocated_ip_range = var.allocated_ip_ranges.replica + dynamic "authorized_networks" { + for_each = var.authorized_networks != null ? var.authorized_networks : {} + iterator = network + content { + name = network.key + value = network.value + } + } + } + + dynamic "database_flags" { + for_each = var.flags != null ? var.flags : {} + iterator = flag + content { + name = flag.key + value = flag.value + } + } + } + deletion_protection = var.deletion_protection +} + +resource "google_sql_database" "databases" { + for_each = var.databases != null ? toset(var.databases) : toset([]) + project = var.project_id + instance = google_sql_database_instance.primary.name + name = each.key +} + +resource "random_password" "passwords" { + for_each = toset([ + for user, password in coalesce(var.users, {}) : + user + if password == null + ]) + length = 16 + special = true +} + +resource "google_sql_user" "users" { + for_each = local.users + project = var.project_id + instance = google_sql_database_instance.primary.name + name = each.value.name + host = each.value.host + password = each.value.password +} + +resource "google_sql_ssl_cert" "postgres_client_certificates" { + for_each = var.postgres_client_certificates != null ? toset(var.postgres_client_certificates) : toset([]) + provider = google-beta + project = var.project_id + instance = google_sql_database_instance.primary.name + common_name = each.key +} diff --git a/assets/modules-fabric/v26/cloudsql-instance/outputs.tf b/assets/modules-fabric/v26/cloudsql-instance/outputs.tf new file mode 100644 index 0000000..1859d84 --- /dev/null +++ b/assets/modules-fabric/v26/cloudsql-instance/outputs.tf @@ -0,0 +1,108 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + _all_instances = merge( + { primary = google_sql_database_instance.primary }, + google_sql_database_instance.replicas + ) +} + +output "connection_name" { + description = "Connection name of the primary instance." + value = google_sql_database_instance.primary.connection_name +} + +output "connection_names" { + description = "Connection names of all instances." + value = { + for id, instance in local._all_instances : + id => instance.connection_name + } +} + +output "id" { + description = "Fully qualified primary instance id." + value = google_sql_database_instance.primary.private_ip_address +} + +output "ids" { + description = "Fully qualified ids of all instances." + value = { + for id, instance in local._all_instances : + id => instance.id + } +} + +output "instances" { + description = "Cloud SQL instance resources." + value = local._all_instances + sensitive = true +} + +output "ip" { + description = "IP address of the primary instance." + value = google_sql_database_instance.primary.private_ip_address +} + +output "ips" { + description = "IP addresses of all instances." + value = { + for id, instance in local._all_instances : + id => instance.private_ip_address + } +} + +output "name" { + description = "Name of the primary instance." + value = google_sql_database_instance.primary.name +} + +output "names" { + description = "Names of all instances." + value = { + for id, instance in local._all_instances : + id => instance.name + } +} + +output "postgres_client_certificates" { + description = "The CA Certificate used to connect to the SQL Instance via SSL." + value = google_sql_ssl_cert.postgres_client_certificates + sensitive = true +} + +output "self_link" { + description = "Self link of the primary instance." + value = google_sql_database_instance.primary.self_link +} + +output "self_links" { + description = "Self links of all instances." + value = { + for id, instance in local._all_instances : + id => instance.self_link + } +} + +output "user_passwords" { + description = "Map of containing the password of all users created through terraform." + value = { + for name, user in google_sql_user.users : + name => user.password + } + sensitive = true +} diff --git a/assets/modules-fabric/v26/cloudsql-instance/variables.tf b/assets/modules-fabric/v26/cloudsql-instance/variables.tf new file mode 100644 index 0000000..e183741 --- /dev/null +++ b/assets/modules-fabric/v26/cloudsql-instance/variables.tf @@ -0,0 +1,208 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +variable "activation_policy" { + description = "This variable specifies when the instance should be active. Can be either ALWAYS, NEVER or ON_DEMAND. Default is ALWAYS." + type = string + default = "ALWAYS" + validation { + condition = var.activation_policy == "NEVER" || var.activation_policy == "ON_DEMAND" || var.activation_policy == "ALWAYS" + error_message = "The variable activation_policy must be ALWAYS, NEVER or ON_DEMAND." + } + nullable = false +} + +variable "allocated_ip_ranges" { + description = "(Optional)The name of the allocated ip range for the private ip CloudSQL instance. For example: \"google-managed-services-default\". If set, the instance ip will be created in the allocated range. The range name must comply with RFC 1035. Specifically, the name must be 1-63 characters long and match the regular expression a-z?." + type = object({ + primary = optional(string) + replica = optional(string) + }) + default = {} + nullable = false +} +variable "authorized_networks" { + description = "Map of NAME=>CIDR_RANGE to allow to connect to the database(s)." + type = map(string) + default = null +} + +variable "availability_type" { + description = "Availability type for the primary replica. Either `ZONAL` or `REGIONAL`." + type = string + default = "ZONAL" +} + +variable "backup_configuration" { + description = "Backup settings for primary instance. Will be automatically enabled if using MySQL with one or more replicas." + nullable = false + type = object({ + enabled = optional(bool, false) + binary_log_enabled = optional(bool, false) + start_time = optional(string, "23:00") + location = optional(string) + log_retention_days = optional(number, 7) + point_in_time_recovery_enabled = optional(bool) + retention_count = optional(number, 7) + }) + default = { + enabled = false + binary_log_enabled = false + start_time = "23:00" + location = null + log_retention_days = 7 + point_in_time_recovery_enabled = null + retention_count = 7 + } +} + +variable "database_version" { + description = "Database type and version to create." + type = string +} + +variable "databases" { + description = "Databases to create once the primary instance is created." + type = list(string) + default = null +} + +variable "deletion_protection" { + description = "Allow terraform to delete instances." + type = bool + default = false +} + +variable "deletion_protection_enabled" { + description = "Set Google's deletion protection attribute which applies across all surfaces (UI, API, & Terraform)." + type = bool + default = false +} + +variable "disk_size" { + description = "Disk size in GB. Set to null to enable autoresize." + type = number + default = null +} + +variable "disk_type" { + description = "The type of data disk: `PD_SSD` or `PD_HDD`." + type = string + default = "PD_SSD" +} + +variable "encryption_key_name" { + description = "The full path to the encryption key used for the CMEK disk encryption of the primary instance." + type = string + default = null +} + +variable "flags" { + description = "Map FLAG_NAME=>VALUE for database-specific tuning." + type = map(string) + default = null +} + +variable "insights_config" { + description = "Query Insights configuration. Defaults to null which disables Query Insights." + type = object({ + query_string_length = optional(number, 1024) + record_application_tags = optional(bool, false) + record_client_address = optional(bool, false) + query_plans_per_minute = optional(number, 5) + }) + default = null +} + +variable "ipv4_enabled" { + description = "Add a public IP address to database instance." + type = bool + default = false +} + +variable "labels" { + description = "Labels to be attached to all instances." + type = map(string) + default = null +} + +variable "name" { + description = "Name of primary instance." + type = string +} + +variable "network" { + description = "VPC self link where the instances will be deployed. Private Service Networking must be enabled and configured in this VPC." + type = string +} + +variable "postgres_client_certificates" { + description = "Map of cert keys connect to the application(s) using public IP." + type = list(string) + default = null +} + +variable "prefix" { + description = "Optional prefix used to generate instance names." + type = string + default = null + validation { + condition = var.prefix != "" + error_message = "Prefix cannot be empty, please use null instead." + } +} + +variable "project_id" { + description = "The ID of the project where this instances will be created." + type = string +} + +variable "region" { + description = "Region of the primary instance." + type = string +} + +variable "replicas" { + description = "Map of NAME=> {REGION, KMS_KEY} for additional read replicas. Set to null to disable replica creation." + type = map(object({ + region = string + encryption_key_name = string + })) + default = {} +} + +variable "require_ssl" { + description = "Enable SSL connections only." + type = bool + default = null +} + +variable "root_password" { + description = "Root password of the Cloud SQL instance. Required for MS SQL Server." + type = string + default = null +} + +variable "tier" { + description = "The machine type to use for the instances." + type = string +} + +variable "users" { + description = "Map of users to create in the primary instance (and replicated to other replicas) in the format USER=>PASSWORD. For MySQL, anything afterr the first `@` (if persent) will be used as the user's host. Set PASSWORD to null if you want to get an autogenerated password." + type = map(string) + default = null +} + diff --git a/assets/modules-fabric/v26/cloudsql-instance/versions.tf b/assets/modules-fabric/v26/cloudsql-instance/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/cloudsql-instance/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/compute-mig/README.md b/assets/modules-fabric/v26/compute-mig/README.md new file mode 100644 index 0000000..5e3dbd8 --- /dev/null +++ b/assets/modules-fabric/v26/compute-mig/README.md @@ -0,0 +1,424 @@ +# GCE Managed Instance Group module + +This module allows creating a managed instance group supporting one or more application versions via instance templates. Optionally, a health check and an autoscaler can be created, and the managed instance group can be configured to be stateful. + +This module can be coupled with the [`compute-vm`](../compute-vm) module which can manage instance templates, and the [`net-lb-int`](../net-lb-int) module to assign the MIG to a backend wired to an Internal Load Balancer. The first use case is shown in the examples below. + +Stateful disks can be created directly, as shown in the last example below. + + +- [Examples](#examples) + - [Simple Example](#simple-example) + - [Multiple Versions](#multiple-versions) + - [Health Check and Autohealing Policies](#health-check-and-autohealing-policies) + - [Autoscaling](#autoscaling) + - [Update Policy](#update-policy) + - [Stateful MIGs - MIG Config](#stateful-migs-mig-config) + - [Stateful MIGs - Instance Config](#stateful-migs-instance-config) +- [Variables](#variables) +- [Outputs](#outputs) + + +## Examples + +### Simple Example + +This example shows how to manage a simple MIG that leverages the `compute-vm` module to manage the underlying instance template. The following sub-examples will only show how to enable specific features of this module, and won't replicate the combined setup. + +```hcl +module "cos-nginx" { + source = "./fabric/modules/cloud-config-container/nginx" +} + +module "nginx-template" { + source = "./fabric/modules/compute-vm" + project_id = var.project_id + name = "nginx-template" + zone = "europe-west1-b" + tags = ["http-server", "ssh"] + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + nat = false + addresses = null + }] + boot_disk = { + initialize_params = { + image = "projects/cos-cloud/global/images/family/cos-stable" + } + } + create_template = true + metadata = { + user-data = module.cos-nginx.cloud_config + } +} + +module "nginx-mig" { + source = "./fabric/modules/compute-mig" + project_id = "my-project" + location = "europe-west1-b" + name = "mig-test" + target_size = 2 + instance_template = module.nginx-template.template.self_link +} +# tftest modules=2 resources=2 inventory=simple.yaml +``` + +### Multiple Versions + +If multiple versions are desired, use more `compute-vm` instances for the additional templates used in each version (not shown here), and reference them like this: + +```hcl +module "cos-nginx" { + source = "./fabric/modules/cloud-config-container/nginx" +} + +module "nginx-template" { + source = "./fabric/modules/compute-vm" + project_id = var.project_id + name = "nginx-template" + zone = "europe-west1-b" + tags = ["http-server", "ssh"] + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + nat = false + addresses = null + }] + boot_disk = { + initialize_params = { + image = "projects/cos-cloud/global/images/family/cos-stable" + } + } + create_template = true + metadata = { + user-data = module.cos-nginx.cloud_config + } +} + +module "nginx-mig" { + source = "./fabric/modules/compute-mig" + project_id = "my-project" + location = "europe-west1-b" + name = "mig-test" + target_size = 3 + instance_template = module.nginx-template.template.self_link + versions = { + canary = { + instance_template = module.nginx-template.template.self_link + target_size = { + fixed = 1 + } + } + } +} +# tftest modules=2 resources=2 +``` + +### Health Check and Autohealing Policies + +Autohealing policies can use an externally defined health check, or have this module auto-create one: + +```hcl +module "cos-nginx" { + source = "./fabric/modules/cloud-config-container/nginx" +} + +module "nginx-template" { + source = "./fabric/modules/compute-vm" + project_id = var.project_id + name = "nginx-template" + zone = "europe-west1-b" + tags = ["http-server", "ssh"] + network_interfaces = [{ + network = var.vpc.self_link, + subnetwork = var.subnet.self_link, + nat = false, + addresses = null + }] + boot_disk = { + initialize_params = { + image = "projects/cos-cloud/global/images/family/cos-stable" + } + } + create_template = true + metadata = { + user-data = module.cos-nginx.cloud_config + } +} + +module "nginx-mig" { + source = "./fabric/modules/compute-mig" + project_id = "my-project" + location = "europe-west1-b" + name = "mig-test" + target_size = 3 + instance_template = module.nginx-template.template.self_link + auto_healing_policies = { + initial_delay_sec = 30 + } + health_check_config = { + enable_logging = true + http = { + port = 80 + } + } +} +# tftest modules=2 resources=3 inventory=health-check.yaml +``` + +### Autoscaling + +The module can create and manage an autoscaler associated with the MIG. When using autoscaling do not set the `target_size` variable or set it to `null`. Here we show a CPU utilization autoscaler, the other available modes are load balancing utilization and custom metric, like the underlying autoscaler resource. + +```hcl +module "cos-nginx" { + source = "./fabric/modules/cloud-config-container/nginx" +} + +module "nginx-template" { + source = "./fabric/modules/compute-vm" + project_id = var.project_id + name = "nginx-template" + zone = "europe-west1-b" + tags = ["http-server", "ssh"] + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + nat = false + addresses = null + }] + boot_disk = { + initialize_params = { + image = "projects/cos-cloud/global/images/family/cos-stable" + } + } + create_template = true + metadata = { + user-data = module.cos-nginx.cloud_config + } +} + +module "nginx-mig" { + source = "./fabric/modules/compute-mig" + project_id = "my-project" + location = "europe-west1-b" + name = "mig-test" + target_size = 3 + instance_template = module.nginx-template.template.self_link + autoscaler_config = { + max_replicas = 3 + min_replicas = 1 + cooldown_period = 30 + scaling_signals = { + cpu_utilization = { + target = 0.65 + } + } + } +} +# tftest modules=2 resources=3 inventory=autoscaling.yaml +``` + +### Update Policy + +```hcl +module "cos-nginx" { + source = "./fabric/modules/cloud-config-container/nginx" +} + +module "nginx-template" { + source = "./fabric/modules/compute-vm" + project_id = var.project_id + name = "nginx-template" + zone = "europe-west1-b" + tags = ["http-server", "ssh"] + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + nat = false + addresses = null + }] + boot_disk = { + initialize_params = { + image = "projects/cos-cloud/global/images/family/cos-stable" + } + } + create_template = true + metadata = { + user-data = module.cos-nginx.cloud_config + } +} + +module "nginx-mig" { + source = "./fabric/modules/compute-mig" + project_id = "my-project" + location = "europe-west1-b" + name = "mig-test" + target_size = 3 + instance_template = module.nginx-template.template.self_link + update_policy = { + minimal_action = "REPLACE" + type = "PROACTIVE" + min_ready_sec = 30 + max_surge = { + fixed = 1 + } + } +} +# tftest modules=2 resources=2 +``` + +### Stateful MIGs - MIG Config + +Stateful MIGs have some limitations documented [here](https://cloud.google.com/compute/docs/instance-groups/configuring-stateful-migs#limitations). Enforcement of these requirements is the responsibility of users of this module. + +You can configure a disk defined in the instance template to be stateful for all instances in the MIG by configuring in the MIG's stateful policy, using the `stateful_disk_mig` variable. Alternatively, you can also configure stateful persistent disks individually per instance of the MIG by setting the `stateful_disk_instance` variable. A discussion on these scenarios can be found in the [docs](https://cloud.google.com/compute/docs/instance-groups/configuring-stateful-disks-in-migs). + +An example using only the configuration at the MIG level can be seen below. + +Note that when referencing the stateful disk, you use `device_name` and not `disk_name`. Specifying an existing disk in the template (and stateful config) only allows a single instance to be managed by the MIG, typically coupled with an autohealing policy (shown in the examples above). + +```hcl +module "cos-nginx" { + source = "./fabric/modules/cloud-config-container/nginx" +} + +module "nginx-template" { + source = "./fabric/modules/compute-vm" + project_id = "my-prj" + name = "nginx-template" + zone = "europe-west8-b" + tags = ["http-server", "ssh"] + instance_type = "e2-small" + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + }] + boot_disk = { + initialize_params = { + image = "projects/cos-cloud/global/images/family/cos-stable" + } + } + attached_disks = [{ + source_type = "attach" + name = "data-1" + size = 10 + source = "test-data-1" + }] + create_template = true + metadata = { + user-data = module.cos-nginx.cloud_config + } +} + +module "nginx-mig" { + source = "./fabric/modules/compute-mig" + project_id = "my-prj" + location = "europe-west8-b" + name = "mig-test-2" + target_size = 1 + instance_template = module.nginx-template.template.self_link + stateful_disks = { + data-1 = false + } +} +# tftest modules=2 resources=2 +``` + +### Stateful MIGs - Instance Config + +Here is an example defining the stateful config at the instance level. As in the example above, specifying an existing disk in the template (and stateful config) only allows a single instance to be managed by the MIG, typically coupled with an autohealing policy (shown in the examples above). + +```hcl +module "cos-nginx" { + source = "./fabric/modules/cloud-config-container/nginx" +} + +module "nginx-template" { + source = "./fabric/modules/compute-vm" + project_id = "my-prj" + name = "nginx-template" + zone = "europe-west8-b" + tags = ["http-server", "ssh"] + instance_type = "e2-small" + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + }] + boot_disk = { + initialize_params = { + image = "projects/cos-cloud/global/images/family/cos-stable" + } + } + attached_disks = [{ + source_type = "attach" + name = "data-1" + size = 10 + source = "test-data-1" + }] + create_template = true + metadata = { + user-data = module.cos-nginx.cloud_config + } +} + +module "nginx-mig" { + source = "./fabric/modules/compute-mig" + project_id = "my-prj" + location = "europe-west8-b" + name = "mig-test" + instance_template = module.nginx-template.template.self_link + stateful_config = { + instance-1 = { + minimal_action = "NONE", + most_disruptive_allowed_action = "REPLACE" + preserved_state = { + disks = { + data-1 = { + source = "projects/my-prj/zones/europe-west8-b/disks/test-data-1" + } + } + metadata = { + foo = "bar" + } + } + } + } +} +# tftest modules=2 resources=3 inventory=stateful.yaml +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [instance_template](variables.tf#L177) | Instance template for the default version. | string | ✓ | | +| [location](variables.tf#L182) | Compute zone or region. | string | ✓ | | +| [name](variables.tf#L187) | Managed group name. | string | ✓ | | +| [project_id](variables.tf#L198) | Project id. | string | ✓ | | +| [all_instances_config](variables.tf#L17) | Metadata and labels set to all instances in the group. | object({…}) | | null | +| [auto_healing_policies](variables.tf#L26) | Auto-healing policies for this group. | object({…}) | | null | +| [autoscaler_config](variables.tf#L35) | Optional autoscaler configuration. | object({…}) | | null | +| [default_version_name](variables.tf#L83) | Name used for the default version. | string | | "default" | +| [description](variables.tf#L89) | Optional description used for all resources managed by this module. | string | | "Terraform managed." | +| [distribution_policy](variables.tf#L95) | DIstribution policy for regional MIG. | object({…}) | | null | +| [health_check_config](variables.tf#L104) | Optional auto-created health check configuration, use the output self-link to set it in the auto healing policy. Refer to examples for usage. | object({…}) | | null | +| [named_ports](variables.tf#L192) | Named ports. | map(number) | | null | +| [stateful_config](variables.tf#L203) | Stateful configuration for individual instances. | map(object({…})) | | {} | +| [stateful_disks](variables.tf#L222) | Stateful disk configuration applied at the MIG level to all instances, in device name => on permanent instance delete rule as boolean. | map(bool) | | {} | +| [target_pools](variables.tf#L229) | Optional list of URLs for target pools to which new instances in the group are added. | list(string) | | [] | +| [target_size](variables.tf#L235) | Group target size, leave null when using an autoscaler. | number | | null | +| [update_policy](variables.tf#L241) | Update policy. Minimal action and type are required. | object({…}) | | null | +| [versions](variables.tf#L262) | Additional application versions, target_size is optional. | map(object({…})) | | {} | +| [wait_for_instances](variables.tf#L275) | Wait for all instances to be created/updated before returning. | object({…}) | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [autoscaler](outputs.tf#L17) | Auto-created autoscaler resource. | | +| [group_manager](outputs.tf#L26) | Instance group resource. | | +| [health_check](outputs.tf#L35) | Auto-created health-check resource. | | +| [id](outputs.tf#L44) | Fully qualified group manager id. | | + diff --git a/assets/modules-fabric/v26/compute-mig/autoscaler.tf b/assets/modules-fabric/v26/compute-mig/autoscaler.tf new file mode 100644 index 0000000..c0f7749 --- /dev/null +++ b/assets/modules-fabric/v26/compute-mig/autoscaler.tf @@ -0,0 +1,231 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Autoscaler resource. + +locals { + as_enabled = true + as_scaling = try(var.autoscaler_config.scaling_control, null) + as_signals = try(var.autoscaler_config.scaling_signals, null) +} + +resource "google_compute_autoscaler" "default" { + provider = google-beta + count = local.is_regional || var.autoscaler_config == null ? 0 : 1 + project = var.project_id + name = var.name + zone = var.location + description = var.description + target = google_compute_instance_group_manager.default.0.id + + autoscaling_policy { + max_replicas = var.autoscaler_config.max_replicas + min_replicas = var.autoscaler_config.min_replicas + cooldown_period = var.autoscaler_config.cooldown_period + mode = var.autoscaler_config.mode + + dynamic "scale_down_control" { + for_each = local.as_scaling.down == null ? [] : [""] + content { + time_window_sec = local.as_scaling.down.time_window_sec + dynamic "max_scaled_down_replicas" { + for_each = ( + local.as_scaling.down.max_replicas_fixed == null && + local.as_scaling.down.max_replicas_percent == null + ? [] + : [""] + ) + content { + fixed = local.as_scaling.down.max_replicas_fixed + percent = local.as_scaling.down.max_replicas_percent + } + } + } + } + + dynamic "scale_in_control" { + for_each = local.as_scaling.in == null ? [] : [""] + content { + time_window_sec = local.as_scaling.in.time_window_sec + dynamic "max_scaled_in_replicas" { + for_each = ( + local.as_scaling.in.max_replicas_fixed == null && + local.as_scaling.in.max_replicas_percent == null + ? [] + : [""] + ) + content { + fixed = local.as_scaling.in.max_replicas_fixed + percent = local.as_scaling.in.max_replicas_percent + } + } + } + } + + dynamic "cpu_utilization" { + for_each = local.as_signals.cpu_utilization == null ? [] : [""] + content { + target = local.as_signals.cpu_utilization.target + predictive_method = ( + local.as_signals.cpu_utilization.optimize_availability == true + ? "OPTIMIZE_AVAILABILITY" + : null + ) + } + } + + dynamic "load_balancing_utilization" { + for_each = local.as_signals.load_balancing_utilization == null ? [] : [""] + content { + target = local.as_signals.load_balancing_utilization.target + } + } + + dynamic "metric" { + for_each = toset( + local.as_signals.metrics == null ? [] : local.as_signals.metrics + ) + content { + name = metric.value.name + type = metric.value.type + target = metric.value.target_value + single_instance_assignment = metric.value.single_instance_assignment + filter = metric.value.time_series_filter + } + } + + dynamic "scaling_schedules" { + for_each = toset( + local.as_signals.schedules == null ? [] : local.as_signals.schedules + ) + iterator = schedule + content { + duration_sec = schedule.value.duration_sec + min_required_replicas = schedule.value.min_required_replicas + name = schedule.value.name + schedule = schedule.value.cron_schedule + description = schedule.value.description + disabled = schedule.value.disabled + time_zone = schedule.value.timezone + } + } + + } +} + +resource "google_compute_region_autoscaler" "default" { + provider = google-beta + count = local.is_regional && var.autoscaler_config != null ? 1 : 0 + project = var.project_id + name = var.name + region = var.location + description = var.description + target = google_compute_region_instance_group_manager.default.0.id + + autoscaling_policy { + max_replicas = var.autoscaler_config.max_replicas + min_replicas = var.autoscaler_config.min_replicas + cooldown_period = var.autoscaler_config.cooldown_period + mode = var.autoscaler_config.mode + + dynamic "scale_down_control" { + for_each = local.as_scaling.down == null ? [] : [""] + content { + time_window_sec = local.as_scaling.down.time_window_sec + dynamic "max_scaled_down_replicas" { + for_each = ( + local.as_scaling.down.max_replicas_fixed == null && + local.as_scaling.down.max_replicas_percent == null + ? [] + : [""] + ) + content { + fixed = local.as_scaling.down.max_replicas_fixed + percent = local.as_scaling.down.max_replicas_percent + } + } + } + } + + dynamic "scale_in_control" { + for_each = local.as_scaling.in == null ? [] : [""] + content { + time_window_sec = local.as_scaling.in.time_window_sec + dynamic "max_scaled_in_replicas" { + for_each = ( + local.as_scaling.in.max_replicas_fixed == null && + local.as_scaling.in.max_replicas_percent == null + ? [] + : [""] + ) + content { + fixed = local.as_scaling.in.max_replicas_fixed + percent = local.as_scaling.in.max_replicas_percent + } + } + } + } + + dynamic "cpu_utilization" { + for_each = local.as_signals.cpu_utilization == null ? [] : [""] + content { + target = local.as_signals.cpu_utilization.target + predictive_method = ( + local.as_signals.cpu_utilization.optimize_availability == true + ? "OPTIMIZE_AVAILABILITY" + : null + ) + } + } + + dynamic "load_balancing_utilization" { + for_each = local.as_signals.load_balancing_utilization == null ? [] : [""] + content { + target = local.as_signals.load_balancing_utilization.target + } + } + + dynamic "metric" { + for_each = toset( + local.as_signals.metrics == null ? [] : local.as_signals.metrics + ) + content { + name = metric.value.name + type = metric.value.type + target = metric.value.target_value + single_instance_assignment = metric.value.single_instance_assignment + filter = metric.value.time_series_filter + } + } + + dynamic "scaling_schedules" { + for_each = toset( + local.as_signals.schedules == null ? [] : local.as_signals.schedules + ) + iterator = schedule + content { + duration_sec = schedule.value.duration_sec + min_required_replicas = schedule.value.min_required_replicas + name = schedule.value.name + schedule = schedule.cron_schedule + description = schedule.value.description + disabled = schedule.value.disabled + time_zone = schedule.value.timezone + } + } + + } +} diff --git a/assets/modules-fabric/v26/compute-mig/health-check.tf b/assets/modules-fabric/v26/compute-mig/health-check.tf new file mode 100644 index 0000000..88f9f6e --- /dev/null +++ b/assets/modules-fabric/v26/compute-mig/health-check.tf @@ -0,0 +1,119 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Health check resource. + +locals { + hc = var.health_check_config + hc_grpc = try(local.hc.grpc, null) != null + hc_http = try(local.hc.http, null) != null + hc_http2 = try(local.hc.http2, null) != null + hc_https = try(local.hc.https, null) != null + hc_ssl = try(local.hc.ssl, null) != null + hc_tcp = try(local.hc.tcp, null) != null +} + +resource "google_compute_health_check" "default" { + provider = google-beta + count = local.hc != null ? 1 : 0 + project = var.project_id + name = var.name + description = local.hc.description + check_interval_sec = local.hc.check_interval_sec + healthy_threshold = local.hc.healthy_threshold + timeout_sec = local.hc.timeout_sec + unhealthy_threshold = local.hc.unhealthy_threshold + + dynamic "grpc_health_check" { + for_each = local.hc_grpc ? [""] : [] + content { + port = local.hc.grpc.port + port_name = local.hc.grpc.port_name + port_specification = local.hc.grpc.port_specification + grpc_service_name = local.hc.grpc.service_name + } + } + + dynamic "http_health_check" { + for_each = local.hc_http ? [""] : [] + content { + host = local.hc.http.host + port = local.hc.http.port + port_name = local.hc.http.port_name + port_specification = local.hc.http.port_specification + proxy_header = local.hc.http.proxy_header + request_path = local.hc.http.request_path + response = local.hc.http.response + } + } + + dynamic "http2_health_check" { + for_each = local.hc_http2 ? [""] : [] + content { + host = local.hc.http.host + port = local.hc.http.port + port_name = local.hc.http.port_name + port_specification = local.hc.http.port_specification + proxy_header = local.hc.http.proxy_header + request_path = local.hc.http.request_path + response = local.hc.http.response + } + } + + dynamic "https_health_check" { + for_each = local.hc_https ? [""] : [] + content { + host = local.hc.https.host + port = local.hc.https.port + port_name = local.hc.https.port_name + port_specification = local.hc.https.port_specification + proxy_header = local.hc.https.proxy_header + request_path = local.hc.https.request_path + response = local.hc.https.response + } + } + + dynamic "ssl_health_check" { + for_each = local.hc_ssl ? [""] : [] + content { + port = local.hc.tcp.port + port_name = local.hc.tcp.port_name + port_specification = local.hc.tcp.port_specification + proxy_header = local.hc.tcp.proxy_header + request = local.hc.tcp.request + response = local.hc.tcp.response + } + } + + dynamic "tcp_health_check" { + for_each = local.hc_tcp ? [""] : [] + content { + port = local.hc.tcp.port + port_name = local.hc.tcp.port_name + port_specification = local.hc.tcp.port_specification + proxy_header = local.hc.tcp.proxy_header + request = local.hc.tcp.request + response = local.hc.tcp.response + } + } + + dynamic "log_config" { + for_each = try(local.hc.enable_logging, null) == true ? [""] : [] + content { + enable = true + } + } +} diff --git a/assets/modules-fabric/v26/compute-mig/main.tf b/assets/modules-fabric/v26/compute-mig/main.tf new file mode 100644 index 0000000..65ce55b --- /dev/null +++ b/assets/modules-fabric/v26/compute-mig/main.tf @@ -0,0 +1,204 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + health_check = ( + try(var.auto_healing_policies.health_check, null) == null + ? try(google_compute_health_check.default.0.self_link, null) + : try(var.auto_healing_policies.health_check, null) + ) + instance_group_manager = ( + local.is_regional ? + google_compute_region_instance_group_manager.default : + google_compute_instance_group_manager.default + ) + is_regional = length(split("-", var.location)) == 2 +} + +resource "google_compute_instance_group_manager" "default" { + provider = google-beta + count = local.is_regional ? 0 : 1 + project = var.project_id + zone = var.location + name = var.name + base_instance_name = var.name + description = var.description + target_size = var.target_size + target_pools = var.target_pools + wait_for_instances = try(var.wait_for_instances.enabled, null) + wait_for_instances_status = try(var.wait_for_instances.status, null) + + dynamic "all_instances_config" { + for_each = var.all_instances_config == null ? [] : [""] + content { + labels = try(var.all_instances_config.labels, null) + metadata = try(var.all_instances_config.metadata, null) + } + } + + dynamic "auto_healing_policies" { + for_each = var.auto_healing_policies == null ? [] : [""] + iterator = config + content { + health_check = local.health_check + initial_delay_sec = var.auto_healing_policies.initial_delay_sec + } + } + + dynamic "named_port" { + for_each = var.named_ports == null ? {} : var.named_ports + iterator = config + content { + name = config.key + port = config.value + } + } + + dynamic "stateful_disk" { + for_each = var.stateful_disks + content { + device_name = stateful_disk.key + delete_rule = stateful_disk.value ? "ON_PERMANENT_INSTANCE_DELETION" : "NEVER" + } + } + + dynamic "update_policy" { + for_each = var.update_policy == null ? [] : [var.update_policy] + iterator = p + content { + minimal_action = p.value.minimal_action + type = p.value.type + max_surge_fixed = try(p.value.max_surge.fixed, null) + max_surge_percent = try(p.value.max_surge.percent, null) + max_unavailable_fixed = try(p.value.max_unavailable.fixed, null) + max_unavailable_percent = try(p.value.max_unavailable.percent, null) + min_ready_sec = p.value.min_ready_sec + most_disruptive_allowed_action = p.value.most_disruptive_action + replacement_method = p.value.replacement_method + } + } + + version { + instance_template = var.instance_template + name = var.default_version_name + } + + dynamic "version" { + for_each = var.versions + content { + name = version.key + instance_template = version.value.instance_template + dynamic "target_size" { + for_each = version.value.target_size == null ? [] : [""] + content { + fixed = version.value.target_size.fixed + percent = version.value.target_size.percent + } + } + } + } +} + +resource "google_compute_region_instance_group_manager" "default" { + provider = google-beta + count = local.is_regional ? 1 : 0 + project = var.project_id + region = var.location + name = var.name + base_instance_name = var.name + description = var.description + distribution_policy_target_shape = try( + var.distribution_policy.target_shape, null + ) + distribution_policy_zones = try( + var.distribution_policy.zones, null + ) + target_size = var.target_size + target_pools = var.target_pools + wait_for_instances = try(var.wait_for_instances.enabled, null) + wait_for_instances_status = try(var.wait_for_instances.status, null) + + dynamic "all_instances_config" { + for_each = var.all_instances_config == null ? [] : [""] + content { + labels = try(var.all_instances_config.labels, null) + metadata = try(var.all_instances_config.metadata, null) + } + } + + dynamic "auto_healing_policies" { + for_each = var.auto_healing_policies == null ? [] : [""] + iterator = config + content { + health_check = local.health_check + initial_delay_sec = var.auto_healing_policies.initial_delay_sec + } + } + + dynamic "named_port" { + for_each = var.named_ports == null ? {} : var.named_ports + iterator = config + content { + name = config.key + port = config.value + } + } + + dynamic "stateful_disk" { + for_each = var.stateful_disks + content { + device_name = stateful_disk.key + delete_rule = stateful_disk.value ? "ON_PERMANENT_INSTANCE_DELETION" : "NEVER" + } + } + + dynamic "update_policy" { + for_each = var.update_policy == null ? [] : [var.update_policy] + iterator = p + content { + minimal_action = p.value.minimal_action + type = p.value.type + instance_redistribution_type = p.value.regional_redistribution_type + max_surge_fixed = try(p.value.max_surge.fixed, null) + max_surge_percent = try(p.value.max_surge.percent, null) + max_unavailable_fixed = try(p.value.max_unavailable.fixed, null) + max_unavailable_percent = try(p.value.max_unavailable.percent, null) + min_ready_sec = p.value.min_ready_sec + most_disruptive_allowed_action = p.value.most_disruptive_action + replacement_method = p.value.replacement_method + } + } + + version { + instance_template = var.instance_template + name = var.default_version_name + } + + dynamic "version" { + for_each = var.versions + content { + name = version.key + instance_template = version.value.instance_template + dynamic "target_size" { + for_each = version.value.target_size == null ? [] : [""] + content { + fixed = version.value.target_size.fixed + percent = version.value.target_size.percent + } + } + } + } +} diff --git a/assets/modules-fabric/v26/compute-mig/outputs.tf b/assets/modules-fabric/v26/compute-mig/outputs.tf new file mode 100644 index 0000000..f761035 --- /dev/null +++ b/assets/modules-fabric/v26/compute-mig/outputs.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "autoscaler" { + description = "Auto-created autoscaler resource." + value = var.autoscaler_config == null ? null : try( + google_compute_autoscaler.default.0, + google_compute_region_autoscaler.default.0, + {} + ) +} + +output "group_manager" { + description = "Instance group resource." + value = try( + google_compute_instance_group_manager.default.0, + google_compute_region_instance_group_manager.default.0, + {} + ) +} + +output "health_check" { + description = "Auto-created health-check resource." + value = ( + var.health_check_config == null + ? null + : google_compute_health_check.default.0 + ) +} + +output "id" { + description = "Fully qualified group manager id." + value = try( + google_compute_instance_group_manager.default.0.id, + google_compute_region_instance_group_manager.default.0.id, + null + ) +} diff --git a/assets/modules-fabric/v26/compute-mig/stateful-config.tf b/assets/modules-fabric/v26/compute-mig/stateful-config.tf new file mode 100644 index 0000000..dc0329d --- /dev/null +++ b/assets/modules-fabric/v26/compute-mig/stateful-config.tf @@ -0,0 +1,91 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Instance-level stateful configuration resources. + +resource "google_compute_per_instance_config" "default" { + for_each = local.is_regional ? {} : var.stateful_config + project = var.project_id + zone = var.location + name = each.key + instance_group_manager = try( + google_compute_instance_group_manager.default.0.name, null + ) + minimal_action = each.value.minimal_action + most_disruptive_allowed_action = each.value.most_disruptive_action + remove_instance_state_on_destroy = each.value.remove_state_on_destroy + + dynamic "preserved_state" { + for_each = each.value.preserved_state == null ? [] : [""] + content { + metadata = each.value.preserved_state.metadata + dynamic "disk" { + for_each = ( + each.value.preserved_state.disks == null + ? {} + : each.value.preserved_state.disks + ) + content { + device_name = disk.key + source = disk.value.source + delete_rule = ( + disk.value.delete_on_instance_deletion == true + ? "ON_PERMANENT_INSTANCE_DELETION" + : "NEVER" + ) + mode = disk.value.read_only == true ? "READ_ONLY" : "READ_WRITE" + } + } + } + } +} + +resource "google_compute_region_per_instance_config" "default" { + for_each = local.is_regional ? var.stateful_config : {} + project = var.project_id + region = var.location + name = each.key + region_instance_group_manager = try( + google_compute_region_instance_group_manager.default.0.name, null + ) + minimal_action = each.value.minimal_action + most_disruptive_allowed_action = each.value.most_disruptive_action + remove_instance_state_on_destroy = each.value.remove_state_on_destroy + + dynamic "preserved_state" { + for_each = each.value.preserved_state == null ? [] : [""] + content { + metadata = each.value.preserved_state.metadata + dynamic "disk" { + for_each = ( + each.value.preserved_state.disks == null + ? {} + : each.value.preserved_state.disks + ) + content { + device_name = disk.key + source = disk.value.source + delete_rule = ( + disk.value.delete_on_instance_deletion == true + ? "ON_PERMANENT_INSTANCE_DELETION" + : "NEVER" + ) + mode = disk.value.read_only == true ? "READ_ONLY" : "READ_WRITE" + } + } + } + } +} diff --git a/assets/modules-fabric/v26/compute-mig/variables.tf b/assets/modules-fabric/v26/compute-mig/variables.tf new file mode 100644 index 0000000..20864d1 --- /dev/null +++ b/assets/modules-fabric/v26/compute-mig/variables.tf @@ -0,0 +1,282 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "all_instances_config" { + description = "Metadata and labels set to all instances in the group." + type = object({ + labels = optional(map(string)) + metadata = optional(map(string)) + }) + default = null +} + +variable "auto_healing_policies" { + description = "Auto-healing policies for this group." + type = object({ + health_check = optional(string) + initial_delay_sec = number + }) + default = null +} + +variable "autoscaler_config" { + description = "Optional autoscaler configuration." + type = object({ + max_replicas = number + min_replicas = number + cooldown_period = optional(number) + mode = optional(string) # OFF, ONLY_UP, ON + scaling_control = optional(object({ + down = optional(object({ + max_replicas_fixed = optional(number) + max_replicas_percent = optional(number) + time_window_sec = optional(number) + })) + in = optional(object({ + max_replicas_fixed = optional(number) + max_replicas_percent = optional(number) + time_window_sec = optional(number) + })) + }), {}) + scaling_signals = optional(object({ + cpu_utilization = optional(object({ + target = number + optimize_availability = optional(bool) + })) + load_balancing_utilization = optional(object({ + target = number + })) + metrics = optional(list(object({ + name = string + type = optional(string) # GAUGE, DELTA_PER_SECOND, DELTA_PER_MINUTE + target_value = optional(number) + single_instance_assignment = optional(number) + time_series_filter = optional(string) + }))) + schedules = optional(list(object({ + duration_sec = number + name = string + min_required_replicas = number + cron_schedule = string + description = optional(bool) + timezone = optional(string) + disabled = optional(bool) + }))) + }), {}) + }) + default = null +} + +variable "default_version_name" { + description = "Name used for the default version." + type = string + default = "default" +} + +variable "description" { + description = "Optional description used for all resources managed by this module." + type = string + default = "Terraform managed." +} + +variable "distribution_policy" { + description = "DIstribution policy for regional MIG." + type = object({ + target_shape = optional(string) + zones = optional(list(string)) + }) + default = null +} + +variable "health_check_config" { + description = "Optional auto-created health check configuration, use the output self-link to set it in the auto healing policy. Refer to examples for usage." + type = object({ + check_interval_sec = optional(number) + description = optional(string, "Terraform managed.") + enable_logging = optional(bool, false) + healthy_threshold = optional(number) + timeout_sec = optional(number) + unhealthy_threshold = optional(number) + grpc = optional(object({ + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + service_name = optional(string) + })) + http = optional(object({ + host = optional(string) + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request_path = optional(string) + response = optional(string) + })) + http2 = optional(object({ + host = optional(string) + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request_path = optional(string) + response = optional(string) + })) + https = optional(object({ + host = optional(string) + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request_path = optional(string) + response = optional(string) + })) + tcp = optional(object({ + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request = optional(string) + response = optional(string) + })) + ssl = optional(object({ + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request = optional(string) + response = optional(string) + })) + }) + default = null + validation { + condition = ( + (try(var.health_check_config.grpc, null) == null ? 0 : 1) + + (try(var.health_check_config.http, null) == null ? 0 : 1) + + (try(var.health_check_config.http2, null) == null ? 0 : 1) + + (try(var.health_check_config.https, null) == null ? 0 : 1) + + (try(var.health_check_config.tcp, null) == null ? 0 : 1) + + (try(var.health_check_config.ssl, null) == null ? 0 : 1) <= 1 + ) + error_message = "Only one health check type can be configured at a time." + } +} + +variable "instance_template" { + description = "Instance template for the default version." + type = string +} + +variable "location" { + description = "Compute zone or region." + type = string +} + +variable "name" { + description = "Managed group name." + type = string +} + +variable "named_ports" { + description = "Named ports." + type = map(number) + default = null +} + +variable "project_id" { + description = "Project id." + type = string +} + +variable "stateful_config" { + description = "Stateful configuration for individual instances." + type = map(object({ + minimal_action = optional(string) + most_disruptive_action = optional(string) + remove_state_on_destroy = optional(bool) + preserved_state = optional(object({ + disks = optional(map(object({ + source = string + delete_on_instance_deletion = optional(bool) + read_only = optional(bool) + }))) + metadata = optional(map(string)) + })) + })) + default = {} + nullable = false +} + +variable "stateful_disks" { + description = "Stateful disk configuration applied at the MIG level to all instances, in device name => on permanent instance delete rule as boolean." + type = map(bool) + default = {} + nullable = false +} + +variable "target_pools" { + description = "Optional list of URLs for target pools to which new instances in the group are added." + type = list(string) + default = [] +} + +variable "target_size" { + description = "Group target size, leave null when using an autoscaler." + type = number + default = null +} + +variable "update_policy" { + description = "Update policy. Minimal action and type are required." + type = object({ + minimal_action = string + type = string + max_surge = optional(object({ + fixed = optional(number) + percent = optional(number) + })) + max_unavailable = optional(object({ + fixed = optional(number) + percent = optional(number) + })) + min_ready_sec = optional(number) + most_disruptive_action = optional(string) + regional_redistribution_type = optional(string) + replacement_method = optional(string) + }) + default = null +} + +variable "versions" { + description = "Additional application versions, target_size is optional." + type = map(object({ + instance_template = string + target_size = optional(object({ + fixed = optional(number) + percent = optional(number) + })) + })) + default = {} + nullable = false +} + +variable "wait_for_instances" { + description = "Wait for all instances to be created/updated before returning." + type = object({ + enabled = bool + status = optional(string) + }) + default = null +} diff --git a/assets/modules-fabric/v26/compute-mig/versions.tf b/assets/modules-fabric/v26/compute-mig/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/compute-mig/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/compute-vm/README.md b/assets/modules-fabric/v26/compute-vm/README.md new file mode 100644 index 0000000..69a6844 --- /dev/null +++ b/assets/modules-fabric/v26/compute-vm/README.md @@ -0,0 +1,733 @@ +# Google Compute Engine VM module + +This module can operate in two distinct modes: + +- instance creation, with optional unmanaged group +- instance template creation + +In both modes, an optional service account can be created and assigned to either instances or template. If you need a managed instance group when using the module in template mode, refer to the [`compute-mig`](../compute-mig) module. + +## Examples + + +- [Examples](#examples) + - [Instance using defaults](#instance-using-defaults) + - [Service account management](#service-account-management) + - [Compute default service account](#compute-default-service-account) + - [Custom service account](#custom-service-account) + - [Custom service account, auto created](#custom-service-account-auto-created) + - [No service account](#no-service-account) + - [Disk management](#disk-management) + - [Disk sources](#disk-sources) + - [Disk types and options](#disk-types-and-options) + - [Boot disk as an independent resource](#boot-disk-as-an-independent-resource) + - [Network interfaces](#network-interfaces) + - [Internal and external IPs](#internal-and-external-ips) + - [Using Alias IPs](#using-alias-ips) + - [Using gVNIC](#using-gvnic) + - [Metadata](#metadata) + - [IAM](#iam) + - [Spot VM](#spot-vm) + - [Confidential compute](#confidential-compute) + - [Disk encryption with Cloud KMS](#disk-encryption-with-cloud-kms) + - [Instance template](#instance-template) + - [Instance group](#instance-group) + - [Instance Schedule](#instance-schedule) + - [Snapshot Schedules](#snapshot-schedules) +- [Variables](#variables) +- [Outputs](#outputs) +- [TODO](#todo) + + +### Instance using defaults + +The simplest example leverages defaults for the boot disk image and size, and uses a service account created by the module. Multiple instances can be managed via the `instance_count` variable. + +```hcl +module "simple-vm-example" { + source = "./fabric/modules/compute-vm" + project_id = var.project_id + zone = "europe-west1-b" + name = "test" + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + }] +} +# tftest modules=1 resources=1 inventory=defaults.yaml +``` + +### Service account management + +VM service accounts can be managed in four different ways: + +- in its default configuration, the module uses the Compute default service account with a basic set of scopes (`devstorage.read_only`, `logging.write`, `monitoring.write`) +- a custom service account can be used by passing its email in the `service_account.email` variable +- a custom service account can be created by the module and used by setting the `service_account.auto_create` variable to `true` +- the instance can be created with no service account by setting the `service_account` variable to `null` + +Scopes for custom service accounts are set by default to `cloud-platform` and `userinfo.email`, and can be further customized regardless of which service account is used by directly setting the `service_account.scopes` variable. + +#### Compute default service account + +```hcl +module "vm-managed-sa-example" { + source = "./fabric/modules/compute-vm" + project_id = var.project_id + zone = "europe-west1-b" + name = "test1" + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + }] +} +# tftest modules=1 resources=1 inventory=sa-default.yaml +``` + +#### Custom service account + +```hcl +module "vm-managed-sa-example2" { + source = "./fabric/modules/compute-vm" + project_id = var.project_id + zone = "europe-west1-b" + name = "test2" + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + }] + service_account = { + email = "sa-0@myproj.iam.gserviceaccount.com" + } +} +# tftest modules=1 resources=1 inventory=sa-custom.yaml +``` + +#### Custom service account, auto created + +```hcl +module "vm-managed-sa-example2" { + source = "./fabric/modules/compute-vm" + project_id = var.project_id + zone = "europe-west1-b" + name = "test2" + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + }] + service_account = { + auto_create = true + } +} +# tftest modules=1 resources=2 inventory=sa-managed.yaml +``` + +#### No service account + +```hcl +module "vm-managed-sa-example2" { + source = "./fabric/modules/compute-vm" + project_id = var.project_id + zone = "europe-west1-b" + name = "test2" + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + }] + service_account = null +} +# tftest modules=1 resources=1 inventory=sa-none.yaml +``` + +### Disk management + +#### Disk sources + +Attached disks can be created and optionally initialized from a pre-existing source, or attached to VMs when pre-existing. The `source` and `source_type` attributes of the `attached_disks` variable allows several modes of operation: + +- `source_type = "image"` can be used with zonal disks in instances and templates, set `source` to the image name or self link +- `source_type = "snapshot"` can be used with instances only, set `source` to the snapshot name or self link +- `source_type = "attach"` can be used for both instances and templates to attach an existing disk, set source to the name (for zonal disks) or self link (for regional disks) of the existing disk to attach; no disk will be created +- `source_type = null` can be used where an empty disk is needed, `source` becomes irrelevant and can be left null + +This is an example of attaching a pre-existing regional PD to a new instance: + +```hcl +module "vm-disks-example" { + source = "./fabric/modules/compute-vm" + project_id = var.project_id + zone = "${var.region}-b" + name = "test" + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + }] + attached_disks = [{ + name = "repd-1" + size = 10 + source_type = "attach" + source = "regions/${var.region}/disks/repd-test-1" + options = { + replica_zone = "${var.region}-c" + } + }] + service_account = { + auto_create = true + } +} +# tftest modules=1 resources=2 +``` + +And the same example for an instance template (where not using the full self link of the disk triggers recreation of the template) + +```hcl +module "vm-disks-example" { + source = "./fabric/modules/compute-vm" + project_id = var.project_id + zone = "${var.region}-b" + name = "test" + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + }] + attached_disks = [{ + name = "repd" + size = 10 + source_type = "attach" + source = "https://www.googleapis.com/compute/v1/projects/${var.project_id}/regions/${var.region}/disks/repd-test-1" + options = { + replica_zone = "${var.region}-c" + } + }] + service_account = { + auto_create = true + } + create_template = true +} +# tftest modules=1 resources=2 +``` + +#### Disk types and options + +The `attached_disks` variable exposes an `option` attribute that can be used to fine tune the configuration of each disk. The following example shows a VM with multiple disks + +```hcl +module "vm-disk-options-example" { + source = "./fabric/modules/compute-vm" + project_id = var.project_id + zone = "europe-west1-b" + name = "test" + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + }] + attached_disks = [ + { + name = "data1" + size = "10" + source_type = "image" + source = "image-1" + options = { + auto_delete = false + replica_zone = "europe-west1-c" + } + }, + { + name = "data2" + size = "20" + source_type = "snapshot" + source = "snapshot-2" + options = { + type = "pd-ssd" + mode = "READ_ONLY" + } + } + ] + service_account = { + auto_create = true + } +} +# tftest modules=1 resources=4 inventory=disk-options.yaml +``` + +#### Boot disk as an independent resource + +To create the boot disk as an independent resources instead of as part of the instance creation flow, set `boot_disk.use_independent_disk` to `true` and optionally configure `boot_disk.initialize_params`. + +This will create the boot disk as its own resource and attach it to the instance, allowing to recreate the instance from Terraform while preserving the boot. + +```hcl +module "simple-vm-example" { + source = "./fabric/modules/compute-vm" + project_id = var.project_id + zone = "europe-west1-b" + name = "test" + boot_disk = { + initialize_params = {} + use_independent_disk = true + } + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + }] + service_account = { + auto_create = true + } +} +# tftest modules=1 resources=3 inventory=independent-boot-disk.yaml +``` + +### Network interfaces + +#### Internal and external IPs + +By default VNs are create with an automatically assigned IP addresses, but you can change it through the `addresses` and `nat` attributes of the `network_interfaces` variable: + +```hcl +module "vm-internal-ip" { + source = "./fabric/modules/compute-vm" + project_id = "my-project" + zone = "europe-west1-b" + name = "vm-internal-ip" + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + addresses = { internal = "10.0.0.2" } + }] +} + +module "vm-external-ip" { + source = "./fabric/modules/compute-vm" + project_id = "my-project" + zone = "europe-west1-b" + name = "vm-external-ip" + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + nat = true + addresses = { external = "8.8.8.8" } + }] +} +# tftest modules=2 resources=2 inventory=ips.yaml +``` + +#### Using Alias IPs + +This example shows how to add additional [Alias IPs](https://cloud.google.com/vpc/docs/alias-ip) to your VM. + +```hcl +module "vm-with-alias-ips" { + source = "./fabric/modules/compute-vm" + project_id = "my-project" + zone = "europe-west1-b" + name = "test" + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + alias_ips = { + alias1 = "10.16.0.10/32" + } + }] +} +# tftest modules=1 resources=1 inventory=alias-ips.yaml +``` + +#### Using gVNIC + +This example shows how to enable [gVNIC](https://cloud.google.com/compute/docs/networking/using-gvnic) on your VM by customizing a `cos` image. Given that gVNIC needs to be enabled as an instance configuration and as a guest os configuration, you'll need to supply a bootable disk with `guest_os_features=GVNIC`. `SEV_CAPABLE`, `UEFI_COMPATIBLE` and `VIRTIO_SCSI_MULTIQUEUE` are enabled implicitly in the `cos`, `rhel`, `centos` and other images. + +```hcl + +resource "google_compute_image" "cos-gvnic" { + project = "my-project" + name = "my-image" + source_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-89-16108-534-18" + + guest_os_features { + type = "GVNIC" + } + guest_os_features { + type = "SEV_CAPABLE" + } + guest_os_features { + type = "UEFI_COMPATIBLE" + } + guest_os_features { + type = "VIRTIO_SCSI_MULTIQUEUE" + } +} + +module "vm-with-gvnic" { + source = "./fabric/modules/compute-vm" + project_id = "my-project" + zone = "europe-west1-b" + name = "test" + boot_disk = { + initialize_params = { + image = google_compute_image.cos-gvnic.self_link + type = "pd-ssd" + } + } + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + nic_type = "GVNIC" + }] + service_account = { + auto_create = true + } +} +# tftest modules=1 resources=3 inventory=gvnic.yaml +``` + +### Metadata + +You can define labels and custom metadata values. Metadata can be leveraged, for example, to define a custom startup script. + +```hcl +module "vm-metadata-example" { + source = "./fabric/modules/compute-vm" + project_id = var.project_id + zone = "europe-west1-b" + name = "nginx-server" + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + }] + labels = { + env = "dev" + system = "crm" + } + metadata = { + startup-script = <<-EOF + #! /bin/bash + apt-get update + apt-get install -y nginx + EOF + } + service_account = { + auto_create = true + } +} +# tftest modules=1 resources=2 inventory=metadata.yaml +``` + +### IAM + +Like most modules, you can assign IAM roles to the instance using the `iam` variable. + +```hcl +module "vm-iam-example" { + source = "./fabric/modules/compute-vm" + project_id = var.project_id + zone = "europe-west1-b" + name = "webserver" + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + }] + iam = { + "roles/compute.instanceAdmin" = [ + "group:webserver@example.com", + "group:admin@example.com" + ] + } +} +# tftest modules=1 resources=2 inventory=iam.yaml + +``` + +### Spot VM + +[Spot VMs](https://cloud.google.com/compute/docs/instances/spot) are ephemeral compute instances suitable for batch jobs and fault-tolerant workloads. Spot VMs provide new features that [preemptible instances](https://cloud.google.com/compute/docs/instances/preemptible) do not support, such as the absence of a maximum runtime. + +```hcl +module "spot-vm-example" { + source = "./fabric/modules/compute-vm" + project_id = var.project_id + zone = "europe-west1-b" + name = "test" + options = { + spot = true + termination_action = "STOP" + } + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + }] +} +# tftest modules=1 resources=1 inventory=spot.yaml +``` + +### Confidential compute + +You can enable confidential compute with the `confidential_compute` variable, which can be used for standalone instances or for instance templates. + +```hcl +module "vm-confidential-example" { + source = "./fabric/modules/compute-vm" + project_id = var.project_id + zone = "europe-west1-b" + name = "confidential-vm" + confidential_compute = true + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + }] + +} + +module "template-confidential-example" { + source = "./fabric/modules/compute-vm" + project_id = var.project_id + zone = "europe-west1-b" + name = "confidential-template" + confidential_compute = true + create_template = true + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + }] +} + +# tftest modules=2 resources=2 inventory=confidential.yaml +``` + +### Disk encryption with Cloud KMS + +This example shows how to control disk encryption via the the `encryption` variable, in this case the self link to a KMS CryptoKey that will be used to encrypt boot and attached disk. Managing the key with the `../kms` module is of course possible, but is not shown here. + +```hcl +module "kms-vm-example" { + source = "./fabric/modules/compute-vm" + project_id = var.project_id + zone = "europe-west1-b" + name = "kms-test" + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + }] + attached_disks = [{ + name = "attached-disk" + size = 10 + }] + service_account = { + auto_create = true + } + encryption = { + encrypt_boot = true + kms_key_self_link = var.kms_key.self_link + } +} +# tftest modules=1 resources=3 inventory=cmek.yaml +``` + +### Instance template + +This example shows how to use the module to manage an instance template that defines an additional attached disk for each instance, and overrides defaults for the boot disk image and service account. + +```hcl +module "cos-test" { + source = "./fabric/modules/compute-vm" + project_id = "my-project" + zone = "europe-west1-b" + name = "test" + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + }] + boot_disk = { + initialize_params = { + image = "projects/cos-cloud/global/images/family/cos-stable" + } + } + attached_disks = [ + { + name = "disk-1" + size = 10 + } + ] + service_account = { + email = "vm-default@my-project.iam.gserviceaccount.com" + } + create_template = true +} +# tftest modules=1 resources=1 inventory=template.yaml +``` + +### Instance group + +If an instance group is needed when operating in instance mode, simply set the `group` variable to a non null map. The map can contain named port declarations, or be empty if named ports are not needed. + +```hcl +locals { + cloud_config = "my cloud config" +} + +module "instance-group" { + source = "./fabric/modules/compute-vm" + project_id = "my-project" + zone = "europe-west1-b" + name = "ilb-test" + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + }] + boot_disk = { + image = "projects/cos-cloud/global/images/family/cos-stable" + } + service_account = { + email = var.service_account.email + scopes = ["https://www.googleapis.com/auth/cloud-platform"] + } + metadata = { + user-data = local.cloud_config + } + group = { named_ports = {} } +} +# tftest modules=1 resources=2 inventory=group.yaml +``` + +### Instance Schedule + +Instance start and stop schedules can be defined via an existing or auto-created resource policy. + +To use an existing policy pass its id to the `instance_schedule` variable: + +```hcl +module "instance" { + source = "./fabric/modules/compute-vm" + project_id = "my-project" + zone = "europe-west1-b" + name = "schedule-test" + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + }] + boot_disk = { + image = "projects/cos-cloud/global/images/family/cos-stable" + } + instance_schedule = { + resource_policy_id = "projects/my-project/regions/europe-west1/resourcePolicies/test" + } +} +# tftest modules=1 resources=1 inventory=instance-schedule-id.yaml +``` + +To create a new policy set its configuration in the `instance_schedule` variable. When removing the policy follow a two-step process by first setting `active = false` in the schedule configuration, which will unattach the policy, then removing the variable so the policy is destroyed. + +```hcl +module "instance" { + source = "./fabric/modules/compute-vm" + project_id = "my-project" + zone = "europe-west1-b" + name = "schedule-test" + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + }] + boot_disk = { + image = "projects/cos-cloud/global/images/family/cos-stable" + } + instance_schedule = { + create_config = { + vm_start = "0 8 * * *" + vm_stop = "0 17 * * *" + } + } +} +# tftest modules=1 resources=2 inventory=instance-schedule-create.yaml +``` + +### Snapshot Schedules + +Snapshot policies can be attached to disks with optional creation managed by the module. + +```hcl +module "instance" { + source = "./fabric/modules/compute-vm" + project_id = "my-project" + zone = "europe-west1-b" + name = "schedule-test" + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + }] + boot_disk = { + image = "projects/cos-cloud/global/images/family/cos-stable" + snapshot_schedule = "boot" + } + attached_disks = [ + { + name = "disk-1" + size = 10 + snapshot_schedule = "generic-vm" + } + ] + snapshot_schedules = { + boot = { + schedule = { + daily = { + days_in_cycle = 1 + start_time = "03:00" + } + } + } + } +} +# tftest modules=1 resources=5 inventory=snapshot-schedule-create.yaml +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [name](variables.tf#L235) | Instance name. | string | ✓ | | +| [network_interfaces](variables.tf#L240) | Network interfaces configuration. Use self links for Shared VPC, set addresses to null if not needed. | list(object({…})) | ✓ | | +| [project_id](variables.tf#L277) | Project id. | string | ✓ | | +| [zone](variables.tf#L369) | Compute zone. | string | ✓ | | +| [attached_disk_defaults](variables.tf#L17) | Defaults for attached disks options. | object({…}) | | {…} | +| [attached_disks](variables.tf#L37) | Additional disks, if options is null defaults will be used in its place. Source type is one of 'image' (zonal disks in vms and template), 'snapshot' (vm), 'existing', and null. | list(object({…})) | | [] | +| [boot_disk](variables.tf#L83) | Boot disk properties. | object({…}) | | {…} | +| [can_ip_forward](variables.tf#L117) | Enable IP forwarding. | bool | | false | +| [confidential_compute](variables.tf#L123) | Enable Confidential Compute for these instances. | bool | | false | +| [create_template](variables.tf#L129) | Create instance template instead of instances. | bool | | false | +| [description](variables.tf#L134) | Description of a Compute Instance. | string | | "Managed by the compute-vm Terraform module." | +| [enable_display](variables.tf#L140) | Enable virtual display on the instances. | bool | | false | +| [encryption](variables.tf#L146) | Encryption options. Only one of kms_key_self_link and disk_encryption_key_raw may be set. If needed, you can specify to encrypt or not the boot disk. | object({…}) | | null | +| [group](variables.tf#L156) | Define this variable to create an instance group for instances. Disabled for template use. | object({…}) | | null | +| [hostname](variables.tf#L164) | Instance FQDN name. | string | | null | +| [iam](variables.tf#L170) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [instance_schedule](variables.tf#L176) | Assign or create and assign an instance schedule policy. Either resource policy id or create_config must be specified if not null. Set active to null to dtach a policy from vm before destroying. | object({…}) | | null | +| [instance_type](variables.tf#L211) | Instance type. | string | | "f1-micro" | +| [labels](variables.tf#L217) | Instance labels. | map(string) | | {} | +| [metadata](variables.tf#L223) | Instance metadata. | map(string) | | {} | +| [min_cpu_platform](variables.tf#L229) | Minimum CPU platform. | string | | null | +| [options](variables.tf#L255) | Instance options. | object({…}) | | {…} | +| [scratch_disks](variables.tf#L282) | Scratch disks configuration. | object({…}) | | {…} | +| [service_account](variables.tf#L294) | Service account email and scopes. If email is null, the default Compute service account will be used unless auto_create is true, in which case a service account will be created. Set the variable to null to avoid attaching a service account. | object({…}) | | {} | +| [shielded_config](variables.tf#L304) | Shielded VM configuration of the instances. | object({…}) | | null | +| [snapshot_schedules](variables.tf#L314) | Snapshot schedule resource policies that can be attached to disks. | map(object({…})) | | {} | +| [tag_bindings](variables.tf#L357) | Tag bindings for this instance, in key => tag value id format. | map(string) | | null | +| [tags](variables.tf#L363) | Instance network tags for firewall rule targets. | list(string) | | [] | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [external_ip](outputs.tf#L17) | Instance main interface external IP addresses. | | +| [group](outputs.tf#L26) | Instance group resource. | | +| [id](outputs.tf#L31) | Fully qualified instance id. | | +| [instance](outputs.tf#L36) | Instance resource. | ✓ | +| [internal_ip](outputs.tf#L42) | Instance main interface internal IP address. | | +| [internal_ips](outputs.tf#L50) | Instance interfaces internal IP addresses. | | +| [self_link](outputs.tf#L58) | Instance self links. | | +| [service_account](outputs.tf#L63) | Service account resource. | | +| [service_account_email](outputs.tf#L68) | Service account email. | | +| [service_account_iam_email](outputs.tf#L73) | Service account email. | | +| [template](outputs.tf#L82) | Template resource. | | +| [template_name](outputs.tf#L87) | Template name. | | + +## TODO + +- [ ] add support for instance groups diff --git a/assets/modules-fabric/v26/compute-vm/main.tf b/assets/modules-fabric/v26/compute-vm/main.tf new file mode 100644 index 0000000..6d20a32 --- /dev/null +++ b/assets/modules-fabric/v26/compute-vm/main.tf @@ -0,0 +1,453 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + attached_disks = { + for disk in var.attached_disks : + (disk.name != null ? disk.name : disk.device_name) => merge(disk, { + options = disk.options == null ? var.attached_disk_defaults : disk.options + }) + } + attached_disks_regional = { + for k, v in local.attached_disks : + k => v if try(v.options.replica_zone, null) != null + } + attached_disks_zonal = { + for k, v in local.attached_disks : + k => v if try(v.options.replica_zone, null) == null + } + on_host_maintenance = ( + var.options.spot || var.confidential_compute + ? "TERMINATE" + : "MIGRATE" + ) + region = join("-", slice(split("-", var.zone), 0, 2)) + service_account = var.service_account == null ? null : { + email = ( + var.service_account.auto_create + ? google_service_account.service_account[0].email + : var.service_account.email + ) + scopes = ( + var.service_account.scopes != null ? var.service_account.scopes : ( + var.service_account.email == null && !var.service_account.auto_create + # default scopes for Compute default SA + ? [ + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring.write" + ] + # default scopes for own SA + : [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/userinfo.email" + ] + ) + ) + } + termination_action = var.options.spot ? coalesce(var.options.termination_action, "STOP") : null +} + +resource "google_compute_disk" "boot" { + count = !var.create_template && var.boot_disk.use_independent_disk ? 1 : 0 + project = var.project_id + zone = var.zone + name = "${var.name}-boot" + type = var.boot_disk.initialize_params.type + size = var.boot_disk.initialize_params.size + image = var.boot_disk.initialize_params.image + labels = merge(var.labels, { + disk_name = "boot" + disk_type = var.boot_disk.initialize_params.type + }) + dynamic "disk_encryption_key" { + for_each = var.encryption != null ? [""] : [] + content { + raw_key = var.encryption.disk_encryption_key_raw + kms_key_self_link = var.encryption.kms_key_self_link + } + } +} + +resource "google_compute_disk" "disks" { + for_each = var.create_template ? {} : { + for k, v in local.attached_disks_zonal : + k => v if v.source_type != "attach" + } + project = var.project_id + zone = var.zone + name = "${var.name}-${each.key}" + type = each.value.options.type + size = each.value.size + image = each.value.source_type == "image" ? each.value.source : null + snapshot = each.value.source_type == "snapshot" ? each.value.source : null + labels = merge(var.labels, { + disk_name = each.value.name + disk_type = each.value.options.type + }) + dynamic "disk_encryption_key" { + for_each = var.encryption != null ? [""] : [] + content { + raw_key = var.encryption.disk_encryption_key_raw + kms_key_self_link = var.encryption.kms_key_self_link + } + } +} + +resource "google_compute_region_disk" "disks" { + provider = google-beta + for_each = var.create_template ? {} : { + for k, v in local.attached_disks_regional : + k => v if v.source_type != "attach" + } + project = var.project_id + region = local.region + replica_zones = [var.zone, each.value.options.replica_zone] + name = "${var.name}-${each.key}" + type = each.value.options.type + size = each.value.size + # image = each.value.source_type == "image" ? each.value.source : null + snapshot = each.value.source_type == "snapshot" ? each.value.source : null + labels = merge(var.labels, { + disk_name = each.value.name + disk_type = each.value.options.type + }) + dynamic "disk_encryption_key" { + for_each = var.encryption != null ? [""] : [] + content { + raw_key = var.encryption.disk_encryption_key_raw + # TODO: check if self link works here + kms_key_name = var.encryption.kms_key_self_link + } + } +} + +resource "google_compute_instance" "default" { + provider = google-beta + count = var.create_template ? 0 : 1 + project = var.project_id + zone = var.zone + name = var.name + hostname = var.hostname + description = var.description + tags = var.tags + machine_type = var.instance_type + min_cpu_platform = var.min_cpu_platform + can_ip_forward = var.can_ip_forward + allow_stopping_for_update = var.options.allow_stopping_for_update + deletion_protection = var.options.deletion_protection + enable_display = var.enable_display + labels = var.labels + metadata = var.metadata + resource_policies = local.ischedule_attach + + dynamic "attached_disk" { + for_each = local.attached_disks_zonal + iterator = config + content { + device_name = ( + config.value.device_name != null + ? config.value.device_name + : config.value.name + ) + mode = config.value.options.mode + source = ( + config.value.source_type == "attach" + ? config.value.source + : google_compute_disk.disks[config.key].name + ) + } + } + + dynamic "attached_disk" { + for_each = local.attached_disks_regional + iterator = config + content { + device_name = ( + config.value.device_name != null + ? config.value.device_name + : config.value.name + ) + mode = config.value.options.mode + source = ( + config.value.source_type == "attach" + ? config.value.source + : google_compute_region_disk.disks[config.key].id + ) + } + } + + boot_disk { + auto_delete = ( + var.boot_disk.use_independent_disk + ? false + : var.boot_disk.auto_delete + ) + source = ( + var.boot_disk.use_independent_disk + ? google_compute_disk.boot.0.id + : var.boot_disk.source + ) + disk_encryption_key_raw = ( + var.encryption != null ? var.encryption.disk_encryption_key_raw : null + ) + kms_key_self_link = ( + var.encryption != null ? var.encryption.kms_key_self_link : null + ) + dynamic "initialize_params" { + for_each = ( + var.boot_disk.initialize_params == null + || + var.boot_disk.use_independent_disk + ? [] + : [""] + ) + content { + image = var.boot_disk.initialize_params.image + size = var.boot_disk.initialize_params.size + type = var.boot_disk.initialize_params.type + } + } + } + + dynamic "confidential_instance_config" { + for_each = var.confidential_compute ? [""] : [] + content { + enable_confidential_compute = true + } + } + + dynamic "network_interface" { + for_each = var.network_interfaces + iterator = config + content { + network = config.value.network + subnetwork = config.value.subnetwork + network_ip = try(config.value.addresses.internal, null) + dynamic "access_config" { + for_each = config.value.nat ? [""] : [] + content { + nat_ip = try(config.value.addresses.external, null) + } + } + dynamic "alias_ip_range" { + for_each = config.value.alias_ips + iterator = config_alias + content { + subnetwork_range_name = config_alias.key + ip_cidr_range = config_alias.value + } + } + nic_type = config.value.nic_type + } + } + + scheduling { + automatic_restart = !var.options.spot + instance_termination_action = local.termination_action + on_host_maintenance = local.on_host_maintenance + preemptible = var.options.spot + provisioning_model = var.options.spot ? "SPOT" : "STANDARD" + } + + dynamic "scratch_disk" { + for_each = [ + for i in range(0, var.scratch_disks.count) : var.scratch_disks.interface + ] + iterator = config + content { + interface = config.value + } + } + + dynamic "service_account" { + for_each = var.service_account == null ? [] : [""] + content { + email = local.service_account.email + scopes = local.service_account.scopes + } + } + + dynamic "shielded_instance_config" { + for_each = var.shielded_config != null ? [var.shielded_config] : [] + iterator = config + content { + enable_secure_boot = config.value.enable_secure_boot + enable_vtpm = config.value.enable_vtpm + enable_integrity_monitoring = config.value.enable_integrity_monitoring + } + } + + # guest_accelerator +} + +resource "google_compute_instance_iam_binding" "default" { + project = var.project_id + for_each = var.iam + zone = var.zone + instance_name = var.name + role = each.key + members = each.value + depends_on = [google_compute_instance.default] +} + +resource "google_compute_instance_template" "default" { + provider = google-beta + count = var.create_template ? 1 : 0 + project = var.project_id + region = local.region + name_prefix = "${var.name}-" + description = var.description + tags = var.tags + machine_type = var.instance_type + min_cpu_platform = var.min_cpu_platform + can_ip_forward = var.can_ip_forward + metadata = var.metadata + labels = var.labels + + disk { + auto_delete = var.boot_disk.auto_delete + boot = true + disk_size_gb = var.boot_disk.initialize_params.size + disk_type = var.boot_disk.initialize_params.type + source_image = var.boot_disk.initialize_params.image + } + + dynamic "confidential_instance_config" { + for_each = var.confidential_compute ? [""] : [] + content { + enable_confidential_compute = true + } + } + + dynamic "disk" { + for_each = local.attached_disks + iterator = config + content { + auto_delete = config.value.options.auto_delete + device_name = config.value.device_name != null ? config.value.device_name : config.value.name + # Cannot use `source` with any of the fields in + # [disk_size_gb disk_name disk_type source_image labels] + disk_type = ( + config.value.source_type != "attach" ? config.value.options.type : null + ) + disk_size_gb = ( + config.value.source_type != "attach" ? config.value.size : null + ) + mode = config.value.options.mode + source_image = ( + config.value.source_type == "image" ? config.value.source : null + ) + source = ( + config.value.source_type == "attach" ? config.value.source : null + ) + disk_name = ( + config.value.source_type != "attach" ? config.value.name : null + ) + type = "PERSISTENT" + dynamic "disk_encryption_key" { + for_each = var.encryption != null ? [""] : [] + content { + kms_key_self_link = var.encryption.kms_key_self_link + } + } + } + } + + dynamic "network_interface" { + for_each = var.network_interfaces + iterator = config + content { + network = config.value.network + subnetwork = config.value.subnetwork + network_ip = try(config.value.addresses.internal, null) + dynamic "access_config" { + for_each = config.value.nat ? [""] : [] + content { + nat_ip = try(config.value.addresses.external, null) + } + } + dynamic "alias_ip_range" { + for_each = config.value.alias_ips + iterator = config_alias + content { + subnetwork_range_name = config_alias.key + ip_cidr_range = config_alias.value + } + } + nic_type = config.value.nic_type + } + } + + scheduling { + automatic_restart = !var.options.spot + instance_termination_action = local.termination_action + on_host_maintenance = local.on_host_maintenance + preemptible = var.options.spot + provisioning_model = var.options.spot ? "SPOT" : "STANDARD" + } + + dynamic "service_account" { + for_each = var.service_account == null ? [] : [""] + content { + email = local.service_account.email + scopes = local.service_account.scopes + } + } + + dynamic "shielded_instance_config" { + for_each = var.shielded_config != null ? [var.shielded_config] : [] + iterator = config + content { + enable_secure_boot = config.value.enable_secure_boot + enable_vtpm = config.value.enable_vtpm + enable_integrity_monitoring = config.value.enable_integrity_monitoring + } + } + + lifecycle { + create_before_destroy = true + } +} + +resource "google_compute_instance_group" "unmanaged" { + count = var.group != null && !var.create_template ? 1 : 0 + project = var.project_id + network = ( + length(var.network_interfaces) > 0 + ? var.network_interfaces.0.network + : "" + ) + zone = var.zone + name = var.name + description = var.description + instances = [google_compute_instance.default.0.self_link] + dynamic "named_port" { + for_each = var.group.named_ports != null ? var.group.named_ports : {} + iterator = config + content { + name = config.key + port = config.value + } + } +} + +resource "google_service_account" "service_account" { + count = try(var.service_account.auto_create, null) == true ? 1 : 0 + project = var.project_id + account_id = "tf-vm-${var.name}" + display_name = "Terraform VM ${var.name}." +} diff --git a/assets/modules-fabric/v26/compute-vm/outputs.tf b/assets/modules-fabric/v26/compute-vm/outputs.tf new file mode 100644 index 0000000..f1df0a3 --- /dev/null +++ b/assets/modules-fabric/v26/compute-vm/outputs.tf @@ -0,0 +1,90 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "external_ip" { + description = "Instance main interface external IP addresses." + value = ( + var.network_interfaces[0].nat + ? try(google_compute_instance.default.0.network_interface.0.access_config.0.nat_ip, null) + : null + ) +} + +output "group" { + description = "Instance group resource." + value = try(google_compute_instance_group.unmanaged.0, null) +} + +output "id" { + description = "Fully qualified instance id." + value = try(google_compute_instance.default.0.id, null) +} + +output "instance" { + description = "Instance resource." + sensitive = true + value = try(google_compute_instance.default.0, null) +} + +output "internal_ip" { + description = "Instance main interface internal IP address." + value = try( + google_compute_instance.default.0.network_interface.0.network_ip, + null + ) +} + +output "internal_ips" { + description = "Instance interfaces internal IP addresses." + value = [ + for nic in try(google_compute_instance.default.0.network_interface, []) + : nic.network_ip + ] +} + +output "self_link" { + description = "Instance self links." + value = try(google_compute_instance.default.0.self_link, null) +} + +output "service_account" { + description = "Service account resource." + value = try(google_service_account.service_account.0, null) +} + +output "service_account_email" { + description = "Service account email." + value = try(local.service_account.email, null) +} + +output "service_account_iam_email" { + description = "Service account email." + value = ( + try(local.service_account.email, null) == null + ? null + : "serviceAccount:${local.service_account.email}" + ) +} + +output "template" { + description = "Template resource." + value = try(google_compute_instance_template.default.0, null) +} + +output "template_name" { + description = "Template name." + value = try(google_compute_instance_template.default.0.name, null) +} diff --git a/assets/modules-fabric/v26/compute-vm/resource-policies.tf b/assets/modules-fabric/v26/compute-vm/resource-policies.tf new file mode 100644 index 0000000..1aaf6ec --- /dev/null +++ b/assets/modules-fabric/v26/compute-vm/resource-policies.tf @@ -0,0 +1,174 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Resource policies. + +locals { + ischedule = try(var.instance_schedule.create_config, null) + ischedule_attach = var.instance_schedule == null ? null : ( + var.instance_schedule.create_config != null + # created policy with optional attach to allow policy destroy + ? ( + var.instance_schedule.create_config.active + ? [google_compute_resource_policy.schedule.0.id] + : null + ) + # externally managed policy + : [var.instance_schedule.resource_policy_id] + ) +} + +resource "google_compute_resource_policy" "schedule" { + count = local.ischedule != null ? 1 : 0 + project = var.project_id + region = substr(var.zone, 0, length(var.zone) - 2) + name = var.name + description = coalesce( + local.ischedule.description, "Schedule policy for ${var.name}." + ) + instance_schedule_policy { + expiration_time = local.ischedule.expiration_time + start_time = local.ischedule.start_time + time_zone = local.ischedule.timezone + dynamic "vm_start_schedule" { + for_each = local.ischedule.vm_start != null ? [""] : [] + content { + schedule = local.ischedule.vm_start + } + } + dynamic "vm_stop_schedule" { + for_each = local.ischedule.vm_stop != null ? [""] : [] + content { + schedule = local.ischedule.vm_stop + } + } + } +} + +resource "google_compute_resource_policy" "snapshot" { + for_each = var.snapshot_schedules + project = var.project_id + region = substr(var.zone, 0, length(var.zone) - 2) + name = "${var.name}-${each.key}" + description = coalesce( + each.value.description, "Schedule policy ${each.key} for ${var.name}." + ) + snapshot_schedule_policy { + schedule { + dynamic "daily_schedule" { + for_each = each.value.schedule.daily != null ? [""] : [] + content { + days_in_cycle = each.value.schedule.daily.days_in_cycle + start_time = each.value.schedule.daily.start_time + } + } + dynamic "hourly_schedule" { + for_each = each.value.schedule.hourly != null ? [""] : [] + content { + hours_in_cycle = each.value.schedule.hourly.hours_in_cycle + start_time = each.value.schedule.hourly.start_time + } + } + dynamic "weekly_schedule" { + for_each = each.value.schedule.weekly != null ? [""] : [] + content { + dynamic "day_of_weeks" { + for_each = each.value.schedule.weekly + content { + day = day_of_weeks.value.day + start_time = day_of_weeks.value.start_time + } + } + } + } + } + dynamic "retention_policy" { + for_each = each.value.retention_policy != null ? [""] : [] + content { + max_retention_days = each.value.retention_policy.max_retention_days + on_source_disk_delete = ( + each.value.retention_policy.on_source_disk_delete_keep == false + ? "APPLY_RETENTION_POLICY" + : "KEEP_AUTO_SNAPSHOTS" + ) + } + } + dynamic "snapshot_properties" { + for_each = each.value.snapshot_properties != null ? [""] : [] + content { + labels = each.value.snapshot_properties.labels + storage_locations = each.value.snapshot_properties.storage_locations + guest_flush = each.value.snapshot_properties.guest_flush + } + } + } +} + +resource "google_compute_disk_resource_policy_attachment" "boot" { + count = var.boot_disk.snapshot_schedule != null ? 1 : 0 + project = var.project_id + zone = var.zone + name = try( + google_compute_resource_policy.snapshot[var.boot_disk.snapshot_schedule].name, + var.boot_disk.snapshot_schedule + ) + disk = var.name + depends_on = [google_compute_instance.default] +} + +resource "google_compute_disk_resource_policy_attachment" "attached" { + for_each = { + for k, v in local.attached_disks_zonal : + k => v if v.snapshot_schedule != null + } + project = var.project_id + zone = var.zone + name = try( + google_compute_resource_policy.snapshot[each.value.snapshot_schedule].name, + each.value.snapshot_schedule + ) + disk = ( + each.value.source_type == "attach" + ? each.value.source + : google_compute_disk.disks[each.key].name + ) + depends_on = [ + google_compute_instance.default, + google_compute_disk.disks + ] +} + +resource "google_compute_region_disk_resource_policy_attachment" "attached" { + for_each = { + for k, v in local.attached_disks_regional : + k => v if v.snapshot_schedule != null + } + project = var.project_id + region = substr(var.zone, 0, length(var.zone) - 2) + name = try( + google_compute_resource_policy.snapshot[each.value.snapshot_schedule].name, + each.value.snapshot_schedule + ) + disk = ( + each.value.source_type == "attach" + ? each.value.source + : google_compute_region_disk.disks[each.key].name + ) + depends_on = [ + google_compute_instance.default, + google_compute_region_disk.disks + ] +} diff --git a/assets/modules-fabric/v26/compute-vm/tags.tf b/assets/modules-fabric/v26/compute-vm/tags.tf new file mode 100644 index 0000000..95be831 --- /dev/null +++ b/assets/modules-fabric/v26/compute-vm/tags.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Tag bindings. + +resource "google_tags_tag_binding" "binding" { + for_each = var.create_template ? {} : coalesce(var.tag_bindings, {}) + parent = "//compute.googleapis.com/${google_compute_instance.default.0.id}" + tag_value = each.value +} diff --git a/assets/modules-fabric/v26/compute-vm/test.tfvars b/assets/modules-fabric/v26/compute-vm/test.tfvars new file mode 100644 index 0000000..5c60eab --- /dev/null +++ b/assets/modules-fabric/v26/compute-vm/test.tfvars @@ -0,0 +1,9 @@ +project_id = "tf-playground-svpc-gce" +zone = "europe-west8-b" +name = "test-sa" +instance_type = "e2-small" +network_interfaces = [{ + network = "https://www.googleapis.com/compute/v1/projects/ldj-dev-net-spoke-0/global/networks/dev-spoke-0" + subnetwork = "https://www.googleapis.com/compute/v1/projects/ldj-dev-net-spoke-0/regions/europe-west8/subnetworks/gce" +}] +# service_account = null diff --git a/assets/modules-fabric/v26/compute-vm/variables.tf b/assets/modules-fabric/v26/compute-vm/variables.tf new file mode 100644 index 0000000..e6a0a4c --- /dev/null +++ b/assets/modules-fabric/v26/compute-vm/variables.tf @@ -0,0 +1,372 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "attached_disk_defaults" { + description = "Defaults for attached disks options." + type = object({ + auto_delete = optional(bool, false) + mode = string + replica_zone = string + type = string + }) + default = { + auto_delete = true + mode = "READ_WRITE" + replica_zone = null + type = "pd-balanced" + } + validation { + condition = var.attached_disk_defaults.mode == "READ_WRITE" || !var.attached_disk_defaults.auto_delete + error_message = "auto_delete can only be specified on READ_WRITE disks." + } +} + +variable "attached_disks" { + description = "Additional disks, if options is null defaults will be used in its place. Source type is one of 'image' (zonal disks in vms and template), 'snapshot' (vm), 'existing', and null." + type = list(object({ + name = string + device_name = optional(string) + # TODO: size can be null when source_type is attach + size = string + snapshot_schedule = optional(string) + source = optional(string) + source_type = optional(string) + options = optional( + object({ + auto_delete = optional(bool, false) + mode = optional(string, "READ_WRITE") + replica_zone = optional(string) + type = optional(string, "pd-balanced") + }), + { + auto_delete = true + mode = "READ_WRITE" + replica_zone = null + type = "pd-balanced" + } + ) + })) + default = [] + validation { + condition = length([ + for d in var.attached_disks : d if( + d.source_type == null + || + contains(["image", "snapshot", "attach"], coalesce(d.source_type, "1")) + ) + ]) == length(var.attached_disks) + error_message = "Source type must be one of 'image', 'snapshot', 'attach', null." + } + + validation { + condition = length([ + for d in var.attached_disks : d if d.options == null || + d.options.mode == "READ_WRITE" || !d.options.auto_delete + ]) == length(var.attached_disks) + error_message = "auto_delete can only be specified on READ_WRITE disks." + } +} + +variable "boot_disk" { + description = "Boot disk properties." + type = object({ + auto_delete = optional(bool, true) + snapshot_schedule = optional(string) + source = optional(string) + initialize_params = optional(object({ + image = optional(string, "projects/debian-cloud/global/images/family/debian-11") + size = optional(number, 10) + type = optional(string, "pd-balanced") + })) + use_independent_disk = optional(bool, false) + }) + default = { + initialize_params = {} + } + nullable = false + validation { + condition = ( + (var.boot_disk.source == null ? 0 : 1) + + (var.boot_disk.initialize_params == null ? 0 : 1) < 2 + ) + error_message = "You can only have one of boot disk source or initialize params." + } + validation { + condition = ( + var.boot_disk.use_independent_disk != true + || + var.boot_disk.initialize_params != null + ) + error_message = "Using an independent disk for boot requires initialize params." + } +} + +variable "can_ip_forward" { + description = "Enable IP forwarding." + type = bool + default = false +} + +variable "confidential_compute" { + description = "Enable Confidential Compute for these instances." + type = bool + default = false +} + +variable "create_template" { + description = "Create instance template instead of instances." + type = bool + default = false +} +variable "description" { + description = "Description of a Compute Instance." + type = string + default = "Managed by the compute-vm Terraform module." +} + +variable "enable_display" { + description = "Enable virtual display on the instances." + type = bool + default = false +} + +variable "encryption" { + description = "Encryption options. Only one of kms_key_self_link and disk_encryption_key_raw may be set. If needed, you can specify to encrypt or not the boot disk." + type = object({ + encrypt_boot = optional(bool, false) + disk_encryption_key_raw = optional(string) + kms_key_self_link = optional(string) + }) + default = null +} + +variable "group" { + description = "Define this variable to create an instance group for instances. Disabled for template use." + type = object({ + named_ports = map(number) + }) + default = null +} + +variable "hostname" { + description = "Instance FQDN name." + type = string + default = null +} + +variable "iam" { + description = "IAM bindings in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} +} + +variable "instance_schedule" { + description = "Assign or create and assign an instance schedule policy. Either resource policy id or create_config must be specified if not null. Set active to null to dtach a policy from vm before destroying." + type = object({ + resource_policy_id = optional(string) + create_config = optional(object({ + active = optional(bool, true) + description = optional(string) + expiration_time = optional(string) + start_time = optional(string) + timezone = optional(string, "UTC") + vm_start = optional(string) + vm_stop = optional(string) + })) + }) + default = null + validation { + condition = ( + var.instance_schedule == null || + try(var.instance_schedule.resource_policy_id, null) != null || + try(var.instance_schedule.create_config, null) != null + ) + error_message = "A resource policy name or configuration must be specified when not null." + } + validation { + condition = ( + try(var.instance_schedule.create_config, null) == null || + length(compact([ + try(var.instance_schedule.create_config.vm_start, null), + try(var.instance_schedule.create_config.vm_stop, null) + ])) > 0 + ) + error_message = "A resource policy configuration must contain at least one schedule." + } +} + +variable "instance_type" { + description = "Instance type." + type = string + default = "f1-micro" +} + +variable "labels" { + description = "Instance labels." + type = map(string) + default = {} +} + +variable "metadata" { + description = "Instance metadata." + type = map(string) + default = {} +} + +variable "min_cpu_platform" { + description = "Minimum CPU platform." + type = string + default = null +} + +variable "name" { + description = "Instance name." + type = string +} + +variable "network_interfaces" { + description = "Network interfaces configuration. Use self links for Shared VPC, set addresses to null if not needed." + type = list(object({ + nat = optional(bool, false) + network = string + subnetwork = string + addresses = optional(object({ + internal = optional(string) + external = optional(string) + }), null) + alias_ips = optional(map(string), {}) + nic_type = optional(string) + })) +} + +variable "options" { + description = "Instance options." + type = object({ + allow_stopping_for_update = optional(bool, true) + deletion_protection = optional(bool, false) + spot = optional(bool, false) + termination_action = optional(string) + }) + default = { + allow_stopping_for_update = true + deletion_protection = false + spot = false + termination_action = null + } + validation { + condition = (var.options.termination_action == null + || + contains(["STOP", "DELETE"], coalesce(var.options.termination_action, "1"))) + error_message = "Allowed values for options.termination_action are 'STOP', 'DELETE' and null." + } +} + +variable "project_id" { + description = "Project id." + type = string +} + +variable "scratch_disks" { + description = "Scratch disks configuration." + type = object({ + count = number + interface = string + }) + default = { + count = 0 + interface = "NVME" + } +} + +variable "service_account" { + description = "Service account email and scopes. If email is null, the default Compute service account will be used unless auto_create is true, in which case a service account will be created. Set the variable to null to avoid attaching a service account." + type = object({ + auto_create = optional(bool, false) + email = optional(string) + scopes = optional(list(string)) + }) + default = {} +} + +variable "shielded_config" { + description = "Shielded VM configuration of the instances." + type = object({ + enable_secure_boot = bool + enable_vtpm = bool + enable_integrity_monitoring = bool + }) + default = null +} + +variable "snapshot_schedules" { + description = "Snapshot schedule resource policies that can be attached to disks." + type = map(object({ + schedule = object({ + daily = optional(object({ + days_in_cycle = number + start_time = string + })) + hourly = optional(object({ + hours_in_cycle = number + start_time = string + })) + weekly = optional(list(object({ + day = string + start_time = string + }))) + }) + description = optional(string) + retention_policy = optional(object({ + max_retention_days = number + on_source_disk_delete_keep = optional(bool) + })) + snapshot_properties = optional(object({ + chain_name = optional(string) + guest_flush = optional(bool) + labels = optional(map(string)) + storage_locations = optional(list(string)) + })) + })) + nullable = false + default = {} + validation { + condition = alltrue([ + for k, v in var.snapshot_schedules : ( + (v.schedule.daily != null ? 1 : 0) + + (v.schedule.hourly != null ? 1 : 0) + + (v.schedule.weekly != null ? 1 : 0) + ) == 1 + ]) + error_message = "Schedule must contain exactly one of daily, hourly, or weekly schedule." + } +} + +variable "tag_bindings" { + description = "Tag bindings for this instance, in key => tag value id format." + type = map(string) + default = null +} + +variable "tags" { + description = "Instance network tags for firewall rule targets." + type = list(string) + default = [] +} + +variable "zone" { + description = "Compute zone." + type = string +} diff --git a/assets/modules-fabric/v26/compute-vm/versions.tf b/assets/modules-fabric/v26/compute-vm/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/compute-vm/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/container-registry/README.md b/assets/modules-fabric/v26/container-registry/README.md new file mode 100644 index 0000000..a5748e2 --- /dev/null +++ b/assets/modules-fabric/v26/container-registry/README.md @@ -0,0 +1,34 @@ +# Google Cloud Container Registry Module + +This module simplifies the creation of GCS buckets used by Google Container Registry. + +## Example + +```hcl +module "container_registry" { + source = "./fabric/modules/container-registry" + project_id = "myproject" + location = "EU" + iam = { + "roles/storage.admin" = ["group:cicd@example.com"] + } +} +# tftest modules=1 resources=2 inventory=simple.yaml +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [project_id](variables.tf#L29) | Registry project id. | string | ✓ | | +| [iam](variables.tf#L17) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [location](variables.tf#L23) | Registry location. Can be US, EU, ASIA or empty. | string | | "" | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [id](outputs.tf#L17) | Fully qualified id of the registry bucket. | | + + diff --git a/assets/modules-fabric/v26/container-registry/main.tf b/assets/modules-fabric/v26/container-registry/main.tf new file mode 100644 index 0000000..7409751 --- /dev/null +++ b/assets/modules-fabric/v26/container-registry/main.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_container_registry" "registry" { + project = var.project_id + location = var.location +} + +resource "google_storage_bucket_iam_binding" "bindings" { + for_each = var.iam + bucket = google_container_registry.registry.id + role = each.key + members = each.value +} diff --git a/assets/modules-fabric/v26/container-registry/outputs.tf b/assets/modules-fabric/v26/container-registry/outputs.tf new file mode 100644 index 0000000..1c2aeb4 --- /dev/null +++ b/assets/modules-fabric/v26/container-registry/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "id" { + description = "Fully qualified id of the registry bucket." + value = google_container_registry.registry.id +} diff --git a/assets/modules-fabric/v26/container-registry/variables.tf b/assets/modules-fabric/v26/container-registry/variables.tf new file mode 100644 index 0000000..9a5709e --- /dev/null +++ b/assets/modules-fabric/v26/container-registry/variables.tf @@ -0,0 +1,32 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "iam" { + description = "IAM bindings for topic in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} +} + +variable "location" { + description = "Registry location. Can be US, EU, ASIA or empty." + type = string + default = "" +} + +variable "project_id" { + description = "Registry project id." + type = string +} diff --git a/assets/modules-fabric/v26/container-registry/versions.tf b/assets/modules-fabric/v26/container-registry/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/container-registry/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/data-catalog-policy-tag/README.md b/assets/modules-fabric/v26/data-catalog-policy-tag/README.md new file mode 100644 index 0000000..8a46478 --- /dev/null +++ b/assets/modules-fabric/v26/data-catalog-policy-tag/README.md @@ -0,0 +1,104 @@ +# Data Catalog Module + +This module simplifies the creation of [Data Catalog](https://cloud.google.com/data-catalog) Policy Tags. Policy Tags can be used to configure [Bigquery column-level access](https://cloud.google.com/bigquery/docs/best-practices-policy-tags). + +Note: Data Catalog is still in beta, hence this module currently uses the beta provider. + + +- [IAM](#iam) +- [Examples](#examples) + - [Simple Taxonomy with policy tags](#simple-taxonomy-with-policy-tags) + - [Taxonomy with IAM binding](#taxonomy-with-iam-binding) +- [Variables](#variables) +- [Outputs](#outputs) +- [TODO](#todo) + + +## IAM + +IAM is managed via several variables that implement different features and levels of control: + +- `iam` and `group_iam` configure authoritative bindings that manage individual roles exclusively, and are internally merged +- `iam_bindings` configure authoritative bindings with optional support for conditions, and are not internally merged with the previous two variables +- `iam_bindings_additive` configure additive bindings via individual role/member pairs with optional support conditions + +The authoritative and additive approaches can be used together, provided different roles are managed by each. Some care must also be taken with the `groups_iam` variable to ensure that variable keys are static values, so that Terraform is able to compute the dependency graph. + +Refer to the [project module](../project/README.md#iam) for examples of the IAM interface. + +## Examples + +### Simple Taxonomy with policy tags + +```hcl +module "cmn-dc" { + source = "./fabric/modules/data-catalog-policy-tag" + name = "my-datacatalog-policy-tags" + project_id = "my-project" + tags = { + low = {} + medium = {} + high = {} + } +} +# tftest modules=1 resources=4 +``` + +### Taxonomy with IAM binding + +```hcl +module "cmn-dc" { + source = "./fabric/modules/data-catalog-policy-tag" + name = "my-datacatalog-policy-tags" + project_id = "my-project" + tags = { + low = {} + medium = {} + high = { + iam = { + "roles/datacatalog.categoryFineGrainedReader" = [ + "group:GROUP_NAME@example.com" + ] + } + } + } + iam = { + "roles/datacatalog.categoryAdmin" = ["group:GROUP_NAME@example.com"] + } + iam_bindings_additive = { + am1-admin = { + member = "user:am1@example.com" + role = "roles/datacatalog.categoryAdmin" + } + } +} +# tftest modules=1 resources=7 +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [name](variables.tf#L77) | Name of this taxonomy. | string | ✓ | | +| [project_id](variables.tf#L92) | GCP project id. | | ✓ | | +| [activated_policy_types](variables.tf#L17) | A list of policy types that are activated for this taxonomy. | list(string) | | ["FINE_GRAINED_ACCESS_CONTROL"] | +| [description](variables.tf#L23) | Description of this taxonomy. | string | | "Taxonomy - Terraform managed" | +| [group_iam](variables.tf#L29) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | +| [iam](variables.tf#L35) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [iam_bindings](variables.tf#L41) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | map(object({…})) | | {} | +| [iam_bindings_additive](variables.tf#L56) | Individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | +| [location](variables.tf#L71) | Data Catalog Taxonomy location. | string | | "eu" | +| [prefix](variables.tf#L82) | Optional prefix used to generate project id and name. | string | | null | +| [tags](variables.tf#L96) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(object({…})) | | {} | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [id](outputs.tf#L17) | Fully qualified taxonomy id. | | +| [tags](outputs.tf#L22) | Policy Tags. | | + +## TODO + +- Support IAM at tag level. +- Support Child policy tags diff --git a/assets/modules-fabric/v26/data-catalog-policy-tag/iam.tf b/assets/modules-fabric/v26/data-catalog-policy-tag/iam.tf new file mode 100644 index 0000000..06c3076 --- /dev/null +++ b/assets/modules-fabric/v26/data-catalog-policy-tag/iam.tf @@ -0,0 +1,92 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Data Catalog Taxonomy IAM definition. + +locals { + _group_iam = { + for r in local._group_iam_roles : r => [ + for k, v in var.group_iam : "group:${k}" if try(index(v, r), null) != null + ] + } + _group_iam_roles = distinct(flatten(values(var.group_iam))) + iam = { + for role in distinct(concat(keys(var.iam), keys(local._group_iam))) : + role => concat( + try(var.iam[role], []), + try(local._group_iam[role], []) + ) + } + tags_iam = flatten([ + for k, v in var.tags : [ + for role, members in v.iam : { + tag = k + role = role + members = members + } + ] + ]) +} + +resource "google_data_catalog_taxonomy_iam_binding" "authoritative" { + provider = google-beta + for_each = local.iam + taxonomy = google_data_catalog_taxonomy.default.id + role = each.key + members = each.value +} + +resource "google_data_catalog_taxonomy_iam_binding" "bindings" { + provider = google-beta + for_each = var.iam_bindings + taxonomy = google_data_catalog_taxonomy.default.id + role = each.value.role + members = each.value.members + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +resource "google_data_catalog_taxonomy_iam_member" "bindings" { + provider = google-beta + for_each = var.iam_bindings_additive + taxonomy = google_data_catalog_taxonomy.default.id + role = each.value.role + member = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +resource "google_data_catalog_policy_tag_iam_binding" "authoritative" { + provider = google-beta + for_each = { + for v in local.tags_iam : "${v.tag}.${v.role}" => v + } + policy_tag = google_data_catalog_policy_tag.default[each.value.tag].name + role = each.value.role + members = each.value.members +} diff --git a/assets/modules-fabric/v26/data-catalog-policy-tag/main.tf b/assets/modules-fabric/v26/data-catalog-policy-tag/main.tf new file mode 100644 index 0000000..0ccd923 --- /dev/null +++ b/assets/modules-fabric/v26/data-catalog-policy-tag/main.tf @@ -0,0 +1,43 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Data Catalog Taxonomy definition + +locals { + name = ( + var.name != null ? var.name : "${local.prefix}taxonomy" + ) + prefix = var.prefix == null ? "" : "${var.prefix}-" +} + +resource "google_data_catalog_taxonomy" "default" { + provider = google-beta + project = var.project_id + region = var.location + display_name = local.name + description = var.description + activated_policy_types = var.activated_policy_types +} + +resource "google_data_catalog_policy_tag" "default" { + for_each = var.tags + provider = google-beta + taxonomy = google_data_catalog_taxonomy.default.id + display_name = each.key + description = coalesce( + each.value.description, "${each.key} - Terraform managed." + ) +} diff --git a/assets/modules-fabric/v26/data-catalog-policy-tag/outputs.tf b/assets/modules-fabric/v26/data-catalog-policy-tag/outputs.tf new file mode 100644 index 0000000..4f579c2 --- /dev/null +++ b/assets/modules-fabric/v26/data-catalog-policy-tag/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "id" { + description = "Fully qualified taxonomy id." + value = google_data_catalog_taxonomy.default.id +} + +output "tags" { + description = "Policy Tags." + value = { for k, v in google_data_catalog_policy_tag.default : k => v.id } +} diff --git a/assets/modules-fabric/v26/data-catalog-policy-tag/variables.tf b/assets/modules-fabric/v26/data-catalog-policy-tag/variables.tf new file mode 100644 index 0000000..0fef9e7 --- /dev/null +++ b/assets/modules-fabric/v26/data-catalog-policy-tag/variables.tf @@ -0,0 +1,104 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "activated_policy_types" { + description = "A list of policy types that are activated for this taxonomy." + type = list(string) + default = ["FINE_GRAINED_ACCESS_CONTROL"] +} + +variable "description" { + description = "Description of this taxonomy." + type = string + default = "Taxonomy - Terraform managed" +} + +variable "group_iam" { + description = "Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable." + type = map(list(string)) + default = {} +} + +variable "iam" { + description = "IAM bindings in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} +} + +variable "iam_bindings" { + description = "Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary." + type = map(object({ + members = list(string) + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} +} + +variable "iam_bindings_additive" { + description = "Individual additive IAM bindings. Keys are arbitrary." + type = map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} +} + +variable "location" { + description = "Data Catalog Taxonomy location." + type = string + default = "eu" +} + +variable "name" { + description = "Name of this taxonomy." + type = string +} + +variable "prefix" { + description = "Optional prefix used to generate project id and name." + type = string + default = null + validation { + condition = var.prefix != "" + error_message = "Prefix cannot be empty, please use null instead." + } +} + +variable "project_id" { + description = "GCP project id." +} + +variable "tags" { + description = "List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format." + type = map(object({ + description = optional(string) + iam = optional(map(list(string)), {}) + })) + nullable = false + default = {} +} diff --git a/assets/modules-fabric/v26/data-catalog-policy-tag/versions.tf b/assets/modules-fabric/v26/data-catalog-policy-tag/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/data-catalog-policy-tag/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/datafusion/README.md b/assets/modules-fabric/v26/datafusion/README.md new file mode 100644 index 0000000..03b65b7 --- /dev/null +++ b/assets/modules-fabric/v26/datafusion/README.md @@ -0,0 +1,68 @@ +# Google Cloud Data Fusion Module + +This module allows simple management of ['Google Data Fusion'](https://cloud.google.com/data-fusion) instances. It supports creating Basic or Enterprise, public or private instances. + +## Examples + +## Auto-managed IP allocation + +```hcl +module "datafusion" { + source = "./fabric/modules/datafusion" + name = "my-datafusion" + region = "europe-west1" + project_id = "my-project" + network = "my-network-name" + # TODO: remove the following line + firewall_create = false +} +# tftest modules=1 resources=3 +``` + +### Externally managed IP allocation + +```hcl +module "datafusion" { + source = "./fabric/modules/datafusion" + name = "my-datafusion" + region = "europe-west1" + project_id = "my-project" + network = "my-network-name" + ip_allocation_create = false + ip_allocation = "10.0.0.0/22" +} +# tftest modules=1 resources=3 +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [name](variables.tf#L63) | Name of the DataFusion instance. | string | ✓ | | +| [network](variables.tf#L68) | Name of the network in the project with which the tenant project will be peered for executing pipelines in the form of projects/{project-id}/global/networks/{network}. | string | ✓ | | +| [project_id](variables.tf#L85) | Project ID. | string | ✓ | | +| [region](variables.tf#L90) | DataFusion region. | string | ✓ | | +| [description](variables.tf#L21) | DataFuzion instance description. | string | | "Terraform managed." | +| [enable_stackdriver_logging](variables.tf#L27) | Option to enable Stackdriver Logging. | bool | | false | +| [enable_stackdriver_monitoring](variables.tf#L33) | Option to enable Stackdriver Monitorig. | bool | | false | +| [firewall_create](variables.tf#L39) | Create Network firewall rules to enable SSH. | bool | | true | +| [ip_allocation](variables.tf#L45) | Ip allocated for datafusion instance when not using the auto created one and created outside of the module. | string | | null | +| [ip_allocation_create](variables.tf#L51) | Create Ip range for datafusion instance. | bool | | true | +| [labels](variables.tf#L57) | The resource labels for instance to use to annotate any related underlying resources, such as Compute Engine VMs. | map(string) | | {} | +| [network_peering](variables.tf#L73) | Create Network peering between project and DataFusion tenant project. | bool | | true | +| [private_instance](variables.tf#L79) | Create private instance. | bool | | true | +| [type](variables.tf#L95) | Datafusion Instance type. It can be BASIC or ENTERPRISE (default value). | string | | "ENTERPRISE" | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [id](outputs.tf#L17) | Fully qualified instance id. | | +| [ip_allocation](outputs.tf#L22) | IP range reserved for Data Fusion instance in case of a private instance. | | +| [resource](outputs.tf#L27) | DataFusion resource. | | +| [service_account](outputs.tf#L32) | DataFusion Service Account. | | +| [service_endpoint](outputs.tf#L37) | DataFusion Service Endpoint. | | +| [version](outputs.tf#L42) | DataFusion version. | | + + diff --git a/assets/modules-fabric/v26/datafusion/main.tf b/assets/modules-fabric/v26/datafusion/main.tf new file mode 100644 index 0000000..d422551 --- /dev/null +++ b/assets/modules-fabric/v26/datafusion/main.tf @@ -0,0 +1,79 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + prefix_length = 22 + ip_allocation = ( + var.ip_allocation_create + ? "${google_compute_global_address.default[0].address}/${local.prefix_length}" + : var.ip_allocation + ) + tenant_project = regex( + "cloud-datafusion-management-sa@([\\w-]+).iam.gserviceaccount.com", + google_data_fusion_instance.default.service_account + )[0] +} + +resource "google_compute_global_address" "default" { + count = var.ip_allocation_create ? 1 : 0 + project = var.project_id + name = "cdf-${var.name}" + address_type = "INTERNAL" + purpose = "VPC_PEERING" + prefix_length = local.prefix_length + network = var.network +} + +resource "google_compute_network_peering" "default" { + count = var.network_peering == true ? 1 : 0 + name = "cdf-${var.name}" + network = "projects/${var.project_id}/global/networks/${var.network}" + peer_network = "projects/${local.tenant_project}/global/networks/${var.region}-${google_data_fusion_instance.default.name}" + export_custom_routes = true + import_custom_routes = true +} + +resource "google_compute_firewall" "default" { + count = var.firewall_create == true ? 1 : 0 + name = "${var.name}-allow-ssh" + project = var.project_id + network = var.network + source_ranges = [local.ip_allocation] + target_tags = ["${var.name}-allow-ssh"] + + allow { + protocol = "tcp" + ports = ["22"] + } +} + +resource "google_data_fusion_instance" "default" { + provider = google-beta + project = var.project_id + name = var.name + type = var.type + description = var.description + labels = var.labels + region = var.region + private_instance = var.private_instance + enable_stackdriver_logging = var.enable_stackdriver_logging + enable_stackdriver_monitoring = var.enable_stackdriver_monitoring + network_config { + network = var.network + ip_allocation = local.ip_allocation + } +} + diff --git a/assets/modules-fabric/v26/datafusion/outputs.tf b/assets/modules-fabric/v26/datafusion/outputs.tf new file mode 100644 index 0000000..a7248c1 --- /dev/null +++ b/assets/modules-fabric/v26/datafusion/outputs.tf @@ -0,0 +1,45 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "id" { + description = "Fully qualified instance id." + value = google_data_fusion_instance.default.id +} + +output "ip_allocation" { + description = "IP range reserved for Data Fusion instance in case of a private instance." + value = local.ip_allocation +} + +output "resource" { + description = "DataFusion resource." + value = google_data_fusion_instance.default +} + +output "service_account" { + description = "DataFusion Service Account." + value = google_data_fusion_instance.default.service_account +} + +output "service_endpoint" { + description = "DataFusion Service Endpoint." + value = google_data_fusion_instance.default.service_endpoint +} + +output "version" { + description = "DataFusion version." + value = google_data_fusion_instance.default.version +} diff --git a/assets/modules-fabric/v26/datafusion/variables.tf b/assets/modules-fabric/v26/datafusion/variables.tf new file mode 100644 index 0000000..19d61f4 --- /dev/null +++ b/assets/modules-fabric/v26/datafusion/variables.tf @@ -0,0 +1,99 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +############################################################################### +# DtaFusion variables # +############################################################################### + +variable "description" { + description = "DataFuzion instance description." + type = string + default = "Terraform managed." +} + +variable "enable_stackdriver_logging" { + description = "Option to enable Stackdriver Logging." + type = bool + default = false +} + +variable "enable_stackdriver_monitoring" { + description = "Option to enable Stackdriver Monitorig." + type = bool + default = false +} + +variable "firewall_create" { + description = "Create Network firewall rules to enable SSH." + type = bool + default = true +} + +variable "ip_allocation" { + description = "Ip allocated for datafusion instance when not using the auto created one and created outside of the module." + type = string + default = null +} + +variable "ip_allocation_create" { + description = "Create Ip range for datafusion instance." + type = bool + default = true +} + +variable "labels" { + description = "The resource labels for instance to use to annotate any related underlying resources, such as Compute Engine VMs." + type = map(string) + default = {} +} + +variable "name" { + description = "Name of the DataFusion instance." + type = string +} + +variable "network" { + description = "Name of the network in the project with which the tenant project will be peered for executing pipelines in the form of projects/{project-id}/global/networks/{network}." + type = string +} + +variable "network_peering" { + description = "Create Network peering between project and DataFusion tenant project." + type = bool + default = true +} + +variable "private_instance" { + description = "Create private instance." + type = bool + default = true +} + +variable "project_id" { + description = "Project ID." + type = string +} + +variable "region" { + description = "DataFusion region." + type = string +} + +variable "type" { + description = "Datafusion Instance type. It can be BASIC or ENTERPRISE (default value)." + type = string + default = "ENTERPRISE" +} diff --git a/assets/modules-fabric/v26/datafusion/versions.tf b/assets/modules-fabric/v26/datafusion/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/datafusion/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/dataplex-datascan/README.md b/assets/modules-fabric/v26/dataplex-datascan/README.md new file mode 100644 index 0000000..4116732 --- /dev/null +++ b/assets/modules-fabric/v26/dataplex-datascan/README.md @@ -0,0 +1,458 @@ +# Dataplex DataScan + +This module manages the creation of Dataplex DataScan resources. + + +- [Data Profiling](#data-profiling) +- [Data Quality](#data-quality) +- [Data Source](#data-source) +- [Execution Schedule](#execution-schedule) +- [IAM](#iam) +- [TODO](#todo) +- [Variables](#variables) +- [Outputs](#outputs) + + +## Data Profiling + +This example shows how to create a Data Profiling scan. To create an Data Profiling scan, provide the `data_profile_spec` input arguments as documented in . + +```hcl +module "dataplex-datascan" { + source = "./fabric/modules/dataplex-datascan" + name = "datascan" + prefix = "test" + project_id = "my-project-name" + region = "us-central1" + labels = { + billing_id = "a" + } + data = { + resource = "//bigquery.googleapis.com/projects/bigquery-public-data/datasets/austin_bikeshare/tables/bikeshare_stations" + } + data_profile_spec = { + sampling_percent = 100 + row_filter = "station_id > 1000" + } + incremental_field = "modified_date" +} +# tftest modules=1 resources=1 inventory=datascan_profiling.yaml +``` + +## Data Quality + +To create an Data Quality scan, provide the `data_quality_spec` input arguments as documented in . + +Documentation for the supported rule types and rule specifications can be found in . + +This example shows how to create a Data Quality scan. + +```hcl +module "dataplex-datascan" { + source = "./fabric/modules/dataplex-datascan" + name = "datascan" + prefix = "test" + project_id = "my-project-name" + region = "us-central1" + labels = { + billing_id = "a" + } + execution_schedule = "TZ=America/New_York 0 1 * * *" + data = { + resource = "//bigquery.googleapis.com/projects/bigquery-public-data/datasets/austin_bikeshare/tables/bikeshare_stations" + } + incremental_field = "modified_date" + data_quality_spec = { + sampling_percent = 100 + row_filter = "station_id > 1000" + rules = [ + { + dimension = "VALIDITY" + non_null_expectation = {} + column = "address" + threshold = 0.99 + }, + { + column = "council_district" + dimension = "VALIDITY" + ignore_null = true + threshold = 0.9 + range_expectation = { + min_value = 1 + max_value = 10 + strict_min_enabled = true + strict_max_enabled = false + } + }, + { + column = "council_district" + dimension = "VALIDITY" + threshold = 0.8 + range_expectation = { + min_value = 3 + max_value = 9 + } + }, + { + column = "power_type" + dimension = "VALIDITY" + ignore_null = false + regex_expectation = { + regex = ".*solar.*" + } + }, + { + column = "property_type" + dimension = "VALIDITY" + ignore_null = false + set_expectation = { + values = ["sidewalk", "parkland"] + } + }, + { + column = "address" + dimension = "UNIQUENESS" + uniqueness_expectation = {} + }, + { + column = "number_of_docks" + dimension = "VALIDITY" + statistic_range_expectation = { + statistic = "MEAN" + min_value = 5 + max_value = 15 + strict_min_enabled = true + strict_max_enabled = true + } + }, + { + column = "footprint_length" + dimension = "VALIDITY" + row_condition_expectation = { + sql_expression = "footprint_length > 0 AND footprint_length <= 10" + } + }, + { + dimension = "VALIDITY" + table_condition_expectation = { + sql_expression = "COUNT(*) > 0" + } + } + ] + } +} +# tftest modules=1 resources=1 inventory=datascan_dq.yaml +``` + +This example shows how you can pass the rules configurations as a separate YAML file into the module. This should produce the same DataScan configuration as the previous example. + +```hcl +module "dataplex-datascan" { + source = "./fabric/modules/dataplex-datascan" + name = "datascan" + prefix = "test" + project_id = "my-project-name" + region = "us-central1" + labels = { + billing_id = "a" + } + execution_schedule = "TZ=America/New_York 0 1 * * *" + data = { + resource = "//bigquery.googleapis.com/projects/bigquery-public-data/datasets/austin_bikeshare/tables/bikeshare_stations" + } + incremental_field = "modified_date" + data_quality_spec_file = { + path = "config/data_quality_spec.yaml" + } +} +# tftest modules=1 resources=1 files=data_quality_spec inventory=datascan_dq.yaml +``` + +The content of the `config/data_quality_spec.yaml` files is as follows: + +```yaml +# tftest-file id=data_quality_spec path=config/data_quality_spec.yaml +sampling_percent: 100 +row_filter: "station_id > 1000" +rules: + - column: address + dimension: VALIDITY + ignore_null: null + non_null_expectation: {} + threshold: 0.99 + - column: council_district + dimension: VALIDITY + ignore_null: true + threshold: 0.9 + range_expectation: + max_value: '10' + min_value: '1' + strict_max_enabled: false + strict_min_enabled: true + - column: council_district + dimension: VALIDITY + range_expectation: + max_value: '9' + min_value: '3' + threshold: 0.8 + - column: power_type + dimension: VALIDITY + ignore_null: false + regex_expectation: + regex: .*solar.* + - column: property_type + dimension: VALIDITY + ignore_null: false + set_expectation: + values: + - sidewalk + - parkland + - column: address + dimension: UNIQUENESS + uniqueness_expectation: {} + - column: number_of_docks + dimension: VALIDITY + statistic_range_expectation: + max_value: '15' + min_value: '5' + statistic: MEAN + strict_max_enabled: true + strict_min_enabled: true + - column: footprint_length + dimension: VALIDITY + row_condition_expectation: + sql_expression: footprint_length > 0 AND footprint_length <= 10 + - dimension: VALIDITY + table_condition_expectation: + sql_expression: COUNT(*) > 0 +``` + +While the module only accepts input in snake_case, the YAML file provided to the `data_quality_spec_file` variable can use either camelCase or snake_case. This example below should also produce the same DataScan configuration as the previous examples. + +```hcl +module "dataplex-datascan" { + source = "./fabric/modules/dataplex-datascan" + name = "datascan" + prefix = "test" + project_id = "my-project-name" + region = "us-central1" + labels = { + billing_id = "a" + } + execution_schedule = "TZ=America/New_York 0 1 * * *" + data = { + resource = "//bigquery.googleapis.com/projects/bigquery-public-data/datasets/austin_bikeshare/tables/bikeshare_stations" + } + incremental_field = "modified_date" + data_quality_spec_file = { + path = "config/data_quality_spec_camel_case.yaml" + } +} +# tftest modules=1 resources=1 files=data_quality_spec_camel_case inventory=datascan_dq.yaml +``` + +The content of the `config/data_quality_spec_camel_case.yaml` files is as follows: + +```yaml +# tftest-file id=data_quality_spec_camel_case path=config/data_quality_spec_camel_case.yaml +samplingPercent: 100 +rowFilter: "station_id > 1000" +rules: + - column: address + dimension: VALIDITY + ignoreNull: null + nonNullExpectation: {} + threshold: 0.99 + - column: council_district + dimension: VALIDITY + ignoreNull: true + threshold: 0.9 + rangeExpectation: + maxValue: '10' + minValue: '1' + strictMaxEnabled: false + strictMinEnabled: true + - column: council_district + dimension: VALIDITY + rangeExpectation: + maxValue: '9' + minValue: '3' + threshold: 0.8 + - column: power_type + dimension: VALIDITY + ignoreNull: false + regexExpectation: + regex: .*solar.* + - column: property_type + dimension: VALIDITY + ignoreNull: false + setExpectation: + values: + - sidewalk + - parkland + - column: address + dimension: UNIQUENESS + uniquenessExpectation: {} + - column: number_of_docks + dimension: VALIDITY + statisticRangeExpectation: + maxValue: '15' + minValue: '5' + statistic: MEAN + strictMaxEnabled: true + strictMinEnabled: true + - column: footprint_length + dimension: VALIDITY + rowConditionExpectation: + sqlExpression: footprint_length > 0 AND footprint_length <= 10 + - dimension: VALIDITY + tableConditionExpectation: + sqlExpression: COUNT(*) > 0 +``` + +## Data Source + +The input variable 'data' is required to create a DataScan. This value is immutable. Once it is set, you cannot change the DataScan to another source. + +The input variable 'data' should be an object containing a single key-value pair that can be one of: + +- `entity`: The Dataplex entity that represents the data source (e.g. BigQuery table) for DataScan, of the form: `projects/{project_number}/locations/{locationId}/lakes/{lakeId}/zones/{zoneId}/entities/{entityId}`. +- `resource`: The service-qualified full resource name of the cloud resource for a DataScan job to scan against. The field could be: BigQuery table of type "TABLE" for DataProfileScan/DataQualityScan format, e.g: `//bigquery.googleapis.com/projects/PROJECT_ID/datasets/DATASET_ID/tables/TABLE_ID`. + +The example below shows how to specify the data source for DataScan of type `resource`: + +```hcl +module "dataplex-datascan" { + source = "./fabric/modules/dataplex-datascan" + name = "datascan" + prefix = "test" + project_id = "my-project-name" + region = "us-central1" + data = { + resource = "//bigquery.googleapis.com/projects/bigquery-public-data/datasets/austin_bikeshare/tables/bikeshare_stations" + } + data_profile_spec = {} +} +# tftest modules=1 resources=1 +``` + +The example below shows how to specify the data source for DataScan of type `entity`: + +```hcl +module "dataplex-datascan" { + source = "./fabric/modules/dataplex-datascan" + name = "datascan" + prefix = "test" + project_id = "my-project-name" + region = "us-central1" + data = { + entity = "projects//locations//lakes//zones//entities/" + } + data_profile_spec = {} +} +# tftest modules=1 resources=1 inventory=datascan_entity.yaml +``` + +## Execution Schedule + +The input variable 'execution_schedule' specifies when a scan should be triggered, based on a cron schedule expression. + +If not specified, the default is `on_demand`, which means the scan will not run until the user calls `dataScans.run` API. + +The following example shows how to schedule the DataScan at 1AM everyday using 'America/New_York' timezone. + +```hcl +module "dataplex-datascan" { + source = "./fabric/modules/dataplex-datascan" + name = "datascan" + prefix = "test" + project_id = "my-project-name" + region = "us-central1" + execution_schedule = "TZ=America/New_York 0 1 * * *" + data = { + resource = "//bigquery.googleapis.com/projects/bigquery-public-data/datasets/austin_bikeshare/tables/bikeshare_stations" + } + data_profile_spec = {} +} + +# tftest modules=1 resources=1 inventory=datascan_cron.yaml +``` + +## IAM + +IAM is managed via several variables that implement different features and levels of control: + +- `iam` and `group_iam` configure authoritative bindings that manage individual roles exclusively, and are internally merged +- `iam_bindings` configure authoritative bindings with optional support for conditions, and are not internally merged with the previous two variables +- `iam_bindings_additive` configure additive bindings via individual role/member pairs with optional support conditions + +The authoritative and additive approaches can be used together, provided different roles are managed by each. Some care must also be taken with the `groups_iam` variable to ensure that variable keys are static values, so that Terraform is able to compute the dependency graph. + +An example is provided below for using some of these variables. Refer to the [project module](../project/README.md#iam) for complete examples of the IAM interface. + +```hcl +module "dataplex-datascan" { + source = "./fabric/modules/dataplex-datascan" + name = "datascan" + prefix = "test" + project_id = "my-project-name" + region = "us-central1" + data = { + resource = "//bigquery.googleapis.com/projects/bigquery-public-data/datasets/austin_bikeshare/tables/bikeshare_stations" + } + data_profile_spec = {} + iam = { + "roles/dataplex.dataScanAdmin" = [ + "serviceAccount:svc-1@project-id.iam.gserviceaccount.com" + ], + "roles/dataplex.dataScanEditor" = [ + "user:admin-user@example.com" + ] + } + group_iam = { + "user-group@example.com" = [ + "roles/dataplex.dataScanViewer" + ] + } + iam_bindings_additive = { + am1-viewer = { + member = "user:am1@example.com" + role = "roles/dataplex.dataScanViewer" + } + } +} +# tftest modules=1 resources=5 inventory=datascan_iam.yaml +``` + +## TODO + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [data](variables.tf#L17) | The data source for DataScan. The source can be either a Dataplex `entity` or a BigQuery `resource`. | object({…}) | ✓ | | +| [name](variables.tf#L157) | Name of Dataplex Scan. | string | ✓ | | +| [project_id](variables.tf#L168) | The ID of the project where the Dataplex DataScan will be created. | string | ✓ | | +| [region](variables.tf#L173) | Region for the Dataplex DataScan. | string | ✓ | | +| [data_profile_spec](variables.tf#L29) | DataProfileScan related setting. Variable descriptions are provided in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataProfileSpec. | object({…}) | | null | +| [data_quality_spec](variables.tf#L38) | DataQualityScan related setting. Variable descriptions are provided in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataQualitySpec. | object({…}) | | null | +| [data_quality_spec_file](variables.tf#L80) | Path to a YAML file containing DataQualityScan related setting. Input content can use either camelCase or snake_case. Variables description are provided in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataQualitySpec. | object({…}) | | null | +| [description](variables.tf#L88) | Custom description for DataScan. | string | | null | +| [execution_schedule](variables.tf#L94) | Schedule DataScan to run periodically based on a cron schedule expression. If not specified, the DataScan is created with `on_demand` schedule, which means it will not run until the user calls `dataScans.run` API. | string | | null | +| [group_iam](variables.tf#L100) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | +| [iam](variables.tf#L107) | Dataplex DataScan IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [iam_bindings](variables.tf#L114) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | map(object({…})) | | {} | +| [iam_bindings_additive](variables.tf#L129) | Individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | +| [incremental_field](variables.tf#L144) | The unnested field (of type Date or Timestamp) that contains values which monotonically increase over time. If not specified, a data scan will run for all data in the table. | string | | null | +| [labels](variables.tf#L150) | Resource labels. | map(string) | | {} | +| [prefix](variables.tf#L162) | Optional prefix used to generate Dataplex DataScan ID. | string | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [data_scan_id](outputs.tf#L17) | Dataplex DataScan ID. | | +| [id](outputs.tf#L22) | A fully qualified Dataplex DataScan identifier for the resource with format projects/{{project}}/locations/{{location}}/dataScans/{{data_scan_id}}. | | +| [name](outputs.tf#L27) | The relative resource name of the scan, of the form: projects/{project}/locations/{locationId}/dataScans/{datascan_id}, where project refers to a project_id or project_number and locationId refers to a GCP region. | | +| [type](outputs.tf#L32) | The type of DataScan. | | + diff --git a/assets/modules-fabric/v26/dataplex-datascan/iam.tf b/assets/modules-fabric/v26/dataplex-datascan/iam.tf new file mode 100644 index 0000000..9ed5914 --- /dev/null +++ b/assets/modules-fabric/v26/dataplex-datascan/iam.tf @@ -0,0 +1,74 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + _group_iam_roles = distinct(flatten(values(var.group_iam))) + _group_iam = { + for r in local._group_iam_roles : r => [ + for k, v in var.group_iam : "group:${k}" if try(index(v, r), null) != null + ] + } + iam = { + for role in distinct(concat(keys(var.iam), keys(local._group_iam))) : + role => concat( + try(var.iam[role], []), + try(local._group_iam[role], []) + ) + } +} + +resource "google_dataplex_datascan_iam_binding" "authoritative_for_role" { + for_each = local.iam + project = google_dataplex_datascan.datascan.project + location = google_dataplex_datascan.datascan.location + data_scan_id = google_dataplex_datascan.datascan.data_scan_id + role = each.key + members = each.value +} + +resource "google_dataplex_datascan_iam_binding" "bindings" { + for_each = var.iam_bindings + project = google_dataplex_datascan.datascan.project + location = google_dataplex_datascan.datascan.location + data_scan_id = google_dataplex_datascan.datascan.data_scan_id + role = each.value.role + members = each.value.members + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +resource "google_dataplex_datascan_iam_member" "bindings" { + for_each = var.iam_bindings_additive + project = google_dataplex_datascan.datascan.project + location = google_dataplex_datascan.datascan.location + data_scan_id = google_dataplex_datascan.datascan.data_scan_id + role = each.value.role + member = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} diff --git a/assets/modules-fabric/v26/dataplex-datascan/main.tf b/assets/modules-fabric/v26/dataplex-datascan/main.tf new file mode 100644 index 0000000..34a140c --- /dev/null +++ b/assets/modules-fabric/v26/dataplex-datascan/main.tf @@ -0,0 +1,178 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + prefix = var.prefix == null || var.prefix == "" ? "" : "${var.prefix}-" + _file_data_quality_spec = var.data_quality_spec_file == null ? null : { + sampling_percent = try(local._file_data_quality_spec_raw.samplingPercent, local._file_data_quality_spec_raw.sampling_percent, null) + row_filter = try(local._file_data_quality_spec_raw.rowFilter, local._file_data_quality_spec_raw.row_filter, null) + rules = local._parsed_rules + } + data_quality_spec = ( + var.data_quality_spec != null || var.data_quality_spec_file != null ? + merge(var.data_quality_spec, local._file_data_quality_spec) : + null + ) +} + +resource "google_dataplex_datascan" "datascan" { + project = var.project_id + location = var.region + data_scan_id = "${local.prefix}${var.name}" + display_name = "${local.prefix}${var.name}" + description = var.description == null ? "Terraform Managed." : "Terraform Managed. ${var.description}" + labels = var.labels + + data { + resource = var.data.resource + entity = var.data.entity + } + + execution_spec { + field = var.incremental_field + trigger { + dynamic "on_demand" { + for_each = var.execution_schedule == null ? [""] : [] + content { + } + } + dynamic "schedule" { + for_each = var.execution_schedule != null ? [""] : [] + content { + cron = var.execution_schedule + } + } + } + } + + dynamic "data_profile_spec" { + for_each = var.data_profile_spec != null ? [""] : [] + content { + sampling_percent = try(var.data_profile_spec.sampling_percent, null) + row_filter = try(var.data_profile_spec.row_filter, null) + } + } + + dynamic "data_quality_spec" { + for_each = local.data_quality_spec != null ? [""] : [] + content { + sampling_percent = try(local.data_quality_spec.sampling_percent, null) + row_filter = try(local.data_quality_spec.row_filter, null) + dynamic "rules" { + for_each = local.data_quality_spec.rules + content { + column = try(rules.value.column, null) + ignore_null = try(rules.value.ignore_null, null) + dimension = rules.value.dimension + threshold = try(rules.value.threshold, null) + + dynamic "non_null_expectation" { + for_each = try(rules.value.non_null_expectation, null) != null ? [""] : [] + content { + } + } + + dynamic "range_expectation" { + for_each = try(rules.value.range_expectation, null) != null ? [""] : [] + content { + min_value = try(rules.value.range_expectation.min_value, null) + max_value = try(rules.value.range_expectation.max_value, null) + strict_min_enabled = try(rules.value.range_expectation.strict_min_enabled, null) + strict_max_enabled = try(rules.value.range_expectation.strict_max_enabled, null) + } + } + + dynamic "set_expectation" { + for_each = try(rules.value.set_expectation, null) != null ? [""] : [] + content { + values = rules.value.set_expectation.values + } + } + + dynamic "uniqueness_expectation" { + for_each = try(rules.value.uniqueness_expectation, null) != null ? [""] : [] + content { + } + } + + dynamic "regex_expectation" { + for_each = try(rules.value.regex_expectation, null) != null ? [""] : [] + content { + regex = rules.value.regex_expectation.regex + } + } + + dynamic "statistic_range_expectation" { + for_each = try(rules.value.statistic_range_expectation, null) != null ? [""] : [] + content { + min_value = try(rules.value.statistic_range_expectation.min_value, null) + max_value = try(rules.value.statistic_range_expectation.max_value, null) + strict_min_enabled = try(rules.value.statistic_range_expectation.strict_min_enabled, null) + strict_max_enabled = try(rules.value.statistic_range_expectation.strict_max_enabled, null) + statistic = rules.value.statistic_range_expectation.statistic + } + } + + dynamic "row_condition_expectation" { + for_each = try(rules.value.row_condition_expectation, null) != null ? [""] : [] + content { + sql_expression = rules.value.row_condition_expectation.sql_expression + } + } + + dynamic "table_condition_expectation" { + for_each = try(rules.value.table_condition_expectation, null) != null ? [""] : [] + content { + sql_expression = rules.value.table_condition_expectation.sql_expression + } + } + + } + } + } + } + + lifecycle { + precondition { + condition = length([for spec in [var.data_profile_spec, var.data_quality_spec, var.data_quality_spec_file] : spec if spec != null]) == 1 + error_message = "DataScan can only contain one of 'data_profile_spec', 'data_quality_spec', 'data_quality_spec_file'." + } + precondition { + condition = alltrue([ + for rule in try(local.data_quality_spec.rules, []) : + contains(["COMPLETENESS", "ACCURACY", "CONSISTENCY", "VALIDITY", "UNIQUENESS", "INTEGRITY"], rule.dimension)]) + error_message = "Datascan 'dimension' field in 'data_quality_spec' must be one of ['COMPLETENESS', 'ACCURACY', 'CONSISTENCY', 'VALIDITY', 'UNIQUENESS', 'INTEGRITY']." + } + precondition { + condition = alltrue([ + for rule in try(local.data_quality_spec.rules, []) : + length([ + for k, v in rule : + v if contains([ + "non_null_expectation", + "range_expectation", + "regex_expectation", + "set_expectation", + "uniqueness_expectation", + "statistic_range_expectation", + "row_condition_expectation", + "table_condition_expectation" + ], k) && v != null + ]) == 1]) + error_message = "Datascan rule must contain a key that is one of ['non_null_expectation', 'range_expectation', 'regex_expectation', 'set_expectation', 'uniqueness_expectation', 'statistic_range_expectation', 'row_condition_expectation', 'table_condition_expectation]." + } + } +} diff --git a/assets/modules-fabric/v26/dataplex-datascan/outputs.tf b/assets/modules-fabric/v26/dataplex-datascan/outputs.tf new file mode 100644 index 0000000..f314c57 --- /dev/null +++ b/assets/modules-fabric/v26/dataplex-datascan/outputs.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "data_scan_id" { + description = "Dataplex DataScan ID." + value = google_dataplex_datascan.datascan.data_scan_id +} + +output "id" { + description = "A fully qualified Dataplex DataScan identifier for the resource with format projects/{{project}}/locations/{{location}}/dataScans/{{data_scan_id}}." + value = google_dataplex_datascan.datascan.id +} + +output "name" { + description = "The relative resource name of the scan, of the form: projects/{project}/locations/{locationId}/dataScans/{datascan_id}, where project refers to a project_id or project_number and locationId refers to a GCP region." + value = google_dataplex_datascan.datascan.name +} + +output "type" { + description = "The type of DataScan." + value = google_dataplex_datascan.datascan.type +} diff --git a/assets/modules-fabric/v26/dataplex-datascan/rules_parsing.tf b/assets/modules-fabric/v26/dataplex-datascan/rules_parsing.tf new file mode 100644 index 0000000..bbdc822 --- /dev/null +++ b/assets/modules-fabric/v26/dataplex-datascan/rules_parsing.tf @@ -0,0 +1,54 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + _file_data_quality_spec_raw = var.data_quality_spec_file != null ? yamldecode(file(var.data_quality_spec_file.path)) : tomap({}) + _parsed_rules = [ + for rule in try(local._file_data_quality_spec_raw.rules, []) : { + column = try(rule.column, null) + ignore_null = try(rule.ignoreNull, rule.ignore_null, null) + dimension = rule.dimension + threshold = try(rule.threshold, null) + non_null_expectation = try(rule.nonNullExpectation, rule.non_null_expectation, null) + range_expectation = can(rule.rangeExpectation) || can(rule.range_expectation) ? { + min_value = try(rule.rangeExpectation.minValue, rule.range_expectation.min_value, null) + max_value = try(rule.rangeExpectation.maxValue, rule.range_expectation.max_value, null) + strict_min_enabled = try(rule.rangeExpectation.strictMinEnabled, rule.range_expectation.strict_min_enabled, null) + strict_max_enabled = try(rule.rangeExpectation.strictMaxEnabled, rule.range_expectation.strict_max_enabled, null) + } : null + regex_expectation = can(rule.regexExpectation) || can(rule.regex_expectation) ? { + regex = try(rule.regexExpectation.regex, rule.regex_expectation.regex, null) + } : null + set_expectation = can(rule.setExpectation) || can(rule.set_expectation) ? { + values = try(rule.setExpectation.values, rule.set_expectation.values, null) + } : null + uniqueness_expectation = try(rule.uniquenessExpectation, rule.uniqueness_expectation, null) + statistic_range_expectation = can(rule.statisticRangeExpectation) || can(rule.statistic_range_expectation) ? { + statistic = try(rule.statisticRangeExpectation.statistic, rule.statistic_range_expectation.statistic) + min_value = try(rule.statisticRangeExpectation.minValue, rule.statistic_range_expectation.min_value, null) + max_value = try(rule.statisticRangeExpectation.maxValue, rule.statistic_range_expectation.max_value, null) + strict_min_enabled = try(rule.statisticRangeExpectation.strictMinEnabled, rule.statistic_range_expectation.strict_min_enabled, null) + strict_max_enabled = try(rule.statisticRangeExpectation.strictMaxEnabled, rule.statistic_range_expectation.strict_max_enabled, null) + } : null + row_condition_expectation = can(rule.rowConditionExpectation) || can(rule.row_condition_expectation) ? { + sql_expression = try(rule.rowConditionExpectation.sqlExpression, rule.row_condition_expectation.sql_expression, null) + } : null + table_condition_expectation = can(rule.tableConditionExpectation) || can(rule.table_condition_expectation) ? { + sql_expression = try(rule.tableConditionExpectation.sqlExpression, rule.table_condition_expectation.sql_expression, null) + } : null + } + ] +} \ No newline at end of file diff --git a/assets/modules-fabric/v26/dataplex-datascan/variables.tf b/assets/modules-fabric/v26/dataplex-datascan/variables.tf new file mode 100644 index 0000000..a13cdc5 --- /dev/null +++ b/assets/modules-fabric/v26/dataplex-datascan/variables.tf @@ -0,0 +1,176 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "data" { + description = "The data source for DataScan. The source can be either a Dataplex `entity` or a BigQuery `resource`." + type = object({ + entity = optional(string) + resource = optional(string) + }) + validation { + condition = length([for k, v in var.data : v if contains(["resource", "entity"], k) && v != null]) == 1 + error_message = "Datascan data must specify one of 'entity', 'resource'." + } +} + +variable "data_profile_spec" { + description = "DataProfileScan related setting. Variable descriptions are provided in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataProfileSpec." + default = null + type = object({ + sampling_percent = optional(number) + row_filter = optional(string) + }) +} + +variable "data_quality_spec" { + description = "DataQualityScan related setting. Variable descriptions are provided in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataQualitySpec." + default = null + type = object({ + sampling_percent = optional(number) + row_filter = optional(string) + rules = list(object({ + column = optional(string) + ignore_null = optional(bool, null) + dimension = string + threshold = optional(number) + non_null_expectation = optional(object({})) + range_expectation = optional(object({ + min_value = optional(number) + max_value = optional(number) + strict_min_enabled = optional(bool) + strict_max_enabled = optional(bool) + })) + regex_expectation = optional(object({ + regex = string + })) + set_expectation = optional(object({ + values = list(string) + })) + uniqueness_expectation = optional(object({})) + statistic_range_expectation = optional(object({ + statistic = string + min_value = optional(number) + max_value = optional(number) + strict_min_enabled = optional(bool) + strict_max_enabled = optional(bool) + })) + row_condition_expectation = optional(object({ + sql_expression = string + })) + table_condition_expectation = optional(object({ + sql_expression = string + })) + })) + }) +} + +variable "data_quality_spec_file" { + description = "Path to a YAML file containing DataQualityScan related setting. Input content can use either camelCase or snake_case. Variables description are provided in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataQualitySpec." + default = null + type = object({ + path = string + }) +} + +variable "description" { + description = "Custom description for DataScan." + default = null + type = string +} + +variable "execution_schedule" { + description = "Schedule DataScan to run periodically based on a cron schedule expression. If not specified, the DataScan is created with `on_demand` schedule, which means it will not run until the user calls `dataScans.run` API." + type = string + default = null +} + +variable "group_iam" { + description = "Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable." + type = map(list(string)) + default = {} + nullable = false +} + +variable "iam" { + description = "Dataplex DataScan IAM bindings in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} + nullable = false +} + +variable "iam_bindings" { + description = "Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary." + type = map(object({ + members = list(string) + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} +} + +variable "iam_bindings_additive" { + description = "Individual additive IAM bindings. Keys are arbitrary." + type = map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} +} + +variable "incremental_field" { + description = "The unnested field (of type Date or Timestamp) that contains values which monotonically increase over time. If not specified, a data scan will run for all data in the table." + type = string + default = null +} + +variable "labels" { + description = "Resource labels." + type = map(string) + default = {} + nullable = false +} + +variable "name" { + description = "Name of Dataplex Scan." + type = string +} + +variable "prefix" { + description = "Optional prefix used to generate Dataplex DataScan ID." + type = string + default = null +} + +variable "project_id" { + description = "The ID of the project where the Dataplex DataScan will be created." + type = string +} + +variable "region" { + description = "Region for the Dataplex DataScan." + type = string +} diff --git a/assets/modules-fabric/v26/dataplex-datascan/versions.tf b/assets/modules-fabric/v26/dataplex-datascan/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/dataplex-datascan/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/dataplex/README.md b/assets/modules-fabric/v26/dataplex/README.md new file mode 100644 index 0000000..c858c1d --- /dev/null +++ b/assets/modules-fabric/v26/dataplex/README.md @@ -0,0 +1,133 @@ +# Dataplex instance with lake, zone & assests + +This module manages the creation of Dataplex instance along with lake, zone & assets in single regions. + +## Simple example + +This example shows how to setup a Dataplex instance, lake, zone & asset creation in GCP project. + +```hcl + +module "dataplex" { + source = "./fabric/modules/dataplex" + name = "terraform-lake" + prefix = "test" + project_id = "myproject" + region = "europe-west2" + zones = { + landing = { + type = "RAW" + discovery = true + assets = { + gcs_1 = { + resource_name = "gcs_bucket" + cron_schedule = "15 15 * * *" + discovery_spec_enabled = true + resource_spec_type = "STORAGE_BUCKET" + } + } + }, + curated = { + type = "CURATED" + discovery = false + assets = { + bq_1 = { + resource_name = "bq_dataset" + cron_schedule = null + discovery_spec_enabled = false + resource_spec_type = "BIGQUERY_DATASET" + } + } + } + } +} + +# tftest modules=1 resources=5 +``` + +## IAM + +This example shows how to setup a Dataplex instance, lake, zone & asset creation in GCP project assigning IAM roles at lake and zone level. + +```hcl + +module "dataplex" { + source = "./fabric/modules/dataplex" + name = "lake" + prefix = "test" + project_id = "myproject" + region = "europe-west2" + iam = { + "roles/dataplex.viewer" = [ + "group:analysts@example.com", + "group:analysts_sensitive@example.com" + ] + } + zones = { + landing = { + type = "RAW" + discovery = true + assets = { + gcs_1 = { + resource_name = "gcs_bucket" + cron_schedule = "15 15 * * *" + discovery_spec_enabled = true + resource_spec_type = "STORAGE_BUCKET" + } + } + }, + curated = { + type = "CURATED" + discovery = false + iam = { + "roles/viewer" = [ + "group:analysts@example.com", + "group:analysts_sensitive@example.com" + ] + "roles/dataplex.dataReader" = [ + "group:analysts@example.com", + "group:analysts_sensitive@example.com" + ] + } + assets = { + bq_1 = { + resource_name = "bq_dataset" + cron_schedule = null + discovery_spec_enabled = false + resource_spec_type = "BIGQUERY_DATASET" + } + } + } + } +} + +# tftest modules=1 resources=8 +``` + +## TODO + +- [ ] support multi-regions + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [name](variables.tf#L30) | Name of Dataplex Lake. | string | ✓ | | +| [project_id](variables.tf#L41) | The ID of the project where this Dataplex Lake will be created. | string | ✓ | | +| [region](variables.tf#L46) | Region of the Dataplax Lake. | string | ✓ | | +| [zones](variables.tf#L51) | Dataplex lake zones, such as `RAW` and `CURATED`. | map(object({…})) | ✓ | | +| [iam](variables.tf#L17) | Dataplex lake IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [location_type](variables.tf#L24) | The location type of the Dataplax Lake. | string | | "SINGLE_REGION" | +| [prefix](variables.tf#L35) | Optional prefix used to generate Dataplex Lake. | string | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [assets](outputs.tf#L17) | Assets attached to the lake of Dataplex Lake. | | +| [id](outputs.tf#L22) | Fully qualified Dataplex Lake id. | | +| [lake](outputs.tf#L27) | The lake name of Dataplex Lake. | | +| [zones](outputs.tf#L32) | The zone name of Dataplex Lake. | | + + diff --git a/assets/modules-fabric/v26/dataplex/main.tf b/assets/modules-fabric/v26/dataplex/main.tf new file mode 100644 index 0000000..b78ca54 --- /dev/null +++ b/assets/modules-fabric/v26/dataplex/main.tf @@ -0,0 +1,120 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + prefix = var.prefix == null ? "" : "${var.prefix}-" + zone_assets = flatten([ + for zone, zones_info in var.zones : [ + for asset, asset_data in zones_info.assets : { + zone_name = zone + asset_name = asset + resource_name = asset_data.resource_name + resource_project = coalesce(asset_data.resource_project, var.project_id) + cron_schedule = asset_data.discovery_spec_enabled ? asset_data.cron_schedule : null + discovery_spec_enabled = asset_data.discovery_spec_enabled + resource_spec_type = asset_data.resource_spec_type + } + ] + ]) + + zone_iam = flatten([ + for zone, zone_details in var.zones : [ + for role, members in zone_details.iam : { + "zone" = zone + "role" = role + "members" = members + } + ] if zone_details.iam != null + ]) + + resource_type_mapping = { + "STORAGE_BUCKET" : "buckets", + "BIGQUERY_DATASET" : "datasets" + } +} + +resource "google_dataplex_lake" "lake" { + name = "${local.prefix}${var.name}" + location = var.region + provider = google-beta + project = var.project_id +} + +resource "google_dataplex_lake_iam_binding" "binding" { + for_each = var.iam + project = var.project_id + location = var.region + lake = google_dataplex_lake.lake.name + role = each.key + members = each.value +} + +resource "google_dataplex_zone" "zone" { + for_each = var.zones + provider = google-beta + project = var.project_id + name = each.key + location = var.region + lake = google_dataplex_lake.lake.name + type = each.value.type + + discovery_spec { + enabled = each.value.discovery + } + + resource_spec { + location_type = var.location_type + } +} + +resource "google_dataplex_zone_iam_binding" "binding" { + for_each = { + for zone_role in local.zone_iam : "${zone_role.zone}-${zone_role.role}" => zone_role + } + project = var.project_id + location = var.region + lake = google_dataplex_lake.lake.name + dataplex_zone = google_dataplex_zone.zone[each.value.zone].name + role = each.value.role + members = each.value.members +} + +resource "google_dataplex_asset" "asset" { + for_each = { + for tm in local.zone_assets : "${tm.zone_name}-${tm.asset_name}" => tm + } + name = each.value.asset_name + location = var.region + provider = google-beta + + lake = google_dataplex_lake.lake.name + dataplex_zone = google_dataplex_zone.zone[each.value.zone_name].name + + discovery_spec { + enabled = each.value.discovery_spec_enabled + schedule = each.value.cron_schedule + } + + resource_spec { + name = format("projects/%s/%s/%s", + each.value.resource_project, + local.resource_type_mapping[each.value.resource_spec_type], + each.value.resource_name + ) + type = each.value.resource_spec_type + } + project = var.project_id +} diff --git a/assets/modules-fabric/v26/dataplex/outputs.tf b/assets/modules-fabric/v26/dataplex/outputs.tf new file mode 100644 index 0000000..0da4fcc --- /dev/null +++ b/assets/modules-fabric/v26/dataplex/outputs.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "assets" { + description = "Assets attached to the lake of Dataplex Lake." + value = local.zone_assets[*] +} + +output "id" { + description = "Fully qualified Dataplex Lake id." + value = google_dataplex_lake.lake.id +} + +output "lake" { + description = "The lake name of Dataplex Lake." + value = google_dataplex_lake.lake.name +} + +output "zones" { + description = "The zone name of Dataplex Lake." + value = distinct(local.zone_assets[*]["zone_name"]) +} + diff --git a/assets/modules-fabric/v26/dataplex/variables.tf b/assets/modules-fabric/v26/dataplex/variables.tf new file mode 100644 index 0000000..fa4e652 --- /dev/null +++ b/assets/modules-fabric/v26/dataplex/variables.tf @@ -0,0 +1,73 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "iam" { + description = "Dataplex lake IAM bindings in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} + nullable = false +} + +variable "location_type" { + description = "The location type of the Dataplax Lake." + type = string + default = "SINGLE_REGION" +} + +variable "name" { + description = "Name of Dataplex Lake." + type = string +} + +variable "prefix" { + description = "Optional prefix used to generate Dataplex Lake." + type = string + default = null +} + +variable "project_id" { + description = "The ID of the project where this Dataplex Lake will be created." + type = string +} + +variable "region" { + description = "Region of the Dataplax Lake." + type = string +} + +variable "zones" { + description = "Dataplex lake zones, such as `RAW` and `CURATED`." + type = map(object({ + type = string + discovery = optional(bool, true) + iam = optional(map(list(string)), null) + assets = map(object({ + resource_name = string + resource_project = optional(string) + cron_schedule = optional(string, "15 15 * * *") + discovery_spec_enabled = optional(bool, true) + resource_spec_type = optional(string, "STORAGE_BUCKET") + })) + })) + validation { + condition = alltrue(flatten([ + for k, v in var.zones : [ + for kk, vv in v.assets : contains(["BIGQUERY_DATASET", "STORAGE_BUCKET"], vv.resource_spec_type) + ] + ])) + error_message = "Asset spect type must be one of 'BIGQUERY_DATASET' or 'STORAGE_BUCKET'." + } +} diff --git a/assets/modules-fabric/v26/dataplex/versions.tf b/assets/modules-fabric/v26/dataplex/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/dataplex/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/dataproc/README.md b/assets/modules-fabric/v26/dataproc/README.md new file mode 100644 index 0000000..5cd220c --- /dev/null +++ b/assets/modules-fabric/v26/dataproc/README.md @@ -0,0 +1,170 @@ +# Google Cloud Dataproc + +This module Manages a Google Cloud [Dataproc](https://cloud.google.com/dataproc) cluster resource, including IAM. + + +- [TODO](#todo) +- [Examples](#examples) + - [Simple](#simple) + - [Cluster configuration](#cluster-configuration) + - [Cluster with CMEK encryption](#cluster-with-cmek-encryption) +- [IAM](#iam) + - [Authoritative IAM](#authoritative-iam) + - [Additive IAM](#additive-iam) +- [Variables](#variables) +- [Outputs](#outputs) + + +## TODO + +- [ ] Add support for Cloud Dataproc [autoscaling policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/dataproc_autoscaling_policy_iam). + +## Examples + +### Simple + +```hcl +module "processing-dp-cluster-2" { + source = "./fabric/modules/dataproc" + project_id = "my-project" + name = "my-cluster" + region = "europe-west1" +} +# tftest modules=1 resources=1 +``` + +### Cluster configuration + +To set cluster configuration use the 'dataproc_config.cluster_config' variable. + +```hcl +module "processing-dp-cluster" { + source = "./fabric/modules/dataproc" + project_id = "my-project" + name = "my-cluster" + region = "europe-west1" + prefix = "prefix" + dataproc_config = { + cluster_config = { + gce_cluster_config = { + subnetwork = "https://www.googleapis.com/compute/v1/projects/PROJECT/regions/europe-west1/subnetworks/SUBNET" + zone = "europe-west1-b" + service_account = "" + service_account_scopes = ["cloud-platform"] + internal_ip_only = true + } + } + } +} +# tftest modules=1 resources=1 +``` + +### Cluster with CMEK encryption + +To set cluster configuration use the Customer Managed Encryption key, set `dataproc_config.encryption_config.` variable. The Compute Engine service agent and the Cloud Storage service agent need to have `CryptoKey Encrypter/Decrypter` role on they configured KMS key ([Documentation](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/customer-managed-encryption)). + +```hcl +module "processing-dp-cluster" { + source = "./fabric/modules/dataproc" + project_id = "my-project" + name = "my-cluster" + region = "europe-west1" + prefix = "prefix" + dataproc_config = { + cluster_config = { + gce_cluster_config = { + subnetwork = "https://www.googleapis.com/compute/v1/projects/PROJECT/regions/europe-west1/subnetworks/SUBNET" + zone = "europe-west1-b" + service_account = "" + service_account_scopes = ["cloud-platform"] + internal_ip_only = true + } + } + encryption_config = { + kms_key_name = "projects/project-id/locations/region/keyRings/key-ring-name/cryptoKeys/key-name" + } + } +} +# tftest modules=1 resources=1 +``` + +## IAM + +IAM is managed via several variables that implement different features and levels of control: + +- `iam` and `group_iam` configure authoritative bindings that manage individual roles exclusively, and are internally merged +- `iam_bindings` configure authoritative bindings with optional support for conditions, and are not internally merged with the previous two variables +- `iam_bindings_additive` configure additive bindings via individual role/member pairs with optional support conditions + +The authoritative and additive approaches can be used together, provided different roles are managed by each. Some care must also be taken with the `groups_iam` variable to ensure that variable keys are static values, so that Terraform is able to compute the dependency graph. + +Refer to the [project module](../project/README.md#iam) for examples of the IAM interface. + +### Authoritative IAM + +```hcl +module "processing-dp-cluster" { + source = "./fabric/modules/dataproc" + project_id = "my-project" + name = "my-cluster" + region = "europe-west1" + prefix = "prefix" + group_iam = { + "gcp-data-engineers@example.net" = [ + "roles/dataproc.viewer" + ] + } + iam = { + "roles/dataproc.viewer" = [ + "serviceAccount:service-account@PROJECT_ID.iam.gserviceaccount.com" + ] + } +} +# tftest modules=1 resources=2 +``` + +### Additive IAM + +```hcl +module "processing-dp-cluster" { + source = "./fabric/modules/dataproc" + project_id = "my-project" + name = "my-cluster" + region = "europe-west1" + prefix = "prefix" + iam_bindings_additive = { + am1-viewer = { + member = "user:am1@example.com" + role = "roles/dataproc.viewer" + } + } +} +# tftest modules=1 resources=2 +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [name](variables.tf#L235) | Cluster name. | string | ✓ | | +| [project_id](variables.tf#L250) | Project ID. | string | ✓ | | +| [region](variables.tf#L255) | Dataproc region. | string | ✓ | | +| [dataproc_config](variables.tf#L17) | Dataproc cluster config. | object({…}) | | {} | +| [group_iam](variables.tf#L185) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | +| [iam](variables.tf#L192) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [iam_bindings](variables.tf#L199) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | map(object({…})) | | {} | +| [iam_bindings_additive](variables.tf#L214) | Individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | +| [labels](variables.tf#L229) | The resource labels for instance to use to annotate any related underlying resources, such as Compute Engine VMs. | map(string) | | {} | +| [prefix](variables.tf#L240) | Optional prefix used to generate project id and name. | string | | null | +| [service_account](variables.tf#L260) | Service account to set on the Dataproc cluster. | string | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [bucket_names](outputs.tf#L19) | List of bucket names which have been assigned to the cluster. | | +| [http_ports](outputs.tf#L24) | The map of port descriptions to URLs. | | +| [id](outputs.tf#L29) | Fully qualified cluster id. | | +| [instance_names](outputs.tf#L34) | List of instance names which have been assigned to the cluster. | | +| [name](outputs.tf#L43) | The name of the cluster. | | + diff --git a/assets/modules-fabric/v26/dataproc/iam.tf b/assets/modules-fabric/v26/dataproc/iam.tf new file mode 100644 index 0000000..ef0428d --- /dev/null +++ b/assets/modules-fabric/v26/dataproc/iam.tf @@ -0,0 +1,76 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# # tfdoc:file:description Generic IAM bindings and roles. + +locals { + _group_iam_roles = distinct(flatten(values(var.group_iam))) + _group_iam = { + for r in local._group_iam_roles : r => [ + for k, v in var.group_iam : "group:${k}" if try(index(v, r), null) != null + ] + } + iam = { + for role in distinct(concat(keys(var.iam), keys(local._group_iam))) : + role => concat( + try(var.iam[role], []), + try(local._group_iam[role], []) + ) + } +} + +resource "google_dataproc_cluster_iam_binding" "authoritative" { + for_each = local.iam + project = var.project_id + cluster = google_dataproc_cluster.cluster.name + region = var.region + role = each.key + members = each.value +} + +resource "google_dataproc_cluster_iam_binding" "bindings" { + for_each = var.iam_bindings + project = var.project_id + cluster = google_dataproc_cluster.cluster.name + region = var.region + role = each.value.role + members = each.value.members + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +resource "google_dataproc_cluster_iam_member" "bindings" { + for_each = var.iam_bindings_additive + project = var.project_id + cluster = google_dataproc_cluster.cluster.name + region = var.region + role = each.value.role + member = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} diff --git a/assets/modules-fabric/v26/dataproc/main.tf b/assets/modules-fabric/v26/dataproc/main.tf new file mode 100644 index 0000000..55bef5c --- /dev/null +++ b/assets/modules-fabric/v26/dataproc/main.tf @@ -0,0 +1,298 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Cloud Dataproc resource definition. + +locals { + prefix = var.prefix == null ? "" : "${var.prefix}-" +} + +resource "google_dataproc_cluster" "cluster" { + name = "${local.prefix}${var.name}" + project = var.project_id + region = var.region + graceful_decommission_timeout = var.dataproc_config.graceful_decommission_timeout + labels = var.labels + dynamic "cluster_config" { + for_each = var.dataproc_config.cluster_config == null ? [] : [""] + content { + staging_bucket = var.dataproc_config.cluster_config.staging_bucket + temp_bucket = var.dataproc_config.cluster_config.temp_bucket + dynamic "gce_cluster_config" { + for_each = var.dataproc_config.cluster_config.gce_cluster_config == null ? [] : [""] + content { + zone = var.dataproc_config.cluster_config.gce_cluster_config.zone + network = var.dataproc_config.cluster_config.gce_cluster_config.network + subnetwork = var.dataproc_config.cluster_config.gce_cluster_config.subnetwork + service_account = var.dataproc_config.cluster_config.gce_cluster_config.service_account + service_account_scopes = var.dataproc_config.cluster_config.gce_cluster_config.service_account_scopes + tags = var.dataproc_config.cluster_config.gce_cluster_config.tags + internal_ip_only = var.dataproc_config.cluster_config.gce_cluster_config.internal_ip_only + metadata = var.dataproc_config.cluster_config.gce_cluster_config.metadata + dynamic "reservation_affinity" { + for_each = var.dataproc_config.cluster_config.gce_cluster_config.reservation_affinity == null ? [] : [""] + content { + consume_reservation_type = var.dataproc_config.cluster_config.gce_cluster_config.reservation_affinity.consume_reservation_type + key = var.dataproc_config.cluster_config.gce_cluster_config.reservation_affinity.key + values = var.dataproc_config.cluster_config.gce_cluster_config.reservation_affinity.value + } + } + dynamic "node_group_affinity" { + for_each = var.dataproc_config.cluster_config.gce_cluster_config.node_group_affinity == null ? [] : [""] + content { + node_group_uri = var.dataproc_config.cluster_config.gce_cluster_config.node_group_uri + } + } + dynamic "shielded_instance_config" { + for_each = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config == null ? [] : [""] + content { + enable_secure_boot = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.enable_secure_boot + enable_vtpm = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.enable_vtpm + enable_integrity_monitoring = var.dataproc_config.cluster_config.gce_cluster_config.shielded_instance_config.enable_integrity_monitoring + } + } + } + } + dynamic "master_config" { + for_each = var.dataproc_config.cluster_config.master_config == null ? [] : [""] + content { + num_instances = var.dataproc_config.cluster_config.master_config.num_instances + machine_type = var.dataproc_config.cluster_config.master_config.machine_type + min_cpu_platform = var.dataproc_config.cluster_config.master_config.min_cpu_platform + image_uri = var.dataproc_config.cluster_config.master_config.image_uri + dynamic "disk_config" { + for_each = var.dataproc_config.cluster_config.master_config.disk_config == null ? [] : [""] + content { + boot_disk_type = var.dataproc_config.cluster_config.master_config.disk_config.boot_disk_type + boot_disk_size_gb = var.dataproc_config.cluster_config.master_config.disk_config.boot_disk_size_gb + num_local_ssds = var.dataproc_config.cluster_config.master_config.disk_config.num_local_ssds + } + } + dynamic "accelerators" { + for_each = var.dataproc_config.cluster_config.master_config.accelerators == null ? [] : [""] + content { + accelerator_type = var.dataproc_config.cluster_config.master_config.accelerators.accelerator_type + accelerator_count = var.dataproc_config.cluster_config.master_config.accelerators.accelerator_count + } + } + } + } + dynamic "worker_config" { + for_each = var.dataproc_config.cluster_config.worker_config == null ? [] : [""] + content { + num_instances = var.dataproc_config.cluster_config.worker_config.num_instances + machine_type = var.dataproc_config.cluster_config.worker_config.machine_type + min_cpu_platform = var.dataproc_config.cluster_config.worker_config.min_cpu_platform + dynamic "disk_config" { + for_each = var.dataproc_config.cluster_config.worker_config.disk_config == null ? [] : [""] + content { + boot_disk_type = var.dataproc_config.cluster_config.worker_config.disk_config.boot_disk_type + boot_disk_size_gb = var.dataproc_config.cluster_config.worker_config.disk_config.boot_disk_size_gb + num_local_ssds = var.dataproc_config.cluster_config.worker_config.disk_config.num_local_ssds + } + } + image_uri = var.dataproc_config.cluster_config.worker_config.image_uri + dynamic "accelerators" { + for_each = var.dataproc_config.cluster_config.worker_config.accelerators == null ? [] : [""] + content { + accelerator_type = var.dataproc_config.cluster_config.accelerators.accelerator_type + accelerator_count = var.dataproc_config.cluster_config.accelerators.accelerator_count + } + } + } + } + dynamic "preemptible_worker_config" { + for_each = var.dataproc_config.cluster_config.preemptible_worker_config == null ? [] : [""] + content { + num_instances = var.dataproc_config.cluster_config.preemptible_worker_config.num_instances + preemptibility = var.dataproc_config.cluster_config.preemptible_worker_config.preemptibility + dynamic "disk_config" { + for_each = var.dataproc_config.cluster_config.preemptible_worker_config.disk_config == null ? [] : [""] + content { + boot_disk_type = var.dataproc_config.cluster_config.disk_config.boot_disk_type + boot_disk_size_gb = var.dataproc_config.cluster_config.disk_config.boot_disk_size_gb + num_local_ssds = var.dataproc_config.cluster_config.disk_config.num_local_ssds + } + } + } + } + dynamic "software_config" { + for_each = var.dataproc_config.cluster_config.software_config == null ? [] : [""] + content { + image_version = var.dataproc_config.cluster_config.software_config.image_version + override_properties = var.dataproc_config.cluster_config.software_config.override_properties + optional_components = var.dataproc_config.cluster_config.software_config.optional_components + } + } + dynamic "security_config" { + for_each = var.dataproc_config.cluster_config.security_config == null ? [] : [""] + content { + dynamic "kerberos_config" { + for_each = try(var.dataproc_config.cluster_config.security_config.kerberos_config == null ? [] : [""], []) + content { + cross_realm_trust_admin_server = var.dataproc_config.cluster_config.kerberos_config.cross_realm_trust_admin_server + cross_realm_trust_kdc = var.dataproc_config.cluster_config.kerberos_config.cross_realm_trust_kdc + cross_realm_trust_realm = var.dataproc_config.cluster_config.kerberos_config.cross_realm_trust_realm + cross_realm_trust_shared_password_uri = var.dataproc_config.cluster_config.kerberos_config.cross_realm_trust_shared_password_uri + enable_kerberos = var.dataproc_config.cluster_config.kerberos_config.enable_kerberos + kdc_db_key_uri = var.dataproc_config.cluster_config.kerberos_config.kdc_db_key_uri + key_password_uri = var.dataproc_config.cluster_config.kerberos_config.key_password_uri + keystore_uri = var.dataproc_config.cluster_config.kerberos_config.keystore_uri + keystore_password_uri = var.dataproc_config.cluster_config.kerberos_config.keystore_password_uri + kms_key_uri = var.dataproc_config.cluster_config.kerberos_config.kms_key_uri + realm = var.dataproc_config.cluster_config.kerberos_config.realm + root_principal_password_uri = var.dataproc_config.cluster_config.kerberos_config.root_principal_password_uri + tgt_lifetime_hours = var.dataproc_config.cluster_config.kerberos_config.tgt_lifetime_hours + truststore_password_uri = var.dataproc_config.cluster_config.kerberos_config.truststore_password_uri + truststore_uri = var.dataproc_config.cluster_config.kerberos_config.truststore_uri + } + } + } + } + dynamic "autoscaling_config" { + for_each = var.dataproc_config.cluster_config.autoscaling_config == null ? [] : [""] + content { + policy_uri = var.dataproc_config.cluster_config.autoscaling_config.policy_uri + } + } + dynamic "initialization_action" { + for_each = var.dataproc_config.cluster_config.initialization_action == null ? [] : [""] + content { + script = var.dataproc_config.cluster_config.initialization_action.script + timeout_sec = var.dataproc_config.cluster_config.initialization_action.timeout_sec + } + } + dynamic "encryption_config" { + for_each = try(var.dataproc_config.cluster_config.encryption_config.kms_key_name == null ? [] : [""], []) + content { + kms_key_name = var.dataproc_config.cluster_config.encryption_config.kms_key_name + } + } + dynamic "dataproc_metric_config" { + for_each = var.dataproc_config.cluster_config.dataproc_metric_config == null ? [] : [""] + content { + dynamic "metrics" { + for_each = var.dataproc_config.cluster_config.dataproc_metric_config.metrics == null ? [] : [""] + content { + metric_source = var.dataproc_config.cluster_config.dataproc_metric_config.metrics.metric_source + metric_overrides = var.dataproc_config.cluster_config.dataproc_metric_config.metrics.metric_overrides + } + } + } + } + dynamic "lifecycle_config" { + for_each = var.dataproc_config.cluster_config.lifecycle_config == null ? [] : [""] + content { + idle_delete_ttl = var.dataproc_config.cluster_config.lifecycle_config.idle_delete_ttl + auto_delete_time = var.dataproc_config.cluster_config.lifecycle_config.auto_delete_time + } + } + dynamic "endpoint_config" { + for_each = var.dataproc_config.cluster_config.endpoint_config == null ? [] : [""] + content { + enable_http_port_access = var.dataproc_config.cluster_config.endpoint_config.enable_http_port_access + } + } + dynamic "metastore_config" { + for_each = var.dataproc_config.cluster_config.metastore_config == null ? [] : [""] + content { + dataproc_metastore_service = var.dataproc_config.cluster_config.metastore_config.dataproc_metastore_service + } + } + + } + } + + dynamic "virtual_cluster_config" { + for_each = var.dataproc_config.virtual_cluster_config == null ? [] : [""] + content { + dynamic "auxiliary_services_config" { + for_each = var.dataproc_config.virtual_cluster_config.auxiliary_services_config == null ? [] : [""] + content { + dynamic "metastore_config" { + for_each = var.dataproc_config.virtual_cluster_config.auxiliary_services_config.metastore_config == null ? [] : [""] + content { + dataproc_metastore_service = var.dataproc_config.virtual_cluster_config.auxiliary_services_config.metastore_config.dataproc_metastore_service + } + } + dynamic "spark_history_server_config" { + for_each = var.dataproc_config.virtual_cluster_config.auxiliary_services_config.spark_history_server_config == null ? [] : [""] + content { + dataproc_cluster = var.dataproc_config.virtual_cluster_config.auxiliary_services_config.spark_history_server_config.dataproc_cluster + } + } + } + } + dynamic "kubernetes_cluster_config" { + for_each = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config == null ? [] : [""] + content { + kubernetes_namespace = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_namespace + dynamic "kubernetes_software_config" { + for_each = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_software_config == null ? [] : [""] + content { + component_version = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_software_config.component_version + properties = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.kubernetes_software_config.properties + } + } + + dynamic "gke_cluster_config" { + for_each = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config == null ? [] : [""] + content { + gke_cluster_target = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.gke_cluster_target + dynamic "node_pool_target" { + for_each = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_target == null ? [] : [""] + content { + node_pool = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_target.node_pool + roles = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_target.roles + dynamic "node_pool_config" { + for_each = try(var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config == null ? [] : [""], []) + content { + dynamic "autoscaling" { + for_each = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.autoscaling == null ? [] : [""] + content { + min_node_count = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.autoscaling.min_node_count + max_node_count = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.autoscaling.max_node_count + } + } + dynamic "config" { + for_each = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.config == null ? [] : [""] + content { + machine_type = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.config.machine_type + local_ssd_count = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.config.local_ssd_count + preemptible = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.config.preemptible + min_cpu_platform = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.config.min_cpu_platform + spot = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.config.spot + } + } + locations = var.dataproc_config.virtual_cluster_config.kubernetes_cluster_config.gke_cluster_config.node_pool_config.locations + } + } + } + } + } + } + } + } + } + } + lifecycle { + ignore_changes = [ + # Some scopes are assigned in addition to the one configured + # https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/service-accounts#dataproc_vm_access_scopes + cluster_config[0].gce_cluster_config[0].service_account_scopes, + ] + } +} diff --git a/assets/modules-fabric/v26/dataproc/outputs.tf b/assets/modules-fabric/v26/dataproc/outputs.tf new file mode 100644 index 0000000..51edb80 --- /dev/null +++ b/assets/modules-fabric/v26/dataproc/outputs.tf @@ -0,0 +1,47 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Cloud Dataproc module output. + +output "bucket_names" { + description = "List of bucket names which have been assigned to the cluster." + value = google_dataproc_cluster.cluster.cluster_config.0.bucket +} + +output "http_ports" { + description = "The map of port descriptions to URLs." + value = google_dataproc_cluster.cluster.cluster_config.0.endpoint_config.0.http_ports +} + +output "id" { + description = "Fully qualified cluster id." + value = google_dataproc_cluster.cluster.id +} + +output "instance_names" { + description = "List of instance names which have been assigned to the cluster." + value = { + master = google_dataproc_cluster.cluster.cluster_config.0.master_config.0.instance_names + worker = google_dataproc_cluster.cluster.cluster_config.0.worker_config.0.instance_names + preemptible_worker = google_dataproc_cluster.cluster.cluster_config.0.preemptible_worker_config.0.instance_names + } +} + +output "name" { + description = "The name of the cluster." + value = google_dataproc_cluster.cluster.name +} + diff --git a/assets/modules-fabric/v26/dataproc/variables.tf b/assets/modules-fabric/v26/dataproc/variables.tf new file mode 100644 index 0000000..8b77c5b --- /dev/null +++ b/assets/modules-fabric/v26/dataproc/variables.tf @@ -0,0 +1,264 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "dataproc_config" { + description = "Dataproc cluster config." + type = object({ + graceful_decommission_timeout = optional(string) + cluster_config = optional(object({ + staging_bucket = optional(string) + temp_bucket = optional(string) + gce_cluster_config = optional(object({ + zone = optional(string) + network = optional(string) + subnetwork = optional(string) + service_account = optional(string) + service_account_scopes = optional(list(string)) + tags = optional(list(string), []) + internal_ip_only = optional(bool) + metadata = optional(map(string), {}) + reservation_affinity = optional(object({ + consume_reservation_type = string + key = string + values = string + })) + node_group_affinity = optional(object({ + node_group_uri = string + })) + + shielded_instance_config = optional(object({ + enable_secure_boot = bool + enable_vtpm = bool + enable_integrity_monitoring = bool + })) + })) + master_config = optional(object({ + num_instances = number + machine_type = string + min_cpu_platform = string + image_uri = string + disk_config = optional(object({ + boot_disk_type = string + boot_disk_size_gb = number + num_local_ssds = number + })) + accelerators = optional(object({ + accelerator_type = string + accelerator_count = number + })) + })) + worker_config = optional(object({ + num_instances = number + machine_type = string + min_cpu_platform = string + disk_config = optional(object({ + boot_disk_type = string + boot_disk_size_gb = number + num_local_ssds = number + })) + image_uri = string + accelerators = optional(object({ + accelerator_type = string + accelerator_count = number + })) + })) + preemptible_worker_config = optional(object({ + num_instances = number + preemptibility = string + disk_config = optional(object({ + boot_disk_type = string + boot_disk_size_gb = number + num_local_ssds = number + })) + })) + software_config = optional(object({ + image_version = optional(string) + override_properties = map(string) + optional_components = optional(list(string)) + })) + security_config = optional(object({ + kerberos_config = object({ + cross_realm_trust_admin_server = optional(string) + cross_realm_trust_kdc = optional(string) + cross_realm_trust_realm = optional(string) + cross_realm_trust_shared_password_uri = optional(string) + enable_kerberos = optional(string) + kdc_db_key_uri = optional(string) + key_password_uri = optional(string) + keystore_uri = optional(string) + keystore_password_uri = optional(string) + kms_key_uri = string + realm = optional(string) + root_principal_password_uri = string + tgt_lifetime_hours = optional(string) + truststore_password_uri = optional(string) + truststore_uri = optional(string) + }) + })) + autoscaling_config = optional(object({ + policy_uri = string + })) + initialization_action = optional(object({ + script = string + timeout_sec = optional(string) + })) + encryption_config = optional(object({ + kms_key_name = string + })) + lifecycle_config = optional(object({ + idle_delete_ttl = optional(string) + auto_delete_time = optional(string) + })) + endpoint_config = optional(object({ + enable_http_port_access = string + })) + dataproc_metric_config = optional(object({ + metrics = list(object({ + metric_source = string + metric_overrides = optional(string) + })) + })) + metastore_config = optional(object({ + dataproc_metastore_service = string + })) + })) + + virtual_cluster_config = optional(object({ + staging_bucket = optional(string) + auxiliary_services_config = optional(object({ + metastore_config = optional(object({ + dataproc_metastore_service = string + })) + spark_history_server_config = optional(object({ + dataproc_cluster = string + })) + })) + kubernetes_cluster_config = object({ + kubernetes_namespace = optional(string) + kubernetes_software_config = object({ + component_version = list(map(string)) + properties = optional(list(map(string))) + }) + + gke_cluster_config = object({ + gke_cluster_target = optional(string) + node_pool_target = optional(object({ + node_pool = string + roles = list(string) + node_pool_config = optional(object({ + autoscaling = optional(object({ + min_node_count = optional(number) + max_node_count = optional(number) + })) + + config = object({ + machine_type = optional(string) + preemptible = optional(bool) + local_ssd_count = optional(number) + min_cpu_platform = optional(string) + spot = optional(bool) + }) + + locations = optional(list(string)) + })) + })) + }) + }) + })) + }) + default = {} +} + +variable "group_iam" { + description = "Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable." + type = map(list(string)) + default = {} + nullable = false +} + +variable "iam" { + description = "IAM bindings in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} + nullable = false +} + +variable "iam_bindings" { + description = "Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary." + type = map(object({ + members = list(string) + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} +} + +variable "iam_bindings_additive" { + description = "Individual additive IAM bindings. Keys are arbitrary." + type = map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} +} + +variable "labels" { + description = "The resource labels for instance to use to annotate any related underlying resources, such as Compute Engine VMs." + type = map(string) + default = {} +} + +variable "name" { + description = "Cluster name." + type = string +} + +variable "prefix" { + description = "Optional prefix used to generate project id and name." + type = string + default = null + validation { + condition = var.prefix != "" + error_message = "Prefix cannot be empty, please use null instead." + } +} + +variable "project_id" { + description = "Project ID." + type = string +} + +variable "region" { + description = "Dataproc region." + type = string +} + +variable "service_account" { + description = "Service account to set on the Dataproc cluster." + type = string + default = null +} diff --git a/assets/modules-fabric/v26/dataproc/versions.tf b/assets/modules-fabric/v26/dataproc/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/dataproc/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/dns-response-policy/README.md b/assets/modules-fabric/v26/dns-response-policy/README.md new file mode 100644 index 0000000..2c77f4e --- /dev/null +++ b/assets/modules-fabric/v26/dns-response-policy/README.md @@ -0,0 +1,149 @@ +# Google Cloud DNS Response Policy + +This module allows management of a [Google Cloud DNS policy and its rules](https://cloud.google.com/dns/docs/zones/manage-response-policies). The policy can already exist and be referenced by name by setting the `policy_create` variable to `false`. + +The module also allows setting rules via a factory. An example is given below. + +## Examples + +### Manage policy and override resolution for specific names + +This example shows how to create a policy with a single rule, that directs a specific Google API name to the restricted VIP addresses. + +```hcl +module "dns-policy" { + source = "./fabric/modules/dns-response-policy" + project_id = "myproject" + name = "googleapis" + networks = { + landing = var.vpc.self_link + } + rules = { + pubsub = { + dns_name = "pubsub.googleapis.com." + local_data = { + A = { + rrdatas = ["199.36.153.4", "199.36.153.5"] + } + } + } + } +} +# tftest modules=1 resources=2 inventory=simple.yaml +``` + +### Use existing policy and override resolution via wildcard with exceptions + +This example shows how to create a policy with a single rule, that directs all Google API names except specific ones to the restricted VIP addresses. + +```hcl +module "dns-policy" { + source = "./fabric/modules/dns-response-policy" + project_id = "myproject" + name = "googleapis" + policy_create = false + networks = { + landing = var.vpc.self_link + } + rules = { + gcr = { + dns_name = "gcr.io." + local_data = { + CNAME = { + rrdatas = ["restricted.googleapis.com."] + } + } + } + googleapis-all = { + dns_name = "*.googleapis.com." + local_data = { + CNAME = { + rrdatas = ["restricted.googleapis.com."] + } + } + } + pubsub = { + dns_name = "pubsub.googleapis.com." + } + restricted = { + dns_name = "restricted.googleapis.com." + local_data = { + A = { + rrdatas = [ + "199.36.153.4", + "199.36.153.5", + "199.36.153.6", + "199.36.153.7" + ] + } + } + } + } +} +# tftest modules=1 resources=4 inventory=complex.yaml +``` + +### Define policy rules via a factory file + +This example shows how to define rules in a factory file, that mirrors the rules defined via variables in the previous example. Rules defined via the variable are merged with factory rules and take precedence over them when using the same rule names. The YAML syntax closely follows the `rules` variable type. + +```hcl +module "dns-policy" { + source = "./fabric/modules/dns-response-policy" + project_id = "myproject" + name = "googleapis" + policy_create = false + networks = { + landing = var.vpc.self_link + } + rules_file = "config/rules.yaml" +} +# tftest modules=1 resources=4 files=rules-file inventory=complex.yaml +``` + +```yaml +gcr: + dns_name: "gcr.io." + local_data: + CNAME: {rrdatas: ["restricted.googleapis.com."]} +googleapis-all: + dns_name: "*.googleapis.com." + local_data: + CNAME: {rrdatas: ["restricted.googleapis.com."]} +pubsub: + dns_name: "pubsub.googleapis.com." +restricted: + dns_name: "restricted.googleapis.com." + local_data: + A: + rrdatas: + - 199.36.153.4 + - 199.36.153.5 + - 199.36.153.6 + - 199.36.153.7 +# tftest-file id=rules-file path=config/rules.yaml +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [name](variables.tf#L30) | Policy name. | string | ✓ | | +| [project_id](variables.tf#L49) | Project id for the zone. | string | ✓ | | +| [clusters](variables.tf#L17) | Map of GKE clusters to which this policy is applied in name => id format. | map(string) | | {} | +| [description](variables.tf#L24) | Policy description. | string | | "Terraform managed." | +| [networks](variables.tf#L35) | Map of VPC self links to which this policy is applied in name => self link format. | map(string) | | {} | +| [policy_create](variables.tf#L42) | Set to false to use the existing policy matching name and only manage rules. | bool | | true | +| [rules](variables.tf#L54) | Map of policy rules in name => rule format. Local data takes precedence over behavior and is in the form record type => attributes. | map(object({…})) | | {} | +| [rules_file](variables.tf#L68) | Optional data file in YAML format listing rules that will be combined with those passed in via the `rules` variable. | string | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [id](outputs.tf#L17) | Fully qualified policy id. | | +| [name](outputs.tf#L22) | Policy name. | | +| [policy](outputs.tf#L27) | Policy resource. | | + + diff --git a/assets/modules-fabric/v26/dns-response-policy/main.tf b/assets/modules-fabric/v26/dns-response-policy/main.tf new file mode 100644 index 0000000..69b7ff4 --- /dev/null +++ b/assets/modules-fabric/v26/dns-response-policy/main.tf @@ -0,0 +1,84 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + _factory_rules = try(yamldecode(file(var.rules_file)), {}) + factory_rules = { + for k, v in local._factory_rules : k => { + dns_name = v.dns_name + behavior = lookup(v, "behavior", "bypassResponsePolicy") + local_data = { + for kk, vv in lookup(v, "local_data", {}) : + kk => merge({ ttl = null, rrdatas = [] }, vv) + } + } + } + policy_name = ( + var.policy_create + ? google_dns_response_policy.default.0.response_policy_name + : var.name + ) +} + +resource "google_dns_response_policy" "default" { + provider = google-beta + count = var.policy_create ? 1 : 0 + project = var.project_id + response_policy_name = var.name + dynamic "networks" { + for_each = var.networks + content { + network_url = networks.value + } + } + dynamic "gke_clusters" { + for_each = var.clusters + content { + gke_cluster_name = gke_clusters.value + } + } +} + +resource "google_dns_response_policy_rule" "default" { + provider = google-beta + for_each = merge(local.factory_rules, var.rules) + project = var.project_id + response_policy = local.policy_name + rule_name = each.key + dns_name = each.value.dns_name + behavior = ( + length(each.value.local_data) == 0 ? each.value.behavior : null + ) + dynamic "local_data" { + for_each = length(each.value.local_data) == 0 ? [] : [""] + content { + dynamic "local_datas" { + for_each = each.value.local_data + iterator = data + content { + # setting name to something different seems to have no effect + # so we comply with the console UI and set it to the rule dns name + # name = split(" ", data.key)[1] + # type = split(" ", data.key)[0] + name = each.value.dns_name + type = data.key + ttl = data.value.ttl + rrdatas = data.value.rrdatas + } + } + } + } +} diff --git a/assets/modules-fabric/v26/dns-response-policy/outputs.tf b/assets/modules-fabric/v26/dns-response-policy/outputs.tf new file mode 100644 index 0000000..95b1086 --- /dev/null +++ b/assets/modules-fabric/v26/dns-response-policy/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "id" { + description = "Fully qualified policy id." + value = try(google_dns_response_policy.default.0.id, null) +} + +output "name" { + description = "Policy name." + value = local.policy_name +} + +output "policy" { + description = "Policy resource." + value = try(google_dns_response_policy.default.0, null) +} diff --git a/assets/modules-fabric/v26/dns-response-policy/variables.tf b/assets/modules-fabric/v26/dns-response-policy/variables.tf new file mode 100644 index 0000000..fa26c3b --- /dev/null +++ b/assets/modules-fabric/v26/dns-response-policy/variables.tf @@ -0,0 +1,72 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "clusters" { + description = "Map of GKE clusters to which this policy is applied in name => id format." + type = map(string) + default = {} + nullable = false +} + +variable "description" { + description = "Policy description." + type = string + default = "Terraform managed." +} + +variable "name" { + description = "Policy name." + type = string +} + +variable "networks" { + description = "Map of VPC self links to which this policy is applied in name => self link format." + type = map(string) + default = {} + nullable = false +} + +variable "policy_create" { + description = "Set to false to use the existing policy matching name and only manage rules." + type = bool + default = true + nullable = false +} + +variable "project_id" { + description = "Project id for the zone." + type = string +} + +variable "rules" { + description = "Map of policy rules in name => rule format. Local data takes precedence over behavior and is in the form record type => attributes." + type = map(object({ + dns_name = string + behavior = optional(string, "bypassResponsePolicy") + local_data = optional(map(object({ + ttl = optional(number) + rrdatas = optional(list(string), []) + })), {}) + })) + default = {} + nullable = false +} + +variable "rules_file" { + description = "Optional data file in YAML format listing rules that will be combined with those passed in via the `rules` variable." + type = string + default = null +} diff --git a/assets/modules-fabric/v26/dns-response-policy/versions.tf b/assets/modules-fabric/v26/dns-response-policy/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/dns-response-policy/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/dns/README.md b/assets/modules-fabric/v26/dns/README.md new file mode 100644 index 0000000..5b29376 --- /dev/null +++ b/assets/modules-fabric/v26/dns/README.md @@ -0,0 +1,164 @@ +# Google Cloud DNS Module + +This module allows simple management of Google Cloud DNS zones and records. It supports creating public, private, forwarding, peering, service directory and reverse-managed based zones. To create inbound/outbound server policies, please have a look at the [net-vpc](../net-vpc/README.md) module. + +For DNSSEC configuration, refer to the [`dns_managed_zone` documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/dns_managed_zone#dnssec_config). + +## Examples + +### Private Zone + +```hcl +module "private-dns" { + source = "./fabric/modules/dns" + project_id = "myproject" + name = "test-example" + zone_config = { + domain = "test.example." + private = { + client_networks = [var.vpc.self_link] + } + } + recordsets = { + "A localhost" = { records = ["127.0.0.1"] } + "A myhost" = { ttl = 600, records = ["10.0.0.120"] } + } + iam = { + "roles/dns.admin" = ["group:dns-administrators@myorg.com"] + } +} +# tftest modules=1 resources=5 inventory=private-zone.yaml +``` + +### Forwarding Zone + +```hcl +module "private-dns" { + source = "./fabric/modules/dns" + project_id = "myproject" + name = "test-example" + zone_config = { + domain = "test.example." + forwarding = { + client_networks = [var.vpc.self_link] + forwarders = { "10.0.1.1" = null, "1.2.3.4" = "private" } + } + } +} +# tftest modules=1 resources=2 inventory=forwarding-zone.yaml +``` + +### Peering Zone + +```hcl +module "private-dns" { + source = "./fabric/modules/dns" + project_id = "myproject" + name = "test-example" + zone_config = { + domain = "." + peering = { + client_networks = [var.vpc.self_link] + peer_network = var.vpc2.self_link + } + } +} +# tftest modules=1 resources=2 inventory=peering-zone.yaml +``` + +### Routing Policies + +```hcl +module "private-dns" { + source = "./fabric/modules/dns" + project_id = "myproject" + name = "test-example" + zone_config = { + domain = "test.example." + private = { + client_networks = [var.vpc.self_link] + } + } + recordsets = { + "A regular" = { records = ["10.20.0.1"] } + "A geo" = { + geo_routing = [ + { location = "europe-west1", records = ["10.0.0.1"] }, + { location = "europe-west2", records = ["10.0.0.2"] }, + { location = "europe-west3", records = ["10.0.0.3"] } + ] + } + + "A wrr" = { + ttl = 600 + wrr_routing = [ + { weight = 0.6, records = ["10.10.0.1"] }, + { weight = 0.2, records = ["10.10.0.2"] }, + { weight = 0.2, records = ["10.10.0.3"] } + ] + } + } +} +# tftest modules=1 resources=5 inventory=routing-policies.yaml +``` + +### Reverse Lookup Zone + +```hcl +module "private-dns" { + source = "./fabric/modules/dns" + project_id = "myproject" + name = "test-example" + zone_config = { + domain = "0.0.10.in-addr.arpa." + private = { + client_networks = [var.vpc.self_link] + } + } +} +# tftest modules=1 resources=2 inventory=reverse-zone.yaml +``` + +### Public Zone + +```hcl +module "public-dns" { + source = "./fabric/modules/dns" + project_id = "myproject" + name = "example" + zone_config = { + domain = "example.com." + public = {} + } + recordsets = { + "A myhost" = { ttl = 300, records = ["127.0.0.1"] } + } + iam = { + "roles/dns.admin" = ["group:dns-administrators@myorg.com"] + } +} +# tftest modules=1 resources=4 inventory=public-zone.yaml +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [name](variables.tf#L29) | Zone name, must be unique within the project. | string | ✓ | | +| [project_id](variables.tf#L34) | Project id for the zone. | string | ✓ | | +| [description](variables.tf#L17) | Domain description. | string | | "Terraform managed." | +| [iam](variables.tf#L23) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | null | +| [recordsets](variables.tf#L39) | Map of DNS recordsets in \"type name\" => {ttl, [records]} format. | map(object({…})) | | {} | +| [zone_config](variables.tf#L74) | DNS zone configuration. | object({…}) | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [dns_keys](outputs.tf#L17) | DNSKEY and DS records of DNSSEC-signed managed zones. | | +| [domain](outputs.tf#L22) | The DNS zone domain. | | +| [id](outputs.tf#L27) | Fully qualified zone id. | | +| [name](outputs.tf#L32) | The DNS zone name. | | +| [name_servers](outputs.tf#L37) | The DNS zone name servers. | | +| [zone](outputs.tf#L42) | DNS zone resource. | | + diff --git a/assets/modules-fabric/v26/dns/main.tf b/assets/modules-fabric/v26/dns/main.tf new file mode 100644 index 0000000..2c4c823 --- /dev/null +++ b/assets/modules-fabric/v26/dns/main.tf @@ -0,0 +1,196 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + managed_zone = (var.zone_config == null ? + data.google_dns_managed_zone.dns_managed_zone.0 + : google_dns_managed_zone.dns_managed_zone.0 + ) + # split record name and type and set as keys in a map + _recordsets_0 = { + for key, attrs in var.recordsets : + key => merge(attrs, zipmap(["type", "name"], split(" ", key))) + } + # compute the final resource name for the recordset + recordsets = { + for key, attrs in local._recordsets_0 : + key => merge(attrs, { + resource_name = ( + attrs.name == "" + ? local.managed_zone.dns_name + : ( + substr(attrs.name, -1, 1) == "." + ? attrs.name + : "${attrs.name}.${local.managed_zone.dns_name}" + ) + ) + }) + } + client_networks = concat( + coalesce(try(var.zone_config.forwarding.client_networks, null), []), + coalesce(try(var.zone_config.peering.client_networks, null), []), + coalesce(try(var.zone_config.private.client_networks, null), []) + ) + visibility = (var.zone_config == null ? + null + : (var.zone_config.forwarding != null || + var.zone_config.peering != null + || var.zone_config.private != null) ? + "private" : + "public" + ) +} + +resource "google_dns_managed_zone" "dns_managed_zone" { + count = (var.zone_config == null) ? 0 : 1 + provider = google-beta + project = var.project_id + name = var.name + dns_name = var.zone_config.domain + description = var.description + visibility = local.visibility + reverse_lookup = try(var.zone_config.private, null) != null && endswith(var.zone_config.domain, ".in-addr.arpa.") + + dynamic "dnssec_config" { + for_each = try(var.zone_config.public.dnssec_config, null) == null ? [] : [""] + iterator = config + content { + kind = "dns#managedZoneDnsSecConfig" + non_existence = var.zone_config.public.dnssec_config.non_existence + state = var.zone_config.public.dnssec_config.state + + default_key_specs { + algorithm = var.zone_config.public.dnssec_config.key_signing_key.algorithm + key_length = var.zone_config.public.dnssec_config.key_signing_key.key_length + key_type = "keySigning" + kind = "dns#dnsKeySpec" + } + + default_key_specs { + algorithm = var.zone_config.public.dnssec_config.zone_signing_key.algorithm + key_length = var.zone_config.public.dnssec_config.zone_signing_key.key_length + key_type = "zoneSigning" + kind = "dns#dnsKeySpec" + } + } + } + + dynamic "forwarding_config" { + for_each = (length(coalesce(try(var.zone_config.forwarding.forwarders, null), {})) > 0 + ? [""] + : [] + ) + content { + dynamic "target_name_servers" { + for_each = var.zone_config.forwarding.forwarders + iterator = forwarder + content { + ipv4_address = forwarder.key + forwarding_path = forwarder.value + } + } + } + } + + dynamic "peering_config" { + for_each = try(var.zone_config.peering.peer_network, null) == null ? [] : [""] + content { + target_network { + network_url = var.zone_config.peering.peer_network + } + } + } + + dynamic "private_visibility_config" { + for_each = length(local.client_networks) > 0 ? [""] : [] + content { + dynamic "networks" { + for_each = local.client_networks + iterator = network + content { + network_url = network.value + } + } + } + } + + dynamic "service_directory_config" { + for_each = (try(var.zone_config.private.service_directory_namespace, null) == null + ? [] + : [""] + ) + content { + namespace { + namespace_url = var.zone_config.private.service_directory_namespace + } + } + } + cloud_logging_config { + enable_logging = try(var.zone_config.public.enable_logging, false) + } +} + +data "google_dns_managed_zone" "dns_managed_zone" { + count = var.zone_config == null ? 1 : 0 + project = var.project_id + name = var.name +} + +resource "google_dns_managed_zone_iam_binding" "iam_bindings" { + for_each = coalesce(var.iam, {}) + project = var.project_id + managed_zone = local.managed_zone.id + role = each.key + members = each.value +} + +data "google_dns_keys" "dns_keys" { + managed_zone = local.managed_zone.id +} + +resource "google_dns_record_set" "dns_record_set" { + for_each = local.recordsets + project = var.project_id + managed_zone = var.name + name = each.value.resource_name + type = each.value.type + ttl = each.value.ttl + rrdatas = each.value.records + + dynamic "routing_policy" { + for_each = (each.value.geo_routing != null || each.value.wrr_routing != null) ? [""] : [] + content { + dynamic "geo" { + for_each = coalesce(each.value.geo_routing, []) + content { + location = geo.value.location + rrdatas = geo.value.records + } + } + dynamic "wrr" { + for_each = coalesce(each.value.wrr_routing, []) + content { + weight = wrr.value.weight + rrdatas = wrr.value.records + } + } + } + } + + depends_on = [ + google_dns_managed_zone.dns_managed_zone + ] +} \ No newline at end of file diff --git a/assets/modules-fabric/v26/dns/outputs.tf b/assets/modules-fabric/v26/dns/outputs.tf new file mode 100644 index 0000000..f8297d8 --- /dev/null +++ b/assets/modules-fabric/v26/dns/outputs.tf @@ -0,0 +1,45 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "dns_keys" { + description = "DNSKEY and DS records of DNSSEC-signed managed zones." + value = data.google_dns_keys.dns_keys +} + +output "domain" { + description = "The DNS zone domain." + value = local.managed_zone.dns_name +} + +output "id" { + description = "Fully qualified zone id." + value = local.managed_zone.id +} + +output "name" { + description = "The DNS zone name." + value = local.managed_zone.name +} + +output "name_servers" { + description = "The DNS zone name servers." + value = local.managed_zone.name_servers +} + +output "zone" { + description = "DNS zone resource." + value = local.managed_zone +} diff --git a/assets/modules-fabric/v26/dns/variables.tf b/assets/modules-fabric/v26/dns/variables.tf new file mode 100644 index 0000000..08395ba --- /dev/null +++ b/assets/modules-fabric/v26/dns/variables.tf @@ -0,0 +1,118 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "description" { + description = "Domain description." + type = string + default = "Terraform managed." +} + +variable "iam" { + description = "IAM bindings in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = null +} + +variable "name" { + description = "Zone name, must be unique within the project." + type = string +} + +variable "project_id" { + description = "Project id for the zone." + type = string +} + +variable "recordsets" { + description = "Map of DNS recordsets in \"type name\" => {ttl, [records]} format." + type = map(object({ + ttl = optional(number, 300) + records = optional(list(string)) + geo_routing = optional(list(object({ + location = string + records = list(string) + }))) + wrr_routing = optional(list(object({ + weight = number + records = list(string) + }))) + })) + default = {} + nullable = false + validation { + condition = alltrue([ + for k, v in coalesce(var.recordsets, {}) : + length(split(" ", k)) == 2 + ]) + error_message = "Recordsets must have keys in the format \"type name\"." + } + validation { + condition = alltrue([ + for k, v in coalesce(var.recordsets, {}) : ( + (v.records != null && v.wrr_routing == null && v.geo_routing == null) || + (v.records == null && v.wrr_routing != null && v.geo_routing == null) || + (v.records == null && v.wrr_routing == null && v.geo_routing != null) + ) + ]) + error_message = "Only one of records, wrr_routing or geo_routing can be defined for each recordset." + } +} + +variable "zone_config" { + description = "DNS zone configuration." + type = object({ + domain = string + forwarding = optional(object({ + forwarders = optional(map(string)) + client_networks = list(string) + })) + peering = optional(object({ + client_networks = list(string) + peer_network = string + })) + public = optional(object({ + dnssec_config = optional(object({ + non_existence = optional(string, "nsec3") + state = string + key_signing_key = optional(object( + { algorithm = string, key_length = number }), + { algorithm = "rsasha256", key_length = 2048 } + ) + zone_signing_key = optional(object( + { algorithm = string, key_length = number }), + { algorithm = "rsasha256", key_length = 1024 } + ) + })) + enable_logging = optional(bool, false) + })) + private = optional(object({ + client_networks = list(string) + service_directory_namespace = optional(string) + })) + }) + validation { + condition = ( + (try(var.zone_config.forwarding, null) == null ? 0 : 1) + + (try(var.zone_config.peering, null) == null ? 0 : 1) + + (try(var.zone_config.public, null) == null ? 0 : 1) + + (try(var.zone_config.private, null) == null ? 0 : 1) <= 1 + ) + error_message = "Only one type of zone can be configured at a time." + } + default = null +} + + diff --git a/assets/modules-fabric/v26/dns/versions.tf b/assets/modules-fabric/v26/dns/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/dns/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/endpoints/README.md b/assets/modules-fabric/v26/endpoints/README.md new file mode 100644 index 0000000..2b68796 --- /dev/null +++ b/assets/modules-fabric/v26/endpoints/README.md @@ -0,0 +1,55 @@ +# Google Cloud Endpoints + +This module allows simple management of ['Google Cloud Endpoints'](https://cloud.google.com/endpoints/) services. It supports creating ['OpenAPI'](https://cloud.google.com/endpoints/docs/openapi) or ['gRPC'](https://cloud.google.com/endpoints/docs/grpc/about-grpc) endpoints. + +## Examples + +### OpenAPI + +```hcl +module "endpoint" { + source = "./fabric/modules/endpoints" + project_id = "my-project" + service_name = "YOUR-API.endpoints.YOUR-PROJECT-ID.cloud.goog" + openapi_config = { "yaml_path" = "configs/endpoints/openapi.yaml" } + iam = { + "servicemanagement.serviceController" = [ + "serviceAccount:123456890-compute@developer.gserviceaccount.com" + ] + } +} +# tftest modules=1 resources=2 files=openapi inventory=simple.yaml +``` + +```yaml +# tftest-file id=openapi path=configs/endpoints/openapi.yaml +swagger: "2.0" +info: + description: "A simple Google Cloud Endpoints API example." + title: "Endpoints Example" + version: "1.0.0" +host: "echo-api.endpoints.YOUR-PROJECT-ID.cloud.goog" +``` + +[Here](https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/endpoints/getting-started/openapi.yaml) you can find an example of an openapi.yaml file. Once created the endpoint, remember to activate the service at project level. + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [openapi_config](variables.tf#L32) | The configuration for an OpenAPI endopoint. Either this or grpc_config must be specified. | object({…}) | ✓ | | +| [service_name](variables.tf#L45) | The name of the service. Usually of the form '$apiname.endpoints.$projectid.cloud.goog'. | string | ✓ | | +| [grpc_config](variables.tf#L17) | The configuration for a gRPC endpoint. Either this or openapi_config must be specified. | object({…}) | | null | +| [iam](variables.tf#L26) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [project_id](variables.tf#L39) | The project ID that the service belongs to. | string | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [endpoints](outputs.tf#L17) | A list of Endpoint objects. | | +| [endpoints_service](outputs.tf#L22) | The Endpoint service resource. | | +| [service_name](outputs.tf#L27) | The name of the service.. | | + + diff --git a/assets/modules-fabric/v26/endpoints/main.tf b/assets/modules-fabric/v26/endpoints/main.tf new file mode 100644 index 0000000..dc8c61b --- /dev/null +++ b/assets/modules-fabric/v26/endpoints/main.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_endpoints_service" "default" { + project = var.project_id + service_name = var.service_name + openapi_config = var.openapi_config != null ? file(var.openapi_config.yaml_path) : null + grpc_config = var.grpc_config != null ? file(var.grpc_config.yaml_path) : null + protoc_output_base64 = var.grpc_config != null ? base64encode(file(var.grpc_config.protoc_output_path)) : null +} + +resource "google_endpoints_service_iam_binding" "default" { + for_each = var.iam + service_name = google_endpoints_service.default.service_name + role = each.key + members = each.value +} diff --git a/assets/modules-fabric/v26/endpoints/outputs.tf b/assets/modules-fabric/v26/endpoints/outputs.tf new file mode 100644 index 0000000..27fe3f6 --- /dev/null +++ b/assets/modules-fabric/v26/endpoints/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "endpoints" { + description = "A list of Endpoint objects." + value = google_endpoints_service.default.endpoints +} + +output "endpoints_service" { + description = "The Endpoint service resource." + value = google_endpoints_service.default +} + +output "service_name" { + description = "The name of the service.." + value = google_endpoints_service.default.service_name +} diff --git a/assets/modules-fabric/v26/endpoints/variables.tf b/assets/modules-fabric/v26/endpoints/variables.tf new file mode 100644 index 0000000..ffd621e --- /dev/null +++ b/assets/modules-fabric/v26/endpoints/variables.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "grpc_config" { + description = "The configuration for a gRPC endpoint. Either this or openapi_config must be specified." + type = object({ + yaml_path = string + protoc_output_path = string + }) + default = null +} + +variable "iam" { + description = "IAM bindings for topic in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} +} + +variable "openapi_config" { + description = "The configuration for an OpenAPI endopoint. Either this or grpc_config must be specified." + type = object({ + yaml_path = string + }) +} + +variable "project_id" { + description = "The project ID that the service belongs to." + type = string + default = null +} + +variable "service_name" { + description = "The name of the service. Usually of the form '$apiname.endpoints.$projectid.cloud.goog'." + type = string +} diff --git a/assets/modules-fabric/v26/endpoints/versions.tf b/assets/modules-fabric/v26/endpoints/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/endpoints/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/folder/README.md b/assets/modules-fabric/v26/folder/README.md new file mode 100644 index 0000000..6566121 --- /dev/null +++ b/assets/modules-fabric/v26/folder/README.md @@ -0,0 +1,313 @@ +# Google Cloud Folder Module + +This module allows the creation and management of folders, including support for IAM bindings, organization policies, and hierarchical firewall rules. + + +- [Basic example with IAM bindings](#basic-example-with-iam-bindings) +- [IAM](#iam) +- [Organization policies](#organization-policies) + - [Organization Policy Factory](#organization-policy-factory) +- [Hierarchical Firewall Policy Attachments](#hierarchical-firewall-policy-attachments) +- [Log Sinks](#log-sinks) +- [Data Access Logs](#data-access-logs) +- [Tags](#tags) +- [Files](#files) +- [Variables](#variables) +- [Outputs](#outputs) + + +## Basic example with IAM bindings + +```hcl +module "folder" { + source = "./fabric/modules/folder" + parent = "organizations/1234567890" + name = "Folder name" + group_iam = { + "cloud-owners@example.org" = [ + "roles/owner", + "roles/resourcemanager.folderAdmin", + "roles/resourcemanager.projectCreator" + ] + } + iam = { + "roles/owner" = ["user:one@example.org"] + } + iam_bindings_additive = { + am1-storage-admin = { + member = "user:am1@example.org" + role = "roles/storage.admin" + } + } +} +# tftest modules=1 resources=5 inventory=iam.yaml +``` + +## IAM + +IAM is managed via several variables that implement different features and levels of control: + +- `iam` and `group_iam` configure authoritative bindings that manage individual roles exclusively, and are internally merged +- `iam_bindings` configure authoritative bindings with optional support for conditions, and are not internally merged with the previous two variables +- `iam_bindings_additive` configure additive bindings via individual role/member pairs with optional support conditions + +The authoritative and additive approaches can be used together, provided different roles are managed by each. Some care must also be taken with the `groups_iam` variable to ensure that variable keys are static values, so that Terraform is able to compute the dependency graph. + +Refer to the [project module](../project/README.md#iam) for examples of the IAM interface. + +## Organization policies + +To manage organization policies, the `orgpolicy.googleapis.com` service should be enabled in the quota project. + +```hcl +module "folder" { + source = "./fabric/modules/folder" + parent = "organizations/1234567890" + name = "Folder name" + org_policies = { + "compute.disableGuestAttributesAccess" = { + rules = [{ enforce = true }] + } + "compute.skipDefaultNetworkCreation" = { + rules = [{ enforce = true }] + } + "iam.disableServiceAccountKeyCreation" = { + rules = [{ enforce = true }] + } + "iam.disableServiceAccountKeyUpload" = { + rules = [ + { + condition = { + expression = "resource.matchTagId('tagKeys/1234', 'tagValues/1234')" + title = "condition" + description = "test condition" + location = "somewhere" + } + enforce = true + }, + { + enforce = false + } + ] + } + "iam.allowedPolicyMemberDomains" = { + rules = [{ + allow = { + values = ["C0xxxxxxx", "C0yyyyyyy"] + } + }] + } + "compute.trustedImageProjects" = { + rules = [{ + allow = { + values = ["projects/my-project"] + } + }] + } + "compute.vmExternalIpAccess" = { + rules = [{ deny = { all = true } }] + } + } +} +# tftest modules=1 resources=8 inventory=org-policies.yaml +``` + +### Organization Policy Factory + +See the [organization policy factory in the project module](../project#organization-policy-factory). + +## Hierarchical Firewall Policy Attachments + +Hierarchical firewall policies can be managed via the [`net-firewall-policy`](../net-firewall-policy/) module, including support for factories. Once a policy is available, attaching it to the organization can be done either in the firewall policy module itself, or here: + +```hcl +module "firewall-policy" { + source = "./fabric/modules/net-firewall-policy" + name = "test-1" + parent_id = module.folder.id + # attachment via the firewall policy module + # attachments = { + # folder-1 = module.folder.id + # } +} + +module "folder" { + source = "./fabric/modules/folder" + parent = "organizations/1234567890" + name = "Folder name" + # attachment via the organization module + firewall_policy = { + name = "test-1" + policy = module.firewall-policy.id + } +} +# tftest modules=2 resources=3 +``` + +## Log Sinks + +```hcl +module "gcs" { + source = "./fabric/modules/gcs" + project_id = "my-project" + name = "gcs_sink" + force_destroy = true +} + +module "dataset" { + source = "./fabric/modules/bigquery-dataset" + project_id = "my-project" + id = "bq_sink" +} + +module "pubsub" { + source = "./fabric/modules/pubsub" + project_id = "my-project" + name = "pubsub_sink" +} + +module "bucket" { + source = "./fabric/modules/logging-bucket" + parent_type = "project" + parent = "my-project" + id = "bucket" +} + +module "folder-sink" { + source = "./fabric/modules/folder" + parent = "folders/657104291943" + name = "my-folder" + logging_sinks = { + warnings = { + destination = module.gcs.id + filter = "severity=WARNING" + type = "storage" + } + info = { + destination = module.dataset.id + filter = "severity=INFO" + type = "bigquery" + } + notice = { + destination = module.pubsub.id + filter = "severity=NOTICE" + type = "pubsub" + } + debug = { + destination = module.bucket.id + filter = "severity=DEBUG" + exclusions = { + no-compute = "logName:compute" + } + type = "logging" + } + } + logging_exclusions = { + no-gce-instances = "resource.type=gce_instance" + } +} +# tftest modules=5 resources=14 inventory=logging.yaml +``` + +## Data Access Logs + +Activation of data access logs can be controlled via the `logging_data_access` variable. If the `iam_bindings_authoritative` variable is used to set a resource-level IAM policy, the data access log configuration will also be authoritative as part of the policy. + +This example shows how to set a non-authoritative access log configuration: + +```hcl +module "folder" { + source = "./fabric/modules/folder" + parent = "folders/657104291943" + name = "my-folder" + logging_data_access = { + allServices = { + # logs for principals listed here will be excluded + ADMIN_READ = ["group:organization-admins@example.org"] + } + "storage.googleapis.com" = { + DATA_READ = [] + DATA_WRITE = [] + } + } +} +# tftest modules=1 resources=3 inventory=logging-data-access.yaml +``` + +## Tags + +Refer to the [Creating and managing tags](https://cloud.google.com/resource-manager/docs/tags/tags-creating-and-managing) documentation for details on usage. + +```hcl +module "org" { + source = "./fabric/modules/organization" + organization_id = var.organization_id + tags = { + environment = { + description = "Environment specification." + iam = null + values = { + dev = null + prod = null + } + } + } +} + +module "folder" { + source = "./fabric/modules/folder" + name = "Test" + parent = module.org.organization_id + tag_bindings = { + env-prod = module.org.tag_values["environment/prod"].id + foo = "tagValues/12345678" + } +} +# tftest modules=2 resources=6 inventory=tags.yaml +``` + + + +## Files + +| name | description | resources | +|---|---|---| +| [iam.tf](./iam.tf) | IAM bindings, roles and audit logging resources. | google_folder_iam_binding · google_folder_iam_member | +| [logging.tf](./logging.tf) | Log sinks and supporting resources. | google_bigquery_dataset_iam_member · google_folder_iam_audit_config · google_logging_folder_exclusion · google_logging_folder_sink · google_project_iam_member · google_pubsub_topic_iam_member · google_storage_bucket_iam_member | +| [main.tf](./main.tf) | Module-level locals and resources. | google_compute_firewall_policy_association · google_essential_contacts_contact · google_folder | +| [organization-policies.tf](./organization-policies.tf) | Folder-level organization policies. | google_org_policy_policy | +| [outputs.tf](./outputs.tf) | Module outputs. | | +| [tags.tf](./tags.tf) | None | google_tags_tag_binding | +| [variables.tf](./variables.tf) | Module variables. | | +| [versions.tf](./versions.tf) | Version pins. | | + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [contacts](variables.tf#L17) | List of essential contacts for this resource. Must be in the form EMAIL -> [NOTIFICATION_TYPES]. Valid notification types are ALL, SUSPENSION, SECURITY, TECHNICAL, BILLING, LEGAL, PRODUCT_UPDATES. | map(list(string)) | | {} | +| [firewall_policy](variables.tf#L24) | Hierarchical firewall policy to associate to this folder. | object({…}) | | null | +| [folder_create](variables.tf#L33) | Create folder. When set to false, uses id to reference an existing folder. | bool | | true | +| [group_iam](variables.tf#L39) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | +| [iam](variables.tf#L46) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [iam_bindings](variables.tf#L53) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | map(object({…})) | | {} | +| [iam_bindings_additive](variables.tf#L68) | Individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | +| [id](variables.tf#L83) | Folder ID in case you use folder_create=false. | string | | null | +| [logging_data_access](variables.tf#L89) | Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services. | map(map(list(string))) | | {} | +| [logging_exclusions](variables.tf#L104) | Logging exclusions for this folder in the form {NAME -> FILTER}. | map(string) | | {} | +| [logging_sinks](variables.tf#L111) | Logging sinks to create for the organization. | map(object({…})) | | {} | +| [name](variables.tf#L141) | Folder name. | string | | null | +| [org_policies](variables.tf#L147) | Organization policies applied to this folder keyed by policy name. | map(object({…})) | | {} | +| [org_policies_data_path](variables.tf#L174) | Path containing org policies in YAML format. | string | | null | +| [parent](variables.tf#L180) | Parent in folders/folder_id or organizations/org_id format. | string | | null | +| [tag_bindings](variables.tf#L190) | Tag bindings for this folder, in key => tag value id format. | map(string) | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [folder](outputs.tf#L17) | Folder resource. | | +| [id](outputs.tf#L22) | Fully qualified folder id. | | +| [name](outputs.tf#L32) | Folder name. | | +| [sink_writer_identities](outputs.tf#L37) | Writer identities created for each sink. | | + diff --git a/assets/modules-fabric/v26/folder/iam.tf b/assets/modules-fabric/v26/folder/iam.tf new file mode 100644 index 0000000..20025b2 --- /dev/null +++ b/assets/modules-fabric/v26/folder/iam.tf @@ -0,0 +1,70 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description IAM bindings, roles and audit logging resources. + +locals { + _group_iam_roles = distinct(flatten(values(var.group_iam))) + _group_iam = { + for r in local._group_iam_roles : r => [ + for k, v in var.group_iam : "group:${k}" if try(index(v, r), null) != null + ] + } + iam = { + for role in distinct(concat(keys(var.iam), keys(local._group_iam))) : + role => concat( + try(var.iam[role], []), + try(local._group_iam[role], []) + ) + } +} + +resource "google_folder_iam_binding" "authoritative" { + for_each = local.iam + folder = local.folder.name + role = each.key + members = each.value +} + +resource "google_folder_iam_binding" "bindings" { + for_each = var.iam_bindings + folder = local.folder.name + role = each.value.role + members = each.value.members + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +resource "google_folder_iam_member" "bindings" { + for_each = var.iam_bindings_additive + folder = local.folder.name + role = each.value.role + member = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} diff --git a/assets/modules-fabric/v26/folder/logging.tf b/assets/modules-fabric/v26/folder/logging.tf new file mode 100644 index 0000000..8000a02 --- /dev/null +++ b/assets/modules-fabric/v26/folder/logging.tf @@ -0,0 +1,117 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Log sinks and supporting resources. + +locals { + sink_bindings = { + for type in ["bigquery", "pubsub", "logging", "storage"] : + type => { + for name, sink in var.logging_sinks : + name => sink + if sink.type == type + } + } +} + +resource "google_folder_iam_audit_config" "default" { + for_each = var.logging_data_access + folder = local.folder.name + service = each.key + dynamic "audit_log_config" { + for_each = each.value + iterator = config + content { + log_type = config.key + exempted_members = config.value + } + } +} + +resource "google_logging_folder_sink" "sink" { + for_each = var.logging_sinks + name = each.key + description = coalesce(each.value.description, "${each.key} (Terraform-managed).") + folder = local.folder.name + destination = "${each.value.type}.googleapis.com/${each.value.destination}" + filter = each.value.filter + include_children = each.value.include_children + disabled = each.value.disabled + + dynamic "bigquery_options" { + for_each = each.value.type == "biquery" && each.value.bq_partitioned_table != false ? [""] : [] + content { + use_partitioned_tables = each.value.bq_partitioned_table + } + } + + dynamic "exclusions" { + for_each = each.value.exclusions + iterator = exclusion + content { + name = exclusion.key + filter = exclusion.value + } + } + + depends_on = [ + google_folder_iam_binding.authoritative + ] +} + +resource "google_storage_bucket_iam_member" "gcs-sinks-binding" { + for_each = local.sink_bindings["storage"] + bucket = each.value.destination + role = "roles/storage.objectCreator" + member = google_logging_folder_sink.sink[each.key].writer_identity +} + +resource "google_bigquery_dataset_iam_member" "bq-sinks-binding" { + for_each = local.sink_bindings["bigquery"] + project = split("/", each.value.destination)[1] + dataset_id = split("/", each.value.destination)[3] + role = "roles/bigquery.dataEditor" + member = google_logging_folder_sink.sink[each.key].writer_identity +} + +resource "google_pubsub_topic_iam_member" "pubsub-sinks-binding" { + for_each = local.sink_bindings["pubsub"] + project = split("/", each.value.destination)[1] + topic = split("/", each.value.destination)[3] + role = "roles/pubsub.publisher" + member = google_logging_folder_sink.sink[each.key].writer_identity +} + +resource "google_project_iam_member" "bucket-sinks-binding" { + for_each = local.sink_bindings["logging"] + project = split("/", each.value.destination)[1] + role = "roles/logging.bucketWriter" + member = google_logging_folder_sink.sink[each.key].writer_identity + + condition { + title = "${each.key} bucket writer" + description = "Grants bucketWriter to ${google_logging_folder_sink.sink[each.key].writer_identity} used by log sink ${each.key} on ${local.folder.id}" + expression = "resource.name.endsWith('${each.value.destination}')" + } +} + +resource "google_logging_folder_exclusion" "logging-exclusion" { + for_each = var.logging_exclusions + name = each.key + folder = local.folder.name + description = "${each.key} (Terraform-managed)." + filter = each.value +} diff --git a/assets/modules-fabric/v26/folder/main.tf b/assets/modules-fabric/v26/folder/main.tf new file mode 100644 index 0000000..853dfde --- /dev/null +++ b/assets/modules-fabric/v26/folder/main.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + folder = ( + var.folder_create + ? try(google_folder.folder.0, null) + : try(data.google_folder.folder.0, null) + ) +} + +data "google_folder" "folder" { + count = var.folder_create ? 0 : 1 + folder = var.id +} + +resource "google_folder" "folder" { + count = var.folder_create ? 1 : 0 + display_name = var.name + parent = var.parent +} + +resource "google_essential_contacts_contact" "contact" { + provider = google-beta + for_each = var.contacts + parent = local.folder.name + email = each.key + language_tag = "en" + notification_category_subscriptions = each.value +} + +resource "google_compute_firewall_policy_association" "default" { + count = var.firewall_policy == null ? 0 : 1 + attachment_target = local.folder.id + name = var.firewall_policy.name + firewall_policy = var.firewall_policy.policy +} diff --git a/assets/modules-fabric/v26/folder/organization-policies.tf b/assets/modules-fabric/v26/folder/organization-policies.tf new file mode 100644 index 0000000..2bf79c4 --- /dev/null +++ b/assets/modules-fabric/v26/folder/organization-policies.tf @@ -0,0 +1,117 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Folder-level organization policies. + +locals { + _factory_data_raw = merge([ + for f in try(fileset(var.org_policies_data_path, "*.yaml"), []) : + yamldecode(file("${var.org_policies_data_path}/${f}")) + ]...) + + # simulate applying defaults to data coming from yaml files + _factory_data = { + for k, v in local._factory_data_raw : + k => { + inherit_from_parent = try(v.inherit_from_parent, null) + reset = try(v.reset, null) + rules = [ + for r in try(v.rules, []) : { + allow = can(r.allow) ? { + all = try(r.allow.all, null) + values = try(r.allow.values, null) + } : null + deny = can(r.deny) ? { + all = try(r.deny.all, null) + values = try(r.deny.values, null) + } : null + enforce = try(r.enforce, null) + condition = { + description = try(r.condition.description, null) + expression = try(r.condition.expression, null) + location = try(r.condition.location, null) + title = try(r.condition.title, null) + } + } + ] + } + } + + _org_policies = merge(local._factory_data, var.org_policies) + + org_policies = { + for k, v in local._org_policies : + k => merge(v, { + name = "${local.folder.name}/policies/${k}" + parent = local.folder.name + is_boolean_policy = ( + alltrue([for r in v.rules : r.allow == null && r.deny == null]) + ) + has_values = ( + length(coalesce(try(v.allow.values, []), [])) > 0 || + length(coalesce(try(v.deny.values, []), [])) > 0 + ) + rules = [ + for r in v.rules : + merge(r, { + has_values = ( + length(coalesce(try(r.allow.values, []), [])) > 0 || + length(coalesce(try(r.deny.values, []), [])) > 0 + ) + }) + ] + }) + } +} + +resource "google_org_policy_policy" "default" { + for_each = local.org_policies + name = each.value.name + parent = each.value.parent + spec { + inherit_from_parent = each.value.inherit_from_parent + reset = each.value.reset + dynamic "rules" { + for_each = each.value.rules + iterator = rule + content { + allow_all = try(rule.value.allow.all, false) == true ? "TRUE" : null + deny_all = try(rule.value.deny.all, false) == true ? "TRUE" : null + enforce = ( + each.value.is_boolean_policy && rule.value.enforce != null + ? upper(tostring(rule.value.enforce)) + : null + ) + dynamic "condition" { + for_each = rule.value.condition.expression != null ? [1] : [] + content { + description = rule.value.condition.description + expression = rule.value.condition.expression + location = rule.value.condition.location + title = rule.value.condition.title + } + } + dynamic "values" { + for_each = rule.value.has_values ? [1] : [] + content { + allowed_values = try(rule.value.allow.values, null) + denied_values = try(rule.value.deny.values, null) + } + } + } + } + } +} diff --git a/assets/modules-fabric/v26/folder/outputs.tf b/assets/modules-fabric/v26/folder/outputs.tf new file mode 100644 index 0000000..3090bbe --- /dev/null +++ b/assets/modules-fabric/v26/folder/outputs.tf @@ -0,0 +1,43 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "folder" { + description = "Folder resource." + value = local.folder +} + +output "id" { + description = "Fully qualified folder id." + value = local.folder.name + depends_on = [ + google_folder_iam_binding.authoritative, + google_folder_iam_binding.bindings, + google_org_policy_policy.default, + ] +} + +output "name" { + description = "Folder name." + value = local.folder.display_name +} + +output "sink_writer_identities" { + description = "Writer identities created for each sink." + value = { + for name, sink in google_logging_folder_sink.sink : + name => sink.writer_identity + } +} diff --git a/assets/modules-fabric/v26/folder/tags.tf b/assets/modules-fabric/v26/folder/tags.tf new file mode 100644 index 0000000..2cd2f2f --- /dev/null +++ b/assets/modules-fabric/v26/folder/tags.tf @@ -0,0 +1,21 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_tags_tag_binding" "binding" { + for_each = coalesce(var.tag_bindings, {}) + parent = "//cloudresourcemanager.googleapis.com/${local.folder.id}" + tag_value = each.value +} diff --git a/assets/modules-fabric/v26/folder/variables.tf b/assets/modules-fabric/v26/folder/variables.tf new file mode 100644 index 0000000..86efc21 --- /dev/null +++ b/assets/modules-fabric/v26/folder/variables.tf @@ -0,0 +1,194 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "contacts" { + description = "List of essential contacts for this resource. Must be in the form EMAIL -> [NOTIFICATION_TYPES]. Valid notification types are ALL, SUSPENSION, SECURITY, TECHNICAL, BILLING, LEGAL, PRODUCT_UPDATES." + type = map(list(string)) + default = {} + nullable = false +} + +variable "firewall_policy" { + description = "Hierarchical firewall policy to associate to this folder." + type = object({ + name = string + policy = string + }) + default = null +} + +variable "folder_create" { + description = "Create folder. When set to false, uses id to reference an existing folder." + type = bool + default = true +} + +variable "group_iam" { + description = "Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable." + type = map(list(string)) + default = {} + nullable = false +} + +variable "iam" { + description = "IAM bindings in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} + nullable = false +} + +variable "iam_bindings" { + description = "Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary." + type = map(object({ + members = list(string) + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} +} + +variable "iam_bindings_additive" { + description = "Individual additive IAM bindings. Keys are arbitrary." + type = map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} +} + +variable "id" { + description = "Folder ID in case you use folder_create=false." + type = string + default = null +} + +variable "logging_data_access" { + description = "Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services." + type = map(map(list(string))) + nullable = false + default = {} + validation { + condition = alltrue(flatten([ + for k, v in var.logging_data_access : [ + for kk, vv in v : contains(["DATA_READ", "DATA_WRITE", "ADMIN_READ"], kk) + ] + ])) + error_message = "Log type keys for each service can only be one of 'DATA_READ', 'DATA_WRITE', 'ADMIN_READ'." + } +} + +variable "logging_exclusions" { + description = "Logging exclusions for this folder in the form {NAME -> FILTER}." + type = map(string) + default = {} + nullable = false +} + +variable "logging_sinks" { + description = "Logging sinks to create for the organization." + type = map(object({ + bq_partitioned_table = optional(bool) + description = optional(string) + destination = string + disabled = optional(bool, false) + exclusions = optional(map(string), {}) + filter = string + include_children = optional(bool, true) + type = string + })) + default = {} + nullable = false + validation { + condition = alltrue([ + for k, v in var.logging_sinks : + contains(["bigquery", "logging", "pubsub", "storage"], v.type) + ]) + error_message = "Type must be one of 'bigquery', 'logging', 'pubsub', 'storage'." + } + validation { + condition = alltrue([ + for k, v in var.logging_sinks : + v.bq_partitioned_table != true || v.type == "bigquery" + ]) + error_message = "Can only set bq_partitioned_table when type is `bigquery`." + } +} + +variable "name" { + description = "Folder name." + type = string + default = null +} + +variable "org_policies" { + description = "Organization policies applied to this folder keyed by policy name." + type = map(object({ + inherit_from_parent = optional(bool) # for list policies only. + reset = optional(bool) + rules = optional(list(object({ + allow = optional(object({ + all = optional(bool) + values = optional(list(string)) + })) + deny = optional(object({ + all = optional(bool) + values = optional(list(string)) + })) + enforce = optional(bool) # for boolean policies only. + condition = optional(object({ + description = optional(string) + expression = optional(string) + location = optional(string) + title = optional(string) + }), {}) + })), []) + })) + default = {} + nullable = false +} + +variable "org_policies_data_path" { + description = "Path containing org policies in YAML format." + type = string + default = null +} + +variable "parent" { + description = "Parent in folders/folder_id or organizations/org_id format." + type = string + default = null + validation { + condition = var.parent == null || can(regex("(organizations|folders)/[0-9]+", var.parent)) + error_message = "Parent must be of the form folders/folder_id or organizations/organization_id." + } +} + +variable "tag_bindings" { + description = "Tag bindings for this folder, in key => tag value id format." + type = map(string) + default = null +} diff --git a/assets/modules-fabric/v26/folder/versions.tf b/assets/modules-fabric/v26/folder/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/folder/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/gcs/README.md b/assets/modules-fabric/v26/gcs/README.md new file mode 100644 index 0000000..e79de52 --- /dev/null +++ b/assets/modules-fabric/v26/gcs/README.md @@ -0,0 +1,146 @@ +# Google Cloud Storage Module + +## Example + +```hcl +module "bucket" { + source = "./fabric/modules/gcs" + project_id = "myproject" + prefix = "test" + name = "my-bucket" + versioning = true + iam = { + "roles/storage.admin" = ["group:storage@example.com"] + } + labels = { + cost-center = "devops" + } +} +# tftest modules=1 resources=2 inventory=simple.yaml +``` + +### Example with Cloud KMS + +```hcl +module "bucket" { + source = "./fabric/modules/gcs" + project_id = "myproject" + name = "my-bucket" + encryption_key = "my-encryption-key" +} +# tftest modules=1 resources=1 inventory=cmek.yaml +``` + +### Example with retention policy and logging + +```hcl +module "bucket" { + source = "./fabric/modules/gcs" + project_id = "myproject" + name = "my-bucket" + retention_policy = { + retention_period = 100 + is_locked = true + } + logging_config = { + log_bucket = "log-bucket" + log_object_prefix = null + } +} +# tftest modules=1 resources=1 inventory=retention-logging.yaml +``` + +### Example with lifecycle rule + +```hcl +module "bucket" { + source = "./fabric/modules/gcs" + project_id = "myproject" + name = "my-bucket" + lifecycle_rules = { + lr-0 = { + action = { + type = "SetStorageClass" + storage_class = "STANDARD" + } + condition = { + age = 30 + } + } + } +} +# tftest modules=1 resources=1 inventory=lifecycle.yaml +``` + +### Minimal example with GCS notifications + +```hcl +module "bucket-gcs-notification" { + source = "./fabric/modules/gcs" + project_id = "myproject" + name = "my-bucket" + notification_config = { + enabled = true + payload_format = "JSON_API_V1" + sa_email = "service-@gs-project-accounts.iam.gserviceaccount.com" # GCS SA email must be passed or fetched from projects module. + topic_name = "gcs-notification-topic" + event_types = ["OBJECT_FINALIZE"] + custom_attributes = {} + } +} +# tftest modules=1 resources=4 inventory=notification.yaml +``` + +### Example with object upload + +```hcl +module "bucket" { + source = "./fabric/modules/gcs" + project_id = "myproject" + name = "my-bucket" + objects_to_upload = { + sample-data = { + name = "example-file.csv" + source = "data/example-file.csv" + content_type = "text/csv" + } + } +} +# tftest modules=1 resources=2 inventory=object-upload.yaml +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [name](variables.tf#L116) | Bucket name suffix. | string | ✓ | | +| [project_id](variables.tf#L171) | Bucket project id. | string | ✓ | | +| [cors](variables.tf#L17) | CORS configuration for the bucket. Defaults to null. | object({…}) | | null | +| [encryption_key](variables.tf#L28) | KMS key that will be used for encryption. | string | | null | +| [force_destroy](variables.tf#L34) | Optional map to set force destroy keyed by name, defaults to false. | bool | | false | +| [iam](variables.tf#L40) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [labels](variables.tf#L46) | Labels to be attached to all buckets. | map(string) | | {} | +| [lifecycle_rules](variables.tf#L52) | Bucket lifecycle rule. | map(object({…})) | | {} | +| [location](variables.tf#L101) | Bucket location. | string | | "EU" | +| [logging_config](variables.tf#L107) | Bucket logging configuration. | object({…}) | | null | +| [notification_config](variables.tf#L121) | GCS Notification configuration. | object({…}) | | null | +| [objects_to_upload](variables.tf#L135) | Objects to be uploaded to bucket. | map(object({…})) | | {} | +| [prefix](variables.tf#L161) | Optional prefix used to generate the bucket name. | string | | null | +| [retention_policy](variables.tf#L176) | Bucket retention policy. | object({…}) | | null | +| [storage_class](variables.tf#L185) | Bucket storage class. | string | | "MULTI_REGIONAL" | +| [uniform_bucket_level_access](variables.tf#L195) | Allow using object ACLs (false) or not (true, this is the recommended behavior) , defaults to true (which is the recommended practice, but not the behavior of storage API). | bool | | true | +| [versioning](variables.tf#L201) | Enable versioning, defaults to false. | bool | | false | +| [website](variables.tf#L207) | Bucket website. | object({…}) | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [bucket](outputs.tf#L17) | Bucket resource. | | +| [id](outputs.tf#L28) | Fully qualified bucket id. | | +| [name](outputs.tf#L37) | Bucket name. | | +| [notification](outputs.tf#L46) | GCS Notification self link. | | +| [objects](outputs.tf#L51) | Objects in GCS bucket. | | +| [topic](outputs.tf#L63) | Topic ID used by GCS. | | +| [url](outputs.tf#L68) | Bucket URL. | | + diff --git a/assets/modules-fabric/v26/gcs/main.tf b/assets/modules-fabric/v26/gcs/main.tf new file mode 100644 index 0000000..a0354fe --- /dev/null +++ b/assets/modules-fabric/v26/gcs/main.tf @@ -0,0 +1,158 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + prefix = var.prefix == null ? "" : "${var.prefix}-" + notification = try(var.notification_config.enabled, false) +} + +resource "google_storage_bucket" "bucket" { + name = "${local.prefix}${lower(var.name)}" + project = var.project_id + location = var.location + storage_class = var.storage_class + force_destroy = var.force_destroy + uniform_bucket_level_access = var.uniform_bucket_level_access + labels = var.labels + versioning { + enabled = var.versioning + } + + dynamic "website" { + for_each = var.website == null ? [] : [""] + + content { + main_page_suffix = var.website.main_page_suffix + not_found_page = var.website.not_found_page + } + } + + dynamic "encryption" { + for_each = var.encryption_key == null ? [] : [""] + + content { + default_kms_key_name = var.encryption_key + } + } + + dynamic "retention_policy" { + for_each = var.retention_policy == null ? [] : [""] + content { + retention_period = var.retention_policy.retention_period + is_locked = var.retention_policy.is_locked + } + } + + dynamic "logging" { + for_each = var.logging_config == null ? [] : [""] + content { + log_bucket = var.logging_config.log_bucket + log_object_prefix = var.logging_config.log_object_prefix + } + } + + dynamic "cors" { + for_each = var.cors == null ? [] : [""] + content { + origin = var.cors.origin + method = var.cors.method + response_header = var.cors.response_header + max_age_seconds = max(3600, var.cors.max_age_seconds) + } + } + + dynamic "lifecycle_rule" { + for_each = var.lifecycle_rules + iterator = rule + content { + action { + type = rule.value.action.type + storage_class = rule.value.action.storage_class + } + condition { + age = rule.value.condition.age + created_before = rule.value.condition.created_before + custom_time_before = rule.value.condition.custom_time_before + days_since_custom_time = rule.value.condition.days_since_custom_time + days_since_noncurrent_time = rule.value.condition.days_since_noncurrent_time + matches_prefix = rule.value.condition.matches_prefix + matches_storage_class = rule.value.condition.matches_storage_class + matches_suffix = rule.value.condition.matches_suffix + noncurrent_time_before = rule.value.condition.noncurrent_time_before + num_newer_versions = rule.value.condition.num_newer_versions + with_state = rule.value.condition.with_state + } + } + } +} + +resource "google_storage_bucket_object" "objects" { + for_each = var.objects_to_upload + + bucket = google_storage_bucket.bucket.id + name = each.value.name + metadata = each.value.metadata + content = each.value.content + source = each.value.source + cache_control = each.value.cache_control + content_disposition = each.value.content_disposition + content_encoding = each.value.content_encoding + content_language = each.value.content_language + content_type = each.value.content_type + event_based_hold = each.value.event_based_hold + temporary_hold = each.value.temporary_hold + detect_md5hash = each.value.detect_md5hash + storage_class = each.value.storage_class + kms_key_name = each.value.kms_key_name + + dynamic "customer_encryption" { + for_each = each.value.customer_encryption == null ? [] : [""] + + content { + encryption_algorithm = each.value.customer_encryption.encryption_algorithm + encryption_key = each.value.customer_encryption.encryption_key + } + } +} + +resource "google_storage_bucket_iam_binding" "bindings" { + for_each = var.iam + bucket = google_storage_bucket.bucket.name + role = each.key + members = each.value +} + +resource "google_storage_notification" "notification" { + count = local.notification ? 1 : 0 + bucket = google_storage_bucket.bucket.name + payload_format = var.notification_config.payload_format + topic = google_pubsub_topic.topic[0].id + custom_attributes = var.notification_config.custom_attributes + event_types = var.notification_config.event_types + object_name_prefix = var.notification_config.object_name_prefix + depends_on = [google_pubsub_topic_iam_binding.binding] +} +resource "google_pubsub_topic_iam_binding" "binding" { + count = local.notification ? 1 : 0 + topic = google_pubsub_topic.topic[0].id + role = "roles/pubsub.publisher" + members = ["serviceAccount:${var.notification_config.sa_email}"] +} +resource "google_pubsub_topic" "topic" { + count = local.notification ? 1 : 0 + project = var.project_id + name = var.notification_config.topic_name +} diff --git a/assets/modules-fabric/v26/gcs/outputs.tf b/assets/modules-fabric/v26/gcs/outputs.tf new file mode 100644 index 0000000..1fd0dc6 --- /dev/null +++ b/assets/modules-fabric/v26/gcs/outputs.tf @@ -0,0 +1,71 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "bucket" { + description = "Bucket resource." + value = google_storage_bucket.bucket +} + +# We add `id` as an alias to `name` to simplify log sink handling. +# Since all other log destinations (pubsub, logging-bucket, bigquery) +# have an id output, it is convenient to have in this module too to +# handle all log destination as homogeneous objects (i.e. you can +# assume any valid log destination has an `id` output). + +output "id" { + description = "Fully qualified bucket id." + value = "${local.prefix}${lower(var.name)}" + depends_on = [ + google_storage_bucket.bucket, + google_storage_bucket_iam_binding.bindings + ] +} + +output "name" { + description = "Bucket name." + value = "${local.prefix}${lower(var.name)}" + depends_on = [ + google_storage_bucket.bucket, + google_storage_bucket_iam_binding.bindings + ] +} + +output "notification" { + description = "GCS Notification self link." + value = local.notification ? google_storage_notification.notification[0].self_link : null +} + +output "objects" { + description = "Objects in GCS bucket." + value = { for k, v in google_storage_bucket_object.objects : k => { + crc32c = v.crc32c + md5hash = v.md5hash + self_link = v.self_link + output_name = v.output_name + media_link = v.media_link + } + } +} + +output "topic" { + description = "Topic ID used by GCS." + value = local.notification ? google_pubsub_topic.topic[0].id : null +} + +output "url" { + description = "Bucket URL." + value = google_storage_bucket.bucket.url +} diff --git a/assets/modules-fabric/v26/gcs/variables.tf b/assets/modules-fabric/v26/gcs/variables.tf new file mode 100644 index 0000000..5f6c5e3 --- /dev/null +++ b/assets/modules-fabric/v26/gcs/variables.tf @@ -0,0 +1,214 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "cors" { + description = "CORS configuration for the bucket. Defaults to null." + type = object({ + origin = optional(list(string)) + method = optional(list(string)) + response_header = optional(list(string)) + max_age_seconds = optional(number) + }) + default = null +} + +variable "encryption_key" { + description = "KMS key that will be used for encryption." + type = string + default = null +} + +variable "force_destroy" { + description = "Optional map to set force destroy keyed by name, defaults to false." + type = bool + default = false +} + +variable "iam" { + description = "IAM bindings in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} +} + +variable "labels" { + description = "Labels to be attached to all buckets." + type = map(string) + default = {} +} + +variable "lifecycle_rules" { + description = "Bucket lifecycle rule." + type = map(object({ + action = object({ + type = string + storage_class = optional(string) + }) + condition = object({ + age = optional(number) + created_before = optional(string) + custom_time_before = optional(string) + days_since_custom_time = optional(number) + days_since_noncurrent_time = optional(number) + matches_prefix = optional(list(string)) + matches_storage_class = optional(list(string)) # STANDARD, MULTI_REGIONAL, REGIONAL, NEARLINE, COLDLINE, ARCHIVE, DURABLE_REDUCED_AVAILABILITY + matches_suffix = optional(list(string)) + noncurrent_time_before = optional(string) + num_newer_versions = optional(number) + with_state = optional(string) # "LIVE", "ARCHIVED", "ANY" + }) + })) + default = {} + nullable = false + validation { + condition = alltrue([ + for k, v in var.lifecycle_rules : v.action != null && v.condition != null + ]) + error_message = "Lifecycle rules action and condition cannot be null." + } + validation { + condition = alltrue([ + for k, v in var.lifecycle_rules : contains( + ["Delete", "SetStorageClass", "AbortIncompleteMultipartUpload"], + v.action.type + ) + ]) + error_message = "Lifecycle rules action type has unsupported value." + } + validation { + condition = alltrue([ + for k, v in var.lifecycle_rules : + v.action.type != "SetStorageClass" + || + v.action.storage_class != null + ]) + error_message = "Lifecycle rules with action type SetStorageClass require a storage class." + } +} + +variable "location" { + description = "Bucket location." + type = string + default = "EU" +} + +variable "logging_config" { + description = "Bucket logging configuration." + type = object({ + log_bucket = string + log_object_prefix = optional(string) + }) + default = null +} + +variable "name" { + description = "Bucket name suffix." + type = string +} + +variable "notification_config" { + description = "GCS Notification configuration." + type = object({ + enabled = bool + payload_format = string + topic_name = string + sa_email = string + event_types = optional(list(string)) + custom_attributes = optional(map(string)) + object_name_prefix = optional(string) + }) + default = null +} + +variable "objects_to_upload" { + description = "Objects to be uploaded to bucket." + type = map(object({ + name = string + metadata = optional(map(string)) + content = optional(string) + source = optional(string) + cache_control = optional(string) + content_disposition = optional(string) + content_encoding = optional(string) + content_language = optional(string) + content_type = optional(string) + event_based_hold = optional(bool) + temporary_hold = optional(bool) + detect_md5hash = optional(string) + storage_class = optional(string) + kms_key_name = optional(string) + customer_encryption = optional(object({ + encryption_algorithm = optional(string) + encryption_key = string + })) + })) + default = {} + nullable = false +} + +variable "prefix" { + description = "Optional prefix used to generate the bucket name." + type = string + default = null + validation { + condition = var.prefix != "" + error_message = "Prefix cannot be empty, please use null instead." + } +} + +variable "project_id" { + description = "Bucket project id." + type = string +} + +variable "retention_policy" { + description = "Bucket retention policy." + type = object({ + retention_period = number + is_locked = optional(bool) + }) + default = null +} + +variable "storage_class" { + description = "Bucket storage class." + type = string + default = "MULTI_REGIONAL" + validation { + condition = contains(["STANDARD", "MULTI_REGIONAL", "REGIONAL", "NEARLINE", "COLDLINE", "ARCHIVE"], var.storage_class) + error_message = "Storage class must be one of STANDARD, MULTI_REGIONAL, REGIONAL, NEARLINE, COLDLINE, ARCHIVE." + } +} + +variable "uniform_bucket_level_access" { + description = "Allow using object ACLs (false) or not (true, this is the recommended behavior) , defaults to true (which is the recommended practice, but not the behavior of storage API)." + type = bool + default = true +} + +variable "versioning" { + description = "Enable versioning, defaults to false." + type = bool + default = false +} + +variable "website" { + description = "Bucket website." + type = object({ + main_page_suffix = optional(string) + not_found_page = optional(string) + }) + default = null +} diff --git a/assets/modules-fabric/v26/gcs/versions.tf b/assets/modules-fabric/v26/gcs/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/gcs/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/gcve-private-cloud/README.md b/assets/modules-fabric/v26/gcve-private-cloud/README.md new file mode 100644 index 0000000..901b307 --- /dev/null +++ b/assets/modules-fabric/v26/gcve-private-cloud/README.md @@ -0,0 +1,100 @@ +# Google Cloud VMWare Engine Private Cloud Module + +This module implements the creation and management of a Google Cloud VMWare Engine Private Cloud with its management cluster. If configured, it also creates the VMWare engine network or it can work with an existing one. The creation of the private connection with the user VPC requires the execution of the [Google SDK command](https://cloud.google.com/sdk/gcloud/reference/vmware/private-connections/create#--routing-mode) the module provides as an output. + +To understand the limits and to propertly configure the vSphere/vSAN subnets CIDR range please refer to the [GCVE public documetation](https://cloud.google.com/vmware-engine/docs/quickstart-networking-requirements). + +Be aware that the deployment of this module might require up to 2 hours depending on the selected private cloud target zone. + + +- [Limitations](#limitations) +- [Basic Private Cloud Creation](#basic-private-cloud-creation) +- [Private Cloud Creation with custom nodes and cores count](#private-cloud-creation-with-custom-nodes-and-cores-count) +- [Variables](#variables) +- [Outputs](#outputs) + + +## Limitations +At the moment this module doesn't support the following use cases: +- Single node private cloud +- Stretched private cloud + +## Basic Private Cloud Creation + +```hcl +module "gcve-pc" { + source = "./fabric/modules/gcve-private-cloud" + name = "gcve-pc" + project_id = "gcve-test-project" + zone = "europe-west8-a" + cidr = "192.168.0.0/24" + + private_connections = { + transit-conn1 = { + name = "transit-conn1" + network_self_link = "projects/test-prj-gcve-01/global/networks/default" + tenant_host_project = "g39a814990532d10ap-tp" + type = "PRIVATE_SERVICE_ACCESS" + routing_mode = "REGIONAL" + } + } +} +# tftest modules=1 resources=2 inventory=basic.yaml +``` +## Private Cloud Creation with custom nodes and cores count + +```hcl +module "gcve-pc" { + source = "./fabric/modules/gcve-private-cloud" + name = "gcve-pc" + project_id = "gcve-test-project" + zone = "europe-west8-a" + cidr = "192.168.0.0/24" + + management_cluster_config = { + node_type_id = "standard-72" + node_count = 6 + custom_core_count = 28 + } + + private_connections = { + transit-conn1 = { + name = "transit-conn1" + network_self_link = "projects/test-prj-gcve-01/global/networks/default" + tenant_host_project = "g39a814990532d10ap-tp" + type = "PRIVATE_SERVICE_ACCESS" + routing_mode = "REGIONAL" + } + } +} +# tftest modules=1 resources=2 inventory=custom.yaml +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [cidr](variables.tf#L16) | vSphere/vSAN subnets CIDR range. To undersatnd the limits, please refer to [GCVE network requirements](https://cloud.google.com/vmware-engine/docs/quickstart-networking-requirements). | string | ✓ | | +| [name](variables.tf#L42) | Private cloud name. | string | ✓ | | +| [project_id](variables.tf#L84) | Project id. | string | ✓ | | +| [zone](variables.tf#L101) | Private cloud zone. | string | ✓ | | +| [description](variables.tf#L21) | Private cloud description. | string | | "Terraform-managed." | +| [management_cluster_config](variables.tf#L27) | Management cluster configuration. | object({…}) | | {…} | +| [private_connections](variables.tf#L47) | VMWare private connections configuration. It is used to create the gcloud command printed as output. | map(object({…})) | | {} | +| [vmw_network_create](variables.tf#L89) | Create the VMware Engine network. When set to false, it uses a data source to reference an existing VMware Engine network. | bool | | true | +| [vmw_network_description](variables.tf#L95) | VMware Engine network description. | string | | "Terraform-managed." | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [hcx](outputs.tf#L17) | Details about a HCX Cloud Manager appliance. | | +| [id](outputs.tf#L22) | ID of the private cloud. | | +| [management_cluster](outputs.tf#L27) | Details of the management cluster of the private cloud. | | +| [network_config](outputs.tf#L32) | Details about the network configuration of the private cloud. | | +| [nsx](outputs.tf#L37) | Details about a NSX Manager appliance. | | +| [private-cloud](outputs.tf#L42) | The private cloud resource. | | +| [private_connections_setup](outputs.tf#L47) | Cloud SDK commands for the private connections manual setup. | | +| [state](outputs.tf#L63) | Details about the state of the private cloud. | | +| [vcenter](outputs.tf#L68) | Details about a vCenter Server management appliance. | | + diff --git a/assets/modules-fabric/v26/gcve-private-cloud/main.tf b/assets/modules-fabric/v26/gcve-private-cloud/main.tf new file mode 100644 index 0000000..3235fb2 --- /dev/null +++ b/assets/modules-fabric/v26/gcve-private-cloud/main.tf @@ -0,0 +1,75 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + region = regex("([a-z]*-[a-z]*[0-9]{1,2})-([a-z])", var.zone)[0] + vmw_network = ( + var.vmw_network_create + ? try(google_vmwareengine_network.private-cloud-network.0, null) + : try(data.google_vmwareengine_network.private-cloud-network.0, null) + ) + tenant_host_project = { + for k, v in var.private_connections : k => v.tenant_host_project == null + ? regex("(.*)/projects/([a-z0-9-]*)/(.*)", "${data.google_compute_network_peering.psa_peering[k].peer_network}")[1] + : v.tenant_host_project + } +} + +data "google_vmwareengine_network" "private-cloud-network" { + count = var.vmw_network_create ? 0 : 1 + provider = google-beta + project = var.project_id + name = "${local.region}-default" + location = local.region +} + +data "google_compute_network_peering" "psa_peering" { + for_each = { for k, v in var.private_connections : k => v if v.tenant_host_project == null } + name = each.value.peering_name + network = each.value.network_self_link +} + +resource "google_vmwareengine_private_cloud" "private-cloud" { + provider = google-beta + project = var.project_id + location = var.zone + name = var.name + description = var.description + + network_config { + management_cidr = var.cidr + vmware_engine_network = local.vmw_network.id + } + + management_cluster { + cluster_id = "${var.name}-mgmt-cluster" + node_type_configs { + node_type_id = var.management_cluster_config.node_type_id + node_count = var.management_cluster_config.node_count + custom_core_count = var.management_cluster_config.custom_core_count + } + } +} + +resource "google_vmwareengine_network" "private-cloud-network" { + count = var.vmw_network_create ? 1 : 0 + provider = google-beta + project = var.project_id + name = "${local.region}-default" + location = local.region + type = "LEGACY" + description = var.vmw_network_description +} diff --git a/assets/modules-fabric/v26/gcve-private-cloud/outputs.tf b/assets/modules-fabric/v26/gcve-private-cloud/outputs.tf new file mode 100644 index 0000000..55578f6 --- /dev/null +++ b/assets/modules-fabric/v26/gcve-private-cloud/outputs.tf @@ -0,0 +1,71 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "hcx" { + description = "Details about a HCX Cloud Manager appliance." + value = google_vmwareengine_private_cloud.private-cloud.hcx +} + +output "id" { + description = "ID of the private cloud." + value = google_vmwareengine_private_cloud.private-cloud.id +} + +output "management_cluster" { + description = "Details of the management cluster of the private cloud." + value = google_vmwareengine_private_cloud.private-cloud.management_cluster +} + +output "network_config" { + description = "Details about the network configuration of the private cloud." + value = google_vmwareengine_private_cloud.private-cloud.network_config +} + +output "nsx" { + description = "Details about a NSX Manager appliance." + value = google_vmwareengine_private_cloud.private-cloud.nsx +} + +output "private-cloud" { + description = "The private cloud resource." + value = google_vmwareengine_private_cloud.private-cloud +} + +output "private_connections_setup" { + description = "Cloud SDK commands for the private connections manual setup." + value = { + for k, v in var.private_connections : k => < +- [Examples](#examples) + - [GKE Autopilot cluster](#gke-autopilot-cluster) + - [Cloud DNS](#cloud-dns) + - [Logging configuration](#logging-configuration) + - [Monitoring configuration](#monitoring-configuration) + - [Backup for GKE](#backup-for-gke) +- [Variables](#variables) +- [Outputs](#outputs) + + +## Examples + +### GKE Autopilot cluster + +This example shows how to [create a GKE cluster in Autopilot mode](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-an-autopilot-cluster). + +```hcl +module "cluster-1" { + source = "./fabric/modules/gke-cluster-autopilot" + project_id = "myproject" + name = "cluster-1" + location = "europe-west1" + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + secondary_range_names = { + pods = "pods" + services = "services" + } + master_authorized_ranges = { + internal-vms = "10.0.0.0/8" + } + master_ipv4_cidr_block = "192.168.0.0/28" + } + private_cluster_config = { + enable_private_endpoint = true + master_global_access = false + } + labels = { + environment = "dev" + } +} +# tftest modules=1 resources=1 inventory=basic.yaml +``` + +### Cloud DNS + +> [!WARNING] +> [Cloud DNS is the only DNS provider for Autopilot clusters](https://cloud.google.com/kubernetes-engine/docs/concepts/service-discovery#cloud_dns) running version `1.25.9-gke.400` and later, and version `1.26.4-gke.500` and later. It is [pre-configured](https://cloud.google.com/kubernetes-engine/docs/resources/autopilot-standard-feature-comparison#feature-comparison) for those clusters. The following example *only* applies to Autopilot clusters running *earlier* versions. + +This example shows how to [use Cloud DNS as a Kubernetes DNS provider](https://cloud.google.com/kubernetes-engine/docs/how-to/cloud-dns). + +```hcl +module "cluster-1" { + source = "./fabric/modules/gke-cluster-autopilot" + project_id = var.project_id + name = "cluster-1" + location = "europe-west1" + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + secondary_range_names = {} # use default names "pods" and "services" + } + enable_features = { + dns = { + provider = "CLOUD_DNS" + scope = "CLUSTER_SCOPE" + domain = "gke.local" + } + } +} +# tftest modules=1 resources=1 inventory=dns.yaml +``` + +### Logging configuration + +> [!NOTE] +> System and workload logs collection is pre-configured for Autopilot clusters and cannot be disabled. + +This example shows how to [collect logs for the Kubernetes control plane components](https://cloud.google.com/stackdriver/docs/solutions/gke/installing). The logs for these components are not collected by default. + +```hcl +module "cluster-1" { + source = "./fabric/modules/gke-cluster-autopilot" + project_id = var.project_id + name = "cluster-1" + location = "europe-west1" + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + secondary_range_names = {} # use default names "pods" and "services" + } + logging_config = { + enable_api_server_logs = true + enable_scheduler_logs = true + enable_controller_manager_logs = true + } +} +# tftest modules=1 resources=1 inventory=logging-config.yaml +``` + +### Monitoring configuration + +> [!NOTE] +> [System metrics](https://cloud.google.com/stackdriver/docs/solutions/gke/managing-metrics#enable-system-metrics) collection is pre-configured for Autopilot clusters and cannot be disabled. + +> [!WARNING] +> GKE **workload metrics** is deprecated and removed in GKE 1.24 and later. Workload metrics is replaced by [Google Cloud Managed Service for Prometheus](https://cloud.google.com/stackdriver/docs/managed-prometheus), which is Google's recommended way to monitor Kubernetes applications by using Cloud Monitoring. + +This example shows how to [configure collection of Kubernetes control plane metrics](https://cloud.google.com/stackdriver/docs/solutions/gke/managing-metrics#enable-control-plane-metrics). These metrics are optional and are not collected by default. + +```hcl +module "cluster-1" { + source = "./fabric/modules/gke-cluster-autopilot" + project_id = var.project_id + name = "cluster-1" + location = "europe-west1" + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + secondary_range_names = {} # use default names "pods" and "services" + } + monitoring_config = { + enable_api_server_metrics = true + enable_controller_manager_metrics = true + enable_scheduler_metrics = true + } +} +# tftest modules=1 resources=1 inventory=monitoring-config-control-plane.yaml +``` + +The next example shows how to [configure collection of kube state metrics](https://cloud.google.com/stackdriver/docs/solutions/gke/managing-metrics#enable-ksm). These metrics are optional and are not collected by default. + +```hcl +module "cluster-1" { + source = "./fabric/modules/gke-cluster-autopilot" + project_id = var.project_id + name = "cluster-1" + location = "europe-west1" + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + secondary_range_names = {} # use default names "pods" and "services" + } + monitoring_config = { + enable_daemonset_metrics = true + enable_deployment_metrics = true + enable_hpa_metrics = true + enable_pod_metrics = true + enable_statefulset_metrics = true + enable_storage_metrics = true + # Kube state metrics collection requires Google Cloud Managed Service for Prometheus, + # which is enabled by default. + # enable_managed_prometheus = true + } +} +# tftest modules=1 resources=1 inventory=monitoring-config-kube-state.yaml +``` + +The *control plane metrics* and *kube state metrics* collection can be configured in a single `monitoring_config` block. + +### Backup for GKE + +> [!NOTE] +> Although Backup for GKE can be enabled as an add-on when configuring your GKE clusters, it is a separate service from GKE. + +[Backup for GKE](https://cloud.google.com/kubernetes-engine/docs/add-on/backup-for-gke/concepts/backup-for-gke) is a service for backing up and restoring workloads in GKE clusters. It has two components: + +* A [Google Cloud API](https://cloud.google.com/kubernetes-engine/docs/add-on/backup-for-gke/reference/rest) that serves as the control plane for the service. +* A GKE add-on (the [Backup for GKE agent](https://cloud.google.com/kubernetes-engine/docs/add-on/backup-for-gke/concepts/backup-for-gke#agent_overview)) that must be enabled in each cluster for which you wish to perform backup and restore operations. + +Backup for GKE is supported in GKE Autopilot clusters with [some restrictions](https://cloud.google.com/kubernetes-engine/docs/add-on/backup-for-gke/concepts/about-autopilot). + +This example shows how to [enable Backup for GKE on a new Autopilot cluster](https://cloud.google.com/kubernetes-engine/docs/add-on/backup-for-gke/how-to/install#enable_on_a_new_cluster_optional) and [plan a set of backups](https://cloud.google.com/kubernetes-engine/docs/add-on/backup-for-gke/how-to/backup-plan). + +```hcl +module "cluster-1" { + source = "./fabric/modules/gke-cluster-autopilot" + project_id = var.project_id + name = "cluster-1" + location = "europe-west1" + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + secondary_range_names = {} + } + backup_configs = { + enable_backup_agent = true + backup_plans = { + "backup-1" = { + region = "europe-west-2" + schedule = "0 9 * * 1" + } + } + } +} +# tftest modules=1 resources=2 inventory=backup.yaml +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [location](variables.tf#L110) | Autopilot clusters are always regional. | string | ✓ | | +| [name](variables.tf#L187) | Cluster name. | string | ✓ | | +| [project_id](variables.tf#L213) | Cluster project ID. | string | ✓ | | +| [vpc_config](variables.tf#L242) | VPC-level configuration. | object({…}) | ✓ | | +| [backup_configs](variables.tf#L17) | Configuration for Backup for GKE. | object({…}) | | {} | +| [description](variables.tf#L37) | Cluster description. | string | | null | +| [enable_addons](variables.tf#L43) | Addons enabled in the cluster (true means enabled). | object({…}) | | {…} | +| [enable_features](variables.tf#L64) | Enable cluster-level features. Certain features allow configuration. | object({…}) | | {} | +| [issue_client_certificate](variables.tf#L98) | Enable issuing client certificate. | bool | | false | +| [labels](variables.tf#L104) | Cluster resource labels. | map(string) | | null | +| [logging_config](variables.tf#L115) | Logging configuration. | object({…}) | | {} | +| [maintenance_config](variables.tf#L126) | Maintenance window configuration. | object({…}) | | {…} | +| [min_master_version](variables.tf#L149) | Minimum version of the master, defaults to the version of the most recent official release. | string | | null | +| [monitoring_config](variables.tf#L155) | Monitoring configuration. System metrics collection cannot be disabled. Control plane metrics are optional. Kube state metrics are optional. Google Cloud Managed Service for Prometheus is enabled by default. | object({…}) | | {} | +| [node_locations](variables.tf#L192) | Zones in which the cluster's nodes are located. | list(string) | | [] | +| [private_cluster_config](variables.tf#L199) | Private cluster configuration. | object({…}) | | null | +| [release_channel](variables.tf#L218) | Release channel for GKE upgrades. Clusters created in the Autopilot mode must use a release channel. Choose between \"RAPID\", \"REGULAR\", and \"STABLE\". | string | | "REGULAR" | +| [service_account](variables.tf#L229) | The Google Cloud Platform Service Account to be used by the node VMs created by GKE Autopilot. | string | | null | +| [tags](variables.tf#L235) | Network tags applied to nodes. | list(string) | | [] | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [ca_certificate](outputs.tf#L17) | Public certificate of the cluster (base64-encoded). | ✓ | +| [cluster](outputs.tf#L23) | Cluster resource. | ✓ | +| [endpoint](outputs.tf#L29) | Cluster endpoint. | | +| [id](outputs.tf#L34) | Fully qualified cluster ID. | | +| [location](outputs.tf#L39) | Cluster location. | | +| [master_version](outputs.tf#L44) | Master version. | | +| [name](outputs.tf#L49) | Cluster name. | | +| [notifications](outputs.tf#L54) | GKE Pub/Sub notifications topic. | | +| [self_link](outputs.tf#L59) | Cluster self link. | ✓ | +| [workload_identity_pool](outputs.tf#L65) | Workload identity pool. | | + diff --git a/assets/modules-fabric/v26/gke-cluster-autopilot/main.tf b/assets/modules-fabric/v26/gke-cluster-autopilot/main.tf new file mode 100644 index 0000000..4ca8ee5 --- /dev/null +++ b/assets/modules-fabric/v26/gke-cluster-autopilot/main.tf @@ -0,0 +1,366 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_container_cluster" "cluster" { + provider = google-beta + project = var.project_id + name = var.name + description = var.description + location = var.location + node_locations = ( + length(var.node_locations) == 0 ? null : var.node_locations + ) + min_master_version = var.min_master_version + network = var.vpc_config.network + subnetwork = var.vpc_config.subnetwork + resource_labels = var.labels + enable_l4_ilb_subsetting = var.enable_features.l4_ilb_subsetting + enable_tpu = var.enable_features.tpu + initial_node_count = 1 + + enable_autopilot = true + allow_net_admin = var.enable_features.allow_net_admin + + addons_config { + http_load_balancing { + disabled = !var.enable_addons.http_load_balancing + } + horizontal_pod_autoscaling { + disabled = !var.enable_addons.horizontal_pod_autoscaling + } + cloudrun_config { + disabled = !var.enable_addons.cloudrun + } + + kalm_config { + enabled = var.enable_addons.kalm + } + config_connector_config { + enabled = var.enable_addons.config_connector + } + gke_backup_agent_config { + enabled = var.backup_configs.enable_backup_agent + } + } + + dynamic "authenticator_groups_config" { + for_each = var.enable_features.groups_for_rbac != null ? [""] : [] + content { + security_group = var.enable_features.groups_for_rbac + } + } + + dynamic "binary_authorization" { + for_each = var.enable_features.binary_authorization ? [""] : [] + content { + evaluation_mode = "PROJECT_SINGLETON_POLICY_ENFORCE" + } + } + + dynamic "cost_management_config" { + for_each = var.enable_features.cost_management == true ? [""] : [] + content { + enabled = true + } + } + + cluster_autoscaling { + dynamic "auto_provisioning_defaults" { + for_each = var.service_account != null ? [""] : [] + content { + service_account = var.service_account + } + } + } + + dynamic "database_encryption" { + for_each = var.enable_features.database_encryption != null ? [""] : [] + content { + state = var.enable_features.database_encryption.state + key_name = var.enable_features.database_encryption.key_name + } + } + + dynamic "dns_config" { + for_each = var.enable_features.dns != null ? [""] : [] + content { + cluster_dns = var.enable_features.dns.provider + cluster_dns_scope = var.enable_features.dns.scope + cluster_dns_domain = var.enable_features.dns.domain + } + } + + dynamic "gateway_api_config" { + for_each = var.enable_features.gateway_api ? [""] : [] + content { + channel = "CHANNEL_STANDARD" + } + } + + dynamic "ip_allocation_policy" { + for_each = var.vpc_config.secondary_range_blocks != null ? [""] : [] + content { + cluster_ipv4_cidr_block = var.vpc_config.secondary_range_blocks.pods + services_ipv4_cidr_block = var.vpc_config.secondary_range_blocks.services + stack_type = var.vpc_config.stack_type + } + } + + dynamic "ip_allocation_policy" { + for_each = var.vpc_config.secondary_range_names != null ? [""] : [] + content { + cluster_secondary_range_name = var.vpc_config.secondary_range_names.pods + services_secondary_range_name = var.vpc_config.secondary_range_names.services + stack_type = var.vpc_config.stack_type + } + } + + logging_config { + enable_components = toset(compact([ + var.logging_config.enable_api_server_logs ? "APISERVER" : null, + var.logging_config.enable_controller_manager_logs ? "CONTROLLER_MANAGER" : null, + var.logging_config.enable_scheduler_logs ? "SCHEDULER" : null, + "SYSTEM_COMPONENTS", + "WORKLOADS", + ])) + } + + maintenance_policy { + dynamic "daily_maintenance_window" { + for_each = ( + try(var.maintenance_config.daily_window_start_time, null) != null + ? [""] + : [] + ) + content { + start_time = var.maintenance_config.daily_window_start_time + } + } + dynamic "recurring_window" { + for_each = ( + try(var.maintenance_config.recurring_window, null) != null + ? [""] + : [] + ) + content { + start_time = var.maintenance_config.recurring_window.start_time + end_time = var.maintenance_config.recurring_window.end_time + recurrence = var.maintenance_config.recurring_window.recurrence + } + } + dynamic "maintenance_exclusion" { + for_each = ( + try(var.maintenance_config.maintenance_exclusions, null) == null + ? [] + : var.maintenance_config.maintenance_exclusions + ) + iterator = exclusion + content { + exclusion_name = exclusion.value.name + start_time = exclusion.value.start_time + end_time = exclusion.value.end_time + } + } + } + + master_auth { + client_certificate_config { + issue_client_certificate = var.issue_client_certificate + } + } + + dynamic "master_authorized_networks_config" { + for_each = var.vpc_config.master_authorized_ranges != null ? [""] : [] + content { + dynamic "cidr_blocks" { + for_each = var.vpc_config.master_authorized_ranges + iterator = range + content { + cidr_block = range.value + display_name = range.key + } + } + } + } + + dynamic "mesh_certificates" { + for_each = var.enable_features.mesh_certificates != null ? [""] : [] + content { + enable_certificates = var.enable_features.mesh_certificates + } + } + + monitoring_config { + enable_components = toset(compact([ + # System metrics collection cannot be disabled for Autopilot clusters. + "SYSTEM_COMPONENTS", + # Control plane metrics: + var.monitoring_config.enable_api_server_metrics ? "APISERVER" : null, + var.monitoring_config.enable_controller_manager_metrics ? "CONTROLLER_MANAGER" : null, + var.monitoring_config.enable_scheduler_metrics ? "SCHEDULER" : null, + # Kube state metrics: + var.monitoring_config.enable_daemonset_metrics ? "DAEMONSET" : null, + var.monitoring_config.enable_deployment_metrics ? "DEPLOYMENT" : null, + var.monitoring_config.enable_hpa_metrics ? "HPA" : null, + var.monitoring_config.enable_pod_metrics ? "POD" : null, + var.monitoring_config.enable_statefulset_metrics ? "STATEFULSET" : null, + var.monitoring_config.enable_storage_metrics ? "STORAGE" : null, + ])) + managed_prometheus { + enabled = var.monitoring_config.enable_managed_prometheus + } + } + + dynamic "notification_config" { + for_each = var.enable_features.upgrade_notifications != null ? [""] : [] + content { + pubsub { + enabled = true + topic = ( + try(var.enable_features.upgrade_notifications.topic_id, null) != null + ? var.enable_features.upgrade_notifications.topic_id + : google_pubsub_topic.notifications[0].id + ) + } + } + } + + dynamic "node_pool_auto_config" { + for_each = length(var.tags) > 0 ? [""] : [] + content { + network_tags { + tags = toset(var.tags) + } + } + } + + dynamic "private_cluster_config" { + for_each = ( + var.private_cluster_config != null ? [""] : [] + ) + content { + enable_private_nodes = true + enable_private_endpoint = var.private_cluster_config.enable_private_endpoint + master_ipv4_cidr_block = try(var.vpc_config.master_ipv4_cidr_block, null) + master_global_access_config { + enabled = var.private_cluster_config.master_global_access + } + } + } + + dynamic "pod_security_policy_config" { + for_each = var.enable_features.pod_security_policy ? [""] : [] + content { + enabled = var.enable_features.pod_security_policy + } + } + + release_channel { + channel = var.release_channel + } + + dynamic "resource_usage_export_config" { + for_each = ( + try(var.enable_features.resource_usage_export.dataset, null) != null + ? [""] + : [] + ) + content { + enable_network_egress_metering = ( + var.enable_features.resource_usage_export.enable_network_egress_metering + ) + enable_resource_consumption_metering = ( + var.enable_features.resource_usage_export.enable_resource_consumption_metering + ) + bigquery_destination { + dataset_id = var.enable_features.resource_usage_export.dataset + } + } + } + + dynamic "vertical_pod_autoscaling" { + for_each = var.enable_features.vertical_pod_autoscaling ? [""] : [] + content { + enabled = var.enable_features.vertical_pod_autoscaling + } + } +} + +resource "google_gke_backup_backup_plan" "backup_plan" { + for_each = var.backup_configs.enable_backup_agent ? var.backup_configs.backup_plans : {} + name = each.key + cluster = google_container_cluster.cluster.id + location = each.value.region + project = var.project_id + retention_policy { + backup_delete_lock_days = try(each.value.retention_policy_delete_lock_days) + backup_retain_days = try(each.value.retention_policy_days) + locked = try(each.value.retention_policy_lock) + } + backup_schedule { + cron_schedule = each.value.schedule + } + + backup_config { + include_volume_data = each.value.include_volume_data + include_secrets = each.value.include_secrets + + dynamic "encryption_key" { + for_each = each.value.encryption_key != null ? [""] : [] + content { + gcp_kms_encryption_key = each.value.encryption_key + } + } + + all_namespaces = lookup(each.value, "namespaces", null) != null ? null : true + dynamic "selected_namespaces" { + for_each = each.value.namespaces != null ? [""] : [] + content { + namespaces = each.value.namespaces + } + } + } +} + +resource "google_compute_network_peering_routes_config" "gke_master" { + count = ( + try(var.private_cluster_config.peering_config, null) != null ? 1 : 0 + ) + project = ( + try(var.private_cluster_config.peering_config, null) == null + ? var.project_id + : var.private_cluster_config.peering_config.project_id + ) + peering = try( + google_container_cluster.cluster.private_cluster_config.0.peering_name, + null + ) + network = element(reverse(split("/", var.vpc_config.network)), 0) + import_custom_routes = var.private_cluster_config.peering_config.import_routes + export_custom_routes = var.private_cluster_config.peering_config.export_routes +} + +resource "google_pubsub_topic" "notifications" { + count = ( + try(var.enable_features.upgrade_notifications, null) != null && + try(var.enable_features.upgrade_notifications.topic_id, null) == null ? 1 : 0 + ) + project = var.project_id + name = "gke-pubsub-notifications" + labels = { + content = "gke-notifications" + } +} diff --git a/assets/modules-fabric/v26/gke-cluster-autopilot/outputs.tf b/assets/modules-fabric/v26/gke-cluster-autopilot/outputs.tf new file mode 100644 index 0000000..7978e55 --- /dev/null +++ b/assets/modules-fabric/v26/gke-cluster-autopilot/outputs.tf @@ -0,0 +1,71 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "ca_certificate" { + description = "Public certificate of the cluster (base64-encoded)." + value = google_container_cluster.cluster.master_auth.0.cluster_ca_certificate + sensitive = true +} + +output "cluster" { + description = "Cluster resource." + sensitive = true + value = google_container_cluster.cluster +} + +output "endpoint" { + description = "Cluster endpoint." + value = google_container_cluster.cluster.endpoint +} + +output "id" { + description = "Fully qualified cluster ID." + value = google_container_cluster.cluster.id +} + +output "location" { + description = "Cluster location." + value = google_container_cluster.cluster.location +} + +output "master_version" { + description = "Master version." + value = google_container_cluster.cluster.master_version +} + +output "name" { + description = "Cluster name." + value = google_container_cluster.cluster.name +} + +output "notifications" { + description = "GKE Pub/Sub notifications topic." + value = try(google_pubsub_topic.notifications[0].id, null) +} + +output "self_link" { + description = "Cluster self link." + sensitive = true + value = google_container_cluster.cluster.self_link +} + +output "workload_identity_pool" { + description = "Workload identity pool." + value = "${var.project_id}.svc.id.goog" + depends_on = [ + google_container_cluster.cluster + ] +} diff --git a/assets/modules-fabric/v26/gke-cluster-autopilot/variables.tf b/assets/modules-fabric/v26/gke-cluster-autopilot/variables.tf new file mode 100644 index 0000000..24f8cd2 --- /dev/null +++ b/assets/modules-fabric/v26/gke-cluster-autopilot/variables.tf @@ -0,0 +1,260 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "backup_configs" { + description = "Configuration for Backup for GKE." + type = object({ + enable_backup_agent = optional(bool, false) + backup_plans = optional(map(object({ + encryption_key = optional(string) + include_secrets = optional(bool, true) + include_volume_data = optional(bool, true) + namespaces = optional(list(string)) + region = string + schedule = string + retention_policy_days = optional(string) + retention_policy_lock = optional(bool, false) + retention_policy_delete_lock_days = optional(string) + })), {}) + }) + default = {} + nullable = false +} + +variable "description" { + description = "Cluster description." + type = string + default = null +} + +variable "enable_addons" { + description = "Addons enabled in the cluster (true means enabled)." + type = object({ + cloudrun = optional(bool, false) + config_connector = optional(bool, false) + dns_cache = optional(bool, false) + horizontal_pod_autoscaling = optional(bool, false) + http_load_balancing = optional(bool, false) + istio = optional(object({ + enable_tls = bool + })) + kalm = optional(bool, false) + network_policy = optional(bool, false) + }) + default = { + horizontal_pod_autoscaling = true + http_load_balancing = true + } + nullable = false +} + +variable "enable_features" { + description = "Enable cluster-level features. Certain features allow configuration." + type = object({ + binary_authorization = optional(bool, false) + cost_management = optional(bool, false) + dns = optional(object({ + provider = optional(string) + scope = optional(string) + domain = optional(string) + })) + database_encryption = optional(object({ + state = string + key_name = string + })) + gateway_api = optional(bool, false) + groups_for_rbac = optional(string) + l4_ilb_subsetting = optional(bool, false) + mesh_certificates = optional(bool) + pod_security_policy = optional(bool, false) + allow_net_admin = optional(bool, false) + resource_usage_export = optional(object({ + dataset = string + enable_network_egress_metering = optional(bool) + enable_resource_consumption_metering = optional(bool) + })) + tpu = optional(bool, false) + upgrade_notifications = optional(object({ + topic_id = optional(string) + })) + vertical_pod_autoscaling = optional(bool, false) + }) + default = {} +} + +variable "issue_client_certificate" { + description = "Enable issuing client certificate." + type = bool + default = false +} + +variable "labels" { + description = "Cluster resource labels." + type = map(string) + default = null +} + +variable "location" { + description = "Autopilot clusters are always regional." + type = string +} + +variable "logging_config" { + description = "Logging configuration." + type = object({ + enable_api_server_logs = optional(bool, false) + enable_scheduler_logs = optional(bool, false) + enable_controller_manager_logs = optional(bool, false) + }) + default = {} + nullable = false +} + +variable "maintenance_config" { + description = "Maintenance window configuration." + type = object({ + daily_window_start_time = optional(string) + recurring_window = optional(object({ + start_time = string + end_time = string + recurrence = string + })) + maintenance_exclusions = optional(list(object({ + name = string + start_time = string + end_time = string + scope = optional(string) + }))) + }) + default = { + daily_window_start_time = "03:00" + recurring_window = null + maintenance_exclusion = [] + } +} + +variable "min_master_version" { + description = "Minimum version of the master, defaults to the version of the most recent official release." + type = string + default = null +} + +variable "monitoring_config" { + description = "Monitoring configuration. System metrics collection cannot be disabled. Control plane metrics are optional. Kube state metrics are optional. Google Cloud Managed Service for Prometheus is enabled by default." + type = object({ + # Control plane metrics + enable_api_server_metrics = optional(bool, false) + enable_controller_manager_metrics = optional(bool, false) + enable_scheduler_metrics = optional(bool, false) + # Kube state metrics. Requires managed Prometheus. Requires provider version >= v4.82.0 + enable_daemonset_metrics = optional(bool, false) + enable_deployment_metrics = optional(bool, false) + enable_hpa_metrics = optional(bool, false) + enable_pod_metrics = optional(bool, false) + enable_statefulset_metrics = optional(bool, false) + enable_storage_metrics = optional(bool, false) + # Google Cloud Managed Service for Prometheus. Autopilot clusters version >= 1.25 must have this on. + enable_managed_prometheus = optional(bool, true) + }) + default = {} + nullable = false + validation { + condition = anytrue([ + var.monitoring_config.enable_daemonset_metrics, + var.monitoring_config.enable_deployment_metrics, + var.monitoring_config.enable_hpa_metrics, + var.monitoring_config.enable_pod_metrics, + var.monitoring_config.enable_statefulset_metrics, + var.monitoring_config.enable_storage_metrics, + ]) ? var.monitoring_config.enable_managed_prometheus : true + error_message = "Kube state metrics collection requires Google Cloud Managed Service for Prometheus to be enabled." + } +} + +variable "name" { + description = "Cluster name." + type = string +} + +variable "node_locations" { + description = "Zones in which the cluster's nodes are located." + type = list(string) + default = [] + nullable = false +} + +variable "private_cluster_config" { + description = "Private cluster configuration." + type = object({ + enable_private_endpoint = optional(bool) + master_global_access = optional(bool) + peering_config = optional(object({ + export_routes = optional(bool) + import_routes = optional(bool) + project_id = optional(string) + })) + }) + default = null +} + +variable "project_id" { + description = "Cluster project ID." + type = string +} + +variable "release_channel" { + description = "Release channel for GKE upgrades. Clusters created in the Autopilot mode must use a release channel. Choose between \"RAPID\", \"REGULAR\", and \"STABLE\"." + type = string + default = "REGULAR" + nullable = false + validation { + condition = contains(["RAPID", "REGULAR", "STABLE"], var.release_channel) + error_message = "Must be one of: RAPID, REGULAR, STABLE." + } +} + +variable "service_account" { + description = "The Google Cloud Platform Service Account to be used by the node VMs created by GKE Autopilot." + type = string + default = null +} + +variable "tags" { + description = "Network tags applied to nodes." + type = list(string) + default = [] + nullable = false +} + +variable "vpc_config" { + description = "VPC-level configuration." + type = object({ + network = string + subnetwork = string + master_ipv4_cidr_block = optional(string) + secondary_range_blocks = optional(object({ + pods = string + services = string + })) + secondary_range_names = optional(object({ + pods = optional(string, "pods") + services = optional(string, "services") + })) + master_authorized_ranges = optional(map(string)) + stack_type = optional(string) + }) + nullable = false +} diff --git a/assets/modules-fabric/v26/gke-cluster-autopilot/versions.tf b/assets/modules-fabric/v26/gke-cluster-autopilot/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/gke-cluster-autopilot/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/gke-cluster-standard/README.md b/assets/modules-fabric/v26/gke-cluster-standard/README.md new file mode 100644 index 0000000..dc2b413 --- /dev/null +++ b/assets/modules-fabric/v26/gke-cluster-standard/README.md @@ -0,0 +1,345 @@ +# GKE Standard cluster module + +This module offers a way to create and manage Google Kubernetes Engine (GKE) [Standard clusters](https://cloud.google.com/kubernetes-engine/docs/concepts/choose-cluster-mode#why-standard). With its sensible default settings based on best practices and authors' experience as Google Cloud practitioners, the module accommodates for many common use cases out-of-the-box, without having to rely on verbose configuration. + +> [!IMPORTANT] +> This module should be used together with the [`gke-nodepool`](../gke-nodepool/) module because the default node pool is deleted upon cluster creation and cannot be re-created. + + +- [Example](#example) + - [GKE Standard cluster](#gke-standard-cluster) + - [Enable Dataplane V2](#enable-dataplane-v2) + - [Managing GKE logs](#managing-gke-logs) + - [Monitoring configuration](#monitoring-configuration) + - [Disable GKE logs or metrics collection](#disable-gke-logs-or-metrics-collection) + - [Cloud DNS](#cloud-dns) + - [Backup for GKE](#backup-for-gke) + - [Automatic creation of new secondary ranges](#automatic-creation-of-new-secondary-ranges) +- [Variables](#variables) +- [Outputs](#outputs) + + +## Example + +### GKE Standard cluster + +This example shows how to [create a zonal GKE cluster in Standard mode](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-zonal-cluster). + +```hcl +module "cluster-1" { + source = "./fabric/modules/gke-cluster-standard" + project_id = "myproject" + name = "cluster-1" + location = "europe-west1-b" + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + secondary_range_names = { + pods = "pods" + services = "services" + } + master_authorized_ranges = { + internal-vms = "10.0.0.0/8" + } + master_ipv4_cidr_block = "192.168.0.0/28" + } + max_pods_per_node = 32 + private_cluster_config = { + enable_private_endpoint = true + master_global_access = false + } + labels = { + environment = "dev" + } +} +# tftest modules=1 resources=1 inventory=basic.yaml +``` + +### Enable Dataplane V2 + +This example shows how to [create a zonal GKE Cluster with Dataplane V2 enabled](https://cloud.google.com/kubernetes-engine/docs/how-to/dataplane-v2). + +```hcl +module "cluster-1" { + source = "./fabric/modules/gke-cluster-standard" + project_id = "myproject" + name = "cluster-dataplane-v2" + location = "europe-west1-b" + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + secondary_range_names = {} # use default names "pods" and "services" + master_authorized_ranges = { + internal-vms = "10.0.0.0/8" + } + master_ipv4_cidr_block = "192.168.0.0/28" + } + private_cluster_config = { + enable_private_endpoint = true + master_global_access = false + } + enable_features = { + dataplane_v2 = true + workload_identity = true + } + labels = { + environment = "dev" + } +} +# tftest modules=1 resources=1 inventory=dataplane-v2.yaml +``` + +### Managing GKE logs + +This example shows you how to [control which logs are sent from your GKE cluster to Cloud Logging](https://cloud.google.com/stackdriver/docs/solutions/gke/installing). + +When you create a new GKE cluster, [Cloud Operations for GKE](https://cloud.google.com/stackdriver/docs/solutions/gke) integration with Cloud Logging is enabled by default and [System logs](https://cloud.google.com/stackdriver/docs/solutions/gke/managing-logs#what_logs) are collected. You can enable collection of several other [types of logs](https://cloud.google.com/stackdriver/docs/solutions/gke/managing-logs#what_logs). The following example enables collection of *all* optional logs. + +```hcl +module "cluster-1" { + source = "./fabric/modules/gke-cluster-standard" + project_id = "myproject" + name = "cluster-1" + location = "europe-west1-b" + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + secondary_range_names = {} + } + logging_config = { + enable_workloads_logs = true + enable_api_server_logs = true + enable_scheduler_logs = true + enable_controller_manager_logs = true + } +} +# tftest modules=1 resources=1 inventory=logging-config-enable-all.yaml +``` + +### Monitoring configuration + +This example shows how to [configure collection of Kubernetes control plane metrics](https://cloud.google.com/stackdriver/docs/solutions/gke/managing-metrics#enable-control-plane-metrics). These metrics are optional and are not collected by default. + +```hcl +module "cluster-1" { + source = "./fabric/modules/gke-cluster-standard" + project_id = "myproject" + name = "cluster-1" + location = "europe-west1-b" + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + secondary_range_names = {} # use default names "pods" and "services" + } + monitoring_config = { + enable_api_server_metrics = true + enable_controller_manager_metrics = true + enable_scheduler_metrics = true + } +} +# tftest modules=1 resources=1 inventory=monitoring-config-control-plane.yaml +``` + +The next example shows how to [configure collection of kube state metrics](https://cloud.google.com/stackdriver/docs/solutions/gke/managing-metrics#enable-ksm). These metrics are optional and are not collected by default. + +```hcl +module "cluster-1" { + source = "./fabric/modules/gke-cluster-standard" + project_id = "myproject" + name = "cluster-1" + location = "europe-west1-b" + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + secondary_range_names = {} # use default names "pods" and "services" + } + monitoring_config = { + enable_daemonset_metrics = true + enable_deployment_metrics = true + enable_hpa_metrics = true + enable_pod_metrics = true + enable_statefulset_metrics = true + enable_storage_metrics = true + # Kube state metrics collection requires Google Cloud Managed Service for Prometheus, + # which is enabled by default. + # enable_managed_prometheus = true + } +} +# tftest modules=1 resources=1 inventory=monitoring-config-kube-state.yaml +``` + +The *control plane metrics* and *kube state metrics* collection can be configured in a single `monitoring_config` block. + +### Disable GKE logs or metrics collection + +> [!WARNING] +> If you've disabled Cloud Logging or Cloud Monitoring, GKE customer support +> is offered on a best-effort basis and might require additional effort +> from your engineering team. + +This example shows how to fully disable logs collection on a zonal GKE Standard cluster. This is not recommended. + +```hcl +module "cluster-1" { + source = "./fabric/modules/gke-cluster-standard" + project_id = "myproject" + name = "cluster-1" + location = "europe-west1-b" + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + secondary_range_names = {} + } + logging_config = { + enable_system_logs = false + } +} +# tftest modules=1 resources=1 inventory=logging-config-disable-all.yaml +``` + +The next example shows how to fully disable metrics collection on a zonal GKE Standard cluster. This is not recommended. + +```hcl +module "cluster-1" { + source = "./fabric/modules/gke-cluster-standard" + project_id = "myproject" + name = "cluster-1" + location = "europe-west1-b" + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + secondary_range_names = {} + } + monitoring_config = { + enable_system_metrics = false + enable_managed_prometheus = false + } +} +# tftest modules=1 resources=1 inventory=monitoring-config-disable-all.yaml +``` + +### Cloud DNS + +This example shows how to [use Cloud DNS as a Kubernetes DNS provider](https://cloud.google.com/kubernetes-engine/docs/how-to/cloud-dns) for GKE Standard clusters. + +```hcl +module "cluster-1" { + source = "./fabric/modules/gke-cluster-standard" + project_id = var.project_id + name = "cluster-1" + location = "europe-west1-b" + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + secondary_range_names = {} + } + enable_features = { + dns = { + provider = "CLOUD_DNS" + scope = "CLUSTER_SCOPE" + domain = "gke.local" + } + } +} +# tftest modules=1 resources=1 inventory=dns.yaml +``` + +### Backup for GKE + +> [!NOTE] +> Although Backup for GKE can be enabled as an add-on when configuring your GKE clusters, it is a separate service from GKE. + +[Backup for GKE](https://cloud.google.com/kubernetes-engine/docs/add-on/backup-for-gke/concepts/backup-for-gke) is a service for backing up and restoring workloads in GKE clusters. It has two components: + +* A [Google Cloud API](https://cloud.google.com/kubernetes-engine/docs/add-on/backup-for-gke/reference/rest) that serves as the control plane for the service. +* A GKE add-on (the [Backup for GKE agent](https://cloud.google.com/kubernetes-engine/docs/add-on/backup-for-gke/concepts/backup-for-gke#agent_overview)) that must be enabled in each cluster for which you wish to perform backup and restore operations. + +This example shows how to [enable Backup for GKE on a new zonal GKE Standard cluster](https://cloud.google.com/kubernetes-engine/docs/add-on/backup-for-gke/how-to/install#enable_on_a_new_cluster_optional) and [plan a set of backups](https://cloud.google.com/kubernetes-engine/docs/add-on/backup-for-gke/how-to/backup-plan). + +```hcl +module "cluster-1" { + source = "./fabric/modules/gke-cluster-standard" + project_id = var.project_id + name = "cluster-1" + location = "europe-west1-b" + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + secondary_range_names = {} + } + backup_configs = { + enable_backup_agent = true + backup_plans = { + "backup-1" = { + region = "europe-west-2" + schedule = "0 9 * * 1" + } + } + } +} +# tftest modules=1 resources=2 inventory=backup.yaml +``` + +### Automatic creation of new secondary ranges + +You can use `var.vpc_config.secondary_range_blocks` to let GKE create new secondary ranges for the cluster. The example below reserves an available /14 block for pods and a /20 for services. + +```hcl +module "cluster-1" { + source = "./fabric/modules/gke-cluster-standard" + project_id = var.project_id + name = "cluster-1" + location = "europe-west1-b" + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + secondary_range_blocks = { + pods = "" + services = "/20" # can be an empty string as well + } + } +} +# tftest modules=1 resources=1 +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [location](variables.tf#L138) | Cluster zone or region. | string | ✓ | | +| [name](variables.tf#L249) | Cluster name. | string | ✓ | | +| [project_id](variables.tf#L275) | Cluster project id. | string | ✓ | | +| [vpc_config](variables.tf#L298) | VPC-level configuration. | object({…}) | ✓ | | +| [backup_configs](variables.tf#L17) | Configuration for Backup for GKE. | object({…}) | | {} | +| [cluster_autoscaling](variables.tf#L37) | Enable and configure limits for Node Auto-Provisioning with Cluster Autoscaler. | object({…}) | | null | +| [description](variables.tf#L58) | Cluster description. | string | | null | +| [enable_addons](variables.tf#L64) | Addons enabled in the cluster (true means enabled). | object({…}) | | {…} | +| [enable_features](variables.tf#L87) | Enable cluster-level features. Certain features allow configuration. | object({…}) | | {…} | +| [issue_client_certificate](variables.tf#L126) | Enable issuing client certificate. | bool | | false | +| [labels](variables.tf#L132) | Cluster resource labels. | map(string) | | null | +| [logging_config](variables.tf#L143) | Logging configuration. | object({…}) | | {} | +| [maintenance_config](variables.tf#L164) | Maintenance window configuration. | object({…}) | | {…} | +| [max_pods_per_node](variables.tf#L187) | Maximum number of pods per node in this cluster. | number | | 110 | +| [min_master_version](variables.tf#L193) | Minimum version of the master, defaults to the version of the most recent official release. | string | | null | +| [monitoring_config](variables.tf#L199) | Monitoring configuration. Google Cloud Managed Service for Prometheus is enabled by default. | object({…}) | | {} | +| [node_locations](variables.tf#L254) | Zones in which the cluster's nodes are located. | list(string) | | [] | +| [private_cluster_config](variables.tf#L261) | Private cluster configuration. | object({…}) | | null | +| [release_channel](variables.tf#L280) | Release channel for GKE upgrades. | string | | null | +| [service_account](variables.tf#L286) | Service account used for the default node pool, only useful if the default GCE service account has been disabled. | string | | null | +| [tags](variables.tf#L292) | Network tags applied to nodes. | list(string) | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [ca_certificate](outputs.tf#L17) | Public certificate of the cluster (base64-encoded). | ✓ | +| [cluster](outputs.tf#L23) | Cluster resource. | ✓ | +| [endpoint](outputs.tf#L29) | Cluster endpoint. | | +| [id](outputs.tf#L34) | FUlly qualified cluster id. | | +| [location](outputs.tf#L39) | Cluster location. | | +| [master_version](outputs.tf#L44) | Master version. | | +| [name](outputs.tf#L49) | Cluster name. | | +| [notifications](outputs.tf#L54) | GKE PubSub notifications topic. | | +| [self_link](outputs.tf#L59) | Cluster self link. | ✓ | +| [workload_identity_pool](outputs.tf#L65) | Workload identity pool. | | + diff --git a/assets/modules-fabric/v26/gke-cluster-standard/main.tf b/assets/modules-fabric/v26/gke-cluster-standard/main.tf new file mode 100644 index 0000000..622c2e4 --- /dev/null +++ b/assets/modules-fabric/v26/gke-cluster-standard/main.tf @@ -0,0 +1,459 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_container_cluster" "cluster" { + provider = google-beta + project = var.project_id + name = var.name + description = var.description + location = var.location + node_locations = ( + length(var.node_locations) == 0 ? null : var.node_locations + ) + min_master_version = var.min_master_version + network = var.vpc_config.network + subnetwork = var.vpc_config.subnetwork + resource_labels = var.labels + default_max_pods_per_node = var.max_pods_per_node + enable_intranode_visibility = var.enable_features.intranode_visibility + enable_l4_ilb_subsetting = var.enable_features.l4_ilb_subsetting + enable_shielded_nodes = var.enable_features.shielded_nodes + enable_tpu = var.enable_features.tpu + initial_node_count = 1 + remove_default_node_pool = true + datapath_provider = ( + var.enable_features.dataplane_v2 + ? "ADVANCED_DATAPATH" + : "DATAPATH_PROVIDER_UNSPECIFIED" + ) + + # the default node pool is deleted here, use the gke-nodepool module instead. + # the default node pool configuration is based on a shielded_nodes variable. + node_config { + service_account = var.service_account + dynamic "shielded_instance_config" { + for_each = var.enable_features.shielded_nodes ? [""] : [] + content { + enable_secure_boot = true + enable_integrity_monitoring = true + } + } + tags = var.tags + } + + addons_config { + dns_cache_config { + enabled = var.enable_addons.dns_cache + } + http_load_balancing { + disabled = !var.enable_addons.http_load_balancing + } + horizontal_pod_autoscaling { + disabled = !var.enable_addons.horizontal_pod_autoscaling + } + network_policy_config { + disabled = !var.enable_addons.network_policy + } + cloudrun_config { + disabled = !var.enable_addons.cloudrun + } + istio_config { + disabled = var.enable_addons.istio == null + auth = ( + try(var.enable_addons.istio.enable_tls, false) ? "AUTH_MUTUAL_TLS" : "AUTH_NONE" + ) + } + gce_persistent_disk_csi_driver_config { + enabled = var.enable_addons.gce_persistent_disk_csi_driver + } + gcp_filestore_csi_driver_config { + enabled = var.enable_addons.gcp_filestore_csi_driver + } + kalm_config { + enabled = var.enable_addons.kalm + } + config_connector_config { + enabled = var.enable_addons.config_connector + } + gke_backup_agent_config { + enabled = var.backup_configs.enable_backup_agent + } + } + + dynamic "authenticator_groups_config" { + for_each = var.enable_features.groups_for_rbac != null ? [""] : [] + content { + security_group = var.enable_features.groups_for_rbac + } + } + + dynamic "binary_authorization" { + for_each = var.enable_features.binary_authorization ? [""] : [] + content { + evaluation_mode = "PROJECT_SINGLETON_POLICY_ENFORCE" + } + } + + dynamic "cost_management_config" { + for_each = var.enable_features.cost_management == true ? [""] : [] + content { + enabled = true + } + } + + dynamic "cluster_autoscaling" { + for_each = var.cluster_autoscaling == null ? [] : [""] + content { + enabled = true + + dynamic "auto_provisioning_defaults" { + for_each = var.cluster_autoscaling.auto_provisioning_defaults != null ? [""] : [] + content { + boot_disk_kms_key = var.cluster_autoscaling.auto_provisioning_defaults.boot_disk_kms_key + image_type = var.cluster_autoscaling.auto_provisioning_defaults.image_type + oauth_scopes = var.cluster_autoscaling.auto_provisioning_defaults.oauth_scopes + service_account = var.cluster_autoscaling.auto_provisioning_defaults.service_account + } + } + dynamic "resource_limits" { + for_each = var.cluster_autoscaling.cpu_limits != null ? [""] : [] + content { + resource_type = "cpu" + minimum = var.cluster_autoscaling.cpu_limits.min + maximum = var.cluster_autoscaling.cpu_limits.max + } + } + dynamic "resource_limits" { + for_each = var.cluster_autoscaling.mem_limits != null ? [""] : [] + content { + resource_type = "memory" + minimum = var.cluster_autoscaling.mem_limits.min + maximum = var.cluster_autoscaling.mem_limits.max + } + } + // TODO: support GPUs too + } + } + + dynamic "database_encryption" { + for_each = var.enable_features.database_encryption != null ? [""] : [] + content { + state = var.enable_features.database_encryption.state + key_name = var.enable_features.database_encryption.key_name + } + } + + dynamic "dns_config" { + for_each = var.enable_features.dns != null ? [""] : [] + content { + cluster_dns = var.enable_features.dns.provider + cluster_dns_scope = var.enable_features.dns.scope + cluster_dns_domain = var.enable_features.dns.domain + } + } + + dynamic "gateway_api_config" { + for_each = var.enable_features.gateway_api ? [""] : [] + content { + channel = "CHANNEL_STANDARD" + } + } + + dynamic "ip_allocation_policy" { + for_each = var.vpc_config.secondary_range_blocks != null ? [""] : [] + content { + cluster_ipv4_cidr_block = var.vpc_config.secondary_range_blocks.pods + services_ipv4_cidr_block = var.vpc_config.secondary_range_blocks.services + stack_type = var.vpc_config.stack_type + } + } + dynamic "ip_allocation_policy" { + for_each = var.vpc_config.secondary_range_names != null ? [""] : [] + content { + cluster_secondary_range_name = var.vpc_config.secondary_range_names.pods + services_secondary_range_name = var.vpc_config.secondary_range_names.services + stack_type = var.vpc_config.stack_type + } + } + + # Send GKE cluster logs from chosen sources to Cloud Logging. + # System logs must be enabled if any other source is enabled. + # This is validated by input variable validation rules. + dynamic "logging_config" { + for_each = var.logging_config.enable_system_logs ? [""] : [] + content { + enable_components = toset(compact([ + var.logging_config.enable_api_server_logs ? "APISERVER" : null, + var.logging_config.enable_controller_manager_logs ? "CONTROLLER_MANAGER" : null, + var.logging_config.enable_scheduler_logs ? "SCHEDULER" : null, + "SYSTEM_COMPONENTS", + var.logging_config.enable_workloads_logs ? "WORKLOADS" : null, + ])) + } + } + # Don't send any GKE cluster logs to Cloud Logging. Input variable validation + # makes sure every other log source is false when enable_system_logs is false. + dynamic "logging_config" { + for_each = var.logging_config.enable_system_logs == false ? [""] : [] + content { + enable_components = [] + } + } + + maintenance_policy { + dynamic "daily_maintenance_window" { + for_each = ( + try(var.maintenance_config.daily_window_start_time, null) != null + ? [""] + : [] + ) + content { + start_time = var.maintenance_config.daily_window_start_time + } + } + dynamic "recurring_window" { + for_each = ( + try(var.maintenance_config.recurring_window, null) != null + ? [""] + : [] + ) + content { + start_time = var.maintenance_config.recurring_window.start_time + end_time = var.maintenance_config.recurring_window.end_time + recurrence = var.maintenance_config.recurring_window.recurrence + } + } + dynamic "maintenance_exclusion" { + for_each = ( + try(var.maintenance_config.maintenance_exclusions, null) == null + ? [] + : var.maintenance_config.maintenance_exclusions + ) + iterator = exclusion + content { + exclusion_name = exclusion.value.name + start_time = exclusion.value.start_time + end_time = exclusion.value.end_time + } + } + } + + master_auth { + client_certificate_config { + issue_client_certificate = var.issue_client_certificate + } + } + + dynamic "master_authorized_networks_config" { + for_each = var.vpc_config.master_authorized_ranges != null ? [""] : [] + content { + dynamic "cidr_blocks" { + for_each = var.vpc_config.master_authorized_ranges + iterator = range + content { + cidr_block = range.value + display_name = range.key + } + } + } + } + + dynamic "mesh_certificates" { + for_each = var.enable_features.mesh_certificates != null ? [""] : [] + content { + enable_certificates = var.enable_features.mesh_certificates + } + } + + monitoring_config { + enable_components = toset(compact([ + # System metrics is the minimum requirement if any other metrics are enabled. This is checked by input var validation. + var.monitoring_config.enable_system_metrics ? "SYSTEM_COMPONENTS" : null, + # Control plane metrics + var.monitoring_config.enable_api_server_metrics ? "APISERVER" : null, + var.monitoring_config.enable_controller_manager_metrics ? "CONTROLLER_MANAGER" : null, + var.monitoring_config.enable_scheduler_metrics ? "SCHEDULER" : null, + # Kube state metrics + var.monitoring_config.enable_daemonset_metrics ? "DAEMONSET" : null, + var.monitoring_config.enable_deployment_metrics ? "DEPLOYMENT" : null, + var.monitoring_config.enable_hpa_metrics ? "HPA" : null, + var.monitoring_config.enable_pod_metrics ? "POD" : null, + var.monitoring_config.enable_statefulset_metrics ? "STATEFULSET" : null, + var.monitoring_config.enable_storage_metrics ? "STORAGE" : null, + ])) + managed_prometheus { + enabled = var.monitoring_config.enable_managed_prometheus + } + } + + # Dataplane V2 has built-in network policies + dynamic "network_policy" { + for_each = ( + var.enable_addons.network_policy && !var.enable_features.dataplane_v2 + ? [""] + : [] + ) + content { + enabled = true + provider = "CALICO" + } + } + + dynamic "notification_config" { + for_each = var.enable_features.upgrade_notifications != null ? [""] : [] + content { + pubsub { + enabled = true + topic = ( + try(var.enable_features.upgrade_notifications.topic_id, null) != null + ? var.enable_features.upgrade_notifications.topic_id + : google_pubsub_topic.notifications[0].id + ) + } + } + } + + dynamic "private_cluster_config" { + for_each = ( + var.private_cluster_config != null ? [""] : [] + ) + content { + enable_private_nodes = true + enable_private_endpoint = var.private_cluster_config.enable_private_endpoint + master_ipv4_cidr_block = try(var.vpc_config.master_ipv4_cidr_block, null) + master_global_access_config { + enabled = var.private_cluster_config.master_global_access + } + } + } + + dynamic "pod_security_policy_config" { + for_each = var.enable_features.pod_security_policy ? [""] : [] + content { + enabled = var.enable_features.pod_security_policy + } + } + + dynamic "release_channel" { + for_each = var.release_channel != null ? [""] : [] + content { + channel = var.release_channel + } + } + + dynamic "resource_usage_export_config" { + for_each = ( + try(var.enable_features.resource_usage_export.dataset, null) != null + ? [""] + : [] + ) + content { + enable_network_egress_metering = ( + var.enable_features.resource_usage_export.enable_network_egress_metering + ) + enable_resource_consumption_metering = ( + var.enable_features.resource_usage_export.enable_resource_consumption_metering + ) + bigquery_destination { + dataset_id = var.enable_features.resource_usage_export.dataset + } + } + } + + dynamic "vertical_pod_autoscaling" { + for_each = var.enable_features.vertical_pod_autoscaling ? [""] : [] + content { + enabled = var.enable_features.vertical_pod_autoscaling + } + } + + dynamic "workload_identity_config" { + for_each = var.enable_features.workload_identity ? [""] : [] + content { + workload_pool = "${var.project_id}.svc.id.goog" + } + } + lifecycle { + ignore_changes = [node_config] + } +} + +resource "google_gke_backup_backup_plan" "backup_plan" { + for_each = var.backup_configs.enable_backup_agent ? var.backup_configs.backup_plans : {} + name = each.key + cluster = google_container_cluster.cluster.id + location = each.value.region + project = var.project_id + retention_policy { + backup_delete_lock_days = try(each.value.retention_policy_delete_lock_days) + backup_retain_days = try(each.value.retention_policy_days) + locked = try(each.value.retention_policy_lock) + } + backup_schedule { + cron_schedule = each.value.schedule + } + + backup_config { + include_volume_data = each.value.include_volume_data + include_secrets = each.value.include_secrets + + dynamic "encryption_key" { + for_each = each.value.encryption_key != null ? [""] : [] + content { + gcp_kms_encryption_key = each.value.encryption_key + } + } + + all_namespaces = lookup(each.value, "namespaces", null) != null ? null : true + dynamic "selected_namespaces" { + for_each = each.value.namespaces != null ? [""] : [] + content { + namespaces = each.value.namespaces + } + } + } +} + + +resource "google_compute_network_peering_routes_config" "gke_master" { + count = ( + try(var.private_cluster_config.peering_config, null) != null ? 1 : 0 + ) + project = ( + try(var.private_cluster_config.peering_config, null) == null + ? var.project_id + : var.private_cluster_config.peering_config.project_id + ) + peering = try( + google_container_cluster.cluster.private_cluster_config.0.peering_name, + null + ) + network = element(reverse(split("/", var.vpc_config.network)), 0) + import_custom_routes = var.private_cluster_config.peering_config.import_routes + export_custom_routes = var.private_cluster_config.peering_config.export_routes +} + +resource "google_pubsub_topic" "notifications" { + count = ( + try(var.enable_features.upgrade_notifications, null) != null && + try(var.enable_features.upgrade_notifications.topic_id, null) == null ? 1 : 0 + ) + project = var.project_id + name = "gke-pubsub-notifications" + labels = { + content = "gke-notifications" + } +} diff --git a/assets/modules-fabric/v26/gke-cluster-standard/outputs.tf b/assets/modules-fabric/v26/gke-cluster-standard/outputs.tf new file mode 100644 index 0000000..f48975c --- /dev/null +++ b/assets/modules-fabric/v26/gke-cluster-standard/outputs.tf @@ -0,0 +1,71 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "ca_certificate" { + description = "Public certificate of the cluster (base64-encoded)." + value = google_container_cluster.cluster.master_auth.0.cluster_ca_certificate + sensitive = true +} + +output "cluster" { + description = "Cluster resource." + sensitive = true + value = google_container_cluster.cluster +} + +output "endpoint" { + description = "Cluster endpoint." + value = google_container_cluster.cluster.endpoint +} + +output "id" { + description = "FUlly qualified cluster id." + value = google_container_cluster.cluster.id +} + +output "location" { + description = "Cluster location." + value = google_container_cluster.cluster.location +} + +output "master_version" { + description = "Master version." + value = google_container_cluster.cluster.master_version +} + +output "name" { + description = "Cluster name." + value = google_container_cluster.cluster.name +} + +output "notifications" { + description = "GKE PubSub notifications topic." + value = try(google_pubsub_topic.notifications[0].id, null) +} + +output "self_link" { + description = "Cluster self link." + sensitive = true + value = google_container_cluster.cluster.self_link +} + +output "workload_identity_pool" { + description = "Workload identity pool." + value = "${var.project_id}.svc.id.goog" + depends_on = [ + google_container_cluster.cluster + ] +} diff --git a/assets/modules-fabric/v26/gke-cluster-standard/variables.tf b/assets/modules-fabric/v26/gke-cluster-standard/variables.tf new file mode 100644 index 0000000..c470dcf --- /dev/null +++ b/assets/modules-fabric/v26/gke-cluster-standard/variables.tf @@ -0,0 +1,316 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "backup_configs" { + description = "Configuration for Backup for GKE." + type = object({ + enable_backup_agent = optional(bool, false) + backup_plans = optional(map(object({ + encryption_key = optional(string) + include_secrets = optional(bool, true) + include_volume_data = optional(bool, true) + namespaces = optional(list(string)) + region = string + schedule = string + retention_policy_days = optional(string) + retention_policy_lock = optional(bool, false) + retention_policy_delete_lock_days = optional(string) + })), {}) + }) + default = {} + nullable = false +} + +variable "cluster_autoscaling" { + description = "Enable and configure limits for Node Auto-Provisioning with Cluster Autoscaler." + type = object({ + auto_provisioning_defaults = optional(object({ + boot_disk_kms_key = optional(string) + image_type = optional(string) + oauth_scopes = optional(list(string)) + service_account = optional(string) + })) + cpu_limits = optional(object({ + min = number + max = number + })) + mem_limits = optional(object({ + min = number + max = number + })) + }) + default = null +} + +variable "description" { + description = "Cluster description." + type = string + default = null +} + +variable "enable_addons" { + description = "Addons enabled in the cluster (true means enabled)." + type = object({ + cloudrun = optional(bool, false) + config_connector = optional(bool, false) + dns_cache = optional(bool, false) + gce_persistent_disk_csi_driver = optional(bool, false) + gcp_filestore_csi_driver = optional(bool, false) + horizontal_pod_autoscaling = optional(bool, false) + http_load_balancing = optional(bool, false) + istio = optional(object({ + enable_tls = bool + })) + kalm = optional(bool, false) + network_policy = optional(bool, false) + }) + default = { + horizontal_pod_autoscaling = true + http_load_balancing = true + } + nullable = false +} + +variable "enable_features" { + description = "Enable cluster-level features. Certain features allow configuration." + type = object({ + binary_authorization = optional(bool, false) + cost_management = optional(bool, false) + dns = optional(object({ + provider = optional(string) + scope = optional(string) + domain = optional(string) + })) + database_encryption = optional(object({ + state = string + key_name = string + })) + dataplane_v2 = optional(bool, false) + gateway_api = optional(bool, false) + groups_for_rbac = optional(string) + intranode_visibility = optional(bool, false) + l4_ilb_subsetting = optional(bool, false) + mesh_certificates = optional(bool) + pod_security_policy = optional(bool, false) + resource_usage_export = optional(object({ + dataset = string + enable_network_egress_metering = optional(bool) + enable_resource_consumption_metering = optional(bool) + })) + shielded_nodes = optional(bool, false) + tpu = optional(bool, false) + upgrade_notifications = optional(object({ + topic_id = optional(string) + })) + vertical_pod_autoscaling = optional(bool, false) + workload_identity = optional(bool, true) + }) + default = { + workload_identity = true + } +} + +variable "issue_client_certificate" { + description = "Enable issuing client certificate." + type = bool + default = false +} + +variable "labels" { + description = "Cluster resource labels." + type = map(string) + default = null +} + +variable "location" { + description = "Cluster zone or region." + type = string +} + +variable "logging_config" { + description = "Logging configuration." + type = object({ + enable_system_logs = optional(bool, true) + enable_workloads_logs = optional(bool, false) + enable_api_server_logs = optional(bool, false) + enable_scheduler_logs = optional(bool, false) + enable_controller_manager_logs = optional(bool, false) + }) + default = {} + nullable = false + # System logs are the minimum required component for enabling log collection. + # So either everything is off (false), or enable_system_logs must be true. + validation { + condition = ( + !anytrue(values(var.logging_config)) || var.logging_config.enable_system_logs + ) + error_message = "System logs are the minimum required component for enabling log collection." + } +} + +variable "maintenance_config" { + description = "Maintenance window configuration." + type = object({ + daily_window_start_time = optional(string) + recurring_window = optional(object({ + start_time = string + end_time = string + recurrence = string + })) + maintenance_exclusions = optional(list(object({ + name = string + start_time = string + end_time = string + scope = optional(string) + }))) + }) + default = { + daily_window_start_time = "03:00" + recurring_window = null + maintenance_exclusion = [] + } +} + +variable "max_pods_per_node" { + description = "Maximum number of pods per node in this cluster." + type = number + default = 110 +} + +variable "min_master_version" { + description = "Minimum version of the master, defaults to the version of the most recent official release." + type = string + default = null +} + +variable "monitoring_config" { + description = "Monitoring configuration. Google Cloud Managed Service for Prometheus is enabled by default." + type = object({ + enable_system_metrics = optional(bool, true) + + # Control plane metrics + enable_api_server_metrics = optional(bool, false) + enable_controller_manager_metrics = optional(bool, false) + enable_scheduler_metrics = optional(bool, false) + + # Kube state metrics + enable_daemonset_metrics = optional(bool, false) + enable_deployment_metrics = optional(bool, false) + enable_hpa_metrics = optional(bool, false) + enable_pod_metrics = optional(bool, false) + enable_statefulset_metrics = optional(bool, false) + enable_storage_metrics = optional(bool, false) + + # Google Cloud Managed Service for Prometheus + enable_managed_prometheus = optional(bool, true) + }) + default = {} + nullable = false + validation { + condition = anytrue([ + var.monitoring_config.enable_api_server_metrics, + var.monitoring_config.enable_controller_manager_metrics, + var.monitoring_config.enable_scheduler_metrics, + var.monitoring_config.enable_daemonset_metrics, + var.monitoring_config.enable_deployment_metrics, + var.monitoring_config.enable_hpa_metrics, + var.monitoring_config.enable_pod_metrics, + var.monitoring_config.enable_statefulset_metrics, + var.monitoring_config.enable_storage_metrics, + ]) ? var.monitoring_config.enable_system_metrics : true + error_message = "System metrics are the minimum required component for enabling metrics collection." + } + validation { + condition = anytrue([ + var.monitoring_config.enable_daemonset_metrics, + var.monitoring_config.enable_deployment_metrics, + var.monitoring_config.enable_hpa_metrics, + var.monitoring_config.enable_pod_metrics, + var.monitoring_config.enable_statefulset_metrics, + var.monitoring_config.enable_storage_metrics, + ]) ? var.monitoring_config.enable_managed_prometheus : true + error_message = "Kube state metrics collection requires Google Cloud Managed Service for Prometheus to be enabled." + } +} + +variable "name" { + description = "Cluster name." + type = string +} + +variable "node_locations" { + description = "Zones in which the cluster's nodes are located." + type = list(string) + default = [] + nullable = false +} + +variable "private_cluster_config" { + description = "Private cluster configuration." + type = object({ + enable_private_endpoint = optional(bool) + master_global_access = optional(bool) + peering_config = optional(object({ + export_routes = optional(bool) + import_routes = optional(bool) + project_id = optional(string) + })) + }) + default = null +} + +variable "project_id" { + description = "Cluster project id." + type = string +} + +variable "release_channel" { + description = "Release channel for GKE upgrades." + type = string + default = null +} + +variable "service_account" { + description = "Service account used for the default node pool, only useful if the default GCE service account has been disabled." + type = string + default = null +} + +variable "tags" { + description = "Network tags applied to nodes." + type = list(string) + default = null +} + +variable "vpc_config" { + description = "VPC-level configuration." + type = object({ + network = string + subnetwork = string + master_ipv4_cidr_block = optional(string) + secondary_range_blocks = optional(object({ + pods = string + services = string + })) + secondary_range_names = optional(object({ + pods = optional(string, "pods") + services = optional(string, "services") + })) + master_authorized_ranges = optional(map(string)) + stack_type = optional(string) + }) + nullable = false +} diff --git a/assets/modules-fabric/v26/gke-cluster-standard/versions.tf b/assets/modules-fabric/v26/gke-cluster-standard/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/gke-cluster-standard/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/gke-hub/README.md b/assets/modules-fabric/v26/gke-hub/README.md new file mode 100644 index 0000000..ec09ad9 --- /dev/null +++ b/assets/modules-fabric/v26/gke-hub/README.md @@ -0,0 +1,338 @@ +# GKE hub module + +This module allows simplified creation and management of a GKE Hub object and its features for a given set of clusters. The given list of clusters will be registered inside the Hub and all the configured features will be activated. + +To use this module you must ensure the following APIs are enabled in the target project: + +- `gkehub.googleapis.com` +- `gkeconnect.googleapis.com` +- `anthosconfigmanagement.googleapis.com` +- `multiclusteringress.googleapis.com` +- `multiclusterservicediscovery.googleapis.com` +- `mesh.googleapis.com` + +## Full GKE Hub example + +```hcl +module "project" { + source = "./fabric/modules/project" + billing_account = var.billing_account_id + name = "gkehub-test" + parent = "folders/12345" + services = [ + "anthosconfigmanagement.googleapis.com", + "container.googleapis.com", + "gkeconnect.googleapis.com", + "gkehub.googleapis.com", + "multiclusteringress.googleapis.com", + "multiclusterservicediscovery.googleapis.com", + "mesh.googleapis.com" + ] +} + +module "vpc" { + source = "./fabric/modules/net-vpc" + project_id = module.project.project_id + name = "network" + subnets = [{ + ip_cidr_range = "10.0.0.0/24" + name = "cluster-1" + region = "europe-west1" + secondary_ip_range = { + pods = "10.1.0.0/16" + services = "10.2.0.0/24" + } + }] +} + +module "cluster_1" { + source = "./fabric/modules/gke-cluster-standard" + project_id = module.project.project_id + name = "cluster-1" + location = "europe-west1" + vpc_config = { + network = module.vpc.self_link + subnetwork = module.vpc.subnet_self_links["europe-west1/cluster-1"] + master_authorized_ranges = { + rfc1918_10_8 = "10.0.0.0/8" + } + master_ipv4_cidr_block = "192.168.0.0/28" + } + enable_features = { + dataplane_v2 = true + workload_identity = true + } + private_cluster_config = { + enable_private_endpoint = true + master_global_access = false + } +} + +module "hub" { + source = "./fabric/modules/gke-hub" + project_id = module.project.project_id + clusters = { + cluster-1 = module.cluster_1.id + } + features = { + appdevexperience = false + configmanagement = true + identityservice = false + multiclusteringress = null + servicemesh = false + multiclusterservicediscovery = false + } + configmanagement_templates = { + default = { + binauthz = false + config_sync = { + git = { + gcp_service_account_email = null + https_proxy = null + policy_dir = "configsync" + secret_type = "none" + source_format = "hierarchy" + sync_branch = "main" + sync_repo = "https://github.com/danielmarzini/configsync-platform-example" + sync_rev = null + sync_wait_secs = null + } + prevent_drift = false + source_format = "hierarchy" + } + hierarchy_controller = { + enable_hierarchical_resource_quota = true + enable_pod_tree_labels = true + } + policy_controller = { + audit_interval_seconds = 120 + exemptable_namespaces = [] + log_denies_enabled = true + referential_rules_enabled = true + template_library_installed = true + } + version = "v1" + } + } + configmanagement_clusters = { + "default" = ["cluster-1"] + } +} + +# tftest modules=4 resources=18 inventory=full.yaml +``` + +## Multi-cluster mesh on GKE + +```hcl +module "project" { + source = "./fabric/modules/project" + billing_account = "123-456-789" + name = "gkehub-test" + parent = "folders/12345" + services = [ + "anthos.googleapis.com", + "container.googleapis.com", + "gkehub.googleapis.com", + "gkeconnect.googleapis.com", + "mesh.googleapis.com", + "meshconfig.googleapis.com", + "meshca.googleapis.com" + ] +} + +resource "google_project_iam_member" "gkehub_fix" { + member = "serviceAccount:${module.project.service_accounts.robots.fleet}" + project = module.project.project_id + role = "roles/gkehub.serviceAgent" +} + + +module "vpc" { + source = "./fabric/modules/net-vpc" + project_id = module.project.project_id + name = "vpc" + mtu = 1500 + subnets = [ + { + ip_cidr_range = "10.0.1.0/24" + name = "subnet-cluster-1" + region = "europe-west1" + secondary_ip_ranges = { + pods = "10.1.0.0/16" + services = "10.2.0.0/24" + } + }, + { + ip_cidr_range = "10.0.2.0/24" + name = "subnet-cluster-2" + region = "europe-west4" + secondary_ip_ranges = { + pods = "10.3.0.0/16" + services = "10.4.0.0/24" + } + }, + { + ip_cidr_range = "10.0.0.0/28" + name = "subnet-mgmt" + region = "europe-west1" + secondary_ip_ranges = null + } + ] +} + +module "firewall" { + source = "./fabric/modules/net-vpc-firewall" + project_id = module.project.project_id + network = module.vpc.name + ingress_rules = { + allow-mesh = { + description = "Allow mesh" + priority = 900 + source_ranges = ["10.1.0.0/16", "10.3.0.0/16"] + targets = ["cluster-1-node", "cluster-2-node"] + }, + "allow-cluster-1-istio" = { + description = "Allow istio sidecar injection, istioctl version and istioctl ps" + source_ranges = ["192.168.1.0/28"] + targets = ["cluster-1-node"] + rules = [ + { protocol = "tcp", ports = [8080, 15014, 15017] } + ] + }, + "allow-cluster-2-istio" = { + description = "Allow istio sidecar injection, istioctl version and istioctl ps" + source_ranges = ["192.168.2.0/28"] + targets = ["cluster-2-node"] + rules = [ + { protocol = "tcp", ports = [8080, 15014, 15017] } + ] + } + } +} + +module "cluster_1" { + source = "./fabric/modules/gke-cluster-standard" + project_id = module.project.project_id + name = "cluster-1" + location = "europe-west1" + vpc_config = { + network = module.vpc.self_link + subnetwork = module.vpc.subnet_self_links["europe-west1/subnet-cluster-1"] + master_authorized_ranges = { + mgmt = "10.0.0.0/28" + pods-cluster-1 = "10.3.0.0/16" + } + master_ipv4_cidr_block = "192.168.1.0/28" + } + private_cluster_config = { + enable_private_endpoint = false + master_global_access = true + } + + release_channel = "REGULAR" + labels = { + mesh_id = "proj-${module.project.number}" + } + enable_features = { + workload_identity = true + dataplane_v2 = true + } +} + +module "cluster_1_nodepool" { + source = "./fabric/modules/gke-nodepool" + project_id = module.project.project_id + cluster_name = module.cluster_1.name + cluster_id = module.cluster_1.id + location = "europe-west1" + name = "cluster-1-nodepool" + node_count = { initial = 1 } + service_account = { create = true } + tags = ["cluster-1-node"] +} + +module "cluster_2" { + source = "./fabric/modules/gke-cluster-standard" + project_id = module.project.project_id + name = "cluster-2" + location = "europe-west4" + vpc_config = { + network = module.vpc.self_link + subnetwork = module.vpc.subnet_self_links["europe-west4/subnet-cluster-2"] + master_authorized_ranges = { + mgmt = "10.0.0.0/28" + pods-cluster-1 = "10.3.0.0/16" + } + master_ipv4_cidr_block = "192.168.2.0/28" + } + private_cluster_config = { + enable_private_endpoint = false + master_global_access = true + } + release_channel = "REGULAR" + labels = { + mesh_id = "proj-${module.project.number}" + } + enable_features = { + workload_identity = true + dataplane_v2 = true + } +} + +module "cluster_2_nodepool" { + source = "./fabric/modules/gke-nodepool" + project_id = module.project.project_id + cluster_name = module.cluster_2.name + cluster_id = module.cluster_2.id + location = "europe-west4" + name = "cluster-2-nodepool" + node_count = { initial = 1 } + service_account = { create = true } + tags = ["cluster-2-node"] +} + +module "hub" { + source = "./fabric/modules/gke-hub" + project_id = module.project.project_id + depends_on = [google_project_iam_member.gkehub_fix] + clusters = { + cluster-1 = module.cluster_1.id + cluster-2 = module.cluster_2.id + } + features = { + appdevexperience = false + configmanagement = false + identityservice = false + multiclusteringress = null + servicemesh = true + multiclusterservicediscovery = false + } + workload_identity_clusters = [ + "cluster-1", + "cluster-2" + ] +} + +# tftest modules=8 resources=34 +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [project_id](variables.tf#L87) | GKE hub project ID. | string | ✓ | | +| [clusters](variables.tf#L17) | Clusters members of this GKE Hub in name => id format. | map(string) | | {} | +| [configmanagement_clusters](variables.tf#L24) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string)) | | {} | +| [configmanagement_templates](variables.tf#L31) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…})) | | {} | +| [features](variables.tf#L66) | Enable and configure fleet features. | object({…}) | | {…} | +| [workload_identity_clusters](variables.tf#L92) | Clusters that will use Fleet Workload Identity. | list(string) | | [] | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [cluster_ids](outputs.tf#L17) | Fully qualified ids of all clusters. | | + + diff --git a/assets/modules-fabric/v26/gke-hub/main.tf b/assets/modules-fabric/v26/gke-hub/main.tf new file mode 100644 index 0000000..a75718e --- /dev/null +++ b/assets/modules-fabric/v26/gke-hub/main.tf @@ -0,0 +1,164 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + _cluster_cm_config = flatten([ + for template, clusters in var.configmanagement_clusters : [ + for cluster in clusters : { + cluster = cluster + template = lookup(var.configmanagement_templates, template, null) + } + ] + ]) + cluster_cm_config = { + for k in local._cluster_cm_config : k.cluster => k.template if( + k.template != null && + var.features.configmanagement == true + ) + } + hub_features = { + for k, v in var.features : k => v if v != null && v != false && v != "" + } +} + +resource "google_gke_hub_membership" "default" { + provider = google-beta + for_each = var.clusters + project = var.project_id + membership_id = each.key + endpoint { + gke_cluster { + resource_link = "//container.googleapis.com/${each.value}" + } + } + dynamic "authority" { + for_each = ( + contains(var.workload_identity_clusters, each.key) ? { 1 = 1 } : {} + ) + content { + issuer = "https://container.googleapis.com/v1/${var.clusters[each.key]}" + } + } +} + +resource "google_gke_hub_feature" "default" { + provider = google-beta + for_each = local.hub_features + project = var.project_id + name = each.key + location = "global" + dynamic "spec" { + for_each = each.key == "multiclusteringress" && each.value != null ? { 1 = 1 } : {} + content { + multiclusteringress { + config_membership = google_gke_hub_membership.default[each.value].id + } + } + } +} + +resource "google_gke_hub_feature_membership" "servicemesh" { + provider = google-beta + for_each = var.features.servicemesh ? var.clusters : {} + project = var.project_id + location = "global" + feature = google_gke_hub_feature.default["servicemesh"].name + membership = google_gke_hub_membership.default[each.key].membership_id + + mesh { + management = "MANAGEMENT_AUTOMATIC" + } +} + +resource "google_gke_hub_feature_membership" "default" { + provider = google-beta + for_each = local.cluster_cm_config + project = var.project_id + location = "global" + feature = google_gke_hub_feature.default["configmanagement"].name + membership = google_gke_hub_membership.default[each.key].membership_id + + configmanagement { + version = each.value.version + + dynamic "binauthz" { + for_each = each.value.binauthz != true ? {} : { 1 = 1 } + content { + enabled = true + } + } + + dynamic "config_sync" { + for_each = each.value.config_sync == null ? {} : { 1 = 1 } + content { + prevent_drift = each.value.config_sync.prevent_drift + source_format = each.value.config_sync.source_format + dynamic "git" { + for_each = ( + try(each.value.config_sync.git, null) == null ? {} : { 1 = 1 } + ) + content { + gcp_service_account_email = ( + each.value.config_sync.git.gcp_service_account_email + ) + https_proxy = each.value.config_sync.git.https_proxy + policy_dir = each.value.config_sync.git.policy_dir + secret_type = each.value.config_sync.git.secret_type + sync_branch = each.value.config_sync.git.sync_branch + sync_repo = each.value.config_sync.git.sync_repo + sync_rev = each.value.config_sync.git.sync_rev + sync_wait_secs = each.value.config_sync.git.sync_wait_secs + } + } + } + } + + dynamic "hierarchy_controller" { + for_each = each.value.hierarchy_controller == null ? {} : { 1 = 1 } + content { + enable_hierarchical_resource_quota = ( + each.value.hierarchy_controller.enable_hierarchical_resource_quota + ) + enable_pod_tree_labels = ( + each.value.hierarchy_controller.enable_pod_tree_labels + ) + enabled = true + } + } + + dynamic "policy_controller" { + for_each = each.value.policy_controller == null ? {} : { 1 = 1 } + content { + audit_interval_seconds = ( + each.value.policy_controller.audit_interval_seconds + ) + exemptable_namespaces = ( + each.value.policy_controller.exemptable_namespaces + ) + log_denies_enabled = ( + each.value.policy_controller.log_denies_enabled + ) + referential_rules_enabled = ( + each.value.policy_controller.referential_rules_enabled + ) + template_library_installed = ( + each.value.policy_controller.template_library_installed + ) + enabled = true + } + } + } +} diff --git a/assets/modules-fabric/v26/gke-hub/outputs.tf b/assets/modules-fabric/v26/gke-hub/outputs.tf new file mode 100644 index 0000000..2e74cda --- /dev/null +++ b/assets/modules-fabric/v26/gke-hub/outputs.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cluster_ids" { + description = "Fully qualified ids of all clusters." + value = { + for k, v in google_gke_hub_membership.default : k => v.id + } + depends_on = [ + google_gke_hub_membership.default, + google_gke_hub_feature.default, + google_gke_hub_feature_membership.default, + ] +} diff --git a/assets/modules-fabric/v26/gke-hub/variables.tf b/assets/modules-fabric/v26/gke-hub/variables.tf new file mode 100644 index 0000000..20641fe --- /dev/null +++ b/assets/modules-fabric/v26/gke-hub/variables.tf @@ -0,0 +1,97 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "clusters" { + description = "Clusters members of this GKE Hub in name => id format." + type = map(string) + default = {} + nullable = false +} + +variable "configmanagement_clusters" { + description = "Config management features enabled on specific sets of member clusters, in config name => [cluster name] format." + type = map(list(string)) + default = {} + nullable = false +} + +variable "configmanagement_templates" { + description = "Sets of config management configurations that can be applied to member clusters, in config name => {options} format." + type = map(object({ + binauthz = bool + config_sync = object({ + git = object({ + gcp_service_account_email = string + https_proxy = string + policy_dir = string + secret_type = string + sync_branch = string + sync_repo = string + sync_rev = string + sync_wait_secs = number + }) + prevent_drift = string + source_format = string + }) + hierarchy_controller = object({ + enable_hierarchical_resource_quota = bool + enable_pod_tree_labels = bool + }) + policy_controller = object({ + audit_interval_seconds = number + exemptable_namespaces = list(string) + log_denies_enabled = bool + referential_rules_enabled = bool + template_library_installed = bool + }) + version = string + })) + default = {} + nullable = false +} + +variable "features" { + description = "Enable and configure fleet features." + type = object({ + appdevexperience = optional(bool, false) + configmanagement = optional(bool, false) + identityservice = optional(bool, false) + multiclusteringress = optional(string, null) + multiclusterservicediscovery = optional(bool, false) + servicemesh = optional(bool, false) + }) + default = { + appdevexperience = false + configmanagement = false + identityservice = false + multiclusteringress = null + servicemesh = false + multiclusterservicediscovery = false + } + nullable = false +} + +variable "project_id" { + description = "GKE hub project ID." + type = string +} + +variable "workload_identity_clusters" { + description = "Clusters that will use Fleet Workload Identity." + type = list(string) + default = [] + nullable = false +} diff --git a/assets/modules-fabric/v26/gke-hub/versions.tf b/assets/modules-fabric/v26/gke-hub/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/gke-hub/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/gke-nodepool/README.md b/assets/modules-fabric/v26/gke-nodepool/README.md new file mode 100644 index 0000000..6a66b2c --- /dev/null +++ b/assets/modules-fabric/v26/gke-nodepool/README.md @@ -0,0 +1,140 @@ +# GKE nodepool module + +This module allows simplified creation and management of individual GKE nodepools, setting sensible defaults (eg a service account is created for nodes if none is set) and allowing for less verbose usage in most use cases. + +## Example usage + +### Module defaults + +If no specific node configuration is set via variables, the module uses the provider's defaults only setting OAuth scopes to a minimal working set and the node machine type to `n1-standard-1`. The service account set by the provider in this case is the GCE default service account. + +```hcl +module "cluster-1-nodepool-1" { + source = "./fabric/modules/gke-nodepool" + project_id = "myproject" + cluster_name = "cluster-1" + location = "europe-west1-b" + name = "nodepool-1" +} +# tftest modules=1 resources=1 inventory=basic.yaml +``` + +### Internally managed service account + +There are three different approaches to defining the nodes service account, all depending on the `service_account` variable where the `create` attribute controls creation of a new service account by this module, and the `email` attribute controls the actual service account to use. + +If you create a new service account, its resource and email (in both plain and IAM formats) are then available in outputs to reference it in other modules or resources. + +#### GCE default service account + +To use the GCE default service account, you can ignore the variable which is equivalent to `{ create = null, email = null }`. This is what the first example of this document does. + +#### Externally defined service account + +To use an existing service account, pass in just the `email` attribute. If you do this, will most likely want to use the `cloud-platform` scope. + +```hcl +module "cluster-1-nodepool-1" { + source = "./fabric/modules/gke-nodepool" + project_id = "myproject" + cluster_name = "cluster-1" + location = "europe-west1-b" + name = "nodepool-1" + service_account = { + email = "foo-bar@myproject.iam.gserviceaccount.com" + oauth_scopes = ["https://www.googleapis.com/auth/cloud-platform"] + } +} +# tftest modules=1 resources=1 inventory=external-sa.yaml +``` + +#### Auto-created service account + +To have the module create a service account, set the `create` attribute to `true` and optionally pass the desired account id in `email`. + +```hcl +module "cluster-1-nodepool-1" { + source = "./fabric/modules/gke-nodepool" + project_id = "myproject" + cluster_name = "cluster-1" + location = "europe-west1-b" + name = "nodepool-1" + service_account = { + create = true + email = "spam-eggs" # optional + oauth_scopes = ["https://www.googleapis.com/auth/cloud-platform"] + } +} +# tftest modules=1 resources=2 inventory=create-sa.yaml +``` +### Node & node pool configuration + +```hcl +module "cluster-1-nodepool-1" { + source = "./fabric/modules/gke-nodepool" + project_id = "myproject" + cluster_name = "cluster-1" + location = "europe-west1-b" + name = "nodepool-1" + labels = { environment = "dev" } + service_account = { + create = true + email = "nodepool-1" # optional + oauth_scopes = ["https://www.googleapis.com/auth/cloud-platform"] + } + node_config = { + machine_type = "n2-standard-2" + disk_size_gb = 50 + disk_type = "pd-ssd" + ephemeral_ssd_count = 1 + gvnic = true + spot = true + } + nodepool_config = { + autoscaling = { + max_node_count = 10 + min_node_count = 1 + } + management = { + auto_repair = true + auto_upgrade = false + } + } +} +# tftest modules=1 resources=2 inventory=config.yaml +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [cluster_name](variables.tf#L23) | Cluster name. | string | ✓ | | +| [location](variables.tf#L41) | Cluster location. | string | ✓ | | +| [project_id](variables.tf#L149) | Cluster project id. | string | ✓ | | +| [cluster_id](variables.tf#L17) | Cluster id. Optional, but providing cluster_id is recommended to prevent cluster misconfiguration in some of the edge cases. | string | | null | +| [gke_version](variables.tf#L28) | Kubernetes nodes version. Ignored if auto_upgrade is set in management_config. | string | | null | +| [labels](variables.tf#L34) | Kubernetes labels applied to each node. | map(string) | | {} | +| [max_pods_per_node](variables.tf#L46) | Maximum number of pods per node. | number | | null | +| [name](variables.tf#L52) | Optional nodepool name. | string | | null | +| [node_config](variables.tf#L58) | Node-level configuration. | object({…}) | | {…} | +| [node_count](variables.tf#L97) | Number of nodes per instance group. Initial value can only be changed by recreation, current is ignored when autoscaling is used. | object({…}) | | {…} | +| [node_locations](variables.tf#L109) | Node locations. | list(string) | | null | +| [nodepool_config](variables.tf#L115) | Nodepool-level configuration. | object({…}) | | null | +| [pod_range](variables.tf#L137) | Pod secondary range configuration. | object({…}) | | null | +| [reservation_affinity](variables.tf#L154) | Configuration of the desired reservation which instances could take capacity from. | object({…}) | | null | +| [service_account](variables.tf#L164) | Nodepool service account. If this variable is set to null, the default GCE service account will be used. If set and email is null, a service account will be created. If scopes are null a default will be used. | object({…}) | | {} | +| [sole_tenant_nodegroup](variables.tf#L175) | Sole tenant node group. | string | | null | +| [tags](variables.tf#L181) | Network tags applied to nodes. | list(string) | | null | +| [taints](variables.tf#L187) | Kubernetes taints applied to all nodes. | list(object({…})) | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [id](outputs.tf#L17) | Fully qualified nodepool id. | | +| [name](outputs.tf#L22) | Nodepool name. | | +| [service_account_email](outputs.tf#L27) | Service account email. | | +| [service_account_iam_email](outputs.tf#L32) | Service account email. | | + + diff --git a/assets/modules-fabric/v26/gke-nodepool/main.tf b/assets/modules-fabric/v26/gke-nodepool/main.tf new file mode 100644 index 0000000..9ae4cf2 --- /dev/null +++ b/assets/modules-fabric/v26/gke-nodepool/main.tf @@ -0,0 +1,227 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + _image = coalesce(var.node_config.image_type, "-") + image = { + is_cos = length(regexall("COS", local._image)) > 0 + is_cos_containerd = ( + var.node_config.image_type == null + || + length(regexall("COS_CONTAINERD", local._image)) > 0 + ) + is_win = length(regexall("WIN", local._image)) > 0 + } + node_metadata = var.node_config.metadata == null ? null : merge( + var.node_config.metadata, + { disable-legacy-endpoints = "true" } + ) + # if no attributes passed for service account, use the GCE default + # if no email specified, create service account + service_account_email = ( + var.service_account.create + ? google_service_account.service_account[0].email + : var.service_account.email + ) + service_account_scopes = ( + var.service_account.oauth_scopes != null + ? var.service_account.oauth_scopes + : [ + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + "https://www.googleapis.com/auth/monitoring.write", + "https://www.googleapis.com/auth/userinfo.email" + ] + ) + taints_windows = ( + local.image.is_win + ? [{ + key = "node.kubernetes.io/os", value = "windows", effect = "NO_EXECUTE" + }] + : [] + ) +} + +resource "google_service_account" "service_account" { + count = var.service_account.create ? 1 : 0 + project = var.project_id + account_id = ( + var.service_account.email != null + ? split("@", var.service_account.email)[0] + : "tf-gke-${var.name}" + ) + display_name = "Terraform GKE ${var.cluster_name} ${var.name}." +} + +resource "google_container_node_pool" "nodepool" { + provider = google-beta + project = var.project_id + cluster = coalesce(var.cluster_id, var.cluster_name) + location = var.location + name = var.name + version = var.gke_version + max_pods_per_node = var.max_pods_per_node + initial_node_count = var.node_count.initial + node_count = var.node_count.current + node_locations = var.node_locations + # placement_policy = var.nodepool_config.placement_policy + + dynamic "autoscaling" { + for_each = ( + try(var.nodepool_config.autoscaling, null) != null + && + !try(var.nodepool_config.autoscaling.use_total_nodes, false) + ? [""] : [] + ) + content { + location_policy = try(var.nodepool_config.autoscaling.location_policy, null) + max_node_count = try(var.nodepool_config.autoscaling.max_node_count, null) + min_node_count = try(var.nodepool_config.autoscaling.min_node_count, null) + } + } + dynamic "autoscaling" { + for_each = ( + try(var.nodepool_config.autoscaling.use_total_nodes, false) ? [""] : [] + ) + content { + location_policy = try(var.nodepool_config.autoscaling.location_policy, null) + total_max_node_count = try(var.nodepool_config.autoscaling.max_node_count, null) + total_min_node_count = try(var.nodepool_config.autoscaling.min_node_count, null) + } + } + + dynamic "management" { + for_each = try(var.nodepool_config.management, null) != null ? [""] : [] + content { + auto_repair = try(var.nodepool_config.management.auto_repair, null) + auto_upgrade = try(var.nodepool_config.management.auto_upgrade, null) + } + } + + dynamic "network_config" { + for_each = var.pod_range != null ? [""] : [] + content { + create_pod_range = var.pod_range.secondary_pod_range.create + pod_ipv4_cidr_block = var.pod_range.secondary_pod_range.cidr + pod_range = var.pod_range.secondary_pod_range.name + } + } + + dynamic "upgrade_settings" { + for_each = try(var.nodepool_config.upgrade_settings, null) != null ? [""] : [] + content { + max_surge = try(var.nodepool_config.upgrade_settings.max_surge, null) + max_unavailable = try(var.nodepool_config.upgrade_settings.max_unavailable, null) + } + } + + node_config { + boot_disk_kms_key = var.node_config.boot_disk_kms_key + disk_size_gb = var.node_config.disk_size_gb + disk_type = var.node_config.disk_type + image_type = var.node_config.image_type + labels = var.labels + local_ssd_count = var.node_config.local_ssd_count + machine_type = var.node_config.machine_type + metadata = local.node_metadata + min_cpu_platform = var.node_config.min_cpu_platform + node_group = var.sole_tenant_nodegroup + oauth_scopes = local.service_account_scopes + preemptible = var.node_config.preemptible + service_account = local.service_account_email + spot = ( + var.node_config.spot == true && var.node_config.preemptible != true + ) + tags = var.tags + taint = ( + var.taints == null ? [] : concat(var.taints, local.taints_windows) + ) + + dynamic "ephemeral_storage_config" { + for_each = var.node_config.ephemeral_ssd_count != null ? [""] : [] + content { + local_ssd_count = var.node_config.ephemeral_ssd_count + } + } + dynamic "gcfs_config" { + for_each = var.node_config.gcfs && local.image.is_cos_containerd ? [""] : [] + content { + enabled = true + } + } + dynamic "guest_accelerator" { + for_each = var.node_config.guest_accelerator != null ? [""] : [] + content { + count = var.node_config.guest_accelerator.count + type = var.node_config.guest_accelerator.type + gpu_partition_size = var.node_config.guest_accelerator.gpu_partition_size + } + } + dynamic "gvnic" { + for_each = var.node_config.gvnic && local.image.is_cos ? [""] : [] + content { + enabled = true + } + } + dynamic "kubelet_config" { + for_each = var.node_config.kubelet_config != null ? [""] : [] + content { + cpu_manager_policy = var.node_config.kubelet_config.cpu_manager_policy + cpu_cfs_quota = var.node_config.kubelet_config.cpu_cfs_quota + cpu_cfs_quota_period = var.node_config.kubelet_config.cpu_cfs_quota_period + } + } + dynamic "linux_node_config" { + for_each = var.node_config.linux_node_config_sysctls != null ? [""] : [] + content { + sysctls = var.node_config.linux_node_config_sysctls + } + } + dynamic "reservation_affinity" { + for_each = var.reservation_affinity != null ? [""] : [] + content { + consume_reservation_type = var.reservation_affinity.consume_reservation_type + key = var.reservation_affinity.key + values = var.reservation_affinity.values + } + } + dynamic "sandbox_config" { + for_each = ( + var.node_config.sandbox_config_gvisor == true && + local.image.is_cos_containerd != null + ? [""] + : [] + ) + content { + sandbox_type = "gvisor" + } + } + dynamic "shielded_instance_config" { + for_each = var.node_config.shielded_instance_config != null ? [""] : [] + content { + enable_secure_boot = var.node_config.shielded_instance_config.enable_secure_boot + enable_integrity_monitoring = var.node_config.shielded_instance_config.enable_integrity_monitoring + } + } + dynamic "workload_metadata_config" { + for_each = var.node_config.workload_metadata_config_mode != null ? [""] : [] + content { + mode = var.node_config.workload_metadata_config_mode + } + } + } +} diff --git a/assets/modules-fabric/v26/gke-nodepool/outputs.tf b/assets/modules-fabric/v26/gke-nodepool/outputs.tf new file mode 100644 index 0000000..4102d9e --- /dev/null +++ b/assets/modules-fabric/v26/gke-nodepool/outputs.tf @@ -0,0 +1,38 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "id" { + description = "Fully qualified nodepool id." + value = google_container_node_pool.nodepool.id +} + +output "name" { + description = "Nodepool name." + value = google_container_node_pool.nodepool.name +} + +output "service_account_email" { + description = "Service account email." + value = local.service_account_email +} + +output "service_account_iam_email" { + description = "Service account email." + value = format( + "serviceAccount:%s", + local.service_account_email == null ? "" : local.service_account_email + ) +} diff --git a/assets/modules-fabric/v26/gke-nodepool/variables.tf b/assets/modules-fabric/v26/gke-nodepool/variables.tf new file mode 100644 index 0000000..1166c34 --- /dev/null +++ b/assets/modules-fabric/v26/gke-nodepool/variables.tf @@ -0,0 +1,195 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "cluster_id" { + description = "Cluster id. Optional, but providing cluster_id is recommended to prevent cluster misconfiguration in some of the edge cases." + type = string + default = null +} + +variable "cluster_name" { + description = "Cluster name." + type = string +} + +variable "gke_version" { + description = "Kubernetes nodes version. Ignored if auto_upgrade is set in management_config." + type = string + default = null +} + +variable "labels" { + description = "Kubernetes labels applied to each node." + type = map(string) + default = {} + nullable = false +} + +variable "location" { + description = "Cluster location." + type = string +} + +variable "max_pods_per_node" { + description = "Maximum number of pods per node." + type = number + default = null +} + +variable "name" { + description = "Optional nodepool name." + type = string + default = null +} + +variable "node_config" { + description = "Node-level configuration." + type = object({ + boot_disk_kms_key = optional(string) + disk_size_gb = optional(number) + disk_type = optional(string) + ephemeral_ssd_count = optional(number) + gcfs = optional(bool, false) + guest_accelerator = optional(object({ + count = number + type = string + gpu_partition_size = optional(string) + })) + gvnic = optional(bool, false) + image_type = optional(string) + kubelet_config = optional(object({ + cpu_manager_policy = string + cpu_cfs_quota = optional(bool) + cpu_cfs_quota_period = optional(string) + })) + linux_node_config_sysctls = optional(map(string)) + local_ssd_count = optional(number) + machine_type = optional(string) + metadata = optional(map(string)) + min_cpu_platform = optional(string) + preemptible = optional(bool) + sandbox_config_gvisor = optional(bool) + shielded_instance_config = optional(object({ + enable_integrity_monitoring = optional(bool) + enable_secure_boot = optional(bool) + })) + spot = optional(bool) + workload_metadata_config_mode = optional(string) + }) + default = { + disk_type = "pd-balanced" + } +} + +variable "node_count" { + description = "Number of nodes per instance group. Initial value can only be changed by recreation, current is ignored when autoscaling is used." + type = object({ + current = optional(number) + initial = number + }) + default = { + initial = 1 + } + nullable = false +} + +variable "node_locations" { + description = "Node locations." + type = list(string) + default = null +} + +variable "nodepool_config" { + description = "Nodepool-level configuration." + type = object({ + autoscaling = optional(object({ + location_policy = optional(string) + max_node_count = optional(number) + min_node_count = optional(number) + use_total_nodes = optional(bool, false) + })) + management = optional(object({ + auto_repair = optional(bool) + auto_upgrade = optional(bool) + })) + # placement_policy = optional(bool) + upgrade_settings = optional(object({ + max_surge = number + max_unavailable = number + })) + }) + default = null +} + +variable "pod_range" { + description = "Pod secondary range configuration." + type = object({ + secondary_pod_range = object({ + cidr = optional(string) + create = optional(bool) + name = string + }) + }) + default = null +} + +variable "project_id" { + description = "Cluster project id." + type = string +} + +variable "reservation_affinity" { + description = "Configuration of the desired reservation which instances could take capacity from." + type = object({ + consume_reservation_type = string + key = optional(string) + values = optional(list(string)) + }) + default = null +} + +variable "service_account" { + description = "Nodepool service account. If this variable is set to null, the default GCE service account will be used. If set and email is null, a service account will be created. If scopes are null a default will be used." + type = object({ + create = optional(bool, false) + email = optional(string) + oauth_scopes = optional(list(string)) + }) + default = {} + nullable = false +} + +variable "sole_tenant_nodegroup" { + description = "Sole tenant node group." + type = string + default = null +} + +variable "tags" { + description = "Network tags applied to nodes." + type = list(string) + default = null +} + +variable "taints" { + description = "Kubernetes taints applied to all nodes." + type = list(object({ + key = string + value = string + effect = string + })) + default = null +} diff --git a/assets/modules-fabric/v26/gke-nodepool/versions.tf b/assets/modules-fabric/v26/gke-nodepool/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/gke-nodepool/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/iam-service-account/README.md b/assets/modules-fabric/v26/iam-service-account/README.md new file mode 100644 index 0000000..ea3362c --- /dev/null +++ b/assets/modules-fabric/v26/iam-service-account/README.md @@ -0,0 +1,77 @@ +# Google Service Account Module + +This module allows simplified creation and management of one a service account and its IAM bindings. + +A key can optionally be generated and will be stored in Terraform state. To use it create a sensitive output in your root modules referencing the `key` output, then extract the private key from the JSON formatted outputs. + +Alternatively, the `key` can be generated with `openssl` library and only the public part uploaded to the Service Account, for more refer to the [Onprem SA Key Management](../../blueprints/cloud-operations/onprem-sa-key-management/) example. + +Note that outputs have no dependencies on IAM bindings to prevent resource cycles. + +## Example + +```hcl +module "myproject-default-service-accounts" { + source = "./fabric/modules/iam-service-account" + project_id = "myproject" + name = "vm-default" + # authoritative roles granted *on* the service accounts to other identities + iam = { + "roles/iam.serviceAccountUser" = ["user:foo@example.com"] + } + # non-authoritative roles granted *to* the service accounts on other resources + iam_project_roles = { + "myproject" = [ + "roles/logging.logWriter", + "roles/monitoring.metricWriter", + ] + } +} +# tftest modules=1 resources=4 inventory=basic.yaml +``` + + +## Files + +| name | description | resources | +|---|---|---| +| [iam.tf](./iam.tf) | IAM bindings. | google_billing_account_iam_member · google_folder_iam_member · google_organization_iam_member · google_project_iam_member · google_service_account_iam_binding · google_service_account_iam_member · google_storage_bucket_iam_member | +| [main.tf](./main.tf) | Module-level locals and resources. | google_service_account · google_service_account_key | +| [outputs.tf](./outputs.tf) | Module outputs. | | +| [variables.tf](./variables.tf) | Module variables. | | +| [versions.tf](./versions.tf) | Version pins. | | + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [name](variables.tf#L114) | Name of the service account to create. | string | ✓ | | +| [project_id](variables.tf#L129) | Project id where service account will be created. | string | ✓ | | +| [description](variables.tf#L17) | Optional description. | string | | null | +| [display_name](variables.tf#L23) | Display name of the service account to create. | string | | "Terraform-managed." | +| [generate_key](variables.tf#L29) | Generate a key for service account. | bool | | false | +| [iam](variables.tf#L35) | IAM bindings on the service account in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [iam_billing_roles](variables.tf#L42) | Billing account roles granted to this service account, by billing account id. Non-authoritative. | map(list(string)) | | {} | +| [iam_bindings](variables.tf#L49) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | map(object({…})) | | {} | +| [iam_bindings_additive](variables.tf#L64) | Individual additive IAM bindings on the service account. Keys are arbitrary. | map(object({…})) | | {} | +| [iam_folder_roles](variables.tf#L79) | Folder roles granted to this service account, by folder id. Non-authoritative. | map(list(string)) | | {} | +| [iam_organization_roles](variables.tf#L86) | Organization roles granted to this service account, by organization id. Non-authoritative. | map(list(string)) | | {} | +| [iam_project_roles](variables.tf#L93) | Project roles granted to this service account, by project id. | map(list(string)) | | {} | +| [iam_sa_roles](variables.tf#L100) | Service account roles granted to this service account, by service account name. | map(list(string)) | | {} | +| [iam_storage_roles](variables.tf#L107) | Storage roles granted to this service account, by bucket name. | map(list(string)) | | {} | +| [prefix](variables.tf#L119) | Prefix applied to service account names. | string | | null | +| [public_keys_directory](variables.tf#L134) | Path to public keys data files to upload to the service account (should have `.pem` extension). | string | | "" | +| [service_account_create](variables.tf#L140) | Create service account. When set to false, uses a data source to reference an existing service account. | bool | | true | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [email](outputs.tf#L17) | Service account email. | | +| [iam_email](outputs.tf#L25) | IAM-format service account email. | | +| [id](outputs.tf#L33) | Fully qualified service account id. | | +| [key](outputs.tf#L42) | Service account key. | ✓ | +| [name](outputs.tf#L48) | Service account name. | | +| [service_account](outputs.tf#L57) | Service account resource. | | +| [service_account_credentials](outputs.tf#L62) | Service account json credential templates for uploaded public keys data. | | + diff --git a/assets/modules-fabric/v26/iam-service-account/iam.tf b/assets/modules-fabric/v26/iam-service-account/iam.tf new file mode 100644 index 0000000..15ae1ac --- /dev/null +++ b/assets/modules-fabric/v26/iam-service-account/iam.tf @@ -0,0 +1,159 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description IAM bindings. + +locals { + iam_billing_pairs = flatten([ + for entity, roles in var.iam_billing_roles : [ + for role in roles : [ + { entity = entity, role = role } + ] + ] + ]) + iam_folder_pairs = flatten([ + for entity, roles in var.iam_folder_roles : [ + for role in roles : [ + { entity = entity, role = role } + ] + ] + ]) + iam_organization_pairs = flatten([ + for entity, roles in var.iam_organization_roles : [ + for role in roles : [ + { entity = entity, role = role } + ] + ] + ]) + iam_project_pairs = flatten([ + for entity, roles in var.iam_project_roles : [ + for role in roles : [ + { entity = entity, role = role } + ] + ] + ]) + iam_sa_pairs = flatten([ + for entity, roles in var.iam_sa_roles : [ + for role in roles : [ + { entity = entity, role = role } + ] + ] + ]) + iam_storage_pairs = flatten([ + for entity, roles in var.iam_storage_roles : [ + for role in roles : [ + { entity = entity, role = role } + ] + ] + ]) +} + +resource "google_service_account_iam_binding" "authoritative" { + for_each = var.iam + service_account_id = local.service_account.name + role = each.key + members = each.value +} + +resource "google_service_account_iam_binding" "bindings" { + for_each = var.iam_bindings + service_account_id = local.service_account.name + role = each.value.role + members = each.value.members + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +resource "google_service_account_iam_member" "bindings" { + for_each = var.iam_bindings_additive + service_account_id = local.service_account.name + role = each.value.role + member = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +resource "google_billing_account_iam_member" "billing-roles" { + for_each = { + for pair in local.iam_billing_pairs : + "${pair.entity}-${pair.role}" => pair + } + billing_account_id = each.value.entity + role = each.value.role + member = local.resource_iam_email +} + +resource "google_folder_iam_member" "folder-roles" { + for_each = { + for pair in local.iam_folder_pairs : + "${pair.entity}-${pair.role}" => pair + } + folder = each.value.entity + role = each.value.role + member = local.resource_iam_email +} + +resource "google_organization_iam_member" "organization-roles" { + for_each = { + for pair in local.iam_organization_pairs : + "${pair.entity}-${pair.role}" => pair + } + org_id = each.value.entity + role = each.value.role + member = local.resource_iam_email +} + +resource "google_project_iam_member" "project-roles" { + for_each = { + for pair in local.iam_project_pairs : + "${pair.entity}-${pair.role}" => pair + } + project = each.value.entity + role = each.value.role + member = local.resource_iam_email +} + +resource "google_service_account_iam_member" "additive" { + for_each = { + for pair in local.iam_sa_pairs : + "${pair.entity}-${pair.role}" => pair + } + service_account_id = each.value.entity + role = each.value.role + member = local.resource_iam_email +} + +resource "google_storage_bucket_iam_member" "bucket-roles" { + for_each = { + for pair in local.iam_storage_pairs : + "${pair.entity}-${pair.role}" => pair + } + bucket = each.value.entity + role = each.value.role + member = local.resource_iam_email +} diff --git a/assets/modules-fabric/v26/iam-service-account/main.tf b/assets/modules-fabric/v26/iam-service-account/main.tf new file mode 100644 index 0000000..2c9ee36 --- /dev/null +++ b/assets/modules-fabric/v26/iam-service-account/main.tf @@ -0,0 +1,87 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # https://github.com/hashicorp/terraform/issues/22405#issuecomment-591917758 + key = try( + var.generate_key + ? google_service_account_key.key["1"] + : map("", null) + , {}) + prefix = var.prefix == null ? "" : "${var.prefix}-" + resource_email_static = "${local.prefix}${var.name}@${var.project_id}.iam.gserviceaccount.com" + resource_iam_email = ( + local.service_account != null + ? "serviceAccount:${local.service_account.email}" + : local.resource_iam_email_static + ) + resource_iam_email_static = "serviceAccount:${local.resource_email_static}" + service_account_id_static = "projects/${var.project_id}/serviceAccounts/${local.resource_email_static}" + service_account = ( + var.service_account_create + ? try(google_service_account.service_account.0, null) + : try(data.google_service_account.service_account.0, null) + ) + service_account_credential_templates = { + for file, _ in local.public_keys_data : file => jsonencode( + { + type : "service_account", + project_id : var.project_id, + private_key_id : split("/", google_service_account_key.upload_key[file].id)[5] + private_key : "REPLACE_ME_WITH_PRIVATE_KEY_DATA" + client_email : local.resource_email_static + client_id : local.service_account.unique_id, + auth_uri : "https://accounts.google.com/o/oauth2/auth", + token_uri : "https://oauth2.googleapis.com/token", + auth_provider_x509_cert_url : "https://www.googleapis.com/oauth2/v1/certs", + client_x509_cert_url : "https://www.googleapis.com/robot/v1/metadata/x509/${urlencode(local.resource_email_static)}" + } + ) + } + public_keys_data = ( + var.public_keys_directory != "" + ? { + for file in fileset("${path.root}/${var.public_keys_directory}", "*.pem") + : file => filebase64("${path.root}/${var.public_keys_directory}/${file}") } + : {} + ) +} + + +data "google_service_account" "service_account" { + count = var.service_account_create ? 0 : 1 + project = var.project_id + account_id = "${local.prefix}${var.name}" +} + +resource "google_service_account" "service_account" { + count = var.service_account_create ? 1 : 0 + project = var.project_id + account_id = "${local.prefix}${var.name}" + display_name = var.display_name + description = var.description +} + +resource "google_service_account_key" "key" { + for_each = var.generate_key ? { 1 = 1 } : {} + service_account_id = local.service_account.email +} + +resource "google_service_account_key" "upload_key" { + for_each = local.public_keys_data + service_account_id = local.service_account.email + public_key_data = each.value +} diff --git a/assets/modules-fabric/v26/iam-service-account/outputs.tf b/assets/modules-fabric/v26/iam-service-account/outputs.tf new file mode 100644 index 0000000..79210ca --- /dev/null +++ b/assets/modules-fabric/v26/iam-service-account/outputs.tf @@ -0,0 +1,65 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "email" { + description = "Service account email." + value = local.resource_email_static + depends_on = [ + local.service_account + ] +} + +output "iam_email" { + description = "IAM-format service account email." + value = local.resource_iam_email_static + depends_on = [ + local.service_account + ] +} + +output "id" { + description = "Fully qualified service account id." + value = local.service_account_id_static + depends_on = [ + data.google_service_account.service_account, + google_service_account.service_account + ] +} + +output "key" { + description = "Service account key." + sensitive = true + value = local.key +} + +output "name" { + description = "Service account name." + value = local.service_account_id_static + depends_on = [ + data.google_service_account.service_account, + google_service_account.service_account + ] +} + +output "service_account" { + description = "Service account resource." + value = local.service_account +} + +output "service_account_credentials" { + description = "Service account json credential templates for uploaded public keys data." + value = local.service_account_credential_templates +} diff --git a/assets/modules-fabric/v26/iam-service-account/variables.tf b/assets/modules-fabric/v26/iam-service-account/variables.tf new file mode 100644 index 0000000..4a75af4 --- /dev/null +++ b/assets/modules-fabric/v26/iam-service-account/variables.tf @@ -0,0 +1,144 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "description" { + description = "Optional description." + type = string + default = null +} + +variable "display_name" { + description = "Display name of the service account to create." + type = string + default = "Terraform-managed." +} + +variable "generate_key" { + description = "Generate a key for service account." + type = bool + default = false +} + +variable "iam" { + description = "IAM bindings on the service account in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} + nullable = false +} + +variable "iam_billing_roles" { + description = "Billing account roles granted to this service account, by billing account id. Non-authoritative." + type = map(list(string)) + default = {} + nullable = false +} + +variable "iam_bindings" { + description = "Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary." + type = map(object({ + members = list(string) + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} +} + +variable "iam_bindings_additive" { + description = "Individual additive IAM bindings on the service account. Keys are arbitrary." + type = map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} +} + +variable "iam_folder_roles" { + description = "Folder roles granted to this service account, by folder id. Non-authoritative." + type = map(list(string)) + default = {} + nullable = false +} + +variable "iam_organization_roles" { + description = "Organization roles granted to this service account, by organization id. Non-authoritative." + type = map(list(string)) + default = {} + nullable = false +} + +variable "iam_project_roles" { + description = "Project roles granted to this service account, by project id." + type = map(list(string)) + default = {} + nullable = false +} + +variable "iam_sa_roles" { + description = "Service account roles granted to this service account, by service account name." + type = map(list(string)) + default = {} + nullable = false +} + +variable "iam_storage_roles" { + description = "Storage roles granted to this service account, by bucket name." + type = map(list(string)) + default = {} + nullable = false +} + +variable "name" { + description = "Name of the service account to create." + type = string +} + +variable "prefix" { + description = "Prefix applied to service account names." + type = string + default = null + validation { + condition = var.prefix != "" + error_message = "Prefix cannot be empty, please use null instead." + } +} + +variable "project_id" { + description = "Project id where service account will be created." + type = string +} + +variable "public_keys_directory" { + description = "Path to public keys data files to upload to the service account (should have `.pem` extension)." + type = string + default = "" +} + +variable "service_account_create" { + description = "Create service account. When set to false, uses a data source to reference an existing service account." + type = bool + default = true +} diff --git a/assets/modules-fabric/v26/iam-service-account/versions.tf b/assets/modules-fabric/v26/iam-service-account/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/iam-service-account/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/kms/README.md b/assets/modules-fabric/v26/kms/README.md new file mode 100644 index 0000000..ddbf4b5 --- /dev/null +++ b/assets/modules-fabric/v26/kms/README.md @@ -0,0 +1,121 @@ +# Google KMS Module + +This module allows creating and managing KMS crypto keys and IAM bindings at both the keyring and crypto key level. An existing keyring can be used, or a new one can be created and managed by the module if needed. + +When using an existing keyring be mindful about applying IAM bindings, as all bindings used by this module are authoritative, and you might inadvertently override bindings managed by the keyring creator. + + +- [Protecting against destroy](#protecting-against-destroy) +- [Examples](#examples) + - [Using an existing keyring](#using-an-existing-keyring) + - [Keyring creation and crypto key rotation and IAM roles](#keyring-creation-and-crypto-key-rotation-and-iam-roles) + - [Crypto key purpose](#crypto-key-purpose) +- [Variables](#variables) +- [Outputs](#outputs) + + +## Protecting against destroy + +In this module **no lifecycle blocks are set on resources to prevent destroy**, in order to allow for experimentation and testing where rapid `apply`/`destroy` cycles are needed. If you plan on using this module to manage non-development resources, **clone it and uncomment the lifecycle blocks** found in `main.tf`. + +## Examples + +### Using an existing keyring + +```hcl +module "kms" { + source = "./fabric/modules/kms" + project_id = "my-project" + iam = { + "roles/cloudkms.admin" = ["user:user1@example.com"] + } + keyring = { location = "europe-west1", name = "test" } + keyring_create = false + keys = { key-a = {}, key-b = {}, key-c = {} } +} +# tftest skip (uses data sources) +``` + +### Keyring creation and crypto key rotation and IAM roles + +```hcl +module "kms" { + source = "./fabric/modules/kms" + project_id = "my-project" + keyring = { + location = "europe-west1" + name = "test" + } + keys = { + key-a = { + iam = { + "roles/cloudkms.admin" = ["user:user3@example.com"] + } + } + key-b = { + rotation_period = "604800s" + iam_bindings_additive = { + key-b-iam1 = { + key = "key-b" + member = "user:am1@example.com" + role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" + } + } + } + key-c = { + labels = { + env = "test" + } + } + } +} +# tftest modules=1 resources=6 inventory=basic.yaml +``` + +### Crypto key purpose + +```hcl +module "kms" { + source = "./fabric/modules/kms" + project_id = "my-project" + keyring = { + location = "europe-west1" + name = "test" + } + keys = { + key-a = { + purpose = "ASYMMETRIC_SIGN" + version_template = { + algorithm = "EC_SIGN_P384_SHA384" + protection_level = "HSM" + } + } + } +} +# tftest modules=1 resources=2 inventory=purpose.yaml +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [keyring](variables.tf#L54) | Keyring attributes. | object({…}) | ✓ | | +| [project_id](variables.tf#L103) | Project id where the keyring will be created. | string | ✓ | | +| [iam](variables.tf#L17) | Keyring IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [iam_bindings](variables.tf#L24) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | map(object({…})) | | {} | +| [iam_bindings_additive](variables.tf#L39) | Keyring individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | +| [keyring_create](variables.tf#L62) | Set to false to manage keys and IAM bindings in an existing keyring. | bool | | true | +| [keys](variables.tf#L68) | Key names and base attributes. Set attributes to null if not needed. | map(object({…})) | | {} | +| [tag_bindings](variables.tf#L108) | Tag bindings for this keyring, in key => tag value id format. | map(string) | | {} | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [id](outputs.tf#L17) | Fully qualified keyring id. | | +| [key_ids](outputs.tf#L26) | Fully qualified key ids. | | +| [keyring](outputs.tf#L38) | Keyring resource. | | +| [keys](outputs.tf#L47) | Key resources. | | +| [location](outputs.tf#L56) | Keyring location. | | +| [name](outputs.tf#L65) | Keyring name. | | + diff --git a/assets/modules-fabric/v26/kms/iam.tf b/assets/modules-fabric/v26/kms/iam.tf new file mode 100644 index 0000000..8ac17a5 --- /dev/null +++ b/assets/modules-fabric/v26/kms/iam.tf @@ -0,0 +1,126 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + key_iam = flatten([ + for k, v in var.keys : [ + for role, members in v.iam : { + key = k + role = role + members = members + } + ] + ]) + key_iam_bindings = merge([ + for k, v in var.keys : { + for binding_key, data in v.iam_bindings : + binding_key => { + key = k + role = data.role + members = data.members + condition = data.condition + } + } + ]...) + key_iam_bindings_additive = merge([ + for k, v in var.keys : { + for binding_key, data in v.iam_bindings_additive : + binding_key => { + key = k + role = data.role + member = data.member + condition = data.condition + } + } + ]...) +} + +resource "google_kms_key_ring_iam_binding" "authoritative" { + for_each = var.iam + key_ring_id = local.keyring.id + role = each.key + members = each.value +} + +resource "google_kms_key_ring_iam_binding" "bindings" { + for_each = var.iam_bindings + key_ring_id = local.keyring.id + role = each.value.role + members = each.value.members + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +resource "google_kms_key_ring_iam_member" "bindings" { + for_each = var.iam_bindings_additive + key_ring_id = local.keyring.id + role = each.value.role + member = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +resource "google_kms_crypto_key_iam_binding" "authoritative" { + for_each = { + for binding in local.key_iam : + "${binding.key}.${binding.role}" => binding + } + role = each.value.role + crypto_key_id = google_kms_crypto_key.default[each.value.key].id + members = each.value.members +} + +resource "google_kms_crypto_key_iam_binding" "bindings" { + for_each = local.key_iam_bindings + role = each.value.role + crypto_key_id = google_kms_crypto_key.default[each.value.key].id + members = each.value.members + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +resource "google_kms_crypto_key_iam_member" "members" { + for_each = local.key_iam_bindings_additive + crypto_key_id = google_kms_crypto_key.default[each.value.key].id + role = each.value.role + member = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} diff --git a/assets/modules-fabric/v26/kms/main.tf b/assets/modules-fabric/v26/kms/main.tf new file mode 100644 index 0000000..6be7c81 --- /dev/null +++ b/assets/modules-fabric/v26/kms/main.tf @@ -0,0 +1,55 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + keyring = ( + var.keyring_create + ? google_kms_key_ring.default.0 + : data.google_kms_key_ring.default.0 + ) +} + +data "google_kms_key_ring" "default" { + count = var.keyring_create ? 0 : 1 + project = var.project_id + name = var.keyring.name + location = var.keyring.location +} + +resource "google_kms_key_ring" "default" { + count = var.keyring_create ? 1 : 0 + project = var.project_id + name = var.keyring.name + location = var.keyring.location +} + +resource "google_kms_crypto_key" "default" { + for_each = var.keys + key_ring = local.keyring.id + name = each.key + rotation_period = each.value.rotation_period + labels = each.value.labels + purpose = each.value.purpose + skip_initial_version_creation = each.value.skip_initial_version_creation + + dynamic "version_template" { + for_each = each.value.version_template == null ? [] : [""] + content { + algorithm = each.value.version_template.algorithm + protection_level = each.value.version_template.protection_level + } + } +} diff --git a/assets/modules-fabric/v26/kms/outputs.tf b/assets/modules-fabric/v26/kms/outputs.tf new file mode 100644 index 0000000..191db82 --- /dev/null +++ b/assets/modules-fabric/v26/kms/outputs.tf @@ -0,0 +1,72 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "id" { + description = "Fully qualified keyring id." + value = local.keyring.id + depends_on = [ + google_kms_key_ring_iam_binding.authoritative, + google_kms_key_ring_iam_binding.bindings + ] +} + +output "key_ids" { + description = "Fully qualified key ids." + value = { + for name, resource in google_kms_crypto_key.default : + name => resource.id + } + depends_on = [ + google_kms_crypto_key_iam_binding.authoritative, + google_kms_crypto_key_iam_binding.bindings + ] +} + +output "keyring" { + description = "Keyring resource." + value = local.keyring + depends_on = [ + google_kms_key_ring_iam_binding.authoritative, + google_kms_key_ring_iam_binding.bindings + ] +} + +output "keys" { + description = "Key resources." + value = google_kms_crypto_key.default + depends_on = [ + google_kms_crypto_key_iam_binding.authoritative, + google_kms_crypto_key_iam_binding.bindings + ] +} + +output "location" { + description = "Keyring location." + value = local.keyring.location + depends_on = [ + google_kms_key_ring_iam_binding.authoritative, + google_kms_key_ring_iam_binding.bindings + ] +} + +output "name" { + description = "Keyring name." + value = local.keyring.name + depends_on = [ + google_kms_key_ring_iam_binding.authoritative, + google_kms_key_ring_iam_binding.bindings + ] +} diff --git a/assets/modules-fabric/v26/kms/tags.tf b/assets/modules-fabric/v26/kms/tags.tf new file mode 100644 index 0000000..c0955c6 --- /dev/null +++ b/assets/modules-fabric/v26/kms/tags.tf @@ -0,0 +1,21 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_tags_tag_binding" "binding" { + for_each = var.tag_bindings + parent = "//cloudresourcemanager.googleapis.com/${local.keyring.id}" + tag_value = each.value +} diff --git a/assets/modules-fabric/v26/kms/variables.tf b/assets/modules-fabric/v26/kms/variables.tf new file mode 100644 index 0000000..3086176 --- /dev/null +++ b/assets/modules-fabric/v26/kms/variables.tf @@ -0,0 +1,113 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "iam" { + description = "Keyring IAM bindings in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} + nullable = false +} + +variable "iam_bindings" { + description = "Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary." + type = map(object({ + members = list(string) + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} +} + +variable "iam_bindings_additive" { + description = "Keyring individual additive IAM bindings. Keys are arbitrary." + type = map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} +} + +variable "keyring" { + description = "Keyring attributes." + type = object({ + location = string + name = string + }) +} + +variable "keyring_create" { + description = "Set to false to manage keys and IAM bindings in an existing keyring." + type = bool + default = true +} + +variable "keys" { + description = "Key names and base attributes. Set attributes to null if not needed." + type = map(object({ + rotation_period = optional(string) + labels = optional(map(string)) + purpose = optional(string, "ENCRYPT_DECRYPT") + skip_initial_version_creation = optional(bool, false) + version_template = optional(object({ + algorithm = string + protection_level = optional(string, "SOFTWARE") + })) + + iam = optional(map(list(string)), {}) + iam_bindings = optional(map(object({ + members = list(string) + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })), {}) + iam_bindings_additive = optional(map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })), {}) + })) + default = {} + nullable = false +} + +variable "project_id" { + description = "Project id where the keyring will be created." + type = string +} + +variable "tag_bindings" { + description = "Tag bindings for this keyring, in key => tag value id format." + type = map(string) + default = {} + nullable = false +} diff --git a/assets/modules-fabric/v26/kms/versions.tf b/assets/modules-fabric/v26/kms/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/kms/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/logging-bucket/README.md b/assets/modules-fabric/v26/logging-bucket/README.md new file mode 100644 index 0000000..8ace8a1 --- /dev/null +++ b/assets/modules-fabric/v26/logging-bucket/README.md @@ -0,0 +1,95 @@ +# Google Cloud Logging Buckets Module + +This module manages [logging buckets](https://cloud.google.com/logging/docs/routing/overview#buckets) for a project, folder, organization or billing account. + +Note that some logging buckets are automatically created for a given folder, project, organization, and billing account cannot be deleted. Creating a resource of this type will acquire and update the resource that already exists at the desired location. These buckets cannot be removed so deleting this resource will remove the bucket config from your terraform state but will leave the logging bucket unchanged. The buckets that are currently automatically created are "_Default" and "_Required". + +See also the `logging_sinks` argument within the [project](../project/), [folder](../folder/) and [organization](../organization) modules. + +## Examples + +### Create custom logging bucket in a project + +```hcl +module "bucket" { + source = "./fabric/modules/logging-bucket" + parent_type = "project" + parent = var.project_id + id = "mybucket" +} +# tftest modules=1 resources=1 inventory=project.yaml +``` + +### Create custom logging bucket in a project enabling Log Analytics and dataset link + +```hcl +module "bucket" { + source = "./fabric/modules/logging-bucket" + parent_type = "project" + parent = var.project_id + id = "mybucket" + log_analytics = { + enable = true + dataset_link_id = "log" + } +} +# tftest modules=1 resources=2 inventory=log_analytics.yaml +``` + +### Change retention period of a folder's _Default bucket + +```hcl +module "folder" { + source = "./fabric/modules/folder" + parent = "folders/657104291943" + name = "my folder" +} + +module "bucket-default" { + source = "./fabric/modules/logging-bucket" + parent_type = "folder" + parent = module.folder.id + id = "_Default" + retention = 10 +} +# tftest modules=2 resources=2 inventory=retention.yaml +``` + +### Organization and billing account buckets + +```hcl +module "bucket-organization" { + source = "./fabric/modules/logging-bucket" + parent_type = "organization" + parent = "organizations/012345" + id = "mybucket" +} + +module "bucket-billing-account" { + source = "./fabric/modules/logging-bucket" + parent_type = "billing_account" + parent = "012345" + id = "mybucket" +} +# tftest modules=2 resources=2 inventory=org-ba.yaml +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [id](variables.tf#L23) | Name of the logging bucket. | string | ✓ | | +| [parent](variables.tf#L51) | ID of the parentresource containing the bucket in the format 'project_id' 'folders/folder_id', 'organizations/organization_id' or 'billing_account_id'. | string | ✓ | | +| [parent_type](variables.tf#L56) | Parent object type for the bucket (project, folder, organization, billing_account). | string | ✓ | | +| [description](variables.tf#L17) | Human-readable description for the logging bucket. | string | | null | +| [kms_key_name](variables.tf#L28) | To enable CMEK for a project logging bucket, set this field to a valid name. The associated service account requires cloudkms.cryptoKeyEncrypterDecrypter roles assigned for the key. | string | | null | +| [location](variables.tf#L34) | Location of the bucket. | string | | "global" | +| [log_analytics](variables.tf#L40) | Enable and configure Analytics Log. | object({…}) | | {} | +| [retention](variables.tf#L61) | Retention time in days for the logging bucket. | number | | 30 | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [id](outputs.tf#L17) | Fully qualified logging bucket id. | | + diff --git a/assets/modules-fabric/v26/logging-bucket/main.tf b/assets/modules-fabric/v26/logging-bucket/main.tf new file mode 100644 index 0000000..697eb43 --- /dev/null +++ b/assets/modules-fabric/v26/logging-bucket/main.tf @@ -0,0 +1,68 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_logging_project_bucket_config" "bucket" { + count = var.parent_type == "project" ? 1 : 0 + project = var.parent + location = var.location + retention_days = var.retention + bucket_id = var.id + description = var.description + enable_analytics = var.log_analytics.enable + + dynamic "cmek_settings" { + for_each = var.kms_key_name == null ? [] : [""] + content { + kms_key_name = var.kms_key_name + } + } +} + +resource "google_logging_folder_bucket_config" "bucket" { + count = var.parent_type == "folder" ? 1 : 0 + folder = var.parent + location = var.location + retention_days = var.retention + bucket_id = var.id + description = var.description +} + +resource "google_logging_linked_dataset" "dataset" { + count = var.log_analytics.dataset_link_id != null && var.parent_type == "project" ? 1 : 0 + link_id = var.log_analytics.dataset_link_id + parent = "projects/${google_logging_project_bucket_config.bucket[0].project}" + bucket = google_logging_project_bucket_config.bucket[0].id + location = var.location + description = var.log_analytics.description +} + +resource "google_logging_organization_bucket_config" "bucket" { + count = var.parent_type == "organization" ? 1 : 0 + organization = var.parent + location = var.location + retention_days = var.retention + bucket_id = var.id + description = var.description +} + +resource "google_logging_billing_account_bucket_config" "bucket" { + count = var.parent_type == "billing_account" ? 1 : 0 + billing_account = var.parent + location = var.location + retention_days = var.retention + bucket_id = var.id + description = var.description +} diff --git a/assets/modules-fabric/v26/logging-bucket/outputs.tf b/assets/modules-fabric/v26/logging-bucket/outputs.tf new file mode 100644 index 0000000..1ea9874 --- /dev/null +++ b/assets/modules-fabric/v26/logging-bucket/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "id" { + description = "Fully qualified logging bucket id." + value = try( + google_logging_project_bucket_config.bucket.0.id, + google_logging_folder_bucket_config.bucket.0.id, + google_logging_organization_bucket_config.bucket.0.id, + google_logging_billing_account_bucket_config.bucket.0.id, + ) +} diff --git a/assets/modules-fabric/v26/logging-bucket/variables.tf b/assets/modules-fabric/v26/logging-bucket/variables.tf new file mode 100644 index 0000000..1720ac4 --- /dev/null +++ b/assets/modules-fabric/v26/logging-bucket/variables.tf @@ -0,0 +1,65 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "description" { + description = "Human-readable description for the logging bucket." + type = string + default = null +} + +variable "id" { + description = "Name of the logging bucket." + type = string +} + +variable "kms_key_name" { + description = "To enable CMEK for a project logging bucket, set this field to a valid name. The associated service account requires cloudkms.cryptoKeyEncrypterDecrypter roles assigned for the key." + type = string + default = null +} + +variable "location" { + description = "Location of the bucket." + type = string + default = "global" +} + +variable "log_analytics" { + description = "Enable and configure Analytics Log." + type = object({ + enable = optional(bool, false) + dataset_link_id = optional(string) + description = optional(string, "Log Analytics Dataset") + }) + nullable = false + default = {} +} + +variable "parent" { + description = "ID of the parentresource containing the bucket in the format 'project_id' 'folders/folder_id', 'organizations/organization_id' or 'billing_account_id'." + type = string +} + +variable "parent_type" { + description = "Parent object type for the bucket (project, folder, organization, billing_account)." + type = string +} + +variable "retention" { + description = "Retention time in days for the logging bucket." + type = number + default = 30 +} diff --git a/assets/modules-fabric/v26/logging-bucket/versions.tf b/assets/modules-fabric/v26/logging-bucket/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/logging-bucket/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/ncc-spoke-ra/README.md b/assets/modules-fabric/v26/ncc-spoke-ra/README.md new file mode 100644 index 0000000..0b19340 --- /dev/null +++ b/assets/modules-fabric/v26/ncc-spoke-ra/README.md @@ -0,0 +1,153 @@ +# NCC Spoke RA Module + +This module allows management of NCC Spokes backed by Router Appliances. Network virtual appliances used as router appliances allow to connect an external network to Google Cloud by using a SD-WAN router or another appliance with BGP capabilities (_site-to-cloud_ connectivity). It is also possible to enable site-to-site data transfer, although this feature is not available in all regions, particularly not in EMEA. + +The module manages a hub (optionally), a spoke, and the corresponding Cloud Router and BGP sessions to the router appliance(s). + +## Examples + +### Simple hub & spoke + +```hcl +module "spoke-ra" { + source = "./fabric/modules/ncc-spoke-ra" + hub = { create = true, name = "ncc-hub" } + name = "spoke-ra" + project_id = "my-project" + region = "europe-west1" + router_appliances = [ + { + internal_ip = "10.0.0.3" + vm_self_link = "projects/my-project/zones/europe-west1-b/instances/router-app" + } + ] + router_config = { + asn = 65000 + ip_interface0 = "10.0.0.14" + ip_interface1 = "10.0.0.15" + peer_asn = 65001 + } + vpc_config = { + network_name = "my-vpc" + subnet_self_link = var.subnet.self_link + } +} +# tftest modules=1 resources=7 +``` + +### Two spokes + +```hcl +module "spoke-ra-a" { + source = "./fabric/modules/ncc-spoke-ra" + hub = { id = "projects/my-project/locations/global/hubs/ncc-hub" } + name = "spoke-ra-a" + project_id = "my-project" + region = "europe-west1" + router_appliances = [ + { + internal_ip = "10.0.0.3" + vm_self_link = "projects/my-project/zones/europe-west1-b/instances/router-app-a" + } + ] + router_config = { + asn = 65000 + ip_interface0 = "10.0.0.14" + ip_interface1 = "10.0.0.15" + peer_asn = 65001 + } + vpc_config = { + network_name = "my-vpc1" + subnet_self_link = "projects/my-project/regions/europe-west1/subnetworks/subnet" + } +} + +module "spoke-ra-b" { + source = "./fabric/modules/ncc-spoke-ra" + hub = { id = "projects/my-project/locations/global/hubs/ncc-hub" } + name = "spoke-ra-b" + project_id = "my-project" + region = "europe-west3" + router_appliances = [ + { + internal_ip = "10.1.0.5" + vm_self_link = "projects/my-project/zones/europe-west3-b/instances/router-app-b" + } + ] + router_config = { + asn = 65000 + ip_interface0 = "10.0.0.14" + ip_interface1 = "10.0.0.15" + peer_asn = 65002 + } + vpc_config = { + network_name = "my-vpc2" + subnet_self_link = "projects/my-project/regions/europe-west3/subnetworks/subnet" + } +} +# tftest modules=2 resources=12 +``` + +### Spoke with load-balanced router appliances + +```hcl +module "spoke-ra" { + source = "./fabric/modules/ncc-spoke-ra" + hub = { id = "projects/my-project/locations/global/hubs/ncc-hub" } + name = "spoke-ra" + project_id = "my-project" + region = "europe-west1" + router_appliances = [ + { + internal_ip = "10.0.0.3" + vm_self_link = "projects/my-project/zones/europe-west1-b/instances/router-app-a" + }, + { + internal_ip = "10.0.0.4" + vm_self_link = "projects/my-project/zones/europe-west1-c/instances/router-app-b" + } + ] + router_config = { + asn = 65000 + custom_advertise = { + all_subnets = true + ip_ranges = { + "10.10.0.0/24" = "peered-vpc" + } + } + ip_interface0 = "10.0.0.14" + ip_interface1 = "10.0.0.15" + peer_asn = 65001 + } + vpc_config = { + network_name = "my-vpc" + subnet_self_link = var.subnet.self_link + } +} +# tftest modules=1 resources=8 +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [hub](variables.tf#L23) | The NCC hub. You should either provide an existing hub id or a hub name if create is true. | object({…}) | ✓ | | +| [name](variables.tf#L37) | The name of the NCC spoke. | string | ✓ | | +| [project_id](variables.tf#L42) | The ID of the project where the NCC hub & spokes will be created. | string | ✓ | | +| [region](variables.tf#L47) | Region where the spoke is located. | string | ✓ | | +| [router_appliances](variables.tf#L52) | List of router appliances this spoke is associated with. | list(object({…})) | ✓ | | +| [router_config](variables.tf#L60) | Configuration of the Cloud Router. | object({…}) | ✓ | | +| [vpc_config](variables.tf#L76) | Network and subnetwork for the CR interfaces. | object({…}) | ✓ | | +| [data_transfer](variables.tf#L17) | Site-to-site data transfer feature, available only in some regions. | bool | | false | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [hub](outputs.tf#L17) | NCC hub resource (only if auto-created). | | +| [id](outputs.tf#L22) | Fully qualified hub id. | | +| [router](outputs.tf#L27) | Cloud Router resource. | | +| [spoke-ra](outputs.tf#L32) | NCC spoke resource. | | + + diff --git a/assets/modules-fabric/v26/ncc-spoke-ra/main.tf b/assets/modules-fabric/v26/ncc-spoke-ra/main.tf new file mode 100644 index 0000000..2ecaae6 --- /dev/null +++ b/assets/modules-fabric/v26/ncc-spoke-ra/main.tf @@ -0,0 +1,133 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + spoke_vms = [ + for ras in var.router_appliances : { + ip = ras.internal_ip + vm = ras.vm_self_link + vm_name = element( + split("/", ras.vm_self_link), length(split("/", ras.vm_self_link)) - 1 + ) + } + ] +} + +resource "google_network_connectivity_hub" "hub" { + count = var.hub.create ? 1 : 0 + project = var.project_id + name = var.hub.name + description = var.hub.description +} + +resource "google_network_connectivity_spoke" "spoke-ra" { + project = var.project_id + hub = try(google_network_connectivity_hub.hub[0].id, var.hub.id) + location = var.region + name = var.name + linked_router_appliance_instances { + dynamic "instances" { + for_each = var.router_appliances + content { + ip_address = instances.value["internal_ip"] + virtual_machine = instances.value["vm_self_link"] + } + } + site_to_site_data_transfer = var.data_transfer + } +} + +resource "google_compute_router" "cr" { + project = var.project_id + name = "${var.name}-cr" + network = var.vpc_config.network_name + region = var.region + bgp { + advertise_mode = ( + var.router_config.custom_advertise != null ? "CUSTOM" : "DEFAULT" + ) + advertised_groups = ( + try(var.router_config.custom_advertise.all_subnets, false) + ? ["ALL_SUBNETS"] : [] + ) + dynamic "advertised_ip_ranges" { + for_each = try(var.router_config.custom_advertise.ip_ranges, {}) + content { + description = advertised_ip_ranges.value + range = advertised_ip_ranges.key + } + } + asn = var.router_config.asn + keepalive_interval = try(var.router_config.keepalive, null) + } +} + +resource "google_compute_router_interface" "intf_0" { + project = var.project_id + name = "${google_compute_router.cr.name}-intf0" + router = google_compute_router.cr.name + region = var.region + subnetwork = var.vpc_config.subnet_self_link + private_ip_address = var.router_config.ip_interface0 +} + +resource "google_compute_router_interface" "intf_1" { + project = var.project_id + name = "${google_compute_router.cr.name}-intf1" + router = google_compute_router.cr.name + region = var.region + subnetwork = var.vpc_config.subnet_self_link + private_ip_address = var.router_config.ip_interface1 + redundant_interface = google_compute_router_interface.intf_0.name +} + +resource "google_compute_router_peer" "peer_0" { + for_each = { + for idx, entry in local.spoke_vms : idx => entry + } + project = var.project_id + name = "${google_compute_router.cr.name}-${each.value.vm_name}-peer1" + router = google_compute_router.cr.name + region = var.region + advertised_route_priority = var.router_config.routes_priority + interface = google_compute_router_interface.intf_0.name + peer_asn = var.router_config.peer_asn + peer_ip_address = each.value.ip + router_appliance_instance = each.value.vm + + depends_on = [ + google_network_connectivity_spoke.spoke-ra + ] +} + +resource "google_compute_router_peer" "peer_1" { + for_each = { + for idx, entry in local.spoke_vms : idx => entry + } + project = var.project_id + name = "${google_compute_router.cr.name}-${each.value.vm_name}-peer2" + router = google_compute_router.cr.name + region = var.region + advertised_route_priority = var.router_config.routes_priority + interface = google_compute_router_interface.intf_1.name + peer_asn = var.router_config.peer_asn + peer_ip_address = each.value.ip + router_appliance_instance = each.value.vm + + depends_on = [ + google_network_connectivity_spoke.spoke-ra + ] +} diff --git a/assets/modules-fabric/v26/ncc-spoke-ra/outputs.tf b/assets/modules-fabric/v26/ncc-spoke-ra/outputs.tf new file mode 100644 index 0000000..fd62c7f --- /dev/null +++ b/assets/modules-fabric/v26/ncc-spoke-ra/outputs.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "hub" { + description = "NCC hub resource (only if auto-created)." + value = try(google_network_connectivity_hub.hub.0, null) +} + +output "id" { + description = "Fully qualified hub id." + value = try(google_network_connectivity_hub.hub.0.id, null) +} + +output "router" { + description = "Cloud Router resource." + value = google_compute_router.cr +} + +output "spoke-ra" { + description = "NCC spoke resource." + value = google_network_connectivity_spoke.spoke-ra +} diff --git a/assets/modules-fabric/v26/ncc-spoke-ra/variables.tf b/assets/modules-fabric/v26/ncc-spoke-ra/variables.tf new file mode 100644 index 0000000..4b01ea1 --- /dev/null +++ b/assets/modules-fabric/v26/ncc-spoke-ra/variables.tf @@ -0,0 +1,82 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "data_transfer" { + description = "Site-to-site data transfer feature, available only in some regions." + type = bool + default = false +} + +variable "hub" { + description = "The NCC hub. You should either provide an existing hub id or a hub name if create is true." + type = object({ + create = optional(bool, false) + description = optional(string) + id = optional(string) + name = optional(string) + }) + validation { + condition = var.hub.create && var.hub.name != null || var.hub.create == false && var.hub.id != null + error_message = "Name is required for configuring new ncc hub while referencing existing hub requires id." + } +} + +variable "name" { + description = "The name of the NCC spoke." + type = string +} + +variable "project_id" { + description = "The ID of the project where the NCC hub & spokes will be created." + type = string +} + +variable "region" { + description = "Region where the spoke is located." + type = string +} + +variable "router_appliances" { + description = "List of router appliances this spoke is associated with." + type = list(object({ + internal_ip = string + vm_self_link = string + })) +} + +variable "router_config" { + description = "Configuration of the Cloud Router." + type = object({ + asn = number + custom_advertise = optional(object({ + all_subnets = bool + ip_ranges = map(string) + })) + ip_interface0 = string + ip_interface1 = string + keepalive = optional(number) + peer_asn = number + routes_priority = optional(number, 100) + }) +} + +variable "vpc_config" { + description = "Network and subnetwork for the CR interfaces." + type = object({ + network_name = string + subnet_self_link = string + }) +} diff --git a/assets/modules-fabric/v26/ncc-spoke-ra/versions.tf b/assets/modules-fabric/v26/ncc-spoke-ra/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/ncc-spoke-ra/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/net-address/README.md b/assets/modules-fabric/v26/net-address/README.md new file mode 100644 index 0000000..9f12235 --- /dev/null +++ b/assets/modules-fabric/v26/net-address/README.md @@ -0,0 +1,127 @@ +# Net Address Reservation Module + +This module allows reserving Compute Engine external, global, and internal addresses. + +## Examples + +### External and global addresses + +```hcl +module "addresses" { + source = "./fabric/modules/net-address" + project_id = var.project_id + external_addresses = { + one = { region = "europe-west1" } + two = { region = "europe-west2" } + } + global_addresses = ["app-1", "app-2"] +} +# tftest modules=1 resources=4 inventory=external.yaml +``` + +### Internal addresses + +```hcl +module "addresses" { + source = "./fabric/modules/net-address" + project_id = var.project_id + internal_addresses = { + ilb-1 = { + purpose = "SHARED_LOADBALANCER_VIP" + region = var.region + subnetwork = var.subnet.self_link + } + ilb-2 = { + address = "10.0.0.2" + region = var.region + subnetwork = var.subnet.self_link + } + } +} +# tftest modules=1 resources=2 inventory=internal.yaml +``` + +### PSA addresses + +```hcl +module "addresses" { + source = "./fabric/modules/net-address" + project_id = var.project_id + psa_addresses = { + cloudsql-mysql = { + address = "10.10.10.0" + network = var.vpc.self_link + prefix_length = 24 + } + } +} +# tftest modules=1 resources=1 inventory=psa.yaml +``` + +### PSC addresses + +```hcl +module "addresses" { + source = "./fabric/modules/net-address" + project_id = var.project_id + psc_addresses = { + one = { + address = null + network = var.vpc.self_link + } + two = { + address = "10.0.0.32" + network = var.vpc.self_link + } + } +} +# tftest modules=1 resources=2 inventory=psc.yaml +``` + +# IPSec Interconnect addresses + +```hcl +module "addresses" { + source = "./fabric/modules/net-address" + project_id = var.project_id + ipsec_interconnect_addresses = { + vpn-gw-range-1 = { + address = "10.255.255.0" + region = var.region + network = var.vpc.self_link + prefix_length = 29 + } + vpn-gw-range-2 = { + address = "10.255.255.8" + region = var.region + network = var.vpc.self_link + prefix_length = 29 + } + } +} +# tftest modules=1 resources=2 inventory=ipsec-interconnect.yaml +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [project_id](variables.tf#L65) | Project where the addresses will be created. | string | ✓ | | +| [external_addresses](variables.tf#L17) | Map of external addresses, keyed by name. | map(object({…})) | | {} | +| [global_addresses](variables.tf#L27) | List of global addresses to create. | list(string) | | [] | +| [internal_addresses](variables.tf#L33) | Map of internal addresses to create, keyed by name. | map(object({…})) | | {} | +| [ipsec_interconnect_addresses](variables.tf#L47) | Map of internal addresses used for HPA VPN over Cloud Interconnect. | map(object({…})) | | {} | +| [psa_addresses](variables.tf#L70) | Map of internal addresses used for Private Service Access. | map(object({…})) | | {} | +| [psc_addresses](variables.tf#L81) | Map of internal addresses used for Private Service Connect. | map(object({…})) | | {} | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [external_addresses](outputs.tf#L17) | Allocated external addresses. | | +| [global_addresses](outputs.tf#L25) | Allocated global external addresses. | | +| [internal_addresses](outputs.tf#L33) | Allocated internal addresses. | | +| [ipsec_interconnect_addresses](outputs.tf#L41) | Allocated internal addresses for HA VPN over Cloud Interconnect. | | +| [psa_addresses](outputs.tf#L49) | Allocated internal addresses for PSA endpoints. | | +| [psc_addresses](outputs.tf#L57) | Allocated internal addresses for PSC endpoints. | | + diff --git a/assets/modules-fabric/v26/net-address/main.tf b/assets/modules-fabric/v26/net-address/main.tf new file mode 100644 index 0000000..b09ba23 --- /dev/null +++ b/assets/modules-fabric/v26/net-address/main.tf @@ -0,0 +1,85 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_compute_global_address" "global" { + for_each = toset(var.global_addresses) + project = var.project_id + name = each.value +} + +resource "google_compute_address" "external" { + provider = google-beta + for_each = var.external_addresses + project = var.project_id + name = each.key + description = each.value.description + address_type = "EXTERNAL" + region = each.value.region + labels = each.value.labels +} + +resource "google_compute_address" "internal" { + provider = google-beta + for_each = var.internal_addresses + project = var.project_id + name = each.key + description = each.value.description + address_type = "INTERNAL" + region = each.value.region + subnetwork = each.value.subnetwork + address = each.value.address + network_tier = each.value.tier + purpose = each.value.purpose + labels = coalesce(each.value.labels, {}) +} + +resource "google_compute_global_address" "psc" { + for_each = var.psc_addresses + project = var.project_id + name = each.key + description = each.value.description + address = try(each.value.address, null) + address_type = "INTERNAL" + network = each.value.network + purpose = "PRIVATE_SERVICE_CONNECT" + # labels = lookup(var.internal_address_labels, each.key, {}) +} + +resource "google_compute_global_address" "psa" { + for_each = var.psa_addresses + project = var.project_id + name = each.key + description = each.value.description + address = each.value.address + address_type = "INTERNAL" + network = each.value.network + prefix_length = each.value.prefix_length + purpose = "VPC_PEERING" + # labels = lookup(var.internal_address_labels, each.key, {}) +} + +resource "google_compute_address" "ipsec_interconnect" { + for_each = var.ipsec_interconnect_addresses + project = var.project_id + name = each.key + description = each.value.description + address = each.value.address + address_type = "INTERNAL" + region = each.value.region + network = each.value.network + prefix_length = each.value.prefix_length + purpose = "IPSEC_INTERCONNECT" +} diff --git a/assets/modules-fabric/v26/net-address/outputs.tf b/assets/modules-fabric/v26/net-address/outputs.tf new file mode 100644 index 0000000..f4f47ef --- /dev/null +++ b/assets/modules-fabric/v26/net-address/outputs.tf @@ -0,0 +1,63 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "external_addresses" { + description = "Allocated external addresses." + value = { + for address in google_compute_address.external : + address.name => address + } +} + +output "global_addresses" { + description = "Allocated global external addresses." + value = { + for address in google_compute_global_address.global : + address.name => address + } +} + +output "internal_addresses" { + description = "Allocated internal addresses." + value = { + for address in google_compute_address.internal : + address.name => address + } +} + +output "ipsec_interconnect_addresses" { + description = "Allocated internal addresses for HA VPN over Cloud Interconnect." + value = { + for address in google_compute_address.ipsec_interconnect : + address.name => address + } +} + +output "psa_addresses" { + description = "Allocated internal addresses for PSA endpoints." + value = { + for address in google_compute_global_address.psa : + address.name => address + } +} + +output "psc_addresses" { + description = "Allocated internal addresses for PSC endpoints." + value = { + for address in google_compute_global_address.psc : + address.name => address + } +} \ No newline at end of file diff --git a/assets/modules-fabric/v26/net-address/variables.tf b/assets/modules-fabric/v26/net-address/variables.tf new file mode 100644 index 0000000..ebcfa5b --- /dev/null +++ b/assets/modules-fabric/v26/net-address/variables.tf @@ -0,0 +1,89 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "external_addresses" { + description = "Map of external addresses, keyed by name." + type = map(object({ + region = string + description = optional(string, "Terraform managed.") + labels = optional(map(string), {}) + })) + default = {} +} + +variable "global_addresses" { + description = "List of global addresses to create." + type = list(string) + default = [] +} + +variable "internal_addresses" { + description = "Map of internal addresses to create, keyed by name." + type = map(object({ + region = string + subnetwork = string + address = optional(string) + description = optional(string, "Terraform managed.") + labels = optional(map(string)) + purpose = optional(string) + tier = optional(string) + })) + default = {} +} + +variable "ipsec_interconnect_addresses" { + description = "Map of internal addresses used for HPA VPN over Cloud Interconnect." + type = map(object({ + region = string + address = string + network = string + description = optional(string, "Terraform managed.") + prefix_length = number + })) + default = {} +} + +# variable "internal_address_labels" { +# description = "Optional labels for internal addresses, keyed by address name." +# type = map(map(string)) +# default = {} +# } + +variable "project_id" { + description = "Project where the addresses will be created." + type = string +} + +variable "psa_addresses" { + description = "Map of internal addresses used for Private Service Access." + type = map(object({ + address = string + network = string + description = optional(string, "Terraform managed.") + prefix_length = number + })) + default = {} +} + +variable "psc_addresses" { + description = "Map of internal addresses used for Private Service Connect." + type = map(object({ + address = string + network = string + description = optional(string, "Terraform managed.") + })) + default = {} +} diff --git a/assets/modules-fabric/v26/net-address/versions.tf b/assets/modules-fabric/v26/net-address/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/net-address/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/net-cloudnat/README.md b/assets/modules-fabric/v26/net-cloudnat/README.md new file mode 100644 index 0000000..9e18ff6 --- /dev/null +++ b/assets/modules-fabric/v26/net-cloudnat/README.md @@ -0,0 +1,95 @@ +# Cloud NAT Module + +Simple Cloud NAT management, with optional router creation. + + +- [Basic Example](#basic-example) +- [Reserved IPs and custom rules](#reserved-ips-and-custom-rules) +- [Variables](#variables) +- [Outputs](#outputs) + + +## Basic Example + +```hcl +module "nat" { + source = "./fabric/modules/net-cloudnat" + project_id = "my-project" + region = "europe-west1" + name = "default" + router_network = "my-vpc" +} +# tftest modules=1 resources=2 +``` + +## Reserved IPs and custom rules + +```hcl +module "addresses" { + source = "./fabric/modules/net-address" + project_id = "my-project" + external_addresses = { + a1 = { region = "europe-west1" } + a2 = { region = "europe-west1" } + a3 = { region = "europe-west1" } + } +} + +module "nat" { + source = "./fabric/modules/net-cloudnat" + project_id = "my-project" + region = "europe-west1" + name = "nat" + router_network = "my-vpc" + addresses = [ + module.addresses.external_addresses["a1"].self_link, + module.addresses.external_addresses["a3"].self_link + ] + + config_port_allocation = { + enable_endpoint_independent_mapping = false + } + + rules = [ + { + description = "rule1" + match = "destination.ip == '8.8.8.8'" + source_ips = [ + module.addresses.external_addresses["a2"].self_link + ] + } + ] +} +# tftest modules=2 resources=5 inventory=rules.yaml +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [name](variables.tf#L63) | Name of the Cloud NAT resource. | string | ✓ | | +| [project_id](variables.tf#L68) | Project where resources will be created. | string | ✓ | | +| [region](variables.tf#L73) | Region where resources will be created. | string | ✓ | | +| [addresses](variables.tf#L17) | Optional list of external address self links. | list(string) | | [] | +| [config_port_allocation](variables.tf#L23) | Configuration for how to assign ports to virtual machines. min_ports_per_vm and max_ports_per_vm have no effect unless enable_dynamic_port_allocation is set to 'true'. | object({…}) | | {} | +| [config_source_subnets](variables.tf#L39) | Subnetwork configuration (ALL_SUBNETWORKS_ALL_IP_RANGES, ALL_SUBNETWORKS_ALL_PRIMARY_IP_RANGES, LIST_OF_SUBNETWORKS). | string | | "ALL_SUBNETWORKS_ALL_IP_RANGES" | +| [config_timeouts](variables.tf#L45) | Timeout configurations. | object({…}) | | {} | +| [logging_filter](variables.tf#L57) | Enables logging if not null, value is one of 'ERRORS_ONLY', 'TRANSLATIONS_ONLY', 'ALL'. | string | | null | +| [router_asn](variables.tf#L78) | Router ASN used for auto-created router. | number | | null | +| [router_create](variables.tf#L84) | Create router. | bool | | true | +| [router_name](variables.tf#L90) | Router name, leave blank if router will be created to use auto generated name. | string | | null | +| [router_network](variables.tf#L96) | Name of the VPC used for auto-created router. | string | | null | +| [rules](variables.tf#L102) | List of rules associated with this NAT. | list(object({…})) | | [] | +| [subnetworks](variables.tf#L113) | Subnetworks to NAT, only used when config_source_subnets equals LIST_OF_SUBNETWORKS. | list(object({…})) | | [] | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [id](outputs.tf#L17) | Fully qualified NAT (router) id. | | +| [name](outputs.tf#L22) | Name of the Cloud NAT. | | +| [nat_ip_allocate_option](outputs.tf#L27) | NAT IP allocation mode. | | +| [region](outputs.tf#L32) | Cloud NAT region. | | +| [router](outputs.tf#L37) | Cloud NAT router resources (if auto created). | | +| [router_name](outputs.tf#L46) | Cloud NAT router name. | | + diff --git a/assets/modules-fabric/v26/net-cloudnat/main.tf b/assets/modules-fabric/v26/net-cloudnat/main.tf new file mode 100644 index 0000000..6d1da01 --- /dev/null +++ b/assets/modules-fabric/v26/net-cloudnat/main.tf @@ -0,0 +1,83 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + router_name = ( + var.router_create + ? try(google_compute_router.router[0].name, null) + : var.router_name + ) +} + +resource "google_compute_router" "router" { + count = var.router_create ? 1 : 0 + name = var.router_name == null ? "${var.name}-nat" : var.router_name + project = var.project_id + region = var.region + network = var.router_network + + dynamic "bgp" { + for_each = var.router_asn == null ? [] : [1] + content { + asn = var.router_asn + } + } +} + +resource "google_compute_router_nat" "nat" { + project = var.project_id + region = var.region + name = var.name + router = local.router_name + nat_ips = var.addresses + nat_ip_allocate_option = length(var.addresses) > 0 ? "MANUAL_ONLY" : "AUTO_ONLY" + source_subnetwork_ip_ranges_to_nat = var.config_source_subnets + icmp_idle_timeout_sec = var.config_timeouts.icmp + udp_idle_timeout_sec = var.config_timeouts.udp + tcp_established_idle_timeout_sec = var.config_timeouts.tcp_established + tcp_transitory_idle_timeout_sec = var.config_timeouts.tcp_transitory + enable_endpoint_independent_mapping = var.config_port_allocation.enable_endpoint_independent_mapping + enable_dynamic_port_allocation = var.config_port_allocation.enable_dynamic_port_allocation + min_ports_per_vm = var.config_port_allocation.min_ports_per_vm + max_ports_per_vm = var.config_port_allocation.max_ports_per_vm + + log_config { + enable = var.logging_filter == null ? false : true + filter = var.logging_filter == null ? "ALL" : var.logging_filter + } + + dynamic "subnetwork" { + for_each = var.subnetworks + content { + name = subnetwork.value.self_link + source_ip_ranges_to_nat = subnetwork.value.config_source_ranges + secondary_ip_range_names = subnetwork.value.secondary_ranges + } + } + + dynamic "rules" { + for_each = { for i, r in var.rules : i => r } + content { + rule_number = rules.key + description = rules.value.description + match = rules.value.match + action { + source_nat_active_ips = rules.value.source_ips + } + } + } +} + diff --git a/assets/modules-fabric/v26/net-cloudnat/outputs.tf b/assets/modules-fabric/v26/net-cloudnat/outputs.tf new file mode 100644 index 0000000..62f6afa --- /dev/null +++ b/assets/modules-fabric/v26/net-cloudnat/outputs.tf @@ -0,0 +1,49 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "id" { + description = "Fully qualified NAT (router) id." + value = google_compute_router_nat.nat.id +} + +output "name" { + description = "Name of the Cloud NAT." + value = google_compute_router_nat.nat.name +} + +output "nat_ip_allocate_option" { + description = "NAT IP allocation mode." + value = google_compute_router_nat.nat.nat_ip_allocate_option +} + +output "region" { + description = "Cloud NAT region." + value = google_compute_router_nat.nat.region +} + +output "router" { + description = "Cloud NAT router resources (if auto created)." + value = ( + var.router_create + ? try(google_compute_router.router[0], null) + : null + ) +} + +output "router_name" { + description = "Cloud NAT router name." + value = local.router_name +} diff --git a/assets/modules-fabric/v26/net-cloudnat/variables.tf b/assets/modules-fabric/v26/net-cloudnat/variables.tf new file mode 100644 index 0000000..ed2649a --- /dev/null +++ b/assets/modules-fabric/v26/net-cloudnat/variables.tf @@ -0,0 +1,121 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "addresses" { + description = "Optional list of external address self links." + type = list(string) + default = [] +} + +variable "config_port_allocation" { + description = "Configuration for how to assign ports to virtual machines. min_ports_per_vm and max_ports_per_vm have no effect unless enable_dynamic_port_allocation is set to 'true'." + type = object({ + enable_endpoint_independent_mapping = optional(bool, true) + enable_dynamic_port_allocation = optional(bool, false) + min_ports_per_vm = optional(number, 64) + max_ports_per_vm = optional(number, 65536) + }) + default = {} + nullable = false + validation { + condition = var.config_port_allocation.enable_dynamic_port_allocation ? var.config_port_allocation.enable_endpoint_independent_mapping == false : true + error_message = "You must set enable_endpoint_independent_mapping to false to set enable_dynamic_port_allocation to true." + } +} + +variable "config_source_subnets" { + description = "Subnetwork configuration (ALL_SUBNETWORKS_ALL_IP_RANGES, ALL_SUBNETWORKS_ALL_PRIMARY_IP_RANGES, LIST_OF_SUBNETWORKS)." + type = string + default = "ALL_SUBNETWORKS_ALL_IP_RANGES" +} + +variable "config_timeouts" { + description = "Timeout configurations." + type = object({ + icmp = optional(number, 30) + tcp_established = optional(number, 1200) + tcp_transitory = optional(number, 30) + udp = optional(number, 30) + }) + default = {} + nullable = false +} + +variable "logging_filter" { + description = "Enables logging if not null, value is one of 'ERRORS_ONLY', 'TRANSLATIONS_ONLY', 'ALL'." + type = string + default = null +} + +variable "name" { + description = "Name of the Cloud NAT resource." + type = string +} + +variable "project_id" { + description = "Project where resources will be created." + type = string +} + +variable "region" { + description = "Region where resources will be created." + type = string +} + +variable "router_asn" { + description = "Router ASN used for auto-created router." + type = number + default = null +} + +variable "router_create" { + description = "Create router." + type = bool + default = true +} + +variable "router_name" { + description = "Router name, leave blank if router will be created to use auto generated name." + type = string + default = null +} + +variable "router_network" { + description = "Name of the VPC used for auto-created router." + type = string + default = null +} + +variable "rules" { + description = "List of rules associated with this NAT." + type = list(object({ + description = optional(string), + match = string + source_ips = list(string) + })) + default = [] + nullable = false +} + +variable "subnetworks" { + description = "Subnetworks to NAT, only used when config_source_subnets equals LIST_OF_SUBNETWORKS." + type = list(object({ + self_link = string, + config_source_ranges = list(string) + secondary_ranges = list(string) + })) + default = [] +} diff --git a/assets/modules-fabric/v26/net-cloudnat/versions.tf b/assets/modules-fabric/v26/net-cloudnat/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/net-cloudnat/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/net-firewall-policy/README.md b/assets/modules-fabric/v26/net-firewall-policy/README.md new file mode 100644 index 0000000..8a71d6b --- /dev/null +++ b/assets/modules-fabric/v26/net-firewall-policy/README.md @@ -0,0 +1,252 @@ +# Firewall Policies + +This module allows creation and management of two different firewall policy types: + +- a [hierarchical policy](https://cloud.google.com/firewall/docs/firewall-policies) in a folder or organization, or +- a [global](https://cloud.google.com/vpc/docs/network-firewall-policies) or [regional](https://cloud.google.com/vpc/docs/regional-firewall-policies) network policy + +The module also manages policy rules via code or a factory, and optional policy attachments. The interface deviates slightly from the [`net-vpc-firewall`](../net-vpc-firewall/) module since the underlying resources and API objects are different. + +The module also makes fewer assumptions about implicit defaults, only using one to set `match.layer4_configs` to `[{ protocol = "all" }]` if no explicit set of protocols and ports has been specified. + + +- [Examples](#examples) + - [Hierarchical Policy](#hierarchical-policy) + - [Global Network policy](#global-network-policy) + - [Regional Network policy](#regional-network-policy) + - [Factory](#factory) +- [Variables](#variables) +- [Outputs](#outputs) + + +## Examples + +### Hierarchical Policy + +```hcl +module "firewall-policy" { + source = "./fabric/modules/net-firewall-policy" + name = "test-1" + parent_id = "folders/1234567890" + attachments = { + test = "folders/4567890123" + } + egress_rules = { + smtp = { + priority = 900 + match = { + destination_ranges = ["0.0.0.0/0"] + layer4_configs = [{ protocol = "tcp", ports = ["25"] }] + } + } + } + ingress_rules = { + icmp = { + priority = 1000 + match = { + source_ranges = ["0.0.0.0/0"] + layer4_configs = [{ protocol = "icmp" }] + } + } + mgmt = { + priority = 1001 + match = { + source_ranges = ["10.1.1.0/24"] + } + } + ssh = { + priority = 1002 + match = { + source_ranges = ["10.0.0.0/8"] + # source_tags = ["tagValues/123456"] + layer4_configs = [{ protocol = "tcp", ports = ["22"] }] + } + } + } +} +# tftest modules=1 resources=6 inventory=hierarchical.yaml +``` + +### Global Network policy + +```hcl +module "vpc" { + source = "./fabric/modules/net-vpc" + project_id = "my-project" + name = "my-network" +} + +module "firewall-policy" { + source = "./fabric/modules/net-firewall-policy" + name = "test-1" + parent_id = "my-project" + region = "global" + attachments = { + my-vpc = module.vpc.self_link + } + egress_rules = { + smtp = { + priority = 900 + match = { + destination_ranges = ["0.0.0.0/0"] + layer4_configs = [{ protocol = "tcp", ports = ["25"] }] + } + } + } + ingress_rules = { + icmp = { + priority = 1000 + match = { + source_ranges = ["0.0.0.0/0"] + layer4_configs = [{ protocol = "icmp" }] + } + } + mgmt = { + priority = 1001 + match = { + source_ranges = ["10.1.1.0/24"] + } + } + ssh = { + priority = 1002 + match = { + source_ranges = ["10.0.0.0/8"] + # source_tags = ["tagValues/123456"] + layer4_configs = [{ protocol = "tcp", ports = ["22"] }] + } + } + } +} +# tftest modules=2 resources=9 inventory=global-net.yaml +``` + +### Regional Network policy + +```hcl +module "vpc" { + source = "./fabric/modules/net-vpc" + project_id = "my-project" + name = "my-network" +} + +module "firewall-policy" { + source = "./fabric/modules/net-firewall-policy" + name = "test-1" + parent_id = "my-project" + region = "europe-west8" + attachments = { + my-vpc = module.vpc.self_link + } + egress_rules = { + smtp = { + priority = 900 + match = { + destination_ranges = ["0.0.0.0/0"] + layer4_configs = [{ protocol = "tcp", ports = ["25"] }] + } + } + } + ingress_rules = { + icmp = { + priority = 1000 + match = { + source_ranges = ["0.0.0.0/0"] + layer4_configs = [{ protocol = "icmp" }] + } + } + } +} +# tftest modules=2 resources=7 inventory=regional-net.yaml +``` + +### Factory + +Similarly to other modules, a rules factory (see [Resource Factories](../../blueprints/factories/)) is also included here to allow route management via descriptive configuration files. + +Factory configuration is via three optional attributes in the `rules_factory_config` variable: + +- `cidr_file_path` specifying the path to a mapping of logical names to CIDR ranges, used for source and destination ranges in rules when available +- `egress_rules_file_path` specifying the path to the egress rules file +- `ingress_rules_file_path` specifying the path to the ingress rules file + +Factory rules are merged with rules declared in code, with the latter taking precedence where both use the same key. + +This is an example of a simple factory: + +```hcl +module "firewall-policy" { + source = "./fabric/modules/net-firewall-policy" + name = "test-1" + parent_id = "folders/1234567890" + attachments = { + test = "folders/4567890123" + } + ingress_rules = { + ssh = { + priority = 1002 + match = { + source_ranges = ["10.0.0.0/8"] + layer4_configs = [{ protocol = "tcp", ports = ["22"] }] + } + } + } + rules_factory_config = { + cidr_file_path = "configs/cidrs.yaml" + egress_rules_file_path = "configs/egress.yaml" + ingress_rules_file_path = "configs/ingress.yaml" + } +} +# tftest modules=1 resources=5 files=cidrs,egress,ingress inventory=factory.yaml +``` + +```yaml +# tftest-file id=cidrs path=configs/cidrs.yaml +rfc1918: + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/24 +``` + +```yaml +# tftest-file id=egress path=configs/egress.yaml +smtp: + priority: 900 + match: + destination_ranges: + - rfc1918 + layer4_configs: + - protocol: tcp + ports: + - 25 +``` + +```yaml +# tftest-file id=ingress path=configs/ingress.yaml +icmp: + priority: 1000 + match: + source_ranges: + - 10.0.0.0/8 + layer4_configs: + - protocol: icmp +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [name](variables.tf#L100) | Policy name. | string | ✓ | | +| [parent_id](variables.tf#L106) | Parent node where the policy will be created, `folders/nnn` or `organizations/nnn` for hierarchical policy, project id for a network policy. | string | ✓ | | +| [attachments](variables.tf#L17) | Ids of the resources to which this policy will be attached, in descriptive name => self link format. Specify folders or organization for hierarchical policy, VPCs for network policy. | map(string) | | {} | +| [description](variables.tf#L24) | Policy description. | string | | null | +| [egress_rules](variables.tf#L30) | List of egress rule definitions, action can be 'allow', 'deny', 'goto_next'. The match.layer4configs map is in protocol => optional [ports] format. | map(object({…})) | | {} | +| [ingress_rules](variables.tf#L65) | List of ingress rule definitions, action can be 'allow', 'deny', 'goto_next'. | map(object({…})) | | {} | +| [region](variables.tf#L112) | Policy region. Leave null for hierarchical policy, set to 'global' for a global network policy. | string | | null | +| [rules_factory_config](variables.tf#L118) | Configuration for the optional rules factory. | object({…}) | | {} | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [id](outputs.tf#L17) | Fully qualified firewall policy id. | | + diff --git a/assets/modules-fabric/v26/net-firewall-policy/factory.tf b/assets/modules-fabric/v26/net-firewall-policy/factory.tf new file mode 100644 index 0000000..4c22775 --- /dev/null +++ b/assets/modules-fabric/v26/net-firewall-policy/factory.tf @@ -0,0 +1,115 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + _factory_egress_rules = try( + yamldecode(file(var.rules_factory_config.egress_rules_file_path)), {} + ) + _factory_ingress_rules = try( + yamldecode(file(var.rules_factory_config.ingress_rules_file_path)), {} + ) + factory_cidrs = try( + yamldecode(file(var.rules_factory_config.cidr_file_path)), {} + ) + factory_egress_rules = { + for k, v in local._factory_egress_rules : "egress/${k}" => { + direction = "EGRESS" + name = k + priority = v.priority + action = lookup(v, "action", "deny") + description = lookup(v, "description", null) + disabled = lookup(v, "disabled", false) + enable_logging = lookup(v, "enable_logging", null) + target_service_accounts = lookup(v, "target_service_accounts", null) + target_tags = lookup(v, "target_tags", null) + match = { + address_groups = lookup(v.match, "address_groups", null) + fqdns = lookup(v.match, "fqdns", null) + region_codes = lookup(v.match, "region_codes", null) + threat_intelligences = lookup(v.match, "threat_intelligences", null) + destination_ranges = ( + lookup(v.match, "destination_ranges", null) == null + ? null + : flatten([ + for r in v.match.destination_ranges : + try(local.factory_cidrs[r], r) + ]) + ) + source_ranges = ( + lookup(v.match, "source_ranges", null) == null + ? null + : flatten([ + for r in v.match.source_ranges : + try(local.factory_cidrs[r], r) + ]) + ) + source_tags = lookup(v.match, "source_tags", null) + layer4_configs = ( + lookup(v.match, "layer4_configs", null) == null + ? [{ protocol = "all", ports = null }] + : [ + for c in v.match.layer4_configs : + merge({ protocol = "all", ports = null }, c) + ] + ) + } + } + } + factory_ingress_rules = { + for k, v in local._factory_ingress_rules : "ingress/${k}" => { + direction = "INGRESS" + name = k + priority = v.priority + action = lookup(v, "action", "allow") + description = lookup(v, "description", null) + disabled = lookup(v, "disabled", false) + enable_logging = lookup(v, "enable_logging", null) + target_service_accounts = lookup(v, "target_service_accounts", null) + target_tags = lookup(v, "target_tags", null) + match = { + address_groups = lookup(v.match, "address_groups", null) + fqdns = lookup(v.match, "fqdns", null) + region_codes = lookup(v.match, "region_codes", null) + threat_intelligences = lookup(v.match, "threat_intelligences", null) + destination_ranges = ( + lookup(v.match, "destination_ranges", null) == null + ? null + : flatten([ + for r in v.match.destination_ranges : + try(local.factory_cidrs[r], r) + ]) + ) + source_ranges = ( + lookup(v.match, "source_ranges", null) == null + ? null + : flatten([ + for r in v.match.source_ranges : + try(local.factory_cidrs[r], r) + ]) + ) + source_tags = lookup(v.match, "source_tags", null) + layer4_configs = ( + lookup(v.match, "layer4_configs", null) == null + ? [{ protocol = "all", ports = null }] + : [ + for c in v.match.layer4_configs : + merge({ protocol = "all", ports = null }, c) + ] + ) + } + } + } +} diff --git a/assets/modules-fabric/v26/net-firewall-policy/hierarchical.tf b/assets/modules-fabric/v26/net-firewall-policy/hierarchical.tf new file mode 100644 index 0000000..6c1decd --- /dev/null +++ b/assets/modules-fabric/v26/net-firewall-policy/hierarchical.tf @@ -0,0 +1,102 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_compute_firewall_policy" "hierarchical" { + count = local.use_hierarchical ? 1 : 0 + parent = var.parent_id + short_name = var.name + description = var.description +} + +resource "google_compute_firewall_policy_association" "hierarchical" { + for_each = local.use_hierarchical ? var.attachments : {} + name = "${var.name}-${each.key}" + attachment_target = each.value + firewall_policy = google_compute_firewall_policy.hierarchical.0.name +} + +output "foo" { + value = { + rules = local.rules + cidrs = local.factory_cidrs + } +} + +resource "google_compute_firewall_policy_rule" "hierarchical" { + # Terraform's type system barfs in the condition if we use the locals map + for_each = toset( + local.use_hierarchical ? keys(local.rules) : [] + ) + firewall_policy = google_compute_firewall_policy.hierarchical.0.name + action = local.rules[each.key].action + description = local.rules[each.key].description + direction = local.rules[each.key].direction + disabled = local.rules[each.key].disabled + enable_logging = local.rules[each.key].enable_logging + priority = local.rules[each.key].priority + target_service_accounts = local.rules[each.key].target_service_accounts + match { + dest_ip_ranges = local.rules[each.key].match.destination_ranges + src_ip_ranges = local.rules[each.key].match.source_ranges + dest_address_groups = ( + local.rules[each.key].direction == "EGRESS" + ? local.rules[each.key].match.address_groups + : null + ) + dest_fqdns = ( + local.rules[each.key].direction == "EGRESS" + ? local.rules[each.key].match.fqdns + : null + ) + dest_region_codes = ( + local.rules[each.key].direction == "EGRESS" + ? local.rules[each.key].match.region_codes + : null + ) + dest_threat_intelligences = ( + local.rules[each.key].direction == "EGRESS" + ? local.rules[each.key].match.threat_intelligences + : null + ) + src_address_groups = ( + local.rules[each.key].direction == "INGRESS" + ? local.rules[each.key].match.address_groups + : null + ) + src_fqdns = ( + local.rules[each.key].direction == "INGRESS" + ? local.rules[each.key].match.fqdns + : null + ) + src_region_codes = ( + local.rules[each.key].direction == "INGRESS" + ? local.rules[each.key].match.region_codes + : null + ) + src_threat_intelligences = ( + local.rules[each.key].direction == "INGRESS" + ? local.rules[each.key].match.threat_intelligences + : null + ) + dynamic "layer4_configs" { + for_each = local.rules[each.key].match.layer4_configs + content { + ip_protocol = layer4_configs.value.protocol + ports = layer4_configs.value.ports + } + } + } +} diff --git a/assets/modules-fabric/v26/net-firewall-policy/main.tf b/assets/modules-fabric/v26/net-firewall-policy/main.tf new file mode 100644 index 0000000..093c8bf --- /dev/null +++ b/assets/modules-fabric/v26/net-firewall-policy/main.tf @@ -0,0 +1,33 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + _rules_egress = { + for name, rule in merge(var.egress_rules) : + "egress/${name}" => merge(rule, { name = name, direction = "EGRESS" }) + } + _rules_ingress = { + for name, rule in merge(var.ingress_rules) : + "ingress/${name}" => merge(rule, { name = name, direction = "INGRESS" }) + } + rules = merge( + local.factory_egress_rules, local.factory_ingress_rules, + local._rules_egress, local._rules_ingress + ) + # do not depend on the parent id as that might be dynamic and prevent count + use_hierarchical = var.region == null + use_regional = !local.use_hierarchical && var.region != "global" +} diff --git a/assets/modules-fabric/v26/net-firewall-policy/net-global.tf b/assets/modules-fabric/v26/net-firewall-policy/net-global.tf new file mode 100644 index 0000000..685c860 --- /dev/null +++ b/assets/modules-fabric/v26/net-firewall-policy/net-global.tf @@ -0,0 +1,118 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_compute_network_firewall_policy" "net-global" { + count = !local.use_hierarchical && !local.use_regional ? 1 : 0 + project = var.parent_id + name = var.name + description = var.description +} + +resource "google_compute_network_firewall_policy_association" "net-global" { + for_each = ( + !local.use_hierarchical && !local.use_regional ? var.attachments : {} + ) + project = var.parent_id + name = "${var.name}-${each.key}" + attachment_target = each.value + firewall_policy = google_compute_network_firewall_policy.net-global.0.name +} + +resource "google_compute_network_firewall_policy_rule" "net-global" { + # Terraform's type system barfs in the condition if we use the locals map + for_each = toset( + !local.use_hierarchical && !local.use_regional + ? keys(local.rules) + : [] + ) + project = var.parent_id + firewall_policy = google_compute_network_firewall_policy.net-global.0.name + rule_name = local.rules[each.key].name + action = local.rules[each.key].action + description = local.rules[each.key].description + direction = local.rules[each.key].direction + disabled = local.rules[each.key].disabled + enable_logging = local.rules[each.key].enable_logging + priority = local.rules[each.key].priority + target_service_accounts = local.rules[each.key].target_service_accounts + match { + dest_ip_ranges = local.rules[each.key].match.destination_ranges + src_ip_ranges = local.rules[each.key].match.source_ranges + dest_address_groups = ( + local.rules[each.key].direction == "EGRESS" + ? local.rules[each.key].match.address_groups + : null + ) + dest_fqdns = ( + local.rules[each.key].direction == "EGRESS" + ? local.rules[each.key].match.fqdns + : null + ) + dest_region_codes = ( + local.rules[each.key].direction == "EGRESS" + ? local.rules[each.key].match.region_codes + : null + ) + dest_threat_intelligences = ( + local.rules[each.key].direction == "EGRESS" + ? local.rules[each.key].match.threat_intelligences + : null + ) + src_address_groups = ( + local.rules[each.key].direction == "INGRESS" + ? local.rules[each.key].match.address_groups + : null + ) + src_fqdns = ( + local.rules[each.key].direction == "INGRESS" + ? local.rules[each.key].match.fqdns + : null + ) + src_region_codes = ( + local.rules[each.key].direction == "INGRESS" + ? local.rules[each.key].match.region_codes + : null + ) + src_threat_intelligences = ( + local.rules[each.key].direction == "INGRESS" + ? local.rules[each.key].match.threat_intelligences + : null + ) + dynamic "layer4_configs" { + for_each = local.rules[each.key].match.layer4_configs + content { + ip_protocol = layer4_configs.value.protocol + ports = layer4_configs.value.ports + } + } + dynamic "src_secure_tags" { + for_each = toset(coalesce(local.rules[each.key].match.source_tags, [])) + content { + name = src_secure_tags.key + } + } + } + dynamic "target_secure_tags" { + for_each = toset( + local.rules[each.key].target_tags == null + ? [] + : local.rules[each.key].target_tags + ) + content { + name = target_secure_tags.value + } + } +} diff --git a/assets/modules-fabric/v26/net-firewall-policy/net-regional.tf b/assets/modules-fabric/v26/net-firewall-policy/net-regional.tf new file mode 100644 index 0000000..a77b30f --- /dev/null +++ b/assets/modules-fabric/v26/net-firewall-policy/net-regional.tf @@ -0,0 +1,121 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_compute_region_network_firewall_policy" "net-regional" { + count = !local.use_hierarchical && local.use_regional ? 1 : 0 + project = var.parent_id + name = var.name + description = var.description + region = var.region +} + +resource "google_compute_region_network_firewall_policy_association" "net-regional" { + for_each = ( + !local.use_hierarchical && local.use_regional ? var.attachments : {} + ) + project = var.parent_id + region = var.region + name = "${var.name}-${each.key}" + attachment_target = each.value + firewall_policy = google_compute_region_network_firewall_policy.net-regional.0.name +} + +resource "google_compute_region_network_firewall_policy_rule" "net-regional" { + # Terraform's type system barfs in the condition if we use the locals map + for_each = toset( + !local.use_hierarchical && local.use_regional + ? keys(local.rules) + : [] + ) + project = var.parent_id + region = var.region + firewall_policy = google_compute_region_network_firewall_policy.net-regional.0.name + rule_name = local.rules[each.key].name + action = local.rules[each.key].action + description = local.rules[each.key].description + direction = local.rules[each.key].direction + disabled = local.rules[each.key].disabled + enable_logging = local.rules[each.key].enable_logging + priority = local.rules[each.key].priority + target_service_accounts = local.rules[each.key].target_service_accounts + match { + dest_ip_ranges = local.rules[each.key].match.destination_ranges + src_ip_ranges = local.rules[each.key].match.source_ranges + dest_address_groups = ( + local.rules[each.key].direction == "EGRESS" + ? local.rules[each.key].match.address_groups + : null + ) + dest_fqdns = ( + local.rules[each.key].direction == "EGRESS" + ? local.rules[each.key].match.fqdns + : null + ) + dest_region_codes = ( + local.rules[each.key].direction == "EGRESS" + ? local.rules[each.key].match.region_codes + : null + ) + dest_threat_intelligences = ( + local.rules[each.key].direction == "EGRESS" + ? local.rules[each.key].match.threat_intelligences + : null + ) + src_address_groups = ( + local.rules[each.key].direction == "INGRESS" + ? local.rules[each.key].match.address_groups + : null + ) + src_fqdns = ( + local.rules[each.key].direction == "INGRESS" + ? local.rules[each.key].match.fqdns + : null + ) + src_region_codes = ( + local.rules[each.key].direction == "INGRESS" + ? local.rules[each.key].match.region_codes + : null + ) + src_threat_intelligences = ( + local.rules[each.key].direction == "INGRESS" + ? local.rules[each.key].match.threat_intelligences + : null + ) + dynamic "layer4_configs" { + for_each = local.rules[each.key].match.layer4_configs + content { + ip_protocol = layer4_configs.value.protocol + ports = layer4_configs.value.ports + } + } + dynamic "src_secure_tags" { + for_each = toset(coalesce(local.rules[each.key].match.source_tags, [])) + content { + name = src_secure_tags.key + } + } + } + dynamic "target_secure_tags" { + for_each = toset( + local.rules[each.key].target_tags == null + ? [] + : local.rules[each.key].target_tags + ) + content { + name = target_secure_tags.value + } + } +} diff --git a/assets/modules-fabric/v26/net-firewall-policy/outputs.tf b/assets/modules-fabric/v26/net-firewall-policy/outputs.tf new file mode 100644 index 0000000..0f16d2a --- /dev/null +++ b/assets/modules-fabric/v26/net-firewall-policy/outputs.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "id" { + description = "Fully qualified firewall policy id." + value = ( + local.use_hierarchical + ? google_compute_firewall_policy.hierarchical.0.id + : ( + local.use_regional + ? google_compute_region_network_firewall_policy.net-regional.0.id + : google_compute_network_firewall_policy.net-global.0.id + ) + ) +} diff --git a/assets/modules-fabric/v26/net-firewall-policy/variables.tf b/assets/modules-fabric/v26/net-firewall-policy/variables.tf new file mode 100644 index 0000000..891c0af --- /dev/null +++ b/assets/modules-fabric/v26/net-firewall-policy/variables.tf @@ -0,0 +1,127 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "attachments" { + description = "Ids of the resources to which this policy will be attached, in descriptive name => self link format. Specify folders or organization for hierarchical policy, VPCs for network policy." + type = map(string) + default = {} + nullable = false +} + +variable "description" { + description = "Policy description." + type = string + default = null +} + +variable "egress_rules" { + description = "List of egress rule definitions, action can be 'allow', 'deny', 'goto_next'. The match.layer4configs map is in protocol => optional [ports] format." + type = map(object({ + priority = number + action = optional(string, "deny") + description = optional(string) + disabled = optional(bool, false) + enable_logging = optional(bool) + target_service_accounts = optional(list(string)) + target_tags = optional(list(string)) + match = object({ + address_groups = optional(list(string)) + fqdns = optional(list(string)) + region_codes = optional(list(string)) + threat_intelligences = optional(list(string)) + destination_ranges = optional(list(string)) + source_ranges = optional(list(string)) + source_tags = optional(list(string)) + layer4_configs = optional(list(object({ + protocol = optional(string, "all") + ports = optional(list(string)) + })), [{}]) + }) + })) + default = {} + nullable = false + validation { + condition = alltrue([ + for k, v in var.egress_rules : + contains(["allow", "deny", "goto_next"], v.action) + ]) + error_message = "Action can only be one of 'allow', 'deny', 'goto_next'." + } +} + +variable "ingress_rules" { + description = "List of ingress rule definitions, action can be 'allow', 'deny', 'goto_next'." + type = map(object({ + priority = number + action = optional(string, "allow") + description = optional(string) + disabled = optional(bool, false) + enable_logging = optional(bool) + target_service_accounts = optional(list(string)) + target_tags = optional(list(string)) + match = object({ + address_groups = optional(list(string)) + fqdns = optional(list(string)) + region_codes = optional(list(string)) + threat_intelligences = optional(list(string)) + destination_ranges = optional(list(string)) + source_ranges = optional(list(string)) + source_tags = optional(list(string)) + layer4_configs = optional(list(object({ + protocol = optional(string, "all") + ports = optional(list(string)) + })), [{}]) + }) + })) + default = {} + nullable = false + validation { + condition = alltrue([ + for k, v in var.ingress_rules : + contains(["allow", "deny", "goto_next"], v.action) + ]) + error_message = "Action can only be one of 'allow', 'deny', 'goto_next'." + } +} + +variable "name" { + description = "Policy name." + type = string + nullable = false +} + +variable "parent_id" { + description = "Parent node where the policy will be created, `folders/nnn` or `organizations/nnn` for hierarchical policy, project id for a network policy." + type = string + nullable = false +} + +variable "region" { + description = "Policy region. Leave null for hierarchical policy, set to 'global' for a global network policy." + type = string + default = null +} + +variable "rules_factory_config" { + description = "Configuration for the optional rules factory." + type = object({ + cidr_file_path = optional(string) + egress_rules_file_path = optional(string) + ingress_rules_file_path = optional(string) + }) + nullable = false + default = {} +} diff --git a/assets/modules-fabric/v26/net-firewall-policy/versions.tf b/assets/modules-fabric/v26/net-firewall-policy/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/net-firewall-policy/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/net-ipsec-over-interconnect/README.md b/assets/modules-fabric/v26/net-ipsec-over-interconnect/README.md new file mode 100644 index 0000000..5cadae3 --- /dev/null +++ b/assets/modules-fabric/v26/net-ipsec-over-interconnect/README.md @@ -0,0 +1,129 @@ +# VLAN Attachment module + +This module allows for the provisioning of [HA VPN over Interconnect](https://cloud.google.com/network-connectivity/docs/interconnect/concepts/ha-vpn-interconnect?hl=it). Specifically, this module creates a VPN gateway, a configurable number of tunnels, and all the resources required to established IPSec and BGP with the peer routers. + +The required pair of encrypted VLAN Attachments can be created leveraging the [net-vlan-attachment](../net-vlan-attachment/) module, as shown in the [IoIC Blueprint](../../blueprints/networking/ha-vpn-over-interconnect/). + +## Examples + +### Single region setup + +```hcl +resource "google_compute_router" "encrypted-interconnect-overlay-router" { + name = "encrypted-interconnect-overlay-router" + project = "myproject" + network = "mynet" + region = "europe-west8" + bgp { + asn = 64514 + advertise_mode = "CUSTOM" + advertised_groups = ["ALL_SUBNETS"] + advertised_ip_ranges { + range = "10.255.255.0/24" + } + advertised_ip_ranges { + range = "192.168.255.0/24" + } + } +} + +resource "google_compute_external_vpn_gateway" "default" { + name = "peer-vpn-gateway" + project = "myproject" + description = "Peer IPSec over Interconnect VPN gateway" + interface { + id = 0 + ip_address = "10.0.0.1" + } + interface { + id = 1 + ip_address = "10.0.0.2" + } +} + +module "vpngw-a" { + source = "./fabric/modules/net-ipsec-over-interconnect" + project_id = "myproject" + network = "mynet" + region = "europe-west8" + name = "vpngw-a" + interconnect_attachments = { + a = "attach-01" + b = "attach-02" + } + peer_gateway_config = { + create = false + id = google_compute_external_vpn_gateway.default.id + } + router_config = { + create = false + name = google_compute_router.encrypted-interconnect-overlay-router.name + } + tunnels = { + remote-0 = { + bgp_peer = { + address = "169.254.1.2" + asn = 64514 + } + bgp_session_range = "169.254.1.1/30" + shared_secret = "foobar" + vpn_gateway_interface = 0 + } + remote-1 = { + bgp_peer = { + address = "169.254.1.6" + asn = 64514 + } + bgp_session_range = "169.254.1.5/30" + shared_secret = "foobar" + vpn_gateway_interface = 1 + } + remote-2 = { + bgp_peer = { + address = "169.254.1.10" + asn = 64514 + } + bgp_session_range = "169.254.1.9/30" + shared_secret = "foobar" + vpn_gateway_interface = 0 + } + remote-3 = { + bgp_peer = { + address = "169.254.1.14" + asn = 64514 + } + bgp_session_range = "169.254.1.13/30" + shared_secret = "foobar" + vpn_gateway_interface = 1 + } + } +} +# tftest modules=1 resources=16 +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [interconnect_attachments](variables.tf#L17) | VLAN attachments used by the VPN Gateway. | object({…}) | ✓ | | +| [name](variables.tf#L25) | Common name to identify the VPN Gateway. | string | ✓ | | +| [network](variables.tf#L30) | The VPC name to which resources are associated to. | string | ✓ | | +| [peer_gateway_config](variables.tf#L35) | IP addresses for the external peer gateway. | object({…}) | ✓ | | +| [project_id](variables.tf#L54) | The project id. | string | ✓ | | +| [region](variables.tf#L59) | GCP Region. | string | ✓ | | +| [router_config](variables.tf#L64) | Cloud Router configuration for the VPN. If you want to reuse an existing router, set create to false and use name to specify the desired router. | object({…}) | ✓ | | +| [tunnels](variables.tf#L79) | VPN tunnel configurations. | map(object({…})) | | {} | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [bgp_peers](outputs.tf#L18) | BGP peer resources. | | +| [external_gateway](outputs.tf#L25) | External VPN gateway resource. | | +| [id](outputs.tf#L30) | Fully qualified VPN gateway id. | | +| [random_secret](outputs.tf#L35) | Generated secret. | | +| [router](outputs.tf#L40) | Router resource (only if auto-created). | | +| [router_name](outputs.tf#L45) | Router name. | | +| [self_link](outputs.tf#L50) | HA VPN gateway self link. | | +| [tunnels](outputs.tf#L55) | VPN tunnel resources. | | + diff --git a/assets/modules-fabric/v26/net-ipsec-over-interconnect/main.tf b/assets/modules-fabric/v26/net-ipsec-over-interconnect/main.tf new file mode 100644 index 0000000..06294ad --- /dev/null +++ b/assets/modules-fabric/v26/net-ipsec-over-interconnect/main.tf @@ -0,0 +1,148 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + peer_gateway_id = ( + var.peer_gateway_config.create + ? try(google_compute_external_vpn_gateway.default[0].id, null) + : var.peer_gateway_config.id + ) + router = ( + var.router_config.create + ? try(google_compute_router.default[0].name, null) + : var.router_config.name + ) + secret = random_id.default.b64_url + +} + +resource "google_compute_ha_vpn_gateway" "default" { + name = "vpn-gw-${var.name}" + network = var.network + project = var.project_id + region = var.region + vpn_interfaces { + id = 0 + interconnect_attachment = var.interconnect_attachments.a + } + vpn_interfaces { + id = 1 + interconnect_attachment = var.interconnect_attachments.b + } +} + +resource "google_compute_external_vpn_gateway" "default" { + count = var.peer_gateway_config.create ? 1 : 0 + name = coalesce(var.peer_gateway_config.name, "peer-vpn-gw-${var.name}") + project = var.project_id + description = var.peer_gateway_config.description + redundancy_type = length(var.peer_gateway_config.interfaces) == 2 ? "TWO_IPS_REDUNDANCY" : "SINGLE_IP_INTERNALLY_REDUNDANT" + dynamic "interface" { + for_each = var.peer_gateway_config.interfaces + content { + id = interface.key + ip_address = interface.value + } + } +} + +resource "google_compute_router" "default" { + count = var.router_config.create ? 1 : 0 + name = coalesce(var.router_config.name, "router-${var.name}") + project = var.project_id + region = var.region + network = var.network + bgp { + advertise_mode = ( + var.router_config.custom_advertise != null + ? "CUSTOM" + : "DEFAULT" + ) + advertised_groups = ( + try(var.router_config.custom_advertise.all_subnets, false) + ? ["ALL_SUBNETS"] + : [] + ) + dynamic "advertised_ip_ranges" { + for_each = try(var.router_config.custom_advertise.ip_ranges, {}) + iterator = range + content { + range = range.key + description = range.value + } + } + keepalive_interval = try(var.router_config.keepalive, null) + asn = var.router_config.asn + } +} + +resource "google_compute_router_peer" "default" { + for_each = var.tunnels + region = var.region + project = var.project_id + name = "${var.name}-${each.key}" + router = local.router + peer_ip_address = each.value.bgp_peer.address + peer_asn = each.value.bgp_peer.asn + advertised_route_priority = each.value.bgp_peer.route_priority + advertise_mode = ( + try(each.value.bgp_peer.custom_advertise, null) != null + ? "CUSTOM" + : "DEFAULT" + ) + advertised_groups = concat( + try(each.value.bgp_peer.custom_advertise.all_subnets, false) ? ["ALL_SUBNETS"] : [], + try(each.value.bgp_peer.custom_advertise.all_vpc_subnets, false) ? ["ALL_VPC_SUBNETS"] : [], + try(each.value.bgp_peer.custom_advertise.all_peer_vpc_subnets, false) ? ["ALL_PEER_VPC_SUBNETS"] : [] + ) + dynamic "advertised_ip_ranges" { + for_each = try(each.value.bgp_peer.custom_advertise.ip_ranges, {}) + iterator = range + content { + range = range.key + description = range.value + } + } + interface = google_compute_router_interface.default[each.key].name +} + +resource "google_compute_router_interface" "default" { + for_each = var.tunnels + project = var.project_id + region = var.region + name = "${var.name}-${each.key}" + router = local.router + ip_range = each.value.bgp_session_range == "" ? null : each.value.bgp_session_range + vpn_tunnel = google_compute_vpn_tunnel.default[each.key].name +} + +resource "google_compute_vpn_tunnel" "default" { + for_each = var.tunnels + project = var.project_id + region = var.region + name = "${var.name}-${each.key}" + vpn_gateway = google_compute_ha_vpn_gateway.default.id + peer_external_gateway = local.peer_gateway_id + shared_secret = coalesce(each.value.shared_secret, local.secret) + router = local.router + vpn_gateway_interface = each.value.vpn_gateway_interface + peer_external_gateway_interface = each.value.peer_external_gateway_interface + ike_version = each.value.ike_version +} + +resource "random_id" "default" { + byte_length = 8 +} diff --git a/assets/modules-fabric/v26/net-ipsec-over-interconnect/outputs.tf b/assets/modules-fabric/v26/net-ipsec-over-interconnect/outputs.tf new file mode 100644 index 0000000..e0ba83f --- /dev/null +++ b/assets/modules-fabric/v26/net-ipsec-over-interconnect/outputs.tf @@ -0,0 +1,65 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +output "bgp_peers" { + description = "BGP peer resources." + value = { + for k, v in google_compute_router_peer.default : k => v + } +} + +output "external_gateway" { + description = "External VPN gateway resource." + value = try(google_compute_external_vpn_gateway.default[0], null) +} + +output "id" { + description = "Fully qualified VPN gateway id." + value = google_compute_ha_vpn_gateway.default.id +} + +output "random_secret" { + description = "Generated secret." + value = local.secret +} + +output "router" { + description = "Router resource (only if auto-created)." + value = one(google_compute_router.default[*]) +} + +output "router_name" { + description = "Router name." + value = local.router +} + +output "self_link" { + description = "HA VPN gateway self link." + value = google_compute_ha_vpn_gateway.default.self_link +} + +output "tunnels" { + description = "VPN tunnel resources." + value = { + for name in keys(var.tunnels) : + name => { + self_link = google_compute_vpn_tunnel.default[name].self_link + name = google_compute_vpn_tunnel.default[name].name + peer_ip = google_compute_vpn_tunnel.default[name].peer_ip + } + } +} diff --git a/assets/modules-fabric/v26/net-ipsec-over-interconnect/variables.tf b/assets/modules-fabric/v26/net-ipsec-over-interconnect/variables.tf new file mode 100644 index 0000000..25cf0cf --- /dev/null +++ b/assets/modules-fabric/v26/net-ipsec-over-interconnect/variables.tf @@ -0,0 +1,105 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "interconnect_attachments" { + description = "VLAN attachments used by the VPN Gateway." + type = object({ + a = string + b = string + }) +} + +variable "name" { + description = "Common name to identify the VPN Gateway." + type = string +} + +variable "network" { + description = "The VPC name to which resources are associated to." + type = string +} + +variable "peer_gateway_config" { + description = "IP addresses for the external peer gateway." + type = object({ + create = optional(bool, false) + description = optional(string, "Terraform managed IPSec over Interconnect VPN gateway") + name = optional(string, null) + id = optional(string, null) + interfaces = optional(list(string), []) + }) + nullable = false + validation { + condition = anytrue([ + var.peer_gateway_config.create == false && var.peer_gateway_config.id != null, + var.peer_gateway_config.create == true && (try(length(var.peer_gateway_config.interfaces) == 1, false) || try(length(var.peer_gateway_config.interfaces) == 2, false)) + ]) + error_message = "When using an existing gateway, an ID must be provided. When not, the gateway can have one or two interfaces." + } +} + +variable "project_id" { + description = "The project id." + type = string +} + +variable "region" { + description = "GCP Region." + type = string +} + +variable "router_config" { + description = "Cloud Router configuration for the VPN. If you want to reuse an existing router, set create to false and use name to specify the desired router." + type = object({ + create = optional(bool, true) + asn = optional(number) + name = optional(string) + keepalive = optional(number) + custom_advertise = optional(object({ + all_subnets = bool + ip_ranges = map(string) + })) + }) + nullable = false +} + +variable "tunnels" { + description = "VPN tunnel configurations." + type = map(object({ + bgp_peer = object({ + address = string + asn = number + route_priority = optional(number, 1000) + custom_advertise = optional(object({ + all_subnets = bool + all_vpc_subnets = bool + all_peer_vpc_subnets = bool + ip_ranges = map(string) + })) + }) + # each BGP session on the same Cloud Router must use a unique /30 CIDR + # from the 169.254.0.0/16 block. + bgp_session_range = string + ike_version = optional(number, 2) + peer_external_gateway_interface = optional(number) + peer_gateway_id = optional(string, "default") + router = optional(string) + shared_secret = optional(string) + vpn_gateway_interface = number + })) + default = {} + nullable = false +} diff --git a/assets/modules-fabric/v26/net-ipsec-over-interconnect/versions.tf b/assets/modules-fabric/v26/net-ipsec-over-interconnect/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/net-ipsec-over-interconnect/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/net-lb-app-ext/.gitignore b/assets/modules-fabric/v26/net-lb-app-ext/.gitignore new file mode 100644 index 0000000..a00202d --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-app-ext/.gitignore @@ -0,0 +1,3 @@ +*backup +*tfstate +tfvars diff --git a/assets/modules-fabric/v26/net-lb-app-ext/README.md b/assets/modules-fabric/v26/net-lb-app-ext/README.md new file mode 100644 index 0000000..159d4c9 --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-app-ext/README.md @@ -0,0 +1,838 @@ +# External Application Load Balancer Module + +This module allows managing Global HTTP/HTTPS Classic Load Balancers (GLBs). It's designed to expose the full configuration of the underlying resources, and to facilitate common usage patterns by providing sensible defaults, and optionally managing prerequisite resources like health checks, instance groups, etc. + +Due to the complexity of the underlying resources, changes to the configuration that involve recreation of resources are best applied in stages, starting by disabling the configuration in the urlmap that references the resources that need recreation, then doing the same for the backend service, etc. + +## Examples + + +- [Examples](#examples) + - [Minimal HTTP Example](#minimal-http-example) + - [Minimal HTTPS examples](#minimal-https-examples) + - [HTTP backends](#http-backends) + - [HTTPS backends](#https-backends) + - [Classic vs Non-classic](#classic-vs-non-classic) + - [Health Checks](#health-checks) + - [Backend Types and Management](#backend-types-and-management) + - [Instance Groups](#instance-groups) + - [Managed Instance Groups](#managed-instance-groups) + - [Storage Buckets](#storage-buckets) + - [Network Endpoint Groups (NEGs)](#network-endpoint-groups-negs) + - [Zonal NEG creation](#zonal-neg-creation) + - [Hybrid NEG creation](#hybrid-neg-creation) + - [Internet NEG creation](#internet-neg-creation) + - [Private Service Connect NEG creation](#private-service-connect-neg-creation) + - [Serverless NEG creation](#serverless-neg-creation) + - [URL Map](#url-map) + - [SSL Certificates](#ssl-certificates) + - [Complex example](#complex-example) +- [Files](#files) +- [Variables](#variables) +- [Outputs](#outputs) + + + +### Minimal HTTP Example + +An HTTP load balancer with a backend service pointing to a GCE instance group: + +```hcl +module "glb-0" { + source = "./fabric/modules/net-lb-app-ext" + project_id = "myprj" + name = "glb-test-0" + backend_service_configs = { + default = { + backends = [ + { backend = "projects/myprj/zones/europe-west8-b/instanceGroups/myig-b" }, + { backend = "projects/myprj/zones/europe-west8-c/instanceGroups/myig-c" }, + ] + } + } +} +# tftest modules=1 resources=5 +``` + +### Minimal HTTPS examples + +#### HTTP backends + +An HTTPS load balancer needs a certificate and backends can be HTTP or HTTPS. THis is an example With HTTP backends and a managed certificate: + +```hcl +module "glb-0" { + source = "./fabric/modules/net-lb-app-ext" + project_id = "myprj" + name = "glb-test-0" + backend_service_configs = { + default = { + backends = [ + { backend = "projects/myprj/zones/europe-west8-b/instanceGroups/myig-b" }, + { backend = "projects/myprj/zones/europe-west8-c/instanceGroups/myig-c" }, + ] + protocol = "HTTP" + } + } + protocol = "HTTPS" + ssl_certificates = { + managed_configs = { + default = { + domains = ["glb-test-0.example.org"] + } + } + } +} +# tftest modules=1 resources=6 +``` + +#### HTTPS backends + +For HTTPS backends the backend service protocol needs to be set to `HTTPS`. The port name if omitted is inferred from the protocol, in this case it is set internally to `https`. The health check also needs to be set to https. This is a complete example: + +```hcl +module "glb-0" { + source = "./fabric/modules/net-lb-app-ext" + project_id = "myprj" + name = "glb-test-0" + backend_service_configs = { + default = { + backends = [ + { backend = "projects/myprj/zones/europe-west8-b/instanceGroups/myig-b" }, + { backend = "projects/myprj/zones/europe-west8-c/instanceGroups/myig-c" }, + ] + protocol = "HTTPS" + } + } + health_check_configs = { + default = { + https = { + port_specification = "USE_SERVING_PORT" + } + } + } + protocol = "HTTPS" + ssl_certificates = { + managed_configs = { + default = { + domains = ["glb-test-0.example.org"] + } + } + } +} +# tftest modules=1 resources=6 +``` + +### Classic vs Non-classic + +The module uses a classic Global Load Balancer by default. To use the non-classic version set the `use_classic_version` variable to `false` as in the following example, note that the module is not enforcing feature sets between the two versions: + +```hcl +module "glb-0" { + source = "./fabric/modules/net-lb-app-ext" + project_id = "myprj" + name = "glb-test-0" + use_classic_version = false + backend_service_configs = { + default = { + backends = [ + { backend = "projects/myprj/zones/europe-west8-b/instanceGroups/myig-b" }, + { backend = "projects/myprj/zones/europe-west8-c/instanceGroups/myig-c" }, + ] + } + } +} +# tftest modules=1 resources=5 +``` + +### Health Checks + +You can leverage externally defined health checks for backend services, or have the module create them for you. + +By default a simple HTTP health check named `default` is created and used in backend services. If you need to override the default, simply define your own health check using the same key (`default`). For more complex configurations you can define your own health checks and reference them via keys in the backend service configurations. + +Health checks created by this module are controlled via the `health_check_configs` variable, which behaves in a similar way to other LB modules in this repository. This is an example that overrides the default health check configuration using a TCP health check: + +```hcl +module "glb-0" { + source = "./fabric/modules/net-lb-app-ext" + project_id = var.project_id + name = "glb-test-0" + backend_service_configs = { + default = { + backends = [{ + backend = "projects/myprj/zones/europe-west1-a/instanceGroups/my-ig" + }] + # no need to reference the hc explicitly when using the `default` key + # health_checks = ["default"] + } + } + health_check_configs = { + default = { + tcp = { port = 80 } + } + } +} +# tftest modules=1 resources=5 +``` + +To leverage existing health checks without having the module create them, simply pass their self links to backend services and set the `health_check_configs` variable to an empty map: + +```hcl +module "glb-0" { + source = "./fabric/modules/net-lb-app-ext" + project_id = var.project_id + name = "glb-test-0" + backend_service_configs = { + default = { + backends = [{ + backend = "projects/myprj/zones/europe-west1-a/instanceGroups/my-ig" + }] + health_checks = ["projects/myprj/global/healthChecks/custom"] + } + } + health_check_configs = {} +} +# tftest modules=1 resources=4 +``` + +### Backend Types and Management + +#### Instance Groups + +The module can optionally create unmanaged instance groups, which can then be referred to in backends via their key. THis is the simple HTTP example above but with instance group creation managed by the module: + +```hcl +module "glb-0" { + source = "./fabric/modules/net-lb-app-ext" + project_id = "myprj" + name = "glb-test-0" + backend_service_configs = { + default = { + backends = [ + { backend = "default-b" } + ] + } + } + group_configs = { + default-b = { + zone = "europe-west8-b" + instances = [ + "projects/myprj/zones/europe-west8-b/instances/vm-a" + ] + named_ports = { http = 80 } + } + } +} +# tftest modules=1 resources=6 +``` + +#### Managed Instance Groups + +This example shows how to use the module with a manage instance group as backend: + +```hcl +module "win-template" { + source = "./fabric/modules/compute-vm" + project_id = "myprj" + zone = "europe-west8-a" + name = "win-template" + instance_type = "n2d-standard-2" + create_template = true + boot_disk = { + initialize_params = { + image = "projects/windows-cloud/global/images/windows-server-2019-dc-v20221214" + size = 70 + } + } + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + nat = false + addresses = null + }] +} + +module "win-mig" { + source = "./fabric/modules/compute-mig" + project_id = "myprj" + location = "europe-west8-a" + name = "win-mig" + instance_template = module.win-template.template.self_link + autoscaler_config = { + max_replicas = 3 + min_replicas = 1 + cooldown_period = 30 + scaling_signals = { + cpu_utilization = { + target = 0.80 + } + } + } + named_ports = { + http = 80 + } +} + +module "glb-0" { + source = "./fabric/modules/net-lb-app-ext" + project_id = "myprj" + name = "glb-test-0" + backend_service_configs = { + default = { + backends = [ + { backend = module.win-mig.group_manager.instance_group } + ] + } + } +} +# tftest modules=3 resources=8 +``` + +#### Storage Buckets + +GCS bucket backends can also be managed and used in this module in a similar way to regular backend services.Multiple GCS bucket backends can be defined and referenced in URL maps by their keys (or self links if defined externally) together with regular backend services, [an example is provided later in this document](#complex-example). This is a simple example that defines a GCS backend as the default for the URL map: + +```hcl +module "glb-0" { + source = "./fabric/modules/net-lb-app-ext" + project_id = "myprj" + name = "glb-test-0" + backend_buckets_config = { + default = { + bucket_name = "tf-playground-svpc-gce-public" + } + } + # with a single GCS backend the implied default health check is not needed + health_check_configs = {} +} +# tftest modules=1 resources=4 +``` + +#### Network Endpoint Groups (NEGs) + +Supported Network Endpoint Groups (NEGs) can also be used as backends. Similarly to groups, you can pass a self link for existing NEGs or have the module manage them for you. A simple example using an existing zonal NEG: + +```hcl +module "glb-0" { + source = "./fabric/modules/net-lb-app-ext" + project_id = "myprj" + name = "glb-test-0" + backend_service_configs = { + default = { + backends = [ + { + backend = "projects/myprj/zones/europe-west8-b/networkEndpointGroups/myneg-b" + balancing_mode = "RATE" + max_rate = { per_endpoint = 10 } + } + ] + } + } +} +# tftest modules=1 resources=5 +``` + +#### Zonal NEG creation + +This example shows how to create and manage zonal NEGs using GCE VMs as endpoints: + +```hcl +module "glb-0" { + source = "./fabric/modules/net-lb-app-ext" + project_id = "myprj" + name = "glb-test-0" + backend_service_configs = { + default = { + backends = [ + { + backend = "neg-0" + balancing_mode = "RATE" + max_rate = { per_endpoint = 10 } + } + ] + } + } + neg_configs = { + neg-0 = { + gce = { + network = "projects/myprj-host/global/networks/svpc" + subnetwork = "projects/myprj-host/regions/europe-west8/subnetworks/gce" + zone = "europe-west8-b" + endpoints = { + e-0 = { + instance = "myinstance-b-0" + ip_address = "10.24.32.25" + port = 80 + } + } + } + } + } +} +# tftest modules=1 resources=7 +``` + +#### Hybrid NEG creation + +This example shows how to create and manage hybrid NEGs: + +```hcl +module "glb-0" { + source = "./fabric/modules/net-lb-app-ext" + project_id = "myprj" + name = "glb-test-0" + backend_service_configs = { + default = { + backends = [ + { + backend = "neg-0" + balancing_mode = "RATE" + max_rate = { per_endpoint = 10 } + } + ] + } + } + neg_configs = { + neg-0 = { + hybrid = { + network = "projects/myprj-host/global/networks/svpc" + zone = "europe-west8-b" + endpoints = { + e-0 = { + ip_address = "10.0.0.10" + port = 80 + } + } + } + } + } +} +# tftest modules=1 resources=7 +``` + +#### Internet NEG creation + +This example shows how to create and manage internet NEGs: + +```hcl +module "glb-0" { + source = "./fabric/modules/net-lb-app-ext" + project_id = "myprj" + name = "glb-test-0" + backend_service_configs = { + default = { + backends = [ + { backend = "neg-0" } + ] + health_checks = [] + } + } + # with a single internet NEG the implied default health check is not needed + health_check_configs = {} + neg_configs = { + neg-0 = { + internet = { + use_fqdn = true + endpoints = { + e-0 = { + destination = "www.example.org" + port = 80 + } + } + } + } + } +} +# tftest modules=1 resources=6 +``` + +#### Private Service Connect NEG creation + +The module supports managing PSC NEGs if the non-classic version of the load balancer is used: + +```hcl +module "glb-0" { + source = "./fabric/modules/net-lb-app-ext" + project_id = "myprj" + name = "glb-test-0" + use_classic_version = false + backend_service_configs = { + default = { + backends = [ + { backend = "neg-0" } + ] + health_checks = [] + } + } + # with a single PSC NEG the implied default health check is not needed + health_check_configs = {} + neg_configs = { + neg-0 = { + psc = { + region = "europe-west8" + target_service = "europe-west8-cloudkms.googleapis.com" + } + } + } +} +# tftest modules=1 resources=5 +``` + +#### Serverless NEG creation + +The module supports managing Serverless NEGs for Cloud Run and Cloud Function. This is an example of a Cloud Run NEG: + +```hcl +module "glb-0" { + source = "./fabric/modules/net-lb-app-ext" + project_id = "myprj" + name = "glb-test-0" + backend_service_configs = { + default = { + backends = [ + { backend = "neg-0" } + ] + health_checks = [] + } + } + # with a single serverless NEG the implied default health check is not needed + health_check_configs = {} + neg_configs = { + neg-0 = { + cloudrun = { + region = "europe-west8" + target_service = { + name = "hello" + } + } + } + } +} +# tftest modules=1 resources=5 +``` + +Serverless NEGs don't use the port name but it should be set to `http`. An HTTPS frontend requires the protocol to be set to `HTTPS`, and the port name field will infer this value if omitted so you need to set it explicitly: + +```hcl +module "glb-0" { + source = "./fabric/modules/net-lb-app-ext" + project_id = "myprj" + name = "glb-test-0" + backend_service_configs = { + default = { + backends = [ + { backend = "neg-0" } + ] + health_checks = [] + port_name = "http" + } + } + # with a single serverless NEG the implied default health check is not needed + health_check_configs = {} + neg_configs = { + neg-0 = { + cloudrun = { + region = "europe-west8" + target_service = { + name = "hello" + } + } + } + } + protocol = "HTTPS" + ssl_certificates = { + managed_configs = { + default = { + domains = ["glb-test-0.example.org"] + } + } + } +} +# tftest modules=1 resources=6 inventory=https-sneg.yaml +``` + +### URL Map + +The module exposes the full URL map resource configuration, with some minor changes to the interface to decrease verbosity, and support for aliasing backend services via keys. + +The default URL map configuration sets the `default` backend service as the default service for the load balancer as a convenience. Just override the `urlmap_config` variable to change the default behaviour: + +```hcl +module "glb-0" { + source = "./fabric/modules/net-lb-app-ext" + project_id = "myprj" + name = "glb-test-0" + backend_service_configs = { + default = { + backends = [{ + backend = "projects/myprj/zones/europe-west8-b/instanceGroups/ig-0" + }] + } + other = { + backends = [{ + backend = "projects/myprj/zones/europe-west8-c/instanceGroups/ig-1" + }] + } + } + urlmap_config = { + default_service = "default" + host_rules = [{ + hosts = ["*"] + path_matcher = "pathmap" + }] + path_matchers = { + pathmap = { + default_service = "default" + path_rules = [{ + paths = ["/other", "/other/*"] + service = "other" + }] + } + } + } +} + +# tftest modules=1 resources=6 +``` + +### SSL Certificates + +The module also allows managing managed and self-managed SSL certificates via the `ssl_certificates` variable. Any certificate defined there will be added to the HTTPS proxy resource. + +THe [HTTPS example above](#minimal-https-examples) shows how to configure manage certificated, the following example shows how to use an unmanaged (or self managed) certificate. The example uses Terraform resource for the key and certificate so that the we don't depend on external files when running tests, in real use the key and certificate are generally provided via external files read by the Terraform `file()` function. + +```hcl +resource "tls_private_key" "default" { + algorithm = "RSA" + rsa_bits = 4096 +} + +resource "tls_self_signed_cert" "default" { + private_key_pem = tls_private_key.default.private_key_pem + subject { + common_name = "example.com" + organization = "ACME Examples, Inc" + } + validity_period_hours = 720 + allowed_uses = [ + "key_encipherment", + "digital_signature", + "server_auth", + ] +} + +module "glb-0" { + source = "./fabric/modules/net-lb-app-ext" + project_id = "myprj" + name = "glb-test-0" + backend_service_configs = { + default = { + backends = [ + { backend = "projects/myprj/zones/europe-west8-b/instanceGroups/myig-b" }, + { backend = "projects/myprj/zones/europe-west8-c/instanceGroups/myig-c" }, + ] + protocol = "HTTP" + } + } + protocol = "HTTPS" + ssl_certificates = { + create_configs = { + default = { + # certificate and key could also be read via file() from external files + certificate = tls_self_signed_cert.default.cert_pem + private_key = tls_private_key.default.private_key_pem + } + } + } +} +# tftest modules=1 resources=8 +``` + +### Complex example + +This example mixes group and NEG backends, and shows how to set HTTPS for specific backends. + +```hcl +module "glb-0" { + source = "./fabric/modules/net-lb-app-ext" + project_id = "myprj" + name = "glb-test-0" + backend_buckets_config = { + gcs-0 = { + bucket_name = "my-bucket" + } + } + backend_service_configs = { + default = { + backends = [ + { backend = "ew8-b" }, + { backend = "ew8-c" }, + ] + } + neg-gce-0 = { + backends = [{ + balancing_mode = "RATE" + backend = "neg-ew8-c" + max_rate = { per_endpoint = 10 } + }] + } + neg-hybrid-0 = { + backends = [{ + backend = "neg-hello" + }] + health_checks = ["neg"] + protocol = "HTTPS" + } + } + group_configs = { + ew8-b = { + zone = "europe-west8-b" + instances = [ + "projects/prj-gce/zones/europe-west8-b/instances/nginx-ew8-b" + ] + named_ports = { http = 80 } + } + ew8-c = { + zone = "europe-west8-c" + instances = [ + "projects/prj-gce/zones/europe-west8-c/instances/nginx-ew8-c" + ] + named_ports = { http = 80 } + } + } + health_check_configs = { + default = { + http = { + port = 80 + } + } + neg = { + https = { + host = "hello.example.com" + port = 443 + } + } + } + neg_configs = { + neg-ew8-c = { + gce = { + network = "projects/myprj-host/global/networks/svpc" + subnetwork = "projects/myprj-host/regions/europe-west8/subnetworks/gce" + zone = "europe-west8-c" + endpoints = { + e-0 = { + instance = "nginx-ew8-c" + ip_address = "10.24.32.26" + port = 80 + } + } + } + } + neg-hello = { + hybrid = { + network = "projects/myprj-host/global/networks/svpc" + zone = "europe-west8-b" + endpoints = { + e-0 = { + ip_address = "192.168.0.3" + port = 443 + } + } + } + } + } + urlmap_config = { + default_service = "default" + host_rules = [ + { + hosts = ["*"] + path_matcher = "gce" + }, + { + hosts = ["hello.example.com"] + path_matcher = "hello" + }, + { + hosts = ["static.example.com"] + path_matcher = "static" + } + ] + path_matchers = { + gce = { + default_service = "default" + path_rules = [ + { + paths = ["/gce-neg", "/gce-neg/*"] + service = "neg-gce-0" + } + ] + } + hello = { + default_service = "neg-hybrid-0" + } + static = { + default_service = "gcs-0" + } + } + } +} +# tftest modules=1 resources=15 +``` + + + + +## Files + +| name | description | resources | +|---|---|---| +| [backend-service.tf](./backend-service.tf) | Backend service resources. | google_compute_backend_service | +| [backends.tf](./backends.tf) | Backend groups and backend buckets resources. | google_compute_backend_bucket | +| [groups.tf](./groups.tf) | None | google_compute_instance_group | +| [health-check.tf](./health-check.tf) | Health check resource. | google_compute_health_check | +| [main.tf](./main.tf) | Module-level locals and resources. | google_compute_global_forwarding_rule · google_compute_managed_ssl_certificate · google_compute_ssl_certificate · google_compute_target_http_proxy · google_compute_target_https_proxy | +| [negs.tf](./negs.tf) | NEG resources. | google_compute_global_network_endpoint · google_compute_global_network_endpoint_group · google_compute_network_endpoint · google_compute_network_endpoint_group · google_compute_region_network_endpoint_group | +| [outputs.tf](./outputs.tf) | Module outputs. | | +| [urlmap.tf](./urlmap.tf) | URL map resources. | google_compute_url_map | +| [variables-backend-service.tf](./variables-backend-service.tf) | Backend services variables. | | +| [variables-health-check.tf](./variables-health-check.tf) | Health check variable. | | +| [variables-urlmap.tf](./variables-urlmap.tf) | URLmap variable. | | +| [variables.tf](./variables.tf) | Module variables. | | +| [versions.tf](./versions.tf) | Version pins. | | + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [name](variables.tf#L91) | Load balancer name. | string | ✓ | | +| [project_id](variables.tf#L193) | Project id. | string | ✓ | | +| [address](variables.tf#L17) | Optional IP address used for the forwarding rule. | string | | null | +| [backend_buckets_config](variables.tf#L23) | Backend buckets configuration. | map(object({…})) | | {} | +| [backend_service_configs](variables-backend-service.tf#L19) | Backend service level configuration. | map(object({…})) | | {} | +| [description](variables.tf#L56) | Optional description used for resources. | string | | "Terraform managed." | +| [group_configs](variables.tf#L62) | Optional unmanaged groups to create. Can be referenced in backends via key or outputs. | map(object({…})) | | {} | +| [health_check_configs](variables-health-check.tf#L19) | Optional auto-created health check configurations, use the output self-link to set it in the auto healing policy. Refer to examples for usage. | map(object({…})) | | {…} | +| [https_proxy_config](variables.tf#L74) | HTTPS proxy connfiguration. | object({…}) | | {} | +| [labels](variables.tf#L85) | Labels set on resources. | map(string) | | {} | +| [neg_configs](variables.tf#L96) | Optional network endpoint groups to create. Can be referenced in backends via key or outputs. | map(object({…})) | | {} | +| [ports](variables.tf#L187) | Optional ports for HTTP load balancer, valid ports are 80 and 8080. | list(string) | | null | +| [protocol](variables.tf#L198) | Protocol supported by this load balancer. | string | | "HTTP" | +| [ssl_certificates](variables.tf#L211) | SSL target proxy certificates (only if protocol is HTTPS) for existing, custom, and managed certificates. | object({…}) | | {} | +| [urlmap_config](variables-urlmap.tf#L19) | The URL map configuration. | object({…}) | | {…} | +| [use_classic_version](variables.tf#L228) | Use classic Global Load Balancer. | bool | | true | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [address](outputs.tf#L17) | Forwarding rule address. | | +| [backend_service_ids](outputs.tf#L22) | Backend service resources. | | +| [backend_service_names](outputs.tf#L29) | Backend service resource names. | | +| [forwarding_rule](outputs.tf#L36) | Forwarding rule resource. | | +| [group_ids](outputs.tf#L41) | Autogenerated instance group ids. | | +| [health_check_ids](outputs.tf#L48) | Autogenerated health check ids. | | +| [id](outputs.tf#L55) | Fully qualified forwarding rule id. | | +| [neg_ids](outputs.tf#L60) | Autogenerated network endpoint group ids. | | + + diff --git a/assets/modules-fabric/v26/net-lb-app-ext/backend-service.tf b/assets/modules-fabric/v26/net-lb-app-ext/backend-service.tf new file mode 100644 index 0000000..acadda3 --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-app-ext/backend-service.tf @@ -0,0 +1,262 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Backend service resources. + +locals { + group_ids = merge( + { + for k, v in google_compute_instance_group.default : k => v.id + }, + { + for k, v in google_compute_global_network_endpoint_group.default : k => v.id + }, + { + for k, v in google_compute_network_endpoint_group.default : k => v.id + }, + { + for k, v in google_compute_region_network_endpoint_group.psc : k => v.id + }, + { + for k, v in google_compute_region_network_endpoint_group.serverless : k => v.id + } + ) + hc_ids = { + for k, v in google_compute_health_check.default : k => v.id + } +} + +# google_compute_backend_bucket + +resource "google_compute_backend_service" "default" { + provider = google-beta + for_each = var.backend_service_configs + project = ( + each.value.project_id == null + ? var.project_id + : each.value.project_id + ) + name = "${var.name}-${each.key}" + description = var.description + affinity_cookie_ttl_sec = each.value.affinity_cookie_ttl_sec + compression_mode = each.value.compression_mode + connection_draining_timeout_sec = each.value.connection_draining_timeout_sec + custom_request_headers = each.value.custom_request_headers + custom_response_headers = each.value.custom_response_headers + enable_cdn = each.value.enable_cdn + health_checks = length(each.value.health_checks) == 0 ? null : [ + for k in each.value.health_checks : lookup(local.hc_ids, k, k) + ] + load_balancing_scheme = var.use_classic_version ? "EXTERNAL" : "EXTERNAL_MANAGED" + port_name = ( + each.value.port_name == null + ? lower(each.value.protocol == null ? var.protocol : each.value.protocol) + : each.value.port_name + ) + protocol = ( + each.value.protocol == null ? var.protocol : each.value.protocol + ) + security_policy = each.value.security_policy + session_affinity = each.value.session_affinity + timeout_sec = each.value.timeout_sec + + dynamic "backend" { + for_each = { for b in coalesce(each.value.backends, []) : b.backend => b } + content { + group = lookup(local.group_ids, backend.key, backend.key) + balancing_mode = backend.value.balancing_mode # UTILIZATION, RATE + capacity_scaler = backend.value.capacity_scaler + description = backend.value.description + max_connections = try( + backend.value.max_connections.per_group, null + ) + max_connections_per_endpoint = try( + backend.value.max_connections.per_endpoint, null + ) + max_connections_per_instance = try( + backend.value.max_connections.per_instance, null + ) + max_rate = try( + backend.value.max_rate.per_group, null + ) + max_rate_per_endpoint = try( + backend.value.max_rate.per_endpoint, null + ) + max_rate_per_instance = try( + backend.value.max_rate.per_instance, null + ) + max_utilization = backend.value.max_utilization + } + } + + dynamic "cdn_policy" { + for_each = ( + each.value.cdn_policy == null ? [] : [each.value.cdn_policy] + ) + iterator = cdn + content { + cache_mode = cdn.value.cache_mode + client_ttl = cdn.value.client_ttl + default_ttl = cdn.value.default_ttl + max_ttl = cdn.value.max_ttl + negative_caching = cdn.value.negative_caching + serve_while_stale = cdn.value.serve_while_stale + signed_url_cache_max_age_sec = cdn.value.signed_url_cache_max_age_sec + dynamic "cache_key_policy" { + for_each = ( + cdn.value.cache_key_policy == null + ? [] + : [cdn.value.cache_key_policy] + ) + iterator = ck + content { + include_host = ck.value.include_host + include_named_cookies = ck.value.include_named_cookies + include_protocol = ck.value.include_protocol + include_query_string = ck.value.include_query_string + query_string_blacklist = ck.value.query_string_blacklist + query_string_whitelist = ck.value.query_string_whitelist + } + } + dynamic "negative_caching_policy" { + for_each = ( + cdn.value.negative_caching_policy == null + ? [] + : [cdn.value.negative_caching_policy] + ) + iterator = nc + content { + code = nc.value.code + ttl = nc.value.ttl + } + } + } + } + + dynamic "circuit_breakers" { + for_each = ( + each.value.circuit_breakers == null ? [] : [each.value.circuit_breakers] + ) + iterator = cb + content { + max_connections = cb.value.max_connections + max_pending_requests = cb.value.max_pending_requests + max_requests = cb.value.max_requests + max_requests_per_connection = cb.value.max_requests_per_connection + max_retries = cb.value.max_retries + dynamic "connect_timeout" { + for_each = ( + cb.value.connect_timeout == null ? [] : [cb.value.connect_timeout] + ) + content { + seconds = connect_timeout.value.seconds + nanos = connect_timeout.value.nanos + } + } + } + } + + dynamic "consistent_hash" { + for_each = ( + each.value.consistent_hash == null ? [] : [each.value.consistent_hash] + ) + iterator = ch + content { + http_header_name = ch.value.http_header_name + minimum_ring_size = ch.value.minimum_ring_size + dynamic "http_cookie" { + for_each = ch.value.http_cookie == null ? [] : [ch.value.http_cookie] + content { + name = http_cookie.value.name + path = http_cookie.value.path + dynamic "ttl" { + for_each = ( + http_cookie.value.ttl == null ? [] : [http_cookie.value.ttl] + ) + content { + seconds = ttl.value.seconds + nanos = ttl.value.nanos + } + } + } + } + } + } + + dynamic "iap" { + for_each = each.value.iap_config == null ? [] : [each.value.iap_config] + content { + oauth2_client_id = iap.value.oauth2_client_id + oauth2_client_secret = iap.value.oauth2_client_secret + oauth2_client_secret_sha256 = iap.value.oauth2_client_secret_sha256 + } + } + + dynamic "log_config" { + for_each = each.value.log_sample_rate == null ? [] : [""] + content { + enable = true + sample_rate = each.value.log_sample_rate + } + } + + dynamic "outlier_detection" { + for_each = ( + each.value.outlier_detection == null ? [] : [each.value.outlier_detection] + ) + iterator = od + content { + consecutive_errors = od.value.consecutive_errors + consecutive_gateway_failure = od.value.consecutive_gateway_failure + enforcing_consecutive_errors = od.value.enforcing_consecutive_errors + enforcing_consecutive_gateway_failure = od.value.enforcing_consecutive_gateway_failure + enforcing_success_rate = od.value.enforcing_success_rate + max_ejection_percent = od.value.max_ejection_percent + success_rate_minimum_hosts = od.value.success_rate_minimum_hosts + success_rate_request_volume = od.value.success_rate_request_volume + success_rate_stdev_factor = od.value.success_rate_stdev_factor + dynamic "base_ejection_time" { + for_each = ( + od.value.base_ejection_time == null ? [] : [od.value.base_ejection_time] + ) + content { + seconds = base_ejection_time.value.seconds + nanos = base_ejection_time.value.nanos + } + } + dynamic "interval" { + for_each = ( + od.value.interval == null ? [] : [od.value.interval] + ) + content { + seconds = interval.value.seconds + nanos = interval.value.nanos + } + } + } + } + + dynamic "security_settings" { + for_each = ( + each.value.security_settings == null ? [] : [each.value.security_settings] + ) + iterator = ss + content { + client_tls_policy = ss.value.client_tls_policy + subject_alt_names = ss.value.subject_alt_names + } + } +} diff --git a/assets/modules-fabric/v26/net-lb-app-ext/backends.tf b/assets/modules-fabric/v26/net-lb-app-ext/backends.tf new file mode 100644 index 0000000..107e2be --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-app-ext/backends.tf @@ -0,0 +1,77 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Backend groups and backend buckets resources. + +resource "google_compute_backend_bucket" "default" { + for_each = var.backend_buckets_config + project = var.project_id + name = "${var.name}-${each.key}" + bucket_name = each.value.bucket_name + compression_mode = each.value.compression_mode + custom_response_headers = each.value.custom_response_headers + description = each.value.description + edge_security_policy = each.value.edge_security_policy + enable_cdn = each.value.enable_cdn + + dynamic "cdn_policy" { + for_each = each.value.cdn_policy == null ? [] : [each.value.cdn_policy] + iterator = p + content { + cache_mode = p.value.cache_mode + client_ttl = p.value.client_ttl + default_ttl = p.value.default_ttl + max_ttl = p.value.max_ttl + negative_caching = p.value.negative_caching + request_coalescing = p.value.request_coalescing + serve_while_stale = p.value.serve_while_stale + signed_url_cache_max_age_sec = p.value.signed_url_cache_max_age_sec + dynamic "bypass_cache_on_request_headers" { + for_each = ( + p.value.bypass_cache_on_request_headers == null + ? [] + : [p.value.bypass_cache_on_request_headers] + ) + iterator = h + content { + header_name = h.value + } + } + dynamic "cache_key_policy" { + for_each = ( + p.value.cache_key_policy == null ? [] : [p.value.cache_key_policy] + ) + iterator = ckp + content { + include_http_headers = ckp.value.include_http_headers + query_string_whitelist = ckp.value.query_string_whitelist + } + } + dynamic "negative_caching_policy" { + for_each = ( + p.value.negative_caching_policy == null + ? [] + : [p.value.negative_caching_policy] + ) + iterator = ncp + content { + code = ncp.value.code + ttl = ncp.value.ttl + } + } + } + } +} diff --git a/assets/modules-fabric/v26/net-lb-app-ext/groups.tf b/assets/modules-fabric/v26/net-lb-app-ext/groups.tf new file mode 100644 index 0000000..285ca79 --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-app-ext/groups.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_compute_instance_group" "default" { + for_each = var.group_configs + project = ( + each.value.project_id == null + ? var.project_id + : each.value.project_id + ) + zone = each.value.zone + name = "${var.name}-${each.key}" + description = var.description + instances = each.value.instances + + dynamic "named_port" { + for_each = each.value.named_ports + content { + name = named_port.key + port = named_port.value + } + } +} diff --git a/assets/modules-fabric/v26/net-lb-app-ext/health-check.tf b/assets/modules-fabric/v26/net-lb-app-ext/health-check.tf new file mode 100644 index 0000000..66ba58c --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-app-ext/health-check.tf @@ -0,0 +1,113 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Health check resource. + +resource "google_compute_health_check" "default" { + provider = google-beta + for_each = var.health_check_configs + project = ( + each.value.project_id == null + ? var.project_id + : each.value.project_id + ) + name = "${var.name}-${each.key}" + description = each.value.description + check_interval_sec = each.value.check_interval_sec + healthy_threshold = each.value.healthy_threshold + timeout_sec = each.value.timeout_sec + unhealthy_threshold = each.value.unhealthy_threshold + + dynamic "grpc_health_check" { + for_each = try(each.value.grpc, null) != null ? [""] : [] + content { + port = each.value.grpc.port + port_name = each.value.grpc.port_name + port_specification = each.value.grpc.port_specification + grpc_service_name = each.value.grpc.service_name + } + } + + dynamic "http_health_check" { + for_each = try(each.value.http, null) != null ? [""] : [] + content { + host = each.value.http.host + port = each.value.http.port + port_name = each.value.http.port_name + port_specification = each.value.http.port_specification + proxy_header = each.value.http.proxy_header + request_path = each.value.http.request_path + response = each.value.http.response + } + } + + dynamic "http2_health_check" { + for_each = try(each.value.http2, null) != null ? [""] : [] + content { + host = each.value.http2.host + port = each.value.http2.port + port_name = each.value.http2.port_name + port_specification = each.value.http2.port_specification + proxy_header = each.value.http2.proxy_header + request_path = each.value.http2.request_path + response = each.value.http2.response + } + } + + dynamic "https_health_check" { + for_each = try(each.value.https, null) != null ? [""] : [] + content { + host = each.value.https.host + port = each.value.https.port + port_name = each.value.https.port_name + port_specification = each.value.https.port_specification + proxy_header = each.value.https.proxy_header + request_path = each.value.https.request_path + response = each.value.https.response + } + } + + dynamic "ssl_health_check" { + for_each = try(each.value.ssl, null) != null ? [""] : [] + content { + port = each.value.ssl.port + port_name = each.value.ssl.port_name + port_specification = each.value.ssl.port_specification + proxy_header = each.value.ssl.proxy_header + request = each.value.ssl.request + response = each.value.ssl.response + } + } + + dynamic "tcp_health_check" { + for_each = try(each.value.tcp, null) != null ? [""] : [] + content { + port = each.value.tcp.port + port_name = each.value.tcp.port_name + port_specification = each.value.tcp.port_specification + proxy_header = each.value.tcp.proxy_header + request = each.value.tcp.request + response = each.value.tcp.response + } + } + + dynamic "log_config" { + for_each = try(each.value.enable_logging, null) == true ? [""] : [] + content { + enable = true + } + } +} diff --git a/assets/modules-fabric/v26/net-lb-app-ext/main.tf b/assets/modules-fabric/v26/net-lb-app-ext/main.tf new file mode 100644 index 0000000..ebe438e --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-app-ext/main.tf @@ -0,0 +1,91 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + fwd_rule_ports = ( + var.protocol == "HTTPS" ? [443] : coalesce(var.ports, [80]) + ) + fwd_rule_target = ( + var.protocol == "HTTPS" + ? google_compute_target_https_proxy.default.0.id + : google_compute_target_http_proxy.default.0.id + ) + proxy_ssl_certificates = concat( + coalesce(var.ssl_certificates.certificate_ids, []), + [for k, v in google_compute_ssl_certificate.default : v.id], + [for k, v in google_compute_managed_ssl_certificate.default : v.id] + ) +} + +resource "google_compute_global_forwarding_rule" "default" { + provider = google-beta + project = var.project_id + name = var.name + description = var.description + ip_address = var.address + ip_protocol = "TCP" + load_balancing_scheme = ( + var.use_classic_version ? "EXTERNAL" : "EXTERNAL_MANAGED" + ) + port_range = join(",", local.fwd_rule_ports) + labels = var.labels + target = local.fwd_rule_target +} + +# certificates + +resource "google_compute_ssl_certificate" "default" { + for_each = var.ssl_certificates.create_configs + project = var.project_id + name = "${var.name}-${each.key}" + certificate = trimspace(each.value.certificate) + private_key = trimspace(each.value.private_key) +} + +resource "google_compute_managed_ssl_certificate" "default" { + for_each = var.ssl_certificates.managed_configs + project = var.project_id + name = "${var.name}-${each.key}" + description = each.value.description + managed { + domains = each.value.domains + } + lifecycle { + create_before_destroy = true + } +} + +# proxies + +resource "google_compute_target_http_proxy" "default" { + count = var.protocol == "HTTPS" ? 0 : 1 + project = var.project_id + name = var.name + description = var.description + url_map = google_compute_url_map.default.id +} + +resource "google_compute_target_https_proxy" "default" { + count = var.protocol == "HTTPS" ? 1 : 0 + project = var.project_id + name = var.name + description = var.description + certificate_map = var.https_proxy_config.certificate_map + quic_override = var.https_proxy_config.quic_override + ssl_certificates = local.proxy_ssl_certificates + ssl_policy = var.https_proxy_config.ssl_policy + url_map = google_compute_url_map.default.id +} diff --git a/assets/modules-fabric/v26/net-lb-app-ext/negs.tf b/assets/modules-fabric/v26/net-lb-app-ext/negs.tf new file mode 100644 index 0000000..0011968 --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-app-ext/negs.tf @@ -0,0 +1,159 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description NEG resources. + +locals { + _neg_endpoints_global = flatten([ + for k, v in local.neg_global : [ + for kk, vv in v.internet.endpoints : merge(vv, { + key = "${k}-${kk}", neg = k, use_fqdn = v.internet.use_fqdn + }) + ] + ]) + _neg_endpoints_zonal = flatten([ + for k, v in local.neg_zonal : [ + for kk, vv in v.endpoints : merge(vv, { + key = "${k}-${kk}", neg = k, zone = v.zone + }) + ] + ]) + neg_endpoints_global = { + for v in local._neg_endpoints_global : (v.key) => v + } + neg_endpoints_zonal = { + for v in local._neg_endpoints_zonal : (v.key) => v + } + neg_global = { + for k, v in var.neg_configs : + k => v if v.internet != null + } + neg_regional_psc = { + for k, v in var.neg_configs : + k => v if v.psc != null + } + neg_regional_serverless = { + for k, v in var.neg_configs : + k => v if v.cloudrun != null || v.cloudfunction != null + } + neg_zonal = { + # we need to rebuild new objects as we cannot merge different types + for k, v in var.neg_configs : k => { + description = v.description + endpoints = v.gce != null ? v.gce.endpoints : v.hybrid.endpoints + network = v.gce != null ? v.gce.network : v.hybrid.network + subnetwork = v.gce != null ? v.gce.subnetwork : null + type = v.gce != null ? "GCE_VM_IP_PORT" : "NON_GCP_PRIVATE_IP_PORT" + zone = v.gce != null ? v.gce.zone : v.hybrid.zone + } if v.gce != null || v.hybrid != null + } +} + + +resource "google_compute_global_network_endpoint_group" "default" { + for_each = local.neg_global + project = var.project_id + name = "${var.name}-${each.key}" + # re-enable once provider properly supports this + # default_port = each.value.default_port + description = coalesce(each.value.description, var.description) + network_endpoint_type = ( + each.value.internet.use_fqdn ? "INTERNET_FQDN_PORT" : "INTERNET_IP_PORT" + ) +} + +resource "google_compute_global_network_endpoint" "default" { + for_each = local.neg_endpoints_global + project = ( + google_compute_global_network_endpoint_group.default[each.value.neg].project + ) + global_network_endpoint_group = ( + google_compute_global_network_endpoint_group.default[each.value.neg].name + ) + fqdn = each.value.use_fqdn ? each.value.destination : null + ip_address = each.value.use_fqdn ? null : each.value.destination + port = each.value.port +} + + +resource "google_compute_network_endpoint_group" "default" { + for_each = local.neg_zonal + project = var.project_id + zone = each.value.zone + name = "${var.name}-${each.key}" + # re-enable once provider properly supports this + # default_port = each.value.default_port + description = coalesce(each.value.description, var.description) + network_endpoint_type = each.value.type + network = each.value.network + subnetwork = ( + each.value.type == "NON_GCP_PRIVATE_IP_PORT" + ? null + : each.value.subnetwork + ) +} + +resource "google_compute_network_endpoint" "default" { + for_each = local.neg_endpoints_zonal + project = ( + google_compute_network_endpoint_group.default[each.value.neg].project + ) + network_endpoint_group = ( + google_compute_network_endpoint_group.default[each.value.neg].name + ) + instance = try(each.value.instance, null) + ip_address = each.value.ip_address + port = each.value.port + zone = each.value.zone +} + +resource "google_compute_region_network_endpoint_group" "psc" { + for_each = local.neg_regional_psc + project = var.project_id + region = each.value.psc.region + name = "${var.name}-${each.key}" + description = coalesce(each.value.description, var.description) + network_endpoint_type = "PRIVATE_SERVICE_CONNECT" + psc_target_service = each.value.psc.target_service + network = each.value.psc.network + subnetwork = each.value.psc.subnetwork +} + +resource "google_compute_region_network_endpoint_group" "serverless" { + for_each = local.neg_regional_serverless + project = var.project_id + region = try( + each.value.cloudrun.region, each.value.cloudfunction.region, null + ) + name = "${var.name}-${each.key}" + description = coalesce(each.value.description, var.description) + network_endpoint_type = "SERVERLESS" + dynamic "cloud_function" { + for_each = each.value.cloudfunction == null ? [] : [""] + content { + function = each.value.cloudfunction.target_function + url_mask = each.value.cloudfunction.target_urlmask + } + } + dynamic "cloud_run" { + for_each = each.value.cloudrun == null ? [] : [""] + content { + service = try(each.value.cloudrun.target_service.name, null) + tag = try(each.value.cloudrun.target_service.tag, null) + url_mask = each.value.cloudrun.target_urlmask + } + } +} diff --git a/assets/modules-fabric/v26/net-lb-app-ext/outputs.tf b/assets/modules-fabric/v26/net-lb-app-ext/outputs.tf new file mode 100644 index 0000000..47f5607 --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-app-ext/outputs.tf @@ -0,0 +1,65 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "address" { + description = "Forwarding rule address." + value = google_compute_global_forwarding_rule.default.ip_address +} + +output "backend_service_ids" { + description = "Backend service resources." + value = { + for k, v in google_compute_backend_service.default : k => v.id + } +} + +output "backend_service_names" { + description = "Backend service resource names." + value = { + for k, v in google_compute_backend_service.default : k => v.name + } +} + +output "forwarding_rule" { + description = "Forwarding rule resource." + value = google_compute_global_forwarding_rule.default +} + +output "group_ids" { + description = "Autogenerated instance group ids." + value = { + for k, v in google_compute_instance_group.default : k => v.id + } +} + +output "health_check_ids" { + description = "Autogenerated health check ids." + value = { + for k, v in google_compute_health_check.default : k => v.id + } +} + +output "id" { + description = "Fully qualified forwarding rule id." + value = google_compute_global_forwarding_rule.default.id +} + +output "neg_ids" { + description = "Autogenerated network endpoint group ids." + value = { + for k, v in google_compute_network_endpoint_group.default : k => v.id + } +} diff --git a/assets/modules-fabric/v26/net-lb-app-ext/urlmap.tf b/assets/modules-fabric/v26/net-lb-app-ext/urlmap.tf new file mode 100644 index 0000000..ec1acaf --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-app-ext/urlmap.tf @@ -0,0 +1,952 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description URL map resources. + +locals { + backend_ids = merge( + { for k, v in google_compute_backend_service.default : k => v.id }, + { for k, v in google_compute_backend_bucket.default : k => v.id } + ) +} + +resource "google_compute_url_map" "default" { + provider = google-beta + project = var.project_id + name = var.name + description = var.description + default_service = ( + var.urlmap_config.default_service == null ? null : lookup( + local.backend_ids, + var.urlmap_config.default_service, + var.urlmap_config.default_service + ) + ) + + dynamic "default_route_action" { + for_each = ( + var.urlmap_config.default_route_action == null + ? [] + : [var.urlmap_config.default_route_action] + ) + iterator = route_action + content { + dynamic "cors_policy" { + for_each = ( + route_action.value.cors_policy == null + ? [] + : [route_action.value.cors_policy] + ) + content { + allow_credentials = cors_policy.value.allow_credentials + allow_headers = cors_policy.value.allow_headers + allow_methods = cors_policy.value.allow_methods + allow_origin_regexes = cors_policy.value.allow_origin_regexes + allow_origins = cors_policy.value.allow_origins + disabled = cors_policy.value.disabled + expose_headers = cors_policy.value.expose_headers + max_age = cors_policy.value.max_age + } + } + dynamic "fault_injection_policy" { + for_each = ( + route_action.value.fault_injection_policy == null + ? [] + : [route_action.value.fault_injection_policy] + ) + content { + dynamic "abort" { + for_each = ( + fault_injection_policy.value.abort == null + ? [] + : [fault_injection_policy.value.abort] + ) + content { + http_status = abort.value.status + percentage = abort.value.percentage + } + } + dynamic "delay" { + for_each = ( + fault_injection_policy.value.delay == null + ? [] + : [fault_injection_policy.value.delay] + ) + content { + percentage = delay.value.percentage + fixed_delay { + nanos = delay.value.fixed.nanos + seconds = delay.value.fixed.seconds + } + } + } + } + } + dynamic "request_mirror_policy" { + for_each = ( + route_action.value.request_mirror_backend == null + ? [] + : [""] + ) + content { + backend_service = lookup( + local.backend_ids, + route_action.value.request_mirror_backend, + route_action.value.request_mirror_backend + ) + } + } + dynamic "retry_policy" { + for_each = ( + route_action.value.retry_policy == null + ? [] + : [route_action.value.retry_policy] + ) + content { + num_retries = retry_policy.value.num_retries + retry_conditions = retry_policy.value.retry_conditions + dynamic "per_try_timeout" { + for_each = ( + retry_policy.value.per_try_timeout == null + ? [] + : [retry_policy.value.per_try_timeout] + ) + content { + nanos = per_try_timeout.value.nanos + seconds = per_try_timeout.value.seconds + } + } + } + } + dynamic "timeout" { + for_each = ( + route_action.value.timeout == null + ? [] + : [route_action.value.timeout] + ) + content { + nanos = timeout.value.nanos + seconds = timeout.value.seconds + } + } + dynamic "url_rewrite" { + for_each = ( + route_action.value.url_rewrite == null + ? [] + : [route_action.value.url_rewrite] + ) + content { + host_rewrite = url_rewrite.value.host + path_prefix_rewrite = url_rewrite.value.path_prefix + } + } + dynamic "weighted_backend_services" { + for_each = coalesce( + route_action.value.weighted_backend_services, {} + ) + iterator = service + content { + backend_service = lookup( + local.backend_ids, service.key, service.key + ) + weight = service.value.weight + dynamic "header_action" { + for_each = ( + service.value.header_action == null + ? [] + : [service.value.header_action] + ) + iterator = h + content { + request_headers_to_remove = h.value.request_remove + response_headers_to_remove = h.value.response_remove + dynamic "request_headers_to_add" { + for_each = coalesce(h.value.request_add, {}) + content { + header_name = request_headers_to_add.key + header_value = request_headers_to_add.value.value + replace = request_headers_to_add.value.replace + } + } + dynamic "response_headers_to_add" { + for_each = coalesce(h.value.response_add, {}) + content { + header_name = response_headers_to_add.key + header_value = response_headers_to_add.value.value + replace = response_headers_to_add.value.replace + } + } + } + } + } + } + } + } + + dynamic "default_url_redirect" { + for_each = ( + var.urlmap_config.default_url_redirect == null + ? [] + : [var.urlmap_config.default_url_redirect] + ) + iterator = r + content { + host_redirect = r.value.host + https_redirect = r.value.https + path_redirect = r.value.path + prefix_redirect = r.value.prefix + redirect_response_code = r.value.response_code + strip_query = r.value.strip_query + } + } + + dynamic "header_action" { + for_each = ( + var.urlmap_config.header_action == null + ? [] + : [var.urlmap_config.header_action] + ) + iterator = h + content { + request_headers_to_remove = h.value.request_remove + response_headers_to_remove = h.value.response_remove + dynamic "request_headers_to_add" { + for_each = coalesce(h.value.request_add, {}) + content { + header_name = request_headers_to_add.key + header_value = request_headers_to_add.value.value + replace = request_headers_to_add.value.replace + } + } + dynamic "response_headers_to_add" { + for_each = coalesce(h.value.response_add, {}) + content { + header_name = response_headers_to_add.key + header_value = response_headers_to_add.value.value + replace = response_headers_to_add.value.replace + } + } + } + } + + dynamic "host_rule" { + for_each = coalesce(var.urlmap_config.host_rules, []) + iterator = r + content { + hosts = r.value.hosts + path_matcher = r.value.path_matcher + description = r.value.description + } + } + + dynamic "path_matcher" { + for_each = coalesce(var.urlmap_config.path_matchers, {}) + iterator = m + content { + default_service = m.value.default_service == null ? null : lookup( + local.backend_ids, m.value.default_service, m.value.default_service + ) + description = m.value.description + name = m.key + dynamic "default_route_action" { + for_each = ( + m.value.default_route_action == null + ? [] + : [m.value.default_route_action] + ) + iterator = route_action + content { + dynamic "cors_policy" { + for_each = ( + route_action.value.cors_policy == null + ? [] + : [route_action.value.cors_policy] + ) + content { + allow_credentials = cors_policy.value.allow_credentials + allow_headers = cors_policy.value.allow_headers + allow_methods = cors_policy.value.allow_methods + allow_origin_regexes = cors_policy.value.allow_origin_regexes + allow_origins = cors_policy.value.allow_origins + disabled = cors_policy.value.disabled + expose_headers = cors_policy.value.expose_headers + max_age = cors_policy.value.max_age + } + } + dynamic "fault_injection_policy" { + for_each = ( + route_action.value.fault_injection_policy == null + ? [] + : [route_action.value.fault_injection_policy] + ) + content { + dynamic "abort" { + for_each = ( + fault_injection_policy.value.abort == null + ? [] + : [fault_injection_policy.value.abort] + ) + content { + http_status = abort.value.status + percentage = abort.value.percentage + } + } + dynamic "delay" { + for_each = ( + fault_injection_policy.value.delay == null + ? [] + : [fault_injection_policy.value.delay] + ) + content { + percentage = delay.value.percentage + fixed_delay { + nanos = delay.value.fixed.nanos + seconds = delay.value.fixed.seconds + } + } + } + } + } + dynamic "request_mirror_policy" { + for_each = ( + route_action.value.request_mirror_backend == null + ? [] + : [""] + ) + content { + backend_service = lookup( + local.backend_ids, + route_action.value.request_mirror_backend, + route_action.value.request_mirror_backend + ) + } + } + dynamic "retry_policy" { + for_each = ( + route_action.value.retry_policy == null + ? [] + : [route_action.value.retry_policy] + ) + content { + num_retries = retry_policy.value.num_retries + retry_conditions = retry_policy.value.retry_conditions + dynamic "per_try_timeout" { + for_each = ( + retry_policy.value.per_try_timeout == null + ? [] + : [retry_policy.value.per_try_timeout] + ) + content { + nanos = per_try_timeout.value.nanos + seconds = per_try_timeout.value.seconds + } + } + } + } + dynamic "timeout" { + for_each = ( + route_action.value.timeout == null + ? [] + : [route_action.value.timeout] + ) + content { + nanos = timeout.value.nanos + seconds = timeout.value.seconds + } + } + dynamic "url_rewrite" { + for_each = ( + route_action.value.url_rewrite == null + ? [] + : [route_action.value.url_rewrite] + ) + content { + host_rewrite = url_rewrite.value.host + path_prefix_rewrite = url_rewrite.value.path_prefix + } + } + dynamic "weighted_backend_services" { + for_each = coalesce( + route_action.value.weighted_backend_services, {} + ) + iterator = service + content { + backend_service = lookup( + local.backend_ids, service.key, service.key + ) + weight = service.value.weight + dynamic "header_action" { + for_each = ( + service.value.header_action == null + ? [] + : [service.value.header_action] + ) + iterator = h + content { + request_headers_to_remove = h.value.request_remove + response_headers_to_remove = h.value.response_remove + dynamic "request_headers_to_add" { + for_each = coalesce(h.value.request_add, {}) + content { + header_name = request_headers_to_add.key + header_value = request_headers_to_add.value.value + replace = request_headers_to_add.value.replace + } + } + dynamic "response_headers_to_add" { + for_each = coalesce(h.value.response_add, {}) + content { + header_name = response_headers_to_add.key + header_value = response_headers_to_add.value.value + replace = response_headers_to_add.value.replace + } + } + } + } + } + } + } + } + dynamic "default_url_redirect" { + for_each = ( + m.value.default_url_redirect == null + ? [] + : [m.value.default_url_redirect] + ) + content { + host_redirect = default_url_redirect.value.host + https_redirect = default_url_redirect.value.https + path_redirect = default_url_redirect.value.path + prefix_redirect = default_url_redirect.value.prefix + redirect_response_code = default_url_redirect.value.response_code + strip_query = default_url_redirect.value.strip_query + } + } + dynamic "header_action" { + for_each = ( + m.value.header_action == null + ? [] + : [m.value.header_action] + ) + iterator = h + content { + request_headers_to_remove = h.value.request_remove + response_headers_to_remove = h.value.response_remove + dynamic "request_headers_to_add" { + for_each = coalesce(h.value.request_add, {}) + content { + header_name = request_headers_to_add.key + header_value = request_headers_to_add.value.value + replace = request_headers_to_add.value.replace + } + } + dynamic "response_headers_to_add" { + for_each = coalesce(h.value.response_add, {}) + content { + header_name = response_headers_to_add.key + header_value = response_headers_to_add.value.value + replace = response_headers_to_add.value.replace + } + } + } + } + dynamic "path_rule" { + for_each = toset(coalesce(m.value.path_rules, [])) + content { + paths = path_rule.value.paths + service = path_rule.value.service == null ? null : lookup( + local.backend_ids, + path_rule.value.service, + path_rule.value.service + ) + dynamic "route_action" { + for_each = ( + path_rule.value.route_action == null + ? [] + : [path_rule.value.route_action] + ) + content { + dynamic "cors_policy" { + for_each = ( + route_action.value.cors_policy == null + ? [] + : [route_action.value.cors_policy] + ) + content { + allow_credentials = cors_policy.value.allow_credentials + allow_headers = cors_policy.value.allow_headers + allow_methods = cors_policy.value.allow_methods + allow_origin_regexes = cors_policy.value.allow_origin_regexes + allow_origins = cors_policy.value.allow_origins + disabled = cors_policy.value.disabled + expose_headers = cors_policy.value.expose_headers + max_age = cors_policy.value.max_age + } + } + dynamic "fault_injection_policy" { + for_each = ( + route_action.value.fault_injection_policy == null + ? [] + : [route_action.value.fault_injection_policy] + ) + content { + dynamic "abort" { + for_each = ( + fault_injection_policy.value.abort == null + ? [] + : [fault_injection_policy.value.abort] + ) + content { + http_status = abort.value.status + percentage = abort.value.percentage + } + } + dynamic "delay" { + for_each = ( + fault_injection_policy.value.delay == null + ? [] + : [fault_injection_policy.value.delay] + ) + content { + percentage = delay.value.percentage + fixed_delay { + nanos = delay.value.fixed.nanos + seconds = delay.value.fixed.seconds + } + } + } + } + } + dynamic "request_mirror_policy" { + for_each = ( + route_action.value.request_mirror_backend == null + ? [] + : [""] + ) + content { + backend_service = lookup( + local.backend_ids, + route_action.value.request_mirror_backend, + route_action.value.request_mirror_backend + ) + } + } + dynamic "retry_policy" { + for_each = ( + route_action.value.retry_policy == null + ? [] + : [route_action.value.retry_policy] + ) + content { + num_retries = retry_policy.value.num_retries + retry_conditions = retry_policy.value.retry_conditions + dynamic "per_try_timeout" { + for_each = ( + retry_policy.value.per_try_timeout == null + ? [] + : [retry_policy.value.per_try_timeout] + ) + content { + nanos = per_try_timeout.value.nanos + seconds = per_try_timeout.value.seconds + } + } + } + } + dynamic "timeout" { + for_each = ( + route_action.value.timeout == null + ? [] + : [route_action.value.timeout] + ) + content { + nanos = timeout.value.nanos + seconds = timeout.value.seconds + } + } + dynamic "url_rewrite" { + for_each = ( + route_action.value.url_rewrite == null + ? [] + : [route_action.value.url_rewrite] + ) + content { + host_rewrite = url_rewrite.value.host + path_prefix_rewrite = url_rewrite.value.path_prefix + } + } + dynamic "weighted_backend_services" { + for_each = coalesce( + route_action.value.weighted_backend_services, {} + ) + iterator = service + content { + backend_service = lookup( + local.backend_ids, service.key, service.key + ) + weight = service.value.weight + dynamic "header_action" { + for_each = ( + service.value.header_action == null + ? [] + : [service.value.header_action] + ) + iterator = h + content { + request_headers_to_remove = h.value.request_remove + response_headers_to_remove = h.value.response_remove + dynamic "request_headers_to_add" { + for_each = coalesce(h.value.request_add, {}) + content { + header_name = request_headers_to_add.key + header_value = request_headers_to_add.value.value + replace = request_headers_to_add.value.replace + } + } + dynamic "response_headers_to_add" { + for_each = coalesce(h.value.response_add, {}) + content { + header_name = response_headers_to_add.key + header_value = response_headers_to_add.value.value + replace = response_headers_to_add.value.replace + } + } + } + } + } + } + } + } + dynamic "url_redirect" { + for_each = ( + path_rule.value.url_redirect == null + ? [] + : [path_rule.value.url_redirect] + ) + content { + host_redirect = url_redirect.value.host + https_redirect = url_redirect.value.https + path_redirect = url_redirect.value.path + prefix_redirect = url_redirect.value.prefix + redirect_response_code = url_redirect.value.response_code + strip_query = url_redirect.value.strip_query + } + } + } + } + dynamic "route_rules" { + for_each = toset(coalesce(m.value.route_rules, [])) + content { + priority = route_rules.value.priority + service = route_rules.value.service == null ? null : lookup( + local.backend_ids, + route_rules.value.service, + route_rules.value.service + ) + dynamic "header_action" { + for_each = ( + route_rules.value.header_action == null + ? [] + : [route_rules.value.header_action] + ) + iterator = h + content { + request_headers_to_remove = h.value.request_remove + response_headers_to_remove = h.value.response_remove + dynamic "request_headers_to_add" { + for_each = coalesce(h.value.request_add, {}) + content { + header_name = request_headers_to_add.key + header_value = request_headers_to_add.value.value + replace = request_headers_to_add.value.replace + } + } + dynamic "response_headers_to_add" { + for_each = coalesce(h.value.response_add, {}) + content { + header_name = response_headers_to_add.key + header_value = response_headers_to_add.value.value + replace = response_headers_to_add.value.replace + } + } + } + } + dynamic "match_rules" { + for_each = toset(coalesce(route_rules.value.match_rules, [])) + content { + ignore_case = match_rules.value.ignore_case + full_path_match = ( + try(match_rules.value.path.type, null) == "full" + ? match_rules.value.path.value + : null + ) + prefix_match = ( + try(match_rules.value.path.type, null) == "prefix" + ? match_rules.value.path.value + : null + ) + regex_match = ( + try(match_rules.value.path.type, null) == "regex" + ? match_rules.value.path.value + : null + ) + dynamic "header_matches" { + for_each = toset(coalesce(match_rules.value.headers, [])) + iterator = h + content { + header_name = h.value.name + exact_match = h.value.type == "exact" ? h.value.value : null + invert_match = h.value.invert_match + prefix_match = h.value.type == "prefix" ? h.value.value : null + present_match = h.value.type == "present" ? h.value.value : null + regex_match = h.value.type == "regex" ? h.value.value : null + suffix_match = h.value.type == "suffix" ? h.value.value : null + dynamic "range_match" { + for_each = ( + h.value.type != "range" || h.value.range_value == null + ? [] + : [""] + ) + content { + range_end = h.value.range_value.end + range_start = h.value.range_value.start + } + } + } + } + dynamic "metadata_filters" { + for_each = toset(coalesce(match_rules.value.metadata_filters, [])) + iterator = m + content { + filter_match_criteria = ( + m.value.match_all ? "MATCH_ALL" : "MATCH_ANY" + ) + dynamic "filter_labels" { + for_each = m.value.labels + content { + name = filter_labels.key + value = filter_labels.value + } + } + } + } + dynamic "query_parameter_matches" { + for_each = toset(coalesce(match_rules.value.query_params, [])) + iterator = q + content { + name = q.value.name + exact_match = ( + q.value.type == "exact" ? q.value.value : null + ) + present_match = ( + q.value.type == "present" ? q.value.value : null + ) + regex_match = ( + q.value.type == "regex" ? q.value.value : null + ) + } + } + } + } + dynamic "route_action" { + for_each = ( + route_rules.value.route_action == null + ? [] + : [route_rules.value.route_action] + ) + content { + dynamic "cors_policy" { + for_each = ( + route_action.value.cors_policy == null + ? [] + : [route_action.value.cors_policy] + ) + content { + allow_credentials = cors_policy.value.allow_credentials + allow_headers = cors_policy.value.allow_headers + allow_methods = cors_policy.value.allow_methods + allow_origin_regexes = cors_policy.value.allow_origin_regexes + allow_origins = cors_policy.value.allow_origins + disabled = cors_policy.value.disabled + expose_headers = cors_policy.value.expose_headers + max_age = cors_policy.value.max_age + } + } + dynamic "fault_injection_policy" { + for_each = ( + route_action.value.fault_injection_policy == null + ? [] + : [route_action.value.fault_injection_policy] + ) + content { + dynamic "abort" { + for_each = ( + fault_injection_policy.value.abort == null + ? [] + : [fault_injection_policy.value.abort] + ) + content { + http_status = abort.value.status + percentage = abort.value.percentage + } + } + dynamic "delay" { + for_each = ( + fault_injection_policy.value.delay == null + ? [] + : [fault_injection_policy.value.delay] + ) + content { + percentage = delay.value.percentage + fixed_delay { + nanos = delay.value.fixed.nanos + seconds = delay.value.fixed.seconds + } + } + } + } + } + dynamic "request_mirror_policy" { + for_each = ( + route_action.value.request_mirror_backend == null + ? [] + : [""] + ) + content { + backend_service = lookup( + local.backend_ids, + route_action.value.request_mirror_backend, + route_action.value.request_mirror_backend + ) + } + } + dynamic "retry_policy" { + for_each = ( + route_action.value.retry_policy == null + ? [] + : [route_action.value.retry_policy] + ) + content { + num_retries = retry_policy.value.num_retries + retry_conditions = retry_policy.value.retry_conditions + dynamic "per_try_timeout" { + for_each = ( + retry_policy.value.per_try_timeout == null + ? [] + : [retry_policy.value.per_try_timeout] + ) + content { + nanos = per_try_timeout.value.nanos + seconds = per_try_timeout.value.seconds + } + } + } + } + dynamic "timeout" { + for_each = ( + route_action.value.timeout == null + ? [] + : [route_action.value.timeout] + ) + content { + nanos = timeout.value.nanos + seconds = timeout.value.seconds + } + } + dynamic "url_rewrite" { + for_each = ( + route_action.value.url_rewrite == null + ? [] + : [route_action.value.url_rewrite] + ) + content { + host_rewrite = url_rewrite.value.host + path_prefix_rewrite = url_rewrite.value.path_prefix + } + } + dynamic "weighted_backend_services" { + for_each = coalesce( + route_action.value.weighted_backend_services, {} + ) + iterator = service + content { + backend_service = lookup( + local.backend_ids, service.key, service.key + ) + weight = service.value.weight + dynamic "header_action" { + for_each = ( + service.value.header_action == null + ? [] + : [service.value.header_action] + ) + iterator = h + content { + request_headers_to_remove = h.value.request_remove + response_headers_to_remove = h.value.response_remove + dynamic "request_headers_to_add" { + for_each = coalesce(h.value.request_add, {}) + content { + header_name = request_headers_to_add.key + header_value = request_headers_to_add.value.value + replace = request_headers_to_add.value.replace + } + } + dynamic "response_headers_to_add" { + for_each = coalesce(h.value.response_add, {}) + content { + header_name = response_headers_to_add.key + header_value = response_headers_to_add.value.value + replace = response_headers_to_add.value.replace + } + } + } + } + } + } + } + } + dynamic "url_redirect" { + for_each = ( + route_rules.value.url_redirect == null + ? [] + : [route_rules.value.url_redirect] + ) + content { + host_redirect = url_redirect.value.host + https_redirect = url_redirect.value.https + path_redirect = url_redirect.value.path + prefix_redirect = url_redirect.value.prefix + redirect_response_code = url_redirect.value.response_code + strip_query = url_redirect.value.strip_query + } + } + } + } + } + } + + dynamic "test" { + for_each = toset(coalesce(var.urlmap_config.test, [])) + content { + host = test.value.host + path = test.value.path + service = test.value.service + description = test.value.description + } + } + +} diff --git a/assets/modules-fabric/v26/net-lb-app-ext/variables-backend-service.tf b/assets/modules-fabric/v26/net-lb-app-ext/variables-backend-service.tf new file mode 100644 index 0000000..51f65df --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-app-ext/variables-backend-service.tf @@ -0,0 +1,150 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Backend services variables. + +variable "backend_service_configs" { + description = "Backend service level configuration." + type = map(object({ + affinity_cookie_ttl_sec = optional(number) + compression_mode = optional(string) + connection_draining_timeout_sec = optional(number) + custom_request_headers = optional(list(string)) + custom_response_headers = optional(list(string)) + enable_cdn = optional(bool) + health_checks = optional(list(string), ["default"]) + log_sample_rate = optional(number) + port_name = optional(string) + project_id = optional(string) + protocol = optional(string) + security_policy = optional(string) + session_affinity = optional(string) + timeout_sec = optional(number) + backends = list(object({ + # group renamed to backend + backend = string + balancing_mode = optional(string, "UTILIZATION") + capacity_scaler = optional(number, 1) + description = optional(string, "Terraform managed.") + failover = optional(bool, false) + max_connections = optional(object({ + per_endpoint = optional(number) + per_group = optional(number) + per_instance = optional(number) + })) + max_rate = optional(object({ + per_endpoint = optional(number) + per_group = optional(number) + per_instance = optional(number) + })) + max_utilization = optional(number) + })) + cdn_policy = optional(object({ + cache_mode = optional(string) + client_ttl = optional(number) + default_ttl = optional(number) + max_ttl = optional(number) + negative_caching = optional(bool) + serve_while_stale = optional(number) + signed_url_cache_max_age_sec = optional(number) + cache_key_policy = optional(object({ + include_host = optional(bool) + include_named_cookies = optional(list(string)) + include_protocol = optional(bool) + include_query_string = optional(bool) + query_string_blacklist = optional(list(string)) + query_string_whitelist = optional(list(string)) + })) + negative_caching_policy = optional(object({ + code = optional(number) + ttl = optional(number) + })) + })) + circuit_breakers = optional(object({ + max_connections = optional(number) + max_pending_requests = optional(number) + max_requests = optional(number) + max_requests_per_connection = optional(number) + max_retries = optional(number) + connect_timeout = optional(object({ + seconds = number + nanos = optional(number) + })) + })) + consistent_hash = optional(object({ + http_header_name = optional(string) + minimum_ring_size = optional(number) + http_cookie = optional(object({ + name = optional(string) + path = optional(string) + ttl = optional(object({ + seconds = number + nanos = optional(number) + })) + })) + })) + iap_config = optional(object({ + oauth2_client_id = string + oauth2_client_secret = string + oauth2_client_secret_sha256 = optional(string) + })) + outlier_detection = optional(object({ + consecutive_errors = optional(number) + consecutive_gateway_failure = optional(number) + enforcing_consecutive_errors = optional(number) + enforcing_consecutive_gateway_failure = optional(number) + enforcing_success_rate = optional(number) + max_ejection_percent = optional(number) + success_rate_minimum_hosts = optional(number) + success_rate_request_volume = optional(number) + success_rate_stdev_factor = optional(number) + base_ejection_time = optional(object({ + seconds = number + nanos = optional(number) + })) + interval = optional(object({ + seconds = number + nanos = optional(number) + })) + })) + security_settings = optional(object({ + client_tls_policy = string + subject_alt_names = list(string) + })) + })) + default = {} + nullable = false + validation { + condition = contains( + [ + "-", "ROUND_ROBIN", "LEAST_REQUEST", "RING_HASH", + "RANDOM", "ORIGINAL_DESTINATION", "MAGLEV" + ], + try(var.backend_service_configs.locality_lb_policy, "-") + ) + error_message = "Invalid locality lb policy value." + } + validation { + condition = contains( + [ + "NONE", "CLIENT_IP", "CLIENT_IP_NO_DESTINATION", + "CLIENT_IP_PORT_PROTO", "CLIENT_IP_PROTO" + ], + try(var.backend_service_configs.session_affinity, "NONE") + ) + error_message = "Invalid session affinity value." + } +} diff --git a/assets/modules-fabric/v26/net-lb-app-ext/variables-health-check.tf b/assets/modules-fabric/v26/net-lb-app-ext/variables-health-check.tf new file mode 100644 index 0000000..be34d77 --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-app-ext/variables-health-check.tf @@ -0,0 +1,109 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Health check variable. + +variable "health_check_configs" { + description = "Optional auto-created health check configurations, use the output self-link to set it in the auto healing policy. Refer to examples for usage." + type = map(object({ + check_interval_sec = optional(number) + description = optional(string, "Terraform managed.") + enable_logging = optional(bool, false) + healthy_threshold = optional(number) + project_id = optional(string) + timeout_sec = optional(number) + unhealthy_threshold = optional(number) + grpc = optional(object({ + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + service_name = optional(string) + })) + http = optional(object({ + host = optional(string) + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request_path = optional(string) + response = optional(string) + })) + http2 = optional(object({ + host = optional(string) + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request_path = optional(string) + response = optional(string) + })) + https = optional(object({ + host = optional(string) + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request_path = optional(string) + response = optional(string) + })) + tcp = optional(object({ + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request = optional(string) + response = optional(string) + })) + ssl = optional(object({ + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request = optional(string) + response = optional(string) + })) + })) + default = { + default = { + http = { + port_specification = "USE_SERVING_PORT" + } + } + } + validation { + condition = alltrue([ + for k, v in var.health_check_configs : ( + (try(v.grpc, null) == null ? 0 : 1) + + (try(v.http, null) == null ? 0 : 1) + + (try(v.http2, null) == null ? 0 : 1) + + (try(v.https, null) == null ? 0 : 1) + + (try(v.tcp, null) == null ? 0 : 1) + + (try(v.ssl, null) == null ? 0 : 1) <= 1 + ) + ]) + error_message = "At most one health check type can be configured at a time." + } + validation { + condition = alltrue(flatten([ + for k, v in var.health_check_configs : [ + for kk, vv in v : contains([ + "-", "USE_FIXED_PORT", "USE_NAMED_PORT", "USE_SERVING_PORT" + ], coalesce(try(vv.port_specification, null), "-")) + ] + ])) + error_message = "Invalid 'port_specification' value. Supported values are 'USE_FIXED_PORT', 'USE_NAMED_PORT', 'USE_SERVING_PORT'." + } +} diff --git a/assets/modules-fabric/v26/net-lb-app-ext/variables-urlmap.tf b/assets/modules-fabric/v26/net-lb-app-ext/variables-urlmap.tf new file mode 100644 index 0000000..e4b72df --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-app-ext/variables-urlmap.tf @@ -0,0 +1,372 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description URLmap variable. + +variable "urlmap_config" { + description = "The URL map configuration." + type = object({ + default_route_action = optional(object({ + request_mirror_backend = optional(string) + cors_policy = optional(object({ + allow_credentials = optional(bool) + allow_headers = optional(string) + allow_methods = optional(string) + allow_origin_regexes = list(string) + allow_origins = list(string) + disabled = optional(bool) + expose_headers = optional(string) + max_age = optional(string) + })) + fault_injection_policy = optional(object({ + abort = optional(object({ + percentage = number + status = number + })) + delay = optional(object({ + fixed = object({ + seconds = number + nanos = number + }) + percentage = number + })) + })) + retry_policy = optional(object({ + num_retries = number + retry_conditions = optional(list(string)) + per_try_timeout = optional(object({ + seconds = number + nanos = optional(number) + })) + })) + timeout = optional(object({ + seconds = number + nanos = optional(number) + })) + url_rewrite = optional(object({ + host = optional(string) + path_prefix = optional(string) + })) + weighted_backend_services = optional(map(object({ + weight = number + header_action = optional(object({ + request_add = optional(map(object({ + value = string + replace = optional(bool, true) + }))) + request_remove = optional(list(string)) + response_add = optional(map(object({ + value = string + replace = optional(bool, true) + }))) + response_remove = optional(list(string)) + })) + }))) + })) + default_service = optional(string) + default_url_redirect = optional(object({ + host = optional(string) + https = optional(bool) + path = optional(string) + prefix = optional(string) + response_code = optional(string) + strip_query = optional(bool) + })) + header_action = optional(object({ + request_add = optional(map(object({ + value = string + replace = optional(bool, true) + }))) + request_remove = optional(list(string)) + response_add = optional(map(object({ + value = string + replace = optional(bool, true) + }))) + response_remove = optional(list(string)) + })) + host_rules = optional(list(object({ + hosts = list(string) + path_matcher = string + description = optional(string) + }))) + path_matchers = optional(map(object({ + description = optional(string) + default_route_action = optional(object({ + request_mirror_backend = optional(string) + cors_policy = optional(object({ + allow_credentials = optional(bool) + allow_headers = optional(string) + allow_methods = optional(string) + allow_origin_regexes = list(string) + allow_origins = list(string) + disabled = optional(bool) + expose_headers = optional(string) + max_age = optional(string) + })) + fault_injection_policy = optional(object({ + abort = optional(object({ + percentage = number + status = number + })) + delay = optional(object({ + fixed = object({ + seconds = number + nanos = number + }) + percentage = number + })) + })) + retry_policy = optional(object({ + num_retries = number + retry_conditions = optional(list(string)) + per_try_timeout = optional(object({ + seconds = number + nanos = optional(number) + })) + })) + timeout = optional(object({ + seconds = number + nanos = optional(number) + })) + url_rewrite = optional(object({ + host = optional(string) + path_prefix = optional(string) + })) + weighted_backend_services = optional(map(object({ + weight = number + header_action = optional(object({ + request_add = optional(map(object({ + value = string + replace = optional(bool, true) + }))) + request_remove = optional(list(string)) + response_add = optional(map(object({ + value = string + replace = optional(bool, true) + }))) + response_remove = optional(list(string)) + })) + }))) + })) + default_service = optional(string) + default_url_redirect = optional(object({ + host = optional(string) + https = optional(bool) + path = optional(string) + prefix = optional(string) + response_code = optional(string) + strip_query = optional(bool) + })) + header_action = optional(object({ + request_add = optional(map(object({ + value = string + replace = optional(bool, true) + }))) + request_remove = optional(list(string)) + response_add = optional(map(object({ + value = string + replace = optional(bool, true) + }))) + response_remove = optional(list(string)) + })) + path_rules = optional(list(object({ + paths = list(string) + service = optional(string) + route_action = optional(object({ + request_mirror_backend = optional(string) + cors_policy = optional(object({ + allow_credentials = optional(bool) + allow_headers = optional(string) + allow_methods = optional(string) + allow_origin_regexes = list(string) + allow_origins = list(string) + disabled = optional(bool) + expose_headers = optional(string) + max_age = optional(string) + })) + fault_injection_policy = optional(object({ + abort = optional(object({ + percentage = number + status = number + })) + delay = optional(object({ + fixed = object({ + seconds = number + nanos = number + }) + percentage = number + })) + })) + retry_policy = optional(object({ + num_retries = number + retry_conditions = optional(list(string)) + per_try_timeout = optional(object({ + seconds = number + nanos = optional(number) + })) + })) + timeout = optional(object({ + seconds = number + nanos = optional(number) + })) + url_rewrite = optional(object({ + host = optional(string) + path_prefix = optional(string) + })) + weighted_backend_services = optional(map(object({ + weight = number + header_action = optional(object({ + request_add = optional(map(object({ + value = string + replace = optional(bool, true) + }))) + request_remove = optional(list(string)) + response_add = optional(map(object({ + value = string + replace = optional(bool, true) + }))) + response_remove = optional(list(string)) + })) + }))) + })) + url_redirect = optional(object({ + host = optional(string) + https = optional(bool) + path = optional(string) + prefix = optional(string) + response_code = optional(string) + strip_query = optional(bool) + })) + }))) + route_rules = optional(list(object({ + priority = number + service = optional(string) + header_action = optional(object({ + request_add = optional(map(object({ + value = string + replace = optional(bool, true) + }))) + request_remove = optional(list(string)) + response_add = optional(map(object({ + value = string + replace = optional(bool, true) + }))) + response_remove = optional(list(string)) + })) + match_rules = optional(list(object({ + ignore_case = optional(bool, false) + headers = optional(list(object({ + name = string + invert_match = optional(bool, false) + type = optional(string, "present") # exact, prefix, suffix, regex, present, range + value = optional(string) + range_value = optional(object({ + end = string + start = string + })) + }))) + metadata_filters = optional(list(object({ + labels = map(string) + match_all = bool # MATCH_ANY, MATCH_ALL + }))) + path = optional(object({ + value = string + type = optional(string, "prefix") # full, prefix, regex + })) + query_params = optional(list(object({ + name = string + value = string + type = optional(string, "present") # exact, present, regex + }))) + }))) + route_action = optional(object({ + request_mirror_backend = optional(string) + cors_policy = optional(object({ + allow_credentials = optional(bool) + allow_headers = optional(string) + allow_methods = optional(string) + allow_origin_regexes = list(string) + allow_origins = list(string) + disabled = optional(bool) + expose_headers = optional(string) + max_age = optional(string) + })) + fault_injection_policy = optional(object({ + abort = optional(object({ + percentage = number + status = number + })) + delay = optional(object({ + fixed = object({ + seconds = number + nanos = number + }) + percentage = number + })) + })) + retry_policy = optional(object({ + num_retries = number + retry_conditions = optional(list(string)) + per_try_timeout = optional(object({ + seconds = number + nanos = optional(number) + })) + })) + timeout = optional(object({ + seconds = number + nanos = optional(number) + })) + url_rewrite = optional(object({ + host = optional(string) + path_prefix = optional(string) + })) + weighted_backend_services = optional(map(object({ + weight = number + header_action = optional(object({ + request_add = optional(map(object({ + value = string + replace = optional(bool, true) + }))) + request_remove = optional(list(string)) + response_add = optional(map(object({ + value = string + replace = optional(bool, true) + }))) + response_remove = optional(list(string)) + })) + }))) + })) + url_redirect = optional(object({ + host = optional(string) + https = optional(bool) + path = optional(string) + prefix = optional(string) + response_code = optional(string) + strip_query = optional(bool) + })) + }))) + }))) + test = optional(list(object({ + host = string + path = string + service = string + description = optional(string) + }))) + }) + default = { + default_service = "default" + } +} diff --git a/assets/modules-fabric/v26/net-lb-app-ext/variables.tf b/assets/modules-fabric/v26/net-lb-app-ext/variables.tf new file mode 100644 index 0000000..bc25409 --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-app-ext/variables.tf @@ -0,0 +1,232 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "address" { + description = "Optional IP address used for the forwarding rule." + type = string + default = null +} + +variable "backend_buckets_config" { + description = "Backend buckets configuration." + type = map(object({ + bucket_name = string + compression_mode = optional(string) + custom_response_headers = optional(list(string)) + description = optional(string) + edge_security_policy = optional(string) + enable_cdn = optional(bool) + cdn_policy = optional(object({ + bypass_cache_on_request_headers = optional(list(string)) + cache_mode = optional(string) + client_ttl = optional(number) + default_ttl = optional(number) + max_ttl = optional(number) + negative_caching = optional(bool) + request_coalescing = optional(bool) + serve_while_stale = optional(number) + signed_url_cache_max_age_sec = optional(number) + cache_key_policy = optional(object({ + include_http_headers = optional(list(string)) + query_string_whitelist = optional(list(string)) + })) + negative_caching_policy = optional(object({ + code = optional(number) + ttl = optional(number) + })) + })) + })) + default = {} + nullable = true +} + +variable "description" { + description = "Optional description used for resources." + type = string + default = "Terraform managed." +} + +variable "group_configs" { + description = "Optional unmanaged groups to create. Can be referenced in backends via key or outputs." + type = map(object({ + zone = string + instances = optional(list(string)) + named_ports = optional(map(number), {}) + project_id = optional(string) + })) + default = {} + nullable = false +} + +variable "https_proxy_config" { + description = "HTTPS proxy connfiguration." + type = object({ + certificate_map = optional(string) + quic_override = optional(string) + ssl_policy = optional(string) + }) + default = {} + nullable = false +} + +variable "labels" { + description = "Labels set on resources." + type = map(string) + default = {} +} + +variable "name" { + description = "Load balancer name." + type = string +} + +variable "neg_configs" { + description = "Optional network endpoint groups to create. Can be referenced in backends via key or outputs." + type = map(object({ + description = optional(string) + cloudfunction = optional(object({ + region = string + target_function = optional(string) + target_urlmask = optional(string) + })) + cloudrun = optional(object({ + region = string + target_service = optional(object({ + name = string + tag = optional(string) + })) + target_urlmask = optional(string) + })) + gce = optional(object({ + network = string + subnetwork = string + zone = string + # default_port = optional(number) + endpoints = optional(map(object({ + instance = string + ip_address = string + port = number + }))) + })) + hybrid = optional(object({ + network = string + zone = string + # re-enable once provider properly support this + # default_port = optional(number) + endpoints = optional(map(object({ + ip_address = string + port = number + }))) + })) + internet = optional(object({ + use_fqdn = optional(bool, true) + # re-enable once provider properly support this + # default_port = optional(number) + endpoints = optional(map(object({ + destination = string + port = number + }))) + })) + psc = optional(object({ + region = string + target_service = string + network = optional(string) + subnetwork = optional(string) + })) + })) + default = {} + nullable = false + validation { + condition = alltrue([ + for k, v in var.neg_configs : ( + (try(v.cloudfunction, null) == null ? 0 : 1) + + (try(v.cloudrun, null) == null ? 0 : 1) + + (try(v.gce, null) == null ? 0 : 1) + + (try(v.hybrid, null) == null ? 0 : 1) + + (try(v.internet, null) == null ? 0 : 1) + + (try(v.psc, null) == null ? 0 : 1) == 1 + ) + ]) + error_message = "Only one type of NEG can be configured at a time." + } + validation { + condition = alltrue([ + for k, v in var.neg_configs : ( + v.cloudrun == null + ? true + : v.cloudrun.target_urlmask != null || v.cloudrun.target_service != null + ) + ]) + error_message = "Cloud Run NEGs need either target service or target urlmask defined." + } + validation { + condition = alltrue([ + for k, v in var.neg_configs : ( + v.cloudfunction == null + ? true + : v.cloudfunction.target_urlmask != null || v.cloudfunction.target_function != null + ) + ]) + error_message = "Cloud Function NEGs need either target function or target urlmask defined." + } +} + +variable "ports" { + description = "Optional ports for HTTP load balancer, valid ports are 80 and 8080." + type = list(string) + default = null +} + +variable "project_id" { + description = "Project id." + type = string +} + +variable "protocol" { + description = "Protocol supported by this load balancer." + type = string + default = "HTTP" + nullable = false + validation { + condition = ( + var.protocol == null || var.protocol == "HTTP" || var.protocol == "HTTPS" + ) + error_message = "Protocol must be HTTP or HTTPS" + } +} + +variable "ssl_certificates" { + description = "SSL target proxy certificates (only if protocol is HTTPS) for existing, custom, and managed certificates." + type = object({ + certificate_ids = optional(list(string), []) + create_configs = optional(map(object({ + certificate = string + private_key = string + })), {}) + managed_configs = optional(map(object({ + domains = list(string) + description = optional(string) + })), {}) + }) + default = {} + nullable = false +} + +variable "use_classic_version" { + description = "Use classic Global Load Balancer." + type = bool + default = true +} diff --git a/assets/modules-fabric/v26/net-lb-app-ext/versions.tf b/assets/modules-fabric/v26/net-lb-app-ext/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-app-ext/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/net-lb-app-int/README.md b/assets/modules-fabric/v26/net-lb-app-int/README.md new file mode 100644 index 0000000..add393c --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-app-int/README.md @@ -0,0 +1,682 @@ +# Internal Application Load Balancer Module + +This module allows managing Internal HTTP/HTTPS Load Balancers (L7 ILBs). It's designed to expose the full configuration of the underlying resources, and to facilitate common usage patterns by providing sensible defaults, and optionally managing prerequisite resources like health checks, instance groups, etc. + +Due to the complexity of the underlying resources, changes to the configuration that involve recreation of resources are best applied in stages, starting by disabling the configuration in the urlmap that references the resources that need recreation, then doing the same for the backend service, etc. + +## Examples + + +- [Examples](#examples) + - [Minimal Example](#minimal-example) + - [Cross-project backend services](#cross-project-backend-services) + - [Health Checks](#health-checks) + - [Instance Groups](#instance-groups) + - [Network Endpoint Groups (NEGs)](#network-endpoint-groups-negs) + - [Zonal NEG creation](#zonal-neg-creation) + - [Hybrid NEG creation](#hybrid-neg-creation) + - [Serverless NEG creation](#serverless-neg-creation) + - [Private Service Connect NEG creation](#private-service-connect-neg-creation) + - [URL Map](#url-map) + - [SSL Certificates](#ssl-certificates) + - [Complex example](#complex-example) +- [Files](#files) +- [Variables](#variables) +- [Outputs](#outputs) + + +### Minimal Example + +An HTTP ILB with a backend service pointing to a GCE instance group: + +```hcl +module "ilb-l7" { + source = "./fabric/modules/net-lb-app-int" + name = "ilb-test" + project_id = var.project_id + region = "europe-west1" + backend_service_configs = { + default = { + backends = [{ + group = "projects/myprj/zones/europe-west1-a/instanceGroups/my-ig" + }] + } + } + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + } +} +# tftest modules=1 resources=5 +``` + +An HTTPS ILB needs a few additional fields: + +```hcl +module "ilb-l7" { + source = "./fabric/modules/net-lb-app-int" + name = "ilb-test" + project_id = var.project_id + region = "europe-west1" + backend_service_configs = { + default = { + backends = [{ + group = "projects/myprj/zones/europe-west1-a/instanceGroups/my-ig" + }] + } + } + protocol = "HTTPS" + ssl_certificates = { + certificate_ids = [ + "projects/myprj/regions/europe-west1/sslCertificates/my-cert" + ] + } + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + } +} +# tftest modules=1 resources=5 +``` + +### Cross-project backend services + +When using Shared VPC, this module also allows configuring [cross-project backend services](https://cloud.google.com/load-balancing/docs/l7-internal/l7-internal-shared-vpc#cross-project): + +```hcl +module "ilb-l7" { + source = "./fabric/modules/net-lb-app-int" + name = "ilb-test" + project_id = "prj-host" + region = "europe-west1" + backend_service_configs = { + default = { + project_id = "prj-svc" + backends = [{ + group = "projects/prj-svc/zones/europe-west1-a/instanceGroups/my-ig" + }] + } + } + health_check_configs = { + default = { + project_id = "prj-svc" + http = { + port_specification = "USE_SERVING_PORT" + } + } + } + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + } +} +# tftest modules=1 resources=5 +``` + +### Health Checks + +You can leverage externally defined health checks for backend services, or have the module create them for you. By default a simple HTTP health check is created, and used in backend services. + +Health check configuration is controlled via the `health_check_configs` variable, which behaves in a similar way to other LB modules in this repository. + +Defining different health checks from the default is very easy. You can for example replace the default HTTP health check with a TCP one and reference it in you backend service: + +```hcl +module "ilb-l7" { + source = "./fabric/modules/net-lb-app-int" + name = "ilb-test" + project_id = var.project_id + region = "europe-west1" + backend_service_configs = { + default = { + backends = [{ + group = "projects/myprj/zones/europe-west1-a/instanceGroups/my-ig" + }] + health_checks = ["custom-tcp"] + } + } + health_check_configs = { + custom-tcp = { + tcp = { port = 80 } + } + } + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + } +} +# tftest modules=1 resources=5 +``` + +To leverage existing health checks without having the module create them, simply pass their self links to backend services and set the `health_check_configs` variable to an empty map: + +```hcl +module "ilb-l7" { + source = "./fabric/modules/net-lb-app-int" + name = "ilb-test" + project_id = var.project_id + region = "europe-west1" + backend_service_configs = { + default = { + backends = [{ + group = "projects/myprj/zones/europe-west1-a/instanceGroups/my-ig" + }] + health_checks = ["projects/myprj/global/healthChecks/custom"] + } + } + health_check_configs = {} + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + } +} +# tftest modules=1 resources=4 +``` + +### Instance Groups + +The module can optionally create unmanaged instance groups, which can then be referred to in backends via their key: + +```hcl +module "ilb-l7" { + source = "./fabric/modules/net-lb-app-int" + name = "ilb-test" + project_id = var.project_id + region = "europe-west1" + backend_service_configs = { + default = { + port_name = "http" + backends = [ + { group = "default" } + ] + } + } + group_configs = { + default = { + zone = "europe-west1-b" + instances = [ + "projects/myprj/zones/europe-west1-b/instances/vm-a" + ] + named_ports = { http = 80 } + } + } + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + } +} +# tftest modules=1 resources=6 +``` + +### Network Endpoint Groups (NEGs) + +Network Endpoint Groups (NEGs) can be used as backends, by passing their id as the backend group in a backends service configuration: + +```hcl +module "ilb-l7" { + source = "./fabric/modules/net-lb-app-int" + name = "ilb-test" + project_id = var.project_id + region = "europe-west1" + backend_service_configs = { + default = { + backends = [{ + balancing_mode = "RATE" + group = "projects/myprj/zones/europe-west1-a/networkEndpointGroups/my-neg" + max_rate = { per_endpoint = 1 } + }] + } + } + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + } +} +# tftest modules=1 resources=5 +``` + +Similarly to instance groups, NEGs can also be managed by this module which supports GCE, hybrid, serverless and Private Service Connect NEGs: + +#### Zonal NEG creation + +```hcl +resource "google_compute_address" "test" { + name = "neg-test" + subnetwork = var.subnet.self_link + address_type = "INTERNAL" + address = "10.0.0.10" + region = "europe-west1" +} + +module "ilb-l7" { + source = "./fabric/modules/net-lb-app-int" + name = "ilb-test" + project_id = var.project_id + region = "europe-west1" + backend_service_configs = { + default = { + backends = [{ + balancing_mode = "RATE" + group = "my-neg" + max_rate = { per_endpoint = 1 } + }] + } + } + neg_configs = { + my-neg = { + gce = { + zone = "europe-west1-b" + endpoints = { + e-0 = { + instance = "test-1" + ip_address = google_compute_address.test.address + # ip_address = "10.0.0.10" + port = 80 + } + } + } + } + } + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + } +} +# tftest modules=1 resources=8 +``` + +#### Hybrid NEG creation + +```hcl +module "ilb-l7" { + source = "./fabric/modules/net-lb-app-int" + name = "ilb-test" + project_id = var.project_id + region = "europe-west1" + backend_service_configs = { + default = { + backends = [{ group = "my-neg" }] + } + } + neg_configs = { + my-neg = { + hybrid = { + zone = "europe-west1-b" + endpoints = { + e-0 = { + ip_address = "10.0.0.10" + port = 80 + } + } + } + } + } + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + } +} +# tftest modules=1 resources=7 +``` + +#### Serverless NEG creation + +```hcl +module "ilb-l7" { + source = "./fabric/modules/net-lb-app-int" + name = "ilb-test" + project_id = var.project_id + region = "europe-west1" + backend_service_configs = { + default = { + backends = [{ + balancing_mode = "RATE" + group = "my-neg" + max_rate = { per_endpoint = 1 } + }] + health_checks = [] + } + } + health_check_configs = {} + neg_configs = { + my-neg = { + cloudrun = { + region = "europe-west1" + target_service = { + name = "my-run-service" + } + } + } + } + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + } +} +# tftest modules=1 resources=5 +``` + +#### Private Service Connect NEG creation + +```hcl +module "ilb-l7" { + source = "./fabric/modules/net-lb-app-int" + name = "ilb-test" + project_id = var.project_id + region = "europe-west1" + backend_service_configs = { + default = { + backends = [{ + group = "my-neg" + }] + health_checks = [] + } + } + health_check_configs = {} + neg_configs = { + my-neg = { + psc = { + region = "europe-west1" + target_service = "europe-west1-cloudkms.googleapis.com" + } + } + } + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + } +} +# tftest modules=1 resources=5 +``` + +### URL Map + +The module exposes the full URL map resource configuration, with some minor changes to the interface to decrease verbosity, and support for aliasing backend services via keys. + +The default URL map configuration sets the `default` backend service as the default service for the load balancer as a convenience. Just override the `urlmap_config` variable to change the default behaviour: + +```hcl +module "ilb-l7" { + source = "./fabric/modules/net-lb-app-int" + name = "ilb-test" + project_id = var.project_id + region = "europe-west1" + backend_service_configs = { + default = { + backends = [{ + group = "projects/myprj/zones/europe-west1-a/instanceGroups/my-ig" + }] + } + video = { + backends = [{ + group = "projects/myprj/zones/europe-west1-a/instanceGroups/my-ig-2" + }] + } + } + urlmap_config = { + default_service = "default" + host_rules = [{ + hosts = ["*"] + path_matcher = "pathmap" + }] + path_matchers = { + pathmap = { + default_service = "default" + path_rules = [{ + paths = ["/video", "/video/*"] + service = "video" + }] + } + } + } + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + } +} + +# tftest modules=1 resources=6 +``` + +### SSL Certificates + +Similarly to health checks, SSL certificates can also be created by the module. In this example we are using private key and certificate resources so that the example test only depends on Terraform providers, but in real use those can be replaced by external files. + +```hcl + +resource "tls_private_key" "default" { + algorithm = "RSA" + rsa_bits = 4096 +} + +resource "tls_self_signed_cert" "default" { + private_key_pem = tls_private_key.default.private_key_pem + subject { + common_name = "example.com" + organization = "ACME Examples, Inc" + } + validity_period_hours = 720 + allowed_uses = [ + "key_encipherment", + "digital_signature", + "server_auth", + ] +} + +module "ilb-l7" { + source = "./fabric/modules/net-lb-app-int" + name = "ilb-test" + project_id = var.project_id + region = "europe-west1" + backend_service_configs = { + default = { + backends = [{ + group = "projects/myprj/zones/europe-west1-a/instanceGroups/my-ig" + }] + } + } + health_check_configs = { + default = { + https = { port = 443 } + } + } + protocol = "HTTPS" + ssl_certificates = { + create_configs = { + default = { + # certificate and key could also be read via file() from external files + certificate = tls_self_signed_cert.default.cert_pem + private_key = tls_private_key.default.private_key_pem + } + } + } + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + } +} +# tftest modules=1 resources=8 +``` + +### Complex example + +This example mixes group and NEG backends, and shows how to set HTTPS for specific backends. + +```hcl +module "ilb-l7" { + source = "./fabric/modules/net-lb-app-int" + name = "ilb-l7-test-0" + project_id = "prj-gce" + region = "europe-west8" + backend_service_configs = { + default = { + backends = [ + { group = "nginx-ew8-b" }, + { group = "nginx-ew8-c" }, + ] + } + gce-neg = { + backends = [{ + balancing_mode = "RATE" + group = "neg-nginx-ew8-c" + max_rate = { per_endpoint = 1 } + }] + } + home = { + backends = [{ + balancing_mode = "RATE" + group = "neg-home-hello" + max_rate = { + per_endpoint = 1 + } + }] + health_checks = ["neg"] + locality_lb_policy = "ROUND_ROBIN" + protocol = "HTTPS" + } + } + group_configs = { + nginx-ew8-b = { + zone = "europe-west8-b" + instances = [ + "projects/prj-gce/zones/europe-west8-b/instances/nginx-ew8-b" + ] + named_ports = { http = 80 } + } + nginx-ew8-c = { + zone = "europe-west8-c" + instances = [ + "projects/prj-gce/zones/europe-west8-c/instances/nginx-ew8-c" + ] + named_ports = { http = 80 } + } + } + health_check_configs = { + default = { + http = { + port = 80 + } + } + neg = { + https = { + host = "hello.home.example.com" + port = 443 + } + } + } + neg_configs = { + neg-nginx-ew8-c = { + gce = { + zone = "europe-west8-c" + endpoints = { + e-0 = { + instance = "nginx-ew8-c" + ip_address = "10.24.32.26" + port = 80 + } + } + } + } + neg-home-hello = { + hybrid = { + zone = "europe-west8-b" + endpoints = { + e-0 = { + ip_address = "192.168.0.3" + port = 443 + } + } + } + } + } + urlmap_config = { + default_service = "default" + host_rules = [ + { + hosts = ["*"] + path_matcher = "gce" + }, + { + hosts = ["hello.home.example.com"] + path_matcher = "home" + } + ] + path_matchers = { + gce = { + default_service = "default" + path_rules = [ + { + paths = ["/gce-neg", "/gce-neg/*"] + service = "gce-neg" + } + ] + } + home = { + default_service = "home" + } + } + } + vpc_config = { + network = "projects/prj-host/global/networks/shared-vpc" + subnetwork = "projects/prj-host/regions/europe-west8/subnetworks/gce" + } +} +# tftest modules=1 resources=14 +``` + + + + +## Files + +| name | description | resources | +|---|---|---| +| [backend-service.tf](./backend-service.tf) | Backend service resources. | google_compute_region_backend_service | +| [groups.tf](./groups.tf) | None | google_compute_instance_group | +| [health-check.tf](./health-check.tf) | Health check resource. | google_compute_health_check | +| [main.tf](./main.tf) | Module-level locals and resources. | google_compute_forwarding_rule · google_compute_network_endpoint · google_compute_network_endpoint_group · google_compute_region_network_endpoint_group · google_compute_region_ssl_certificate · google_compute_region_target_http_proxy · google_compute_region_target_https_proxy | +| [outputs.tf](./outputs.tf) | Module outputs. | | +| [urlmap.tf](./urlmap.tf) | URL map resources. | google_compute_region_url_map | +| [variables-backend-service.tf](./variables-backend-service.tf) | Backend services variables. | | +| [variables-health-check.tf](./variables-health-check.tf) | Health check variable. | | +| [variables-urlmap.tf](./variables-urlmap.tf) | URLmap variable. | | +| [variables.tf](./variables.tf) | Module variables. | | +| [versions.tf](./versions.tf) | Version pins. | | + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [name](variables.tf#L54) | Load balancer name. | string | ✓ | | +| [project_id](variables.tf#L138) | Project id. | string | ✓ | | +| [region](variables.tf#L156) | The region where to allocate the ILB resources. | string | ✓ | | +| [vpc_config](variables.tf#L183) | VPC-level configuration. | object({…}) | ✓ | | +| [address](variables.tf#L17) | Optional IP address used for the forwarding rule. | string | | null | +| [backend_service_configs](variables-backend-service.tf#L19) | Backend service level configuration. | map(object({…})) | | {} | +| [description](variables.tf#L23) | Optional description used for resources. | string | | "Terraform managed." | +| [global_access](variables.tf#L30) | Allow client access from all regions. | bool | | null | +| [group_configs](variables.tf#L36) | Optional unmanaged groups to create. Can be referenced in backends via key or outputs. | map(object({…})) | | {} | +| [health_check_configs](variables-health-check.tf#L19) | Optional auto-created health check configurations, use the output self-link to set it in the auto healing policy. Refer to examples for usage. | map(object({…})) | | {…} | +| [labels](variables.tf#L48) | Labels set on resources. | map(string) | | {} | +| [neg_configs](variables.tf#L59) | Optional network endpoint groups to create. Can be referenced in backends via key or outputs. | map(object({…})) | | {} | +| [network_tier_premium](variables.tf#L125) | Use premium network tier. Defaults to true. | bool | | true | +| [ports](variables.tf#L132) | Optional ports for HTTP load balancer, valid ports are 80 and 8080. | list(string) | | null | +| [protocol](variables.tf#L143) | Protocol supported by this load balancer. | string | | "HTTP" | +| [service_directory_registration](variables.tf#L161) | Service directory namespace and service used to register this load balancer. | object({…}) | | null | +| [ssl_certificates](variables.tf#L170) | SSL target proxy certificates (only if protocol is HTTPS). | object({…}) | | {} | +| [urlmap_config](variables-urlmap.tf#L19) | The URL map configuration. | object({…}) | | {…} | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [address](outputs.tf#L17) | Forwarding rule address. | | +| [backend_service_ids](outputs.tf#L22) | Backend service resources. | | +| [backend_service_names](outputs.tf#L29) | Backend service resource names. | | +| [forwarding_rule](outputs.tf#L36) | Forwarding rule resource. | | +| [group_ids](outputs.tf#L41) | Autogenerated instance group ids. | | +| [health_check_ids](outputs.tf#L48) | Autogenerated health check ids. | | +| [id](outputs.tf#L55) | Fully qualified forwarding rule id. | | +| [neg_ids](outputs.tf#L60) | Autogenerated network endpoint group ids. | | + + diff --git a/assets/modules-fabric/v26/net-lb-app-int/backend-service.tf b/assets/modules-fabric/v26/net-lb-app-int/backend-service.tf new file mode 100644 index 0000000..669a291 --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-app-int/backend-service.tf @@ -0,0 +1,237 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Backend service resources. + +locals { + group_ids = merge( + { + for k, v in google_compute_instance_group.default : k => v.id + }, + { + for k, v in google_compute_network_endpoint_group.default : k => v.id + }, + { + for k, v in google_compute_region_network_endpoint_group.default : k => v.id + }, + { + for k, v in google_compute_region_network_endpoint_group.psc : k => v.id + } + ) + hc_ids = { + for k, v in google_compute_health_check.default : k => v.id + } +} + +resource "google_compute_region_backend_service" "default" { + provider = google-beta + for_each = var.backend_service_configs + project = ( + each.value.project_id == null + ? var.project_id + : each.value.project_id + ) + region = var.region + name = "${var.name}-${each.key}" + description = var.description + affinity_cookie_ttl_sec = each.value.affinity_cookie_ttl_sec + connection_draining_timeout_sec = each.value.connection_draining_timeout_sec + health_checks = length(each.value.health_checks) == 0 ? null : [ + for k in each.value.health_checks : lookup(local.hc_ids, k, k) + ] # not for internet / serverless NEGs + locality_lb_policy = each.value.locality_lb_policy + load_balancing_scheme = "INTERNAL_MANAGED" + port_name = each.value.port_name # defaults to http, not for NEGs + protocol = ( + each.value.protocol == null ? var.protocol : each.value.protocol + ) + session_affinity = each.value.session_affinity + timeout_sec = each.value.timeout_sec + + dynamic "backend" { + for_each = { for b in coalesce(each.value.backends, []) : b.group => b } + content { + group = lookup(local.group_ids, backend.key, backend.key) + balancing_mode = backend.value.balancing_mode + capacity_scaler = backend.value.capacity_scaler + description = backend.value.description + failover = backend.value.failover + max_connections = try( + backend.value.max_connections.per_group, null + ) + max_connections_per_endpoint = try( + backend.value.max_connections.per_endpoint, null + ) + max_connections_per_instance = try( + backend.value.max_connections.per_instance, null + ) + max_rate = try( + backend.value.max_rate.per_group, null + ) + max_rate_per_endpoint = try( + backend.value.max_rate.per_endpoint, null + ) + max_rate_per_instance = try( + backend.value.max_rate.per_instance, null + ) + max_utilization = backend.value.max_utilization + } + } + + dynamic "circuit_breakers" { + for_each = ( + each.value.circuit_breakers == null ? [] : [each.value.circuit_breakers] + ) + iterator = cb + content { + max_connections = cb.value.max_connections + max_pending_requests = cb.value.max_pending_requests + max_requests = cb.value.max_requests + max_requests_per_connection = cb.value.max_requests_per_connection + max_retries = cb.value.max_retries + dynamic "connect_timeout" { + for_each = ( + cb.value.connect_timeout == null ? [] : [cb.value.connect_timeout] + ) + content { + seconds = connect_timeout.value.seconds + nanos = connect_timeout.value.nanos + } + } + } + } + + dynamic "connection_tracking_policy" { + for_each = ( + each.value.connection_tracking == null + ? [] + : [each.value.connection_tracking] + ) + iterator = cb + content { + connection_persistence_on_unhealthy_backends = ( + cb.value.persist_conn_on_unhealthy != null + ? cb.value.persist_conn_on_unhealthy + : null + ) + idle_timeout_sec = cb.value.idle_timeout_sec + tracking_mode = ( + cb.value.track_per_session != null + ? cb.value.track_per_session + : null + ) + } + } + + dynamic "consistent_hash" { + for_each = ( + each.value.consistent_hash == null ? [] : [each.value.consistent_hash] + ) + iterator = ch + content { + http_header_name = ch.value.http_header_name + minimum_ring_size = ch.value.minimum_ring_size + dynamic "http_cookie" { + for_each = ch.value.http_cookie == null ? [] : [ch.value.http_cookie] + content { + name = http_cookie.value.name + path = http_cookie.value.path + dynamic "ttl" { + for_each = ( + http_cookie.value.ttl == null ? [] : [http_cookie.value.ttl] + ) + content { + seconds = ttl.value.seconds + nanos = ttl.value.nanos + } + } + } + } + } + } + + dynamic "failover_policy" { + for_each = ( + each.value.failover_config == null ? [] : [each.value.failover_config] + ) + iterator = fc + content { + disable_connection_drain_on_failover = fc.value.disable_conn_drain + drop_traffic_if_unhealthy = fc.value.drop_traffic_if_unhealthy + failover_ratio = fc.value.ratio + } + } + + dynamic "iap" { + for_each = each.value.iap_config == null ? [] : [each.value.iap_config] + content { + oauth2_client_id = iap.value.oauth2_client_id + oauth2_client_secret = iap.value.oauth2_client_secret + oauth2_client_secret_sha256 = iap.value.oauth2_client_secret_sha256 + } + } + + dynamic "log_config" { + for_each = each.value.log_sample_rate == null ? [] : [""] + content { + enable = true + sample_rate = each.value.log_sample_rate + } + } + + dynamic "outlier_detection" { + for_each = ( + each.value.outlier_detection == null ? [] : [each.value.outlier_detection] + ) + iterator = od + content { + consecutive_errors = od.value.consecutive_errors + consecutive_gateway_failure = od.value.consecutive_gateway_failure + enforcing_consecutive_errors = od.value.enforcing_consecutive_errors + enforcing_consecutive_gateway_failure = od.value.enforcing_consecutive_gateway_failure + enforcing_success_rate = od.value.enforcing_success_rate + max_ejection_percent = od.value.max_ejection_percent + success_rate_minimum_hosts = od.value.success_rate_minimum_hosts + success_rate_request_volume = od.value.success_rate_request_volume + success_rate_stdev_factor = od.value.success_rate_stdev_factor + dynamic "base_ejection_time" { + for_each = ( + od.value.base_ejection_time == null ? [] : [od.value.base_ejection_time] + ) + content { + seconds = base_ejection_time.value.seconds + nanos = base_ejection_time.value.nanos + } + } + dynamic "interval" { + for_each = ( + od.value.interval == null ? [] : [od.value.interval] + ) + content { + seconds = interval.value.seconds + nanos = interval.value.nanos + } + } + } + } + + dynamic "subsetting" { + for_each = each.value.enable_subsetting == true ? [""] : [] + content { + policy = "CONSISTENT_HASH_SUBSETTING" + } + } +} diff --git a/assets/modules-fabric/v26/net-lb-app-int/groups.tf b/assets/modules-fabric/v26/net-lb-app-int/groups.tf new file mode 100644 index 0000000..53ba6b2 --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-app-int/groups.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_compute_instance_group" "default" { + for_each = var.group_configs + project = ( + each.value.project_id == null + ? var.project_id + : each.value.project_id + ) + zone = each.value.zone + name = "${var.name}-${each.key}" + description = var.description + instances = each.value.instances + + dynamic "named_port" { + for_each = each.value.named_ports + content { + name = named_port.key + port = named_port.value + } + } +} + diff --git a/assets/modules-fabric/v26/net-lb-app-int/health-check.tf b/assets/modules-fabric/v26/net-lb-app-int/health-check.tf new file mode 100644 index 0000000..66ba58c --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-app-int/health-check.tf @@ -0,0 +1,113 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Health check resource. + +resource "google_compute_health_check" "default" { + provider = google-beta + for_each = var.health_check_configs + project = ( + each.value.project_id == null + ? var.project_id + : each.value.project_id + ) + name = "${var.name}-${each.key}" + description = each.value.description + check_interval_sec = each.value.check_interval_sec + healthy_threshold = each.value.healthy_threshold + timeout_sec = each.value.timeout_sec + unhealthy_threshold = each.value.unhealthy_threshold + + dynamic "grpc_health_check" { + for_each = try(each.value.grpc, null) != null ? [""] : [] + content { + port = each.value.grpc.port + port_name = each.value.grpc.port_name + port_specification = each.value.grpc.port_specification + grpc_service_name = each.value.grpc.service_name + } + } + + dynamic "http_health_check" { + for_each = try(each.value.http, null) != null ? [""] : [] + content { + host = each.value.http.host + port = each.value.http.port + port_name = each.value.http.port_name + port_specification = each.value.http.port_specification + proxy_header = each.value.http.proxy_header + request_path = each.value.http.request_path + response = each.value.http.response + } + } + + dynamic "http2_health_check" { + for_each = try(each.value.http2, null) != null ? [""] : [] + content { + host = each.value.http2.host + port = each.value.http2.port + port_name = each.value.http2.port_name + port_specification = each.value.http2.port_specification + proxy_header = each.value.http2.proxy_header + request_path = each.value.http2.request_path + response = each.value.http2.response + } + } + + dynamic "https_health_check" { + for_each = try(each.value.https, null) != null ? [""] : [] + content { + host = each.value.https.host + port = each.value.https.port + port_name = each.value.https.port_name + port_specification = each.value.https.port_specification + proxy_header = each.value.https.proxy_header + request_path = each.value.https.request_path + response = each.value.https.response + } + } + + dynamic "ssl_health_check" { + for_each = try(each.value.ssl, null) != null ? [""] : [] + content { + port = each.value.ssl.port + port_name = each.value.ssl.port_name + port_specification = each.value.ssl.port_specification + proxy_header = each.value.ssl.proxy_header + request = each.value.ssl.request + response = each.value.ssl.response + } + } + + dynamic "tcp_health_check" { + for_each = try(each.value.tcp, null) != null ? [""] : [] + content { + port = each.value.tcp.port + port_name = each.value.tcp.port_name + port_specification = each.value.tcp.port_specification + proxy_header = each.value.tcp.proxy_header + request = each.value.tcp.request + response = each.value.tcp.response + } + } + + dynamic "log_config" { + for_each = try(each.value.enable_logging, null) == true ? [""] : [] + content { + enable = true + } + } +} diff --git a/assets/modules-fabric/v26/net-lb-app-int/main.tf b/assets/modules-fabric/v26/net-lb-app-int/main.tf new file mode 100644 index 0000000..79757f5 --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-app-int/main.tf @@ -0,0 +1,185 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # we need keys in the endpoint type to address issue #1055 + _neg_endpoints = flatten([ + for k, v in local.neg_zonal : [ + for kk, vv in v.endpoints : merge(vv, { + key = "${k}-${kk}", neg = k, zone = v.zone + }) + ] + ]) + fwd_rule_ports = ( + var.protocol == "HTTPS" ? [443] : coalesce(var.ports, [80]) + ) + fwd_rule_target = ( + var.protocol == "HTTPS" + ? google_compute_region_target_https_proxy.default.0.id + : google_compute_region_target_http_proxy.default.0.id + ) + neg_endpoints = { + for v in local._neg_endpoints : (v.key) => v + } + neg_regional = { + for k, v in var.neg_configs : + k => merge(v.cloudrun, { project_id = v.project_id }) if v.cloudrun != null + } + neg_zonal = { + # we need to rebuild new objects as we cannot merge different types + for k, v in var.neg_configs : k => { + endpoints = v.gce != null ? v.gce.endpoints : v.hybrid.endpoints + network = v.gce != null ? v.gce.network : v.hybrid.network + project_id = v.project_id + subnetwork = v.gce != null ? v.gce.subnetwork : null + type = v.gce != null ? "GCE_VM_IP_PORT" : "NON_GCP_PRIVATE_IP_PORT" + zone = v.gce != null ? v.gce.zone : v.hybrid.zone + } if v.gce != null || v.hybrid != null + } + neg_regional_psc = { + for k, v in var.neg_configs : + k => v if v.psc != null + } + proxy_ssl_certificates = concat( + coalesce(var.ssl_certificates.certificate_ids, []), + [for k, v in google_compute_region_ssl_certificate.default : v.id] + ) +} + +resource "google_compute_forwarding_rule" "default" { + provider = google-beta + project = var.project_id + region = var.region + name = var.name + description = var.description + ip_address = var.address + ip_protocol = "TCP" + load_balancing_scheme = "INTERNAL_MANAGED" + network = var.vpc_config.network + network_tier = var.network_tier_premium ? "PREMIUM" : "STANDARD" + port_range = join(",", local.fwd_rule_ports) + subnetwork = var.vpc_config.subnetwork + labels = var.labels + target = local.fwd_rule_target + # during the preview phase you cannot change this attribute on an existing rule + allow_global_access = var.global_access + dynamic "service_directory_registrations" { + for_each = var.service_directory_registration == null ? [] : [""] + content { + namespace = var.service_directory_registration.namespace + service = var.service_directory_registration.service + } + } +} + +resource "google_compute_region_ssl_certificate" "default" { + for_each = var.ssl_certificates.create_configs + project = var.project_id + region = var.region + name = "${var.name}-${each.key}" + certificate = each.value.certificate + private_key = each.value.private_key + + lifecycle { + create_before_destroy = true + } +} + +resource "google_compute_region_target_http_proxy" "default" { + count = var.protocol == "HTTPS" ? 0 : 1 + project = var.project_id + region = var.region + name = var.name + description = var.description + url_map = google_compute_region_url_map.default.id +} + +resource "google_compute_region_target_https_proxy" "default" { + count = var.protocol == "HTTPS" ? 1 : 0 + project = var.project_id + region = var.region + name = var.name + description = var.description + ssl_certificates = local.proxy_ssl_certificates + url_map = google_compute_region_url_map.default.id +} + +resource "google_compute_network_endpoint_group" "default" { + for_each = local.neg_zonal + project = ( + each.value.project_id == null + ? var.project_id + : each.value.project_id + ) + zone = each.value.zone + name = "${var.name}-${each.key}" + # re-enable once provider properly supports this + # default_port = each.value.default_port + description = var.description + network_endpoint_type = each.value.type + network = ( + each.value.network != null ? each.value.network : var.vpc_config.network + ) + subnetwork = ( + each.value.type == "NON_GCP_PRIVATE_IP_PORT" + ? null + : try(each.value.subnetwork, var.vpc_config.subnetwork) + ) +} + +resource "google_compute_network_endpoint" "default" { + for_each = local.neg_endpoints + project = ( + google_compute_network_endpoint_group.default[each.value.neg].project + ) + network_endpoint_group = ( + google_compute_network_endpoint_group.default[each.value.neg].name + ) + instance = try(each.value.instance, null) + ip_address = each.value.ip_address + port = each.value.port + zone = each.value.zone +} + +resource "google_compute_region_network_endpoint_group" "default" { + for_each = local.neg_regional + project = ( + each.value.project_id == null + ? var.project_id + : each.value.project_id + ) + region = each.value.region + name = "${var.name}-${each.key}" + description = var.description + network_endpoint_type = "SERVERLESS" + cloud_run { + service = try(each.value.target_service.name, null) + tag = try(each.value.target_service.tag, null) + url_mask = each.value.target_urlmask + } +} + +resource "google_compute_region_network_endpoint_group" "psc" { + for_each = local.neg_regional_psc + project = var.project_id + region = each.value.psc.region + name = "${var.name}-${each.key}" + //description = coalesce(each.value.description, var.description) + network_endpoint_type = "PRIVATE_SERVICE_CONNECT" + psc_target_service = each.value.psc.target_service + network = each.value.psc.network + subnetwork = each.value.psc.subnetwork +} diff --git a/assets/modules-fabric/v26/net-lb-app-int/outputs.tf b/assets/modules-fabric/v26/net-lb-app-int/outputs.tf new file mode 100644 index 0000000..1491d8c --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-app-int/outputs.tf @@ -0,0 +1,65 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "address" { + description = "Forwarding rule address." + value = google_compute_forwarding_rule.default.ip_address +} + +output "backend_service_ids" { + description = "Backend service resources." + value = { + for k, v in google_compute_region_backend_service.default : k => v.id + } +} + +output "backend_service_names" { + description = "Backend service resource names." + value = { + for k, v in google_compute_region_backend_service.default : k => v.name + } +} + +output "forwarding_rule" { + description = "Forwarding rule resource." + value = google_compute_forwarding_rule.default +} + +output "group_ids" { + description = "Autogenerated instance group ids." + value = { + for k, v in google_compute_instance_group.default : k => v.id + } +} + +output "health_check_ids" { + description = "Autogenerated health check ids." + value = { + for k, v in google_compute_health_check.default : k => v.id + } +} + +output "id" { + description = "Fully qualified forwarding rule id." + value = google_compute_forwarding_rule.default.id +} + +output "neg_ids" { + description = "Autogenerated network endpoint group ids." + value = { + for k, v in google_compute_network_endpoint_group.default : k => v.id + } +} diff --git a/assets/modules-fabric/v26/net-lb-app-int/urlmap.tf b/assets/modules-fabric/v26/net-lb-app-int/urlmap.tf new file mode 100644 index 0000000..21f7e59 --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-app-int/urlmap.tf @@ -0,0 +1,576 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description URL map resources. + +locals { + backend_ids = { + for k, v in google_compute_region_backend_service.default : k => v.id + } +} + +resource "google_compute_region_url_map" "default" { + provider = google-beta + project = var.project_id + region = var.region + name = var.name + description = var.description + default_service = ( + var.urlmap_config.default_service == null ? null : lookup( + local.backend_ids, + var.urlmap_config.default_service, + var.urlmap_config.default_service + ) + ) + + dynamic "default_url_redirect" { + for_each = ( + var.urlmap_config.default_url_redirect == null + ? [] + : [var.urlmap_config.default_url_redirect] + ) + iterator = r + content { + host_redirect = r.value.host + https_redirect = r.value.https + path_redirect = r.value.path + prefix_redirect = r.value.prefix + redirect_response_code = r.value.response_code + strip_query = r.value.strip_query + } + } + + dynamic "host_rule" { + for_each = coalesce(var.urlmap_config.host_rules, []) + iterator = r + content { + hosts = r.value.hosts + path_matcher = r.value.path_matcher + description = r.value.description + } + } + + dynamic "path_matcher" { + for_each = coalesce(var.urlmap_config.path_matchers, {}) + iterator = m + content { + default_service = m.value.default_service == null ? null : lookup( + local.backend_ids, m.value.default_service, m.value.default_service + ) + description = m.value.description + name = m.key + dynamic "default_url_redirect" { + for_each = ( + m.value.default_url_redirect == null + ? [] + : [m.value.default_url_redirect] + ) + content { + host_redirect = default_url_redirect.value.host + https_redirect = default_url_redirect.value.https + path_redirect = default_url_redirect.value.path + prefix_redirect = default_url_redirect.value.prefix + redirect_response_code = default_url_redirect.value.response_code + strip_query = default_url_redirect.value.strip_query + } + } + dynamic "path_rule" { + for_each = toset(coalesce(m.value.path_rules, [])) + content { + paths = path_rule.value.paths + service = path_rule.value.service == null ? null : lookup( + local.backend_ids, + path_rule.value.service, + path_rule.value.service + ) + dynamic "route_action" { + for_each = ( + path_rule.value.route_action == null + ? [] + : [path_rule.value.route_action] + ) + content { + dynamic "cors_policy" { + for_each = ( + route_action.value.cors_policy == null + ? [] + : [route_action.value.cors_policy] + ) + content { + allow_credentials = cors_policy.value.allow_credentials + allow_headers = cors_policy.value.allow_headers + allow_methods = cors_policy.value.allow_methods + allow_origin_regexes = cors_policy.value.allow_origin_regexes + allow_origins = cors_policy.value.allow_origins + disabled = cors_policy.value.disabled + expose_headers = cors_policy.value.expose_headers + max_age = cors_policy.value.max_age + } + } + dynamic "fault_injection_policy" { + for_each = ( + route_action.value.fault_injection_policy == null + ? [] + : [route_action.value.fault_injection_policy] + ) + content { + dynamic "abort" { + for_each = ( + fault_injection_policy.value.abort == null + ? [] + : [fault_injection_policy.value.abort] + ) + content { + http_status = abort.value.status + percentage = abort.value.percentage + } + } + dynamic "delay" { + for_each = ( + fault_injection_policy.value.delay == null + ? [] + : [fault_injection_policy.value.delay] + ) + content { + percentage = delay.value.percentage + fixed_delay { + nanos = delay.value.fixed.nanos + seconds = delay.value.fixed.seconds + } + } + } + } + } + dynamic "request_mirror_policy" { + for_each = ( + route_action.value.request_mirror_backend == null + ? [] + : [""] + ) + content { + backend_service = lookup( + local.backend_ids, + route_action.value.request_mirror_backend, + route_action.value.request_mirror_backend + ) + } + } + dynamic "retry_policy" { + for_each = ( + route_action.value.retry_policy == null + ? [] + : [route_action.value.retry_policy] + ) + content { + num_retries = retry_policy.value.num_retries + retry_conditions = retry_policy.value.retry_conditions + dynamic "per_try_timeout" { + for_each = ( + retry_policy.value.per_try_timeout == null + ? [] + : [retry_policy.value.per_try_timeout] + ) + content { + nanos = per_try_timeout.value.nanos + seconds = per_try_timeout.value.seconds + } + } + } + } + dynamic "timeout" { + for_each = ( + route_action.value.timeout == null + ? [] + : [route_action.value.timeout] + ) + content { + nanos = timeout.value.nanos + seconds = timeout.value.seconds + } + } + dynamic "url_rewrite" { + for_each = ( + route_action.value.url_rewrite == null + ? [] + : [route_action.value.url_rewrite] + ) + content { + host_rewrite = url_rewrite.value.host + path_prefix_rewrite = url_rewrite.value.path_prefix + } + } + dynamic "weighted_backend_services" { + for_each = coalesce( + route_action.value.weighted_backend_services, {} + ) + iterator = service + content { + backend_service = lookup( + local.backend_ids, service.key, service.key + ) + weight = service.value.weight + dynamic "header_action" { + for_each = ( + service.value.header_action == null + ? [] + : [service.value.header_action] + ) + iterator = h + content { + request_headers_to_remove = h.value.request_remove + response_headers_to_remove = h.value.response_remove + dynamic "request_headers_to_add" { + for_each = coalesce(h.value.request_add, {}) + content { + header_name = request_headers_to_add.key + header_value = request_headers_to_add.value.value + replace = request_headers_to_add.value.replace + } + } + dynamic "response_headers_to_add" { + for_each = coalesce(h.value.response_add, {}) + content { + header_name = response_headers_to_add.key + header_value = response_headers_to_add.value.value + replace = response_headers_to_add.value.replace + } + } + } + } + } + } + } + } + dynamic "url_redirect" { + for_each = ( + path_rule.value.url_redirect == null + ? [] + : [path_rule.value.url_redirect] + ) + content { + host_redirect = url_redirect.value.host + https_redirect = url_redirect.value.https + path_redirect = url_redirect.value.path + prefix_redirect = url_redirect.value.prefix + redirect_response_code = url_redirect.value.response_code + strip_query = url_redirect.value.strip_query + } + } + } + } + dynamic "route_rules" { + for_each = toset(coalesce(m.value.route_rules, [])) + content { + priority = route_rules.value.priority + service = route_rules.value.service == null ? null : lookup( + local.backend_ids, + route_rules.value.service, + route_rules.value.service + ) + dynamic "header_action" { + for_each = ( + route_rules.value.header_action == null + ? [] + : [route_rules.value.header_action] + ) + iterator = h + content { + request_headers_to_remove = h.value.request_remove + response_headers_to_remove = h.value.response_remove + dynamic "request_headers_to_add" { + for_each = coalesce(h.value.request_add, {}) + content { + header_name = request_headers_to_add.key + header_value = request_headers_to_add.value.value + replace = request_headers_to_add.value.replace + } + } + dynamic "response_headers_to_add" { + for_each = coalesce(h.value.response_add, {}) + content { + header_name = response_headers_to_add.key + header_value = response_headers_to_add.value.value + replace = response_headers_to_add.value.replace + } + } + } + } + dynamic "match_rules" { + for_each = toset(coalesce(route_rules.value.match_rules, [])) + content { + ignore_case = match_rules.value.ignore_case + full_path_match = ( + try(match_rules.value.path.type, null) == "full" + ? match_rules.value.path.value + : null + ) + prefix_match = ( + try(match_rules.value.path.type, null) == "prefix" + ? match_rules.value.path.value + : null + ) + regex_match = ( + try(match_rules.value.path.type, null) == "regex" + ? match_rules.value.path.value + : null + ) + dynamic "header_matches" { + for_each = toset(coalesce(match_rules.value.headers, [])) + iterator = h + content { + header_name = h.value.name + exact_match = h.value.type == "exact" ? h.value.value : null + invert_match = h.value.invert_match + prefix_match = h.value.type == "prefix" ? h.value.value : null + present_match = h.value.type == "present" ? h.value.value : null + regex_match = h.value.type == "regex" ? h.value.value : null + suffix_match = h.value.type == "suffix" ? h.value.value : null + dynamic "range_match" { + for_each = ( + h.value.type != "range" || h.value.range_value == null + ? [] + : [""] + ) + content { + range_end = h.value.range_value.end + range_start = h.value.range_value.start + } + } + } + } + dynamic "metadata_filters" { + for_each = toset(coalesce(match_rules.value.metadata_filters, [])) + iterator = m + content { + filter_match_criteria = ( + m.value.match_all ? "MATCH_ALL" : "MATCH_ANY" + ) + dynamic "filter_labels" { + for_each = m.value.labels + content { + name = filter_labels.key + value = filter_labels.value + } + } + } + } + dynamic "query_parameter_matches" { + for_each = toset(coalesce(match_rules.value.query_params, [])) + iterator = q + content { + name = q.value.name + exact_match = ( + q.value.type == "exact" ? q.value.value : null + ) + present_match = ( + q.value.type == "present" ? q.value.value : null + ) + regex_match = ( + q.value.type == "regex" ? q.value.value : null + ) + } + } + } + } + dynamic "route_action" { + for_each = ( + route_rules.value.route_action == null + ? [] + : [route_rules.value.route_action] + ) + content { + dynamic "cors_policy" { + for_each = ( + route_action.value.cors_policy == null + ? [] + : [route_action.value.cors_policy] + ) + content { + allow_credentials = cors_policy.value.allow_credentials + allow_headers = cors_policy.value.allow_headers + allow_methods = cors_policy.value.allow_methods + allow_origin_regexes = cors_policy.value.allow_origin_regexes + allow_origins = cors_policy.value.allow_origins + disabled = cors_policy.value.disabled + expose_headers = cors_policy.value.expose_headers + max_age = cors_policy.value.max_age + } + } + dynamic "fault_injection_policy" { + for_each = ( + route_action.value.fault_injection_policy == null + ? [] + : [route_action.value.fault_injection_policy] + ) + content { + dynamic "abort" { + for_each = ( + fault_injection_policy.value.abort == null + ? [] + : [fault_injection_policy.value.abort] + ) + content { + http_status = abort.value.status + percentage = abort.value.percentage + } + } + dynamic "delay" { + for_each = ( + fault_injection_policy.value.delay == null + ? [] + : [fault_injection_policy.value.delay] + ) + content { + percentage = delay.value.percentage + fixed_delay { + nanos = delay.value.fixed.nanos + seconds = delay.value.fixed.seconds + } + } + } + } + } + dynamic "request_mirror_policy" { + for_each = ( + route_action.value.request_mirror_backend == null + ? [] + : [""] + ) + content { + backend_service = lookup( + local.backend_ids, + route_action.value.request_mirror_backend, + route_action.value.request_mirror_backend + ) + } + } + dynamic "retry_policy" { + for_each = ( + route_action.value.retry_policy == null + ? [] + : [route_action.value.retry_policy] + ) + content { + num_retries = retry_policy.value.num_retries + retry_conditions = retry_policy.value.retry_conditions + dynamic "per_try_timeout" { + for_each = ( + retry_policy.value.per_try_timeout == null + ? [] + : [retry_policy.value.per_try_timeout] + ) + content { + nanos = per_try_timeout.value.nanos + seconds = per_try_timeout.value.seconds + } + } + } + } + dynamic "timeout" { + for_each = ( + route_action.value.timeout == null + ? [] + : [route_action.value.timeout] + ) + content { + nanos = timeout.value.nanos + seconds = timeout.value.seconds + } + } + dynamic "url_rewrite" { + for_each = ( + route_action.value.url_rewrite == null + ? [] + : [route_action.value.url_rewrite] + ) + content { + host_rewrite = url_rewrite.value.host + path_prefix_rewrite = url_rewrite.value.path_prefix + } + } + dynamic "weighted_backend_services" { + for_each = coalesce( + route_action.value.weighted_backend_services, {} + ) + iterator = service + content { + backend_service = lookup( + local.backend_ids, service.key, service.key + ) + weight = service.value.weight + dynamic "header_action" { + for_each = ( + service.value.header_action == null + ? [] + : [service.value.header_action] + ) + iterator = h + content { + request_headers_to_remove = h.value.request_remove + response_headers_to_remove = h.value.response_remove + dynamic "request_headers_to_add" { + for_each = coalesce(h.value.request_add, {}) + content { + header_name = request_headers_to_add.key + header_value = request_headers_to_add.value.value + replace = request_headers_to_add.value.replace + } + } + dynamic "response_headers_to_add" { + for_each = coalesce(h.value.response_add, {}) + content { + header_name = response_headers_to_add.key + header_value = response_headers_to_add.value.value + replace = response_headers_to_add.value.replace + } + } + } + } + } + } + } + } + dynamic "url_redirect" { + for_each = ( + route_rules.value.url_redirect == null + ? [] + : [route_rules.value.url_redirect] + ) + content { + host_redirect = url_redirect.value.host + https_redirect = url_redirect.value.https + path_redirect = url_redirect.value.path + prefix_redirect = url_redirect.value.prefix + redirect_response_code = url_redirect.value.response_code + strip_query = url_redirect.value.strip_query + } + } + } + } + } + } + + dynamic "test" { + for_each = toset(coalesce(var.urlmap_config.test, [])) + content { + host = test.value.host + path = test.value.path + service = test.value.service + description = test.value.description + } + } + +} diff --git a/assets/modules-fabric/v26/net-lb-app-int/variables-backend-service.tf b/assets/modules-fabric/v26/net-lb-app-int/variables-backend-service.tf new file mode 100644 index 0000000..0119d1b --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-app-int/variables-backend-service.tf @@ -0,0 +1,131 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Backend services variables. + +variable "backend_service_configs" { + description = "Backend service level configuration." + type = map(object({ + affinity_cookie_ttl_sec = optional(number) + connection_draining_timeout_sec = optional(number) + health_checks = optional(list(string), ["default"]) + locality_lb_policy = optional(string) + log_sample_rate = optional(number) + port_name = optional(string) + project_id = optional(string) + protocol = optional(string) + session_affinity = optional(string) + timeout_sec = optional(number) + backends = list(object({ + group = string + balancing_mode = optional(string, "UTILIZATION") + capacity_scaler = optional(number, 1) + description = optional(string, "Terraform managed.") + failover = optional(bool, false) + max_connections = optional(object({ + per_endpoint = optional(number) + per_group = optional(number) + per_instance = optional(number) + })) + max_rate = optional(object({ + per_endpoint = optional(number) + per_group = optional(number) + per_instance = optional(number) + })) + max_utilization = optional(number) + })) + circuit_breakers = optional(object({ + max_connections = optional(number) + max_pending_requests = optional(number) + max_requests = optional(number) + max_requests_per_connection = optional(number) + max_retries = optional(number) + connect_timeout = optional(object({ + seconds = number + nanos = optional(number) + })) + })) + connection_tracking = optional(object({ + idle_timeout_sec = optional(number) + persist_conn_on_unhealthy = optional(string) + track_per_session = optional(bool) + })) + consistent_hash = optional(object({ + http_header_name = optional(string) + minimum_ring_size = optional(number) + http_cookie = optional(object({ + name = optional(string) + path = optional(string) + ttl = optional(object({ + seconds = number + nanos = optional(number) + })) + })) + })) + enable_subsetting = optional(bool) + failover_config = optional(object({ + disable_conn_drain = optional(bool) + drop_traffic_if_unhealthy = optional(bool) + ratio = optional(number) + })) + iap_config = optional(object({ + oauth2_client_id = string + oauth2_client_secret = string + oauth2_client_secret_sha256 = optional(string) + })) + outlier_detection = optional(object({ + consecutive_errors = optional(number) + consecutive_gateway_failure = optional(number) + enforcing_consecutive_errors = optional(number) + enforcing_consecutive_gateway_failure = optional(number) + enforcing_success_rate = optional(number) + max_ejection_percent = optional(number) + success_rate_minimum_hosts = optional(number) + success_rate_request_volume = optional(number) + success_rate_stdev_factor = optional(number) + base_ejection_time = optional(object({ + seconds = number + nanos = optional(number) + })) + interval = optional(object({ + seconds = number + nanos = optional(number) + })) + })) + })) + default = {} + nullable = false + validation { + condition = contains( + [ + "-", "ROUND_ROBIN", "LEAST_REQUEST", "RING_HASH", + "RANDOM", "ORIGINAL_DESTINATION", "MAGLEV" + ], + try(var.backend_service_configs.locality_lb_policy, "-") + ) + error_message = "Invalid locality lb policy value." + } + validation { + condition = contains( + [ + "NONE", "CLIENT_IP", "CLIENT_IP_NO_DESTINATION", + "CLIENT_IP_PORT_PROTO", "CLIENT_IP_PROTO" + ], + try(var.backend_service_configs.session_affinity, "NONE") + ) + error_message = "Invalid session affinity value." + } +} diff --git a/assets/modules-fabric/v26/net-lb-app-int/variables-health-check.tf b/assets/modules-fabric/v26/net-lb-app-int/variables-health-check.tf new file mode 100644 index 0000000..a51994e --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-app-int/variables-health-check.tf @@ -0,0 +1,109 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Health check variable. + +variable "health_check_configs" { + description = "Optional auto-created health check configurations, use the output self-link to set it in the auto healing policy. Refer to examples for usage." + type = map(object({ + check_interval_sec = optional(number) + description = optional(string, "Terraform managed.") + enable_logging = optional(bool, false) + healthy_threshold = optional(number) + project_id = optional(string) + timeout_sec = optional(number) + unhealthy_threshold = optional(number) + grpc = optional(object({ + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + service_name = optional(string) + })) + http = optional(object({ + host = optional(string) + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request_path = optional(string) + response = optional(string) + })) + http2 = optional(object({ + host = optional(string) + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request_path = optional(string) + response = optional(string) + })) + https = optional(object({ + host = optional(string) + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request_path = optional(string) + response = optional(string) + })) + tcp = optional(object({ + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request = optional(string) + response = optional(string) + })) + ssl = optional(object({ + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request = optional(string) + response = optional(string) + })) + })) + default = { + default = { + http = { + port_specification = "USE_SERVING_PORT" + } + } + } + validation { + condition = alltrue([ + for k, v in var.health_check_configs : ( + (try(v.grpc, null) == null ? 0 : 1) + + (try(v.http, null) == null ? 0 : 1) + + (try(v.http2, null) == null ? 0 : 1) + + (try(v.https, null) == null ? 0 : 1) + + (try(v.tcp, null) == null ? 0 : 1) + + (try(v.ssl, null) == null ? 0 : 1) <= 1 + ) + ]) + error_message = "Only one health check type can be configured at a time." + } + validation { + condition = alltrue(flatten([ + for k, v in var.health_check_configs : [ + for kk, vv in v : contains([ + "-", "USE_FIXED_PORT", "USE_NAMED_PORT", "USE_SERVING_PORT" + ], coalesce(try(vv.port_specification, null), "-")) + ] + ])) + error_message = "Invalid 'port_specification' value. Supported values are 'USE_FIXED_PORT', 'USE_NAMED_PORT', 'USE_SERVING_PORT'." + } +} diff --git a/assets/modules-fabric/v26/net-lb-app-int/variables-urlmap.tf b/assets/modules-fabric/v26/net-lb-app-int/variables-urlmap.tf new file mode 100644 index 0000000..cd1869f --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-app-int/variables-urlmap.tf @@ -0,0 +1,234 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description URLmap variable. + +variable "urlmap_config" { + description = "The URL map configuration." + type = object({ + default_service = optional(string) + default_url_redirect = optional(object({ + host = optional(string) + https = optional(bool) + path = optional(string) + prefix = optional(string) + response_code = optional(string) + strip_query = optional(bool) + })) + host_rules = optional(list(object({ + hosts = list(string) + path_matcher = string + description = optional(string) + }))) + path_matchers = optional(map(object({ + description = optional(string) + default_service = optional(string) + default_url_redirect = optional(object({ + host = optional(string) + https = optional(bool) + path = optional(string) + prefix = optional(string) + response_code = optional(string) + strip_query = optional(bool) + })) + path_rules = optional(list(object({ + paths = list(string) + service = optional(string) + route_action = optional(object({ + request_mirror_backend = optional(string) + cors_policy = optional(object({ + allow_credentials = optional(bool) + allow_headers = optional(string) + allow_methods = optional(string) + allow_origin_regexes = list(string) + allow_origins = list(string) + disabled = optional(bool) + expose_headers = optional(string) + max_age = optional(string) + })) + fault_injection_policy = optional(object({ + abort = optional(object({ + percentage = number + status = number + })) + delay = optional(object({ + fixed = object({ + seconds = number + nanos = number + }) + percentage = number + })) + })) + retry_policy = optional(object({ + num_retries = number + retry_conditions = optional(list(string)) + per_try_timeout = optional(object({ + seconds = number + nanos = optional(number) + })) + })) + timeout = optional(object({ + seconds = number + nanos = optional(number) + })) + url_rewrite = optional(object({ + host = optional(string) + path_prefix = optional(string) + })) + weighted_backend_services = optional(map(object({ + weight = number + header_action = optional(object({ + request_add = optional(map(object({ + value = string + replace = optional(bool, true) + }))) + request_remove = optional(list(string)) + response_add = optional(map(object({ + value = string + replace = optional(bool, true) + }))) + response_remove = optional(list(string)) + })) + }))) + })) + url_redirect = optional(object({ + host = optional(string) + https = optional(bool) + path = optional(string) + prefix = optional(string) + response_code = optional(string) + strip_query = optional(bool) + })) + }))) + route_rules = optional(list(object({ + priority = number + service = optional(string) + header_action = optional(object({ + request_add = optional(map(object({ + value = string + replace = optional(bool, true) + }))) + request_remove = optional(list(string)) + response_add = optional(map(object({ + value = string + replace = optional(bool, true) + }))) + response_remove = optional(list(string)) + })) + match_rules = optional(list(object({ + ignore_case = optional(bool, false) + headers = optional(list(object({ + name = string + invert_match = optional(bool, false) + type = optional(string, "present") # exact, prefix, suffix, regex, present, range + value = optional(string) + range_value = optional(object({ + end = string + start = string + })) + }))) + metadata_filters = optional(list(object({ + labels = map(string) + match_all = bool # MATCH_ANY, MATCH_ALL + }))) + path = optional(object({ + value = string + type = optional(string, "prefix") # full, prefix, regex + })) + query_params = optional(list(object({ + name = string + value = string + type = optional(string, "present") # exact, present, regex + }))) + }))) + route_action = optional(object({ + request_mirror_backend = optional(string) + cors_policy = optional(object({ + allow_credentials = optional(bool) + allow_headers = optional(string) + allow_methods = optional(string) + allow_origin_regexes = list(string) + allow_origins = list(string) + disabled = optional(bool) + expose_headers = optional(string) + max_age = optional(string) + })) + fault_injection_policy = optional(object({ + abort = optional(object({ + percentage = number + status = number + })) + delay = optional(object({ + fixed = object({ + seconds = number + nanos = number + }) + percentage = number + })) + })) + retry_policy = optional(object({ + num_retries = number + retry_conditions = optional(list(string)) + per_try_timeout = optional(object({ + seconds = number + nanos = optional(number) + })) + })) + timeout = optional(object({ + seconds = number + nanos = optional(number) + })) + url_rewrite = optional(object({ + host = optional(string) + path_prefix = optional(string) + })) + weighted_backend_services = optional(map(object({ + weight = number + header_action = optional(object({ + request_add = optional(map(object({ + value = string + replace = optional(bool, true) + }))) + request_remove = optional(list(string)) + response_add = optional(map(object({ + value = string + replace = optional(bool, true) + }))) + response_remove = optional(list(string)) + })) + }))) + })) + url_redirect = optional(object({ + host = optional(string) + https = optional(bool) + path = optional(string) + prefix = optional(string) + response_code = optional(string) + strip_query = optional(bool) + })) + }))) + }))) + test = optional(list(object({ + host = string + path = string + service = string + description = optional(string) + }))) + }) + default = { + default_service = "default" + } +} diff --git a/assets/modules-fabric/v26/net-lb-app-int/variables.tf b/assets/modules-fabric/v26/net-lb-app-int/variables.tf new file mode 100644 index 0000000..64127c0 --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-app-int/variables.tf @@ -0,0 +1,190 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "address" { + description = "Optional IP address used for the forwarding rule." + type = string + default = null +} + +variable "description" { + description = "Optional description used for resources." + type = string + default = "Terraform managed." +} + +# during the preview phase you cannot change this attribute on an existing rule +variable "global_access" { + description = "Allow client access from all regions." + type = bool + default = null +} + +variable "group_configs" { + description = "Optional unmanaged groups to create. Can be referenced in backends via key or outputs." + type = map(object({ + zone = string + instances = optional(list(string)) + named_ports = optional(map(number), {}) + project_id = optional(string) + })) + default = {} + nullable = false +} + +variable "labels" { + description = "Labels set on resources." + type = map(string) + default = {} +} + +variable "name" { + description = "Load balancer name." + type = string +} + +variable "neg_configs" { + description = "Optional network endpoint groups to create. Can be referenced in backends via key or outputs." + type = map(object({ + project_id = optional(string) + cloudrun = optional(object({ + region = string + target_service = optional(object({ + name = string + tag = optional(string) + })) + target_urlmask = optional(string) + })) + gce = optional(object({ + zone = string + # default_port = optional(number) + network = optional(string) + subnetwork = optional(string) + endpoints = optional(map(object({ + instance = string + ip_address = string + port = number + }))) + + })) + hybrid = optional(object({ + zone = string + network = optional(string) + # re-enable once provider properly support this + # default_port = optional(number) + endpoints = optional(map(object({ + ip_address = string + port = number + }))) + })) + psc = optional(object({ + region = string + target_service = string + network = optional(string) + subnetwork = optional(string) + })) + })) + default = {} + nullable = false + validation { + condition = alltrue([ + for k, v in var.neg_configs : ( + (try(v.cloudrun, null) == null ? 0 : 1) + + (try(v.gce, null) == null ? 0 : 1) + + (try(v.hybrid, null) == null ? 0 : 1) + + (try(v.psc, null) == null ? 0 : 1) == 1 + ) + ]) + error_message = "Only one type of neg can be configured at a time." + } + validation { + condition = alltrue([ + for k, v in var.neg_configs : ( + v.cloudrun == null + ? true + : v.cloudrun.target_urlmask != null || v.cloudrun.target_service != null + ) + ]) + error_message = "Cloud Run negs need either target type or target urlmask defined." + } +} + +variable "network_tier_premium" { + description = "Use premium network tier. Defaults to true." + type = bool + default = true + nullable = false +} + +variable "ports" { + description = "Optional ports for HTTP load balancer, valid ports are 80 and 8080." + type = list(string) + default = null +} + +variable "project_id" { + description = "Project id." + type = string +} + +variable "protocol" { + description = "Protocol supported by this load balancer." + type = string + default = "HTTP" + nullable = false + validation { + condition = ( + var.protocol == null || var.protocol == "HTTP" || var.protocol == "HTTPS" + ) + error_message = "Protocol must be HTTP or HTTPS" + } +} + +variable "region" { + description = "The region where to allocate the ILB resources." + type = string +} + +variable "service_directory_registration" { + description = "Service directory namespace and service used to register this load balancer." + type = object({ + namespace = string + service = string + }) + default = null +} + +variable "ssl_certificates" { + description = "SSL target proxy certificates (only if protocol is HTTPS)." + type = object({ + certificate_ids = optional(list(string), []) + create_configs = optional(map(object({ + certificate = string + private_key = string + })), {}) + }) + default = {} + nullable = false +} + +variable "vpc_config" { + description = "VPC-level configuration." + type = object({ + network = string + subnetwork = string + }) + nullable = false +} diff --git a/assets/modules-fabric/v26/net-lb-app-int/versions.tf b/assets/modules-fabric/v26/net-lb-app-int/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-app-int/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/net-lb-ext/README.md b/assets/modules-fabric/v26/net-lb-ext/README.md new file mode 100644 index 0000000..c63f3ac --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-ext/README.md @@ -0,0 +1,188 @@ +# External Passthrough Network Load Balancer Module + +This module allows managing a GCE Network Load Balancer and integrates the forwarding rule, regional backend, and optional health check resources. It's designed to be a simple match for the [`compute-vm`](../compute-vm) module, which can be used to manage instance templates and instance groups. + +## Examples + +- [Referencing existing MIGs](#referencing-existing-migs) +- [Externally manages instances](#externally-managed-instances) +- [End to end example](#end-to-end-example) + +### Referencing existing MIGs + +This example shows how to reference existing Managed Infrastructure Groups (MIGs). + +```hcl +module "instance_template" { + source = "./fabric/modules/compute-vm" + project_id = var.project_id + zone = "europe-west1-b" + name = "vm-test" + create_template = true + service_account = { + auto_create = true + } + network_interfaces = [ + { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + } + ] + tags = [ + "http-server" + ] +} + +module "mig" { + source = "./fabric/modules/compute-mig" + project_id = var.project_id + location = "europe-west1" + name = "mig-test" + target_size = 1 + instance_template = module.instance_template.template.self_link +} + +module "nlb" { + source = "./fabric/modules/net-lb-ext" + project_id = var.project_id + region = "europe-west1" + name = "nlb-test" + backends = [{ + group = module.mig.group_manager.instance_group + }] + health_check_config = { + http = { + port = 80 + } + } +} +# tftest modules=3 resources=6 +``` + +### Externally managed instances + +This examples shows how to create an NLB by combining externally managed instances (in a custom module or even outside of the current root module) in an unmanaged group. When using internally managed groups, remember to run `terraform apply` each time group instances change. + +```hcl +module "nlb" { + source = "./fabric/modules/net-lb-ext" + project_id = var.project_id + region = "europe-west1" + name = "nlb-test" + group_configs = { + my-group = { + zone = "europe-west1-b" + instances = [ + "instance-1-self-link", + "instance-2-self-link" + ] + } + } + backends = [{ + group = module.nlb.groups.my-group.self_link + }] + health_check_config = { + http = { + port = 80 + } + } +} +# tftest modules=1 resources=4 +``` + +### End to end example + +This example spins up a simple HTTP server and combines four modules: + +- [`nginx`](../cloud-config-container/nginx) from the `cloud-config-container` collection, to manage instance configuration +- [`compute-vm`](../compute-vm) to manage the instance template and unmanaged instance group +- this module to create a Network Load Balancer in front of the managed instance group + +Note that the example uses the GCE default service account. You might want to create an ad-hoc service account by combining the [`iam-service-account`](../iam-service-account) module, or by having the GCE VM module create one for you. In both cases, remember to set at least logging write permissions for the service account, or the container on the instances won't be able to start. + +```hcl +module "cos-nginx" { + source = "./fabric/modules/cloud-config-container/nginx" +} + +module "instance-group" { + source = "./fabric/modules/compute-vm" + for_each = toset(["b", "c"]) + project_id = var.project_id + zone = "europe-west1-${each.key}" + name = "nlb-test-${each.key}" + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + nat = false + addresses = null + }] + boot_disk = { + initialize_params = { + image = "projects/cos-cloud/global/images/family/cos-stable" + type = "pd-ssd" + size = 10 + } + } + tags = ["http-server", "ssh"] + metadata = { + user-data = module.cos-nginx.cloud_config + } + group = { named_ports = {} } +} + +module "nlb" { + source = "./fabric/modules/net-lb-ext" + project_id = var.project_id + region = "europe-west1" + name = "nlb-test" + ports = [80] + backends = [ + for z, mod in module.instance-group : { + group = mod.group.self_link + } + ] + health_check_config = { + http = { + port = 80 + } + } +} +# tftest modules=3 resources=7 +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [name](variables.tf#L189) | Name used for all resources. | string | ✓ | | +| [project_id](variables.tf#L200) | Project id where resources will be created. | string | ✓ | | +| [region](variables.tf#L216) | GCP region. | string | ✓ | | +| [address](variables.tf#L17) | Optional IP address used for the forwarding rule. | string | | null | +| [backend_service_config](variables.tf#L23) | Backend service level configuration. | object({…}) | | {} | +| [backends](variables.tf#L72) | Load balancer backends, balancing mode is one of 'CONNECTION' or 'UTILIZATION'. | list(object({…})) | | [] | +| [description](variables.tf#L83) | Optional description used for resources. | string | | "Terraform managed." | +| [group_configs](variables.tf#L89) | Optional unmanaged groups to create. Can be referenced in backends via outputs. | map(object({…})) | | {} | +| [health_check](variables.tf#L100) | Name of existing health check to use, disables auto-created health check. | string | | null | +| [health_check_config](variables.tf#L106) | Optional auto-created health check configuration, use the output self-link to set it in the auto healing policy. Refer to examples for usage. | object({…}) | | {…} | +| [labels](variables.tf#L183) | Labels set on resources. | map(string) | | {} | +| [ports](variables.tf#L194) | Comma-separated ports, leave null to use all ports. | list(string) | | null | +| [protocol](variables.tf#L205) | IP protocol used, defaults to TCP. UDP or L3_DEFAULT can also be used. | string | | "TCP" | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [backend_service](outputs.tf#L17) | Backend resource. | | +| [backend_service_id](outputs.tf#L22) | Backend id. | | +| [backend_service_self_link](outputs.tf#L27) | Backend self link. | | +| [forwarding_rule](outputs.tf#L32) | Forwarding rule resource. | | +| [forwarding_rule_address](outputs.tf#L37) | Forwarding rule address. | | +| [forwarding_rule_self_link](outputs.tf#L42) | Forwarding rule self link. | | +| [group_self_links](outputs.tf#L47) | Optional unmanaged instance group self links. | | +| [groups](outputs.tf#L54) | Optional unmanaged instance group resources. | | +| [health_check](outputs.tf#L59) | Auto-created health-check resource. | | +| [health_check_self_id](outputs.tf#L64) | Auto-created health-check self id. | | +| [health_check_self_link](outputs.tf#L69) | Auto-created health-check self link. | | +| [id](outputs.tf#L74) | Fully qualified forwarding rule id. | | + diff --git a/assets/modules-fabric/v26/net-lb-ext/groups.tf b/assets/modules-fabric/v26/net-lb-ext/groups.tf new file mode 100644 index 0000000..f3fcaa8 --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-ext/groups.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Optional instance group resources. + +resource "google_compute_instance_group" "default" { + for_each = var.group_configs + project = var.project_id + zone = each.value.zone + name = "${var.name}-${each.key}" + description = var.description + instances = each.value.instances + + dynamic "named_port" { + for_each = each.value.named_ports + content { + name = named_port.key + port = named_port.value + } + } +} diff --git a/assets/modules-fabric/v26/net-lb-ext/health-check.tf b/assets/modules-fabric/v26/net-lb-ext/health-check.tf new file mode 100644 index 0000000..08ea016 --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-ext/health-check.tf @@ -0,0 +1,120 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Health check resource. + +locals { + hc = var.health_check_config + hc_grpc = try(local.hc.grpc, null) != null + hc_http = try(local.hc.http, null) != null + hc_http2 = try(local.hc.http2, null) != null + hc_https = try(local.hc.https, null) != null + hc_ssl = try(local.hc.ssl, null) != null + hc_tcp = try(local.hc.tcp, null) != null +} + +resource "google_compute_region_health_check" "default" { + provider = google-beta + count = local.hc != null ? 1 : 0 + project = var.project_id + region = var.region + name = var.name + description = local.hc.description + check_interval_sec = local.hc.check_interval_sec + healthy_threshold = local.hc.healthy_threshold + timeout_sec = local.hc.timeout_sec + unhealthy_threshold = local.hc.unhealthy_threshold + + dynamic "grpc_health_check" { + for_each = local.hc_grpc ? [""] : [] + content { + port = local.hc.grpc.port + port_name = local.hc.grpc.port_name + port_specification = local.hc.grpc.port_specification + grpc_service_name = local.hc.grpc.service_name + } + } + + dynamic "http_health_check" { + for_each = local.hc_http ? [""] : [] + content { + host = local.hc.http.host + port = local.hc.http.port + port_name = local.hc.http.port_name + port_specification = local.hc.http.port_specification + proxy_header = local.hc.http.proxy_header + request_path = local.hc.http.request_path + response = local.hc.http.response + } + } + + dynamic "http2_health_check" { + for_each = local.hc_http2 ? [""] : [] + content { + host = local.hc.http.host + port = local.hc.http.port + port_name = local.hc.http.port_name + port_specification = local.hc.http.port_specification + proxy_header = local.hc.http.proxy_header + request_path = local.hc.http.request_path + response = local.hc.http.response + } + } + + dynamic "https_health_check" { + for_each = local.hc_https ? [""] : [] + content { + host = local.hc.https.host + port = local.hc.https.port + port_name = local.hc.https.port_name + port_specification = local.hc.https.port_specification + proxy_header = local.hc.https.proxy_header + request_path = local.hc.https.request_path + response = local.hc.https.response + } + } + + dynamic "ssl_health_check" { + for_each = local.hc_ssl ? [""] : [] + content { + port = local.hc.tcp.port + port_name = local.hc.tcp.port_name + port_specification = local.hc.tcp.port_specification + proxy_header = local.hc.tcp.proxy_header + request = local.hc.tcp.request + response = local.hc.tcp.response + } + } + + dynamic "tcp_health_check" { + for_each = local.hc_tcp ? [""] : [] + content { + port = local.hc.tcp.port + port_name = local.hc.tcp.port_name + port_specification = local.hc.tcp.port_specification + proxy_header = local.hc.tcp.proxy_header + request = local.hc.tcp.request + response = local.hc.tcp.response + } + } + + dynamic "log_config" { + for_each = try(local.hc.enable_logging, null) == true ? [""] : [] + content { + enable = true + } + } +} diff --git a/assets/modules-fabric/v26/net-lb-ext/main.tf b/assets/modules-fabric/v26/net-lb-ext/main.tf new file mode 100644 index 0000000..68619b6 --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-ext/main.tf @@ -0,0 +1,103 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + bs_conntrack = var.backend_service_config.connection_tracking + bs_failover = var.backend_service_config.failover_config + health_check = ( + var.health_check != null + ? var.health_check + : google_compute_region_health_check.default.0.self_link + ) +} + +resource "google_compute_forwarding_rule" "default" { + provider = google-beta + project = var.project_id + region = var.region + name = var.name + description = var.description + ip_address = var.address + ip_protocol = var.protocol + backend_service = ( + google_compute_region_backend_service.default.self_link + ) + load_balancing_scheme = "EXTERNAL" + ports = var.ports # "nnnnn" or "nnnnn,nnnnn,nnnnn" max 5 + all_ports = var.ports == null ? true : null + labels = var.labels + # is_mirroring_collector = false +} + +resource "google_compute_region_backend_service" "default" { + provider = google-beta + project = var.project_id + region = var.region + name = var.name + description = var.description + load_balancing_scheme = "EXTERNAL" + protocol = var.backend_service_config.protocol + health_checks = [local.health_check] + connection_draining_timeout_sec = var.backend_service_config.connection_draining_timeout_sec + locality_lb_policy = var.backend_service_config.locality_lb_policy + port_name = var.backend_service_config.port_name + session_affinity = var.backend_service_config.session_affinity + timeout_sec = var.backend_service_config.timeout_sec + + dynamic "backend" { + for_each = { for b in var.backends : b.group => b } + content { + balancing_mode = "CONNECTION" + description = backend.value.description + failover = backend.value.failover + group = backend.key + } + } + + dynamic "connection_tracking_policy" { + for_each = local.bs_conntrack == null ? [] : [""] + content { + connection_persistence_on_unhealthy_backends = ( + local.bs_conntrack.persist_conn_on_unhealthy != null + ? local.bs_conntrack.persist_conn_on_unhealthy + : null + ) + idle_timeout_sec = local.bs_conntrack.idle_timeout_sec + tracking_mode = try( + local.bs_conntrack.track_per_session + ? "PER_SESSION" + : "PER_CONNECTION", null + ) + } + } + + dynamic "failover_policy" { + for_each = local.bs_failover == null ? [] : [""] + content { + disable_connection_drain_on_failover = local.bs_failover.disable_conn_drain + drop_traffic_if_unhealthy = local.bs_failover.drop_traffic_if_unhealthy + failover_ratio = local.bs_failover.ratio + } + } + + dynamic "log_config" { + for_each = var.backend_service_config.log_sample_rate == null ? [] : [""] + content { + enable = true + sample_rate = var.backend_service_config.log_sample_rate + } + } +} diff --git a/assets/modules-fabric/v26/net-lb-ext/outputs.tf b/assets/modules-fabric/v26/net-lb-ext/outputs.tf new file mode 100644 index 0000000..f7bb543 --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-ext/outputs.tf @@ -0,0 +1,77 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "backend_service" { + description = "Backend resource." + value = google_compute_region_backend_service.default +} + +output "backend_service_id" { + description = "Backend id." + value = google_compute_region_backend_service.default.id +} + +output "backend_service_self_link" { + description = "Backend self link." + value = google_compute_region_backend_service.default.self_link +} + +output "forwarding_rule" { + description = "Forwarding rule resource." + value = google_compute_forwarding_rule.default +} + +output "forwarding_rule_address" { + description = "Forwarding rule address." + value = google_compute_forwarding_rule.default.ip_address +} + +output "forwarding_rule_self_link" { + description = "Forwarding rule self link." + value = google_compute_forwarding_rule.default.self_link +} + +output "group_self_links" { + description = "Optional unmanaged instance group self links." + value = { + for k, v in google_compute_instance_group.default : k => v.self_link + } +} + +output "groups" { + description = "Optional unmanaged instance group resources." + value = google_compute_instance_group.default +} + +output "health_check" { + description = "Auto-created health-check resource." + value = try(google_compute_region_health_check.default.0, null) +} + +output "health_check_self_id" { + description = "Auto-created health-check self id." + value = try(google_compute_region_health_check.default.0.id, null) +} + +output "health_check_self_link" { + description = "Auto-created health-check self link." + value = try(google_compute_region_health_check.default.0.self_link, null) +} + +output "id" { + description = "Fully qualified forwarding rule id." + value = google_compute_forwarding_rule.default.id +} diff --git a/assets/modules-fabric/v26/net-lb-ext/variables.tf b/assets/modules-fabric/v26/net-lb-ext/variables.tf new file mode 100644 index 0000000..dbc9b54 --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-ext/variables.tf @@ -0,0 +1,219 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "address" { + description = "Optional IP address used for the forwarding rule." + type = string + default = null +} + +variable "backend_service_config" { + description = "Backend service level configuration." + type = object({ + connection_draining_timeout_sec = optional(number) + connection_tracking = optional(object({ + idle_timeout_sec = optional(number) + persist_conn_on_unhealthy = optional(string) + track_per_session = optional(bool) + })) + failover_config = optional(object({ + disable_conn_drain = optional(bool) + drop_traffic_if_unhealthy = optional(bool) + ratio = optional(number) + })) + locality_lb_policy = optional(string) + log_sample_rate = optional(number) + port_name = optional(string) + protocol = optional(string, "UNSPECIFIED") + session_affinity = optional(string) + timeout_sec = optional(number) + }) + default = {} + nullable = false + validation { + condition = contains( + ["TCP", "UDP", "UNSPECIFIED"], + coalesce(var.backend_service_config.protocol, "TCP") + ) + error_message = "Protocol can be 'TCP', 'UDP', 'UNSPECIFIED'." + } + validation { + condition = contains( + ["MAGLEV", "WEIGHTED_MAGLEV"], + coalesce(var.backend_service_config.locality_lb_policy, "MAGLEV") + ) + error_message = "Locality LB policy can be 'MAGLEV', 'WEIGHTED_MAGLEV'." + } + validation { + condition = contains( + [ + "NONE", "CLIENT_IP", "CLIENT_IP_NO_DESTINATION", + "CLIENT_IP_PORT_PROTO", "CLIENT_IP_PROTO" + ], + coalesce(var.backend_service_config.session_affinity, "NONE") + ) + error_message = "Invalid session affinity value." + } +} + +variable "backends" { + description = "Load balancer backends, balancing mode is one of 'CONNECTION' or 'UTILIZATION'." + type = list(object({ + group = string + description = optional(string, "Terraform managed.") + failover = optional(bool, false) + })) + default = [] + nullable = false +} + +variable "description" { + description = "Optional description used for resources." + type = string + default = "Terraform managed." +} + +variable "group_configs" { + description = "Optional unmanaged groups to create. Can be referenced in backends via outputs." + type = map(object({ + zone = string + instances = optional(list(string)) + named_ports = optional(map(number), {}) + })) + default = {} + nullable = false +} + +variable "health_check" { + description = "Name of existing health check to use, disables auto-created health check." + type = string + default = null +} + +variable "health_check_config" { + description = "Optional auto-created health check configuration, use the output self-link to set it in the auto healing policy. Refer to examples for usage." + type = object({ + check_interval_sec = optional(number) + description = optional(string, "Terraform managed.") + enable_logging = optional(bool, false) + healthy_threshold = optional(number) + timeout_sec = optional(number) + unhealthy_threshold = optional(number) + grpc = optional(object({ + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + service_name = optional(string) + })) + http = optional(object({ + host = optional(string) + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request_path = optional(string) + response = optional(string) + })) + http2 = optional(object({ + host = optional(string) + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request_path = optional(string) + response = optional(string) + })) + https = optional(object({ + host = optional(string) + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request_path = optional(string) + response = optional(string) + })) + tcp = optional(object({ + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request = optional(string) + response = optional(string) + })) + ssl = optional(object({ + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request = optional(string) + response = optional(string) + })) + }) + default = { + tcp = { + port_specification = "USE_SERVING_PORT" + } + } + validation { + condition = ( + (try(var.health_check_config.grpc, null) == null ? 0 : 1) + + (try(var.health_check_config.http, null) == null ? 0 : 1) + + (try(var.health_check_config.http2, null) == null ? 0 : 1) + + (try(var.health_check_config.https, null) == null ? 0 : 1) + + (try(var.health_check_config.tcp, null) == null ? 0 : 1) + + (try(var.health_check_config.ssl, null) == null ? 0 : 1) <= 1 + ) + error_message = "Only one health check type can be configured at a time." + } +} + +variable "labels" { + description = "Labels set on resources." + type = map(string) + default = {} +} + +variable "name" { + description = "Name used for all resources." + type = string +} + +variable "ports" { + description = "Comma-separated ports, leave null to use all ports." + type = list(string) + default = null +} + +variable "project_id" { + description = "Project id where resources will be created." + type = string +} + +variable "protocol" { + description = "IP protocol used, defaults to TCP. UDP or L3_DEFAULT can also be used." + type = string + default = "TCP" + nullable = false + validation { + condition = contains(["L3_DEFAULT", "TCP", "UDP"], var.protocol) + error_message = "Allowed values are 'TCP', 'UDP', 'L3_DEFAULT'." + } +} + +variable "region" { + description = "GCP region." + type = string +} diff --git a/assets/modules-fabric/v26/net-lb-ext/versions.tf b/assets/modules-fabric/v26/net-lb-ext/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-ext/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/net-lb-int/README.md b/assets/modules-fabric/v26/net-lb-int/README.md new file mode 100644 index 0000000..c577388 --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-int/README.md @@ -0,0 +1,215 @@ +# Internal Passthrough Network Load Balancer Module + +This module allows managing a GCE Internal Load Balancer and integrates the forwarding rule, regional backend, and optional health check resources. It's designed to be a simple match for the [`compute-vm`](../compute-vm) module, which can be used to manage instance templates and instance groups. + +## Issues + +There are some corner cases where Terraform raises a cycle error on apply, for example when using the entire ILB module as a value in `for_each` counts used to create static routes in the VPC module. These are easily fixed by using forwarding rule ids instead of modules as values in the `for_each` loop. + + + +## Examples + +- [Referencing existing MIGs](#referencing-existing-migs) +- [Externally managed instances](#externally-managed-instances) +- [End to end example](#end-to-end-example) + +### Referencing existing MIGs + +This example shows how to reference existing Managed Infrastructure Groups (MIGs). + +```hcl +module "instance_template" { + source = "./fabric/modules/compute-vm" + project_id = var.project_id + zone = "europe-west1-b" + name = "vm-test" + create_template = true + service_account = { + auto_create = true + } + network_interfaces = [ + { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + } + ] + tags = [ + "http-server" + ] +} + +module "mig" { + source = "./fabric/modules/compute-mig" + project_id = var.project_id + location = "europe-west1" + name = "mig-test" + target_size = 1 + instance_template = module.instance_template.template.self_link +} + +module "ilb" { + source = "./fabric/modules/net-lb-int" + project_id = var.project_id + region = "europe-west1" + name = "ilb-test" + service_label = "ilb-test" + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + } + backends = [{ + group = module.mig.group_manager.instance_group + }] + health_check_config = { + http = { + port = 80 + } + } +} +# tftest modules=3 resources=6 +``` + +### Externally managed instances + +This examples shows how to create an ILB by combining externally managed instances (in a custom module or even outside of the current root module) in an unmanaged group. When using internally managed groups, remember to run `terraform apply` each time group instances change. + +```hcl +module "ilb" { + source = "./fabric/modules/net-lb-int" + project_id = var.project_id + region = "europe-west1" + name = "ilb-test" + service_label = "ilb-test" + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + } + group_configs = { + my-group = { + zone = "europe-west1-b" + instances = [ + "instance-1-self-link", + "instance-2-self-link" + ] + } + } + backends = [{ + group = module.ilb.groups.my-group.self_link + }] + health_check_config = { + http = { + port = 80 + } + } +} +# tftest modules=1 resources=4 +``` + +### End to end example + +This example spins up a simple HTTP server and combines four modules: + +- [`nginx`](../cloud-config-container/nginx) from the `cloud-config-container` collection, to manage instance configuration +- [`compute-vm`](../compute-vm) to manage the instance template and unmanaged instance group +- this module to create an Internal Load Balancer in front of the managed instance group + +Note that the example uses the GCE default service account. You might want to create an ad-hoc service account by combining the [`iam-service-account`](../iam-service-account) module, or by having the GCE VM module create one for you. In both cases, remember to set at least logging write permissions for the service account, or the container on the instances won't be able to start. + +```hcl +module "cos-nginx" { + source = "./fabric/modules/cloud-config-container/nginx" +} + +module "instance-group" { + source = "./fabric/modules/compute-vm" + for_each = toset(["b", "c"]) + project_id = var.project_id + zone = "europe-west1-${each.key}" + name = "ilb-test-${each.key}" + network_interfaces = [{ + network = var.vpc.self_link + subnetwork = var.subnet.self_link + nat = false + addresses = null + }] + boot_disk = { + initialize_params = { + image = "projects/cos-cloud/global/images/family/cos-stable" + type = "pd-ssd" + size = 10 + } + } + tags = ["http-server", "ssh"] + metadata = { + user-data = module.cos-nginx.cloud_config + } + group = { named_ports = {} } +} + +module "ilb" { + source = "./fabric/modules/net-lb-int" + project_id = var.project_id + region = "europe-west1" + name = "ilb-test" + service_label = "ilb-test" + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + } + ports = [80] + backends = [ + for z, mod in module.instance-group : { + group = mod.group.self_link + balancing_mode = "UTILIZATION" + } + ] + health_check_config = { + http = { + port = 80 + } + } +} +# tftest modules=3 resources=7 +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [name](variables.tf#L188) | Name used for all resources. | string | ✓ | | +| [project_id](variables.tf#L199) | Project id where resources will be created. | string | ✓ | | +| [region](variables.tf#L210) | GCP region. | string | ✓ | | +| [vpc_config](variables.tf#L221) | VPC-level configuration. | object({…}) | ✓ | | +| [address](variables.tf#L17) | Optional IP address used for the forwarding rule. | string | | null | +| [backend_service_config](variables.tf#L23) | Backend service level configuration. | object({…}) | | {} | +| [backends](variables.tf#L56) | Load balancer backends, balancing mode is one of 'CONNECTION' or 'UTILIZATION'. | list(object({…})) | | [] | +| [description](variables.tf#L75) | Optional description used for resources. | string | | "Terraform managed." | +| [global_access](variables.tf#L81) | Global access, defaults to false if not set. | bool | | null | +| [group_configs](variables.tf#L87) | Optional unmanaged groups to create. Can be referenced in backends via outputs. | map(object({…})) | | {} | +| [health_check](variables.tf#L99) | Name of existing health check to use, disables auto-created health check. | string | | null | +| [health_check_config](variables.tf#L105) | Optional auto-created health check configuration, use the output self-link to set it in the auto healing policy. Refer to examples for usage. | object({…}) | | {…} | +| [labels](variables.tf#L182) | Labels set on resources. | map(string) | | {} | +| [ports](variables.tf#L193) | Comma-separated ports, leave null to use all ports. | list(string) | | null | +| [protocol](variables.tf#L204) | IP protocol used, defaults to TCP. | string | | "TCP" | +| [service_label](variables.tf#L215) | Optional prefix of the fully qualified forwarding rule name. | string | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [backend_service](outputs.tf#L17) | Backend resource. | | +| [backend_service_id](outputs.tf#L22) | Backend id. | | +| [backend_service_self_link](outputs.tf#L27) | Backend self link. | | +| [forwarding_rule](outputs.tf#L32) | Forwarding rule resource. | | +| [forwarding_rule_address](outputs.tf#L37) | Forwarding rule address. | | +| [forwarding_rule_self_link](outputs.tf#L42) | Forwarding rule self link. | | +| [group_self_links](outputs.tf#L47) | Optional unmanaged instance group self links. | | +| [groups](outputs.tf#L54) | Optional unmanaged instance group resources. | | +| [health_check](outputs.tf#L59) | Auto-created health-check resource. | | +| [health_check_self_id](outputs.tf#L64) | Auto-created health-check self id. | | +| [health_check_self_link](outputs.tf#L69) | Auto-created health-check self link. | | +| [id](outputs.tf#L74) | Fully qualified forwarding rule id. | | + diff --git a/assets/modules-fabric/v26/net-lb-int/groups.tf b/assets/modules-fabric/v26/net-lb-int/groups.tf new file mode 100644 index 0000000..5bb7197 --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-int/groups.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Optional instance group resources. + +resource "google_compute_instance_group" "default" { + for_each = var.group_configs + project = var.project_id + zone = each.value.zone + name = "${var.name}-${each.key}" + description = each.value.description + instances = each.value.instances + + dynamic "named_port" { + for_each = each.value.named_ports + content { + name = named_port.key + port = named_port.value + } + } +} diff --git a/assets/modules-fabric/v26/net-lb-int/health-check.tf b/assets/modules-fabric/v26/net-lb-int/health-check.tf new file mode 100644 index 0000000..88f9f6e --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-int/health-check.tf @@ -0,0 +1,119 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Health check resource. + +locals { + hc = var.health_check_config + hc_grpc = try(local.hc.grpc, null) != null + hc_http = try(local.hc.http, null) != null + hc_http2 = try(local.hc.http2, null) != null + hc_https = try(local.hc.https, null) != null + hc_ssl = try(local.hc.ssl, null) != null + hc_tcp = try(local.hc.tcp, null) != null +} + +resource "google_compute_health_check" "default" { + provider = google-beta + count = local.hc != null ? 1 : 0 + project = var.project_id + name = var.name + description = local.hc.description + check_interval_sec = local.hc.check_interval_sec + healthy_threshold = local.hc.healthy_threshold + timeout_sec = local.hc.timeout_sec + unhealthy_threshold = local.hc.unhealthy_threshold + + dynamic "grpc_health_check" { + for_each = local.hc_grpc ? [""] : [] + content { + port = local.hc.grpc.port + port_name = local.hc.grpc.port_name + port_specification = local.hc.grpc.port_specification + grpc_service_name = local.hc.grpc.service_name + } + } + + dynamic "http_health_check" { + for_each = local.hc_http ? [""] : [] + content { + host = local.hc.http.host + port = local.hc.http.port + port_name = local.hc.http.port_name + port_specification = local.hc.http.port_specification + proxy_header = local.hc.http.proxy_header + request_path = local.hc.http.request_path + response = local.hc.http.response + } + } + + dynamic "http2_health_check" { + for_each = local.hc_http2 ? [""] : [] + content { + host = local.hc.http.host + port = local.hc.http.port + port_name = local.hc.http.port_name + port_specification = local.hc.http.port_specification + proxy_header = local.hc.http.proxy_header + request_path = local.hc.http.request_path + response = local.hc.http.response + } + } + + dynamic "https_health_check" { + for_each = local.hc_https ? [""] : [] + content { + host = local.hc.https.host + port = local.hc.https.port + port_name = local.hc.https.port_name + port_specification = local.hc.https.port_specification + proxy_header = local.hc.https.proxy_header + request_path = local.hc.https.request_path + response = local.hc.https.response + } + } + + dynamic "ssl_health_check" { + for_each = local.hc_ssl ? [""] : [] + content { + port = local.hc.tcp.port + port_name = local.hc.tcp.port_name + port_specification = local.hc.tcp.port_specification + proxy_header = local.hc.tcp.proxy_header + request = local.hc.tcp.request + response = local.hc.tcp.response + } + } + + dynamic "tcp_health_check" { + for_each = local.hc_tcp ? [""] : [] + content { + port = local.hc.tcp.port + port_name = local.hc.tcp.port_name + port_specification = local.hc.tcp.port_specification + proxy_header = local.hc.tcp.proxy_header + request = local.hc.tcp.request + response = local.hc.tcp.response + } + } + + dynamic "log_config" { + for_each = try(local.hc.enable_logging, null) == true ? [""] : [] + content { + enable = true + } + } +} diff --git a/assets/modules-fabric/v26/net-lb-int/main.tf b/assets/modules-fabric/v26/net-lb-int/main.tf new file mode 100644 index 0000000..698293a --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-int/main.tf @@ -0,0 +1,111 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +locals { + bs_conntrack = var.backend_service_config.connection_tracking + bs_failover = var.backend_service_config.failover_config + health_check = ( + var.health_check != null + ? var.health_check + : google_compute_health_check.default.0.self_link + ) +} + +resource "google_compute_forwarding_rule" "default" { + provider = google-beta + project = var.project_id + region = var.region + name = var.name + description = var.description + ip_address = var.address + ip_protocol = var.protocol # TCP | UDP + backend_service = ( + google_compute_region_backend_service.default.self_link + ) + load_balancing_scheme = "INTERNAL" + network = var.vpc_config.network + ports = var.ports # "nnnnn" or "nnnnn,nnnnn,nnnnn" max 5 + subnetwork = var.vpc_config.subnetwork + allow_global_access = var.global_access + labels = var.labels + all_ports = var.ports == null ? true : null + service_label = var.service_label + # is_mirroring_collector = false +} + +resource "google_compute_region_backend_service" "default" { + provider = google-beta + project = var.project_id + region = var.region + name = var.name + description = var.description + load_balancing_scheme = "INTERNAL" + protocol = var.protocol + network = var.vpc_config.network + health_checks = [local.health_check] + connection_draining_timeout_sec = var.backend_service_config.connection_draining_timeout_sec + session_affinity = var.backend_service_config.session_affinity + timeout_sec = var.backend_service_config.timeout_sec + + dynamic "backend" { + for_each = { for b in var.backends : b.group => b } + content { + balancing_mode = backend.value.balancing_mode + description = backend.value.description + failover = backend.value.failover + group = backend.key + } + } + + dynamic "connection_tracking_policy" { + for_each = local.bs_conntrack == null ? [] : [""] + content { + connection_persistence_on_unhealthy_backends = ( + local.bs_conntrack.persist_conn_on_unhealthy != null + ? local.bs_conntrack.persist_conn_on_unhealthy + : null + ) + idle_timeout_sec = local.bs_conntrack.idle_timeout_sec + tracking_mode = try(local.bs_conntrack.track_per_session ? "PER_SESSION" : "PER_CONNECTION", null) + } + } + + dynamic "failover_policy" { + for_each = local.bs_failover == null ? [] : [""] + content { + disable_connection_drain_on_failover = local.bs_failover.disable_conn_drain + drop_traffic_if_unhealthy = local.bs_failover.drop_traffic_if_unhealthy + failover_ratio = local.bs_failover.ratio + } + } + + dynamic "log_config" { + for_each = var.backend_service_config.log_sample_rate == null ? [] : [""] + content { + enable = true + sample_rate = var.backend_service_config.log_sample_rate + } + } + + dynamic "subsetting" { + for_each = var.backend_service_config.enable_subsetting == true ? [""] : [] + content { + policy = "CONSISTENT_HASH_SUBSETTING" + } + } + +} diff --git a/assets/modules-fabric/v26/net-lb-int/outputs.tf b/assets/modules-fabric/v26/net-lb-int/outputs.tf new file mode 100644 index 0000000..bab17b9 --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-int/outputs.tf @@ -0,0 +1,77 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "backend_service" { + description = "Backend resource." + value = google_compute_region_backend_service.default +} + +output "backend_service_id" { + description = "Backend id." + value = google_compute_region_backend_service.default.id +} + +output "backend_service_self_link" { + description = "Backend self link." + value = google_compute_region_backend_service.default.self_link +} + +output "forwarding_rule" { + description = "Forwarding rule resource." + value = google_compute_forwarding_rule.default +} + +output "forwarding_rule_address" { + description = "Forwarding rule address." + value = google_compute_forwarding_rule.default.ip_address +} + +output "forwarding_rule_self_link" { + description = "Forwarding rule self link." + value = google_compute_forwarding_rule.default.self_link +} + +output "group_self_links" { + description = "Optional unmanaged instance group self links." + value = { + for k, v in google_compute_instance_group.default : k => v.self_link + } +} + +output "groups" { + description = "Optional unmanaged instance group resources." + value = google_compute_instance_group.default +} + +output "health_check" { + description = "Auto-created health-check resource." + value = try(google_compute_health_check.default.0, null) +} + +output "health_check_self_id" { + description = "Auto-created health-check self id." + value = try(google_compute_health_check.default.0.id, null) +} + +output "health_check_self_link" { + description = "Auto-created health-check self link." + value = try(google_compute_health_check.default.0.self_link, null) +} + +output "id" { + description = "Fully qualified forwarding rule id." + value = google_compute_forwarding_rule.default.id +} diff --git a/assets/modules-fabric/v26/net-lb-int/variables.tf b/assets/modules-fabric/v26/net-lb-int/variables.tf new file mode 100644 index 0000000..9e90c1d --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-int/variables.tf @@ -0,0 +1,228 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "address" { + description = "Optional IP address used for the forwarding rule." + type = string + default = null +} + +variable "backend_service_config" { + description = "Backend service level configuration." + type = object({ + connection_draining_timeout_sec = optional(number) + connection_tracking = optional(object({ + idle_timeout_sec = optional(number) + persist_conn_on_unhealthy = optional(string) + track_per_session = optional(bool) + })) + enable_subsetting = optional(bool) + failover_config = optional(object({ + disable_conn_drain = optional(bool) + drop_traffic_if_unhealthy = optional(bool) + ratio = optional(number) + })) + log_sample_rate = optional(number) + session_affinity = optional(string) + timeout_sec = optional(number) + }) + default = {} + nullable = false + validation { + condition = contains( + [ + "NONE", "CLIENT_IP", "CLIENT_IP_NO_DESTINATION", + "CLIENT_IP_PORT_PROTO", "CLIENT_IP_PROTO" + ], + coalesce(var.backend_service_config.session_affinity, "NONE") + ) + error_message = "Invalid session affinity value." + } +} + +variable "backends" { + description = "Load balancer backends, balancing mode is one of 'CONNECTION' or 'UTILIZATION'." + type = list(object({ + group = string + balancing_mode = optional(string, "CONNECTION") + description = optional(string, "Terraform managed.") + failover = optional(bool, false) + })) + default = [] + nullable = false + validation { + condition = alltrue([ + for b in var.backends : contains( + ["CONNECTION", "UTILIZATION"], coalesce(b.balancing_mode, "CONNECTION") + )]) + error_message = "When specified balancing mode needs to be 'CONNECTION' or 'UTILIZATION'." + } +} + +variable "description" { + description = "Optional description used for resources." + type = string + default = "Terraform managed." +} + +variable "global_access" { + description = "Global access, defaults to false if not set." + type = bool + default = null +} + +variable "group_configs" { + description = "Optional unmanaged groups to create. Can be referenced in backends via outputs." + type = map(object({ + zone = string + description = optional(string, "Terraform managed.") + instances = optional(list(string)) + named_ports = optional(map(number), {}) + })) + default = {} + nullable = false +} + +variable "health_check" { + description = "Name of existing health check to use, disables auto-created health check." + type = string + default = null +} + +variable "health_check_config" { + description = "Optional auto-created health check configuration, use the output self-link to set it in the auto healing policy. Refer to examples for usage." + type = object({ + check_interval_sec = optional(number) + description = optional(string, "Terraform managed.") + enable_logging = optional(bool, false) + healthy_threshold = optional(number) + timeout_sec = optional(number) + unhealthy_threshold = optional(number) + grpc = optional(object({ + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + service_name = optional(string) + })) + http = optional(object({ + host = optional(string) + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request_path = optional(string) + response = optional(string) + })) + http2 = optional(object({ + host = optional(string) + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request_path = optional(string) + response = optional(string) + })) + https = optional(object({ + host = optional(string) + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request_path = optional(string) + response = optional(string) + })) + tcp = optional(object({ + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request = optional(string) + response = optional(string) + })) + ssl = optional(object({ + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request = optional(string) + response = optional(string) + })) + }) + default = { + tcp = { + port_specification = "USE_SERVING_PORT" + } + } + validation { + condition = ( + (try(var.health_check_config.grpc, null) == null ? 0 : 1) + + (try(var.health_check_config.http, null) == null ? 0 : 1) + + (try(var.health_check_config.http2, null) == null ? 0 : 1) + + (try(var.health_check_config.https, null) == null ? 0 : 1) + + (try(var.health_check_config.tcp, null) == null ? 0 : 1) + + (try(var.health_check_config.ssl, null) == null ? 0 : 1) <= 1 + ) + error_message = "Only one health check type can be configured at a time." + } +} + +variable "labels" { + description = "Labels set on resources." + type = map(string) + default = {} +} + +variable "name" { + description = "Name used for all resources." + type = string +} + +variable "ports" { + description = "Comma-separated ports, leave null to use all ports." + type = list(string) + default = null +} + +variable "project_id" { + description = "Project id where resources will be created." + type = string +} + +variable "protocol" { + description = "IP protocol used, defaults to TCP." + type = string + default = "TCP" +} + +variable "region" { + description = "GCP region." + type = string +} + +variable "service_label" { + description = "Optional prefix of the fully qualified forwarding rule name." + type = string + default = null +} + +variable "vpc_config" { + description = "VPC-level configuration." + type = object({ + network = string + subnetwork = string + }) + nullable = false +} diff --git a/assets/modules-fabric/v26/net-lb-int/versions.tf b/assets/modules-fabric/v26/net-lb-int/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-int/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/net-lb-proxy-int/README.md b/assets/modules-fabric/v26/net-lb-proxy-int/README.md new file mode 100644 index 0000000..e606b6b --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-proxy-int/README.md @@ -0,0 +1,321 @@ +# Internal Proxy Network Load Balancer Module + +This module allows managing Internal HTTP/HTTPS Load Balancers (L7 ILBs). It's designed to expose the full configuration of the underlying resources, and to facilitate common usage patterns by providing sensible defaults, and optionally managing prerequisite resources like health checks, instance groups, etc. + +Due to the complexity of the underlying resources, changes to the configuration that involve recreation of resources are best applied in stages, starting by disabling the configuration in the urlmap that references the resources that need recreation, then doing the same for the backend service, etc. + +## Examples + + +- [Examples](#examples) + - [Minimal Example](#minimal-example) + - [Health Checks](#health-checks) + - [Instance Groups](#instance-groups) + - [Network Endpoint Groups (NEGs)](#network-endpoint-groups-negs) + - [Zonal NEG creation](#zonal-neg-creation) + - [Hybrid NEG creation](#hybrid-neg-creation) + - [Private Service Connect NEG creation](#private-service-connect-neg-creation) +- [Files](#files) +- [Variables](#variables) +- [Outputs](#outputs) + + +### Minimal Example + +An Regional internal proxy Network Load Balancer with a backend service pointing to an existing GCE instance group: + +```hcl +module "tcp-proxy" { + source = "./fabric/modules/net-lb-proxy-int" + name = "ilb-test" + project_id = var.project_id + region = "europe-west1" + backend_service_config = { + backends = [{ + group = "projects/myprj/zones/europe-west1-a/instanceGroups/my-ig" + }] + } + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + } +} +# tftest modules=1 resources=4 +``` + +### Health Checks + +You can leverage externally defined health checks for backend services, or have the module create them for you. By default a simple TCP health check on port 80 is created, and used in backend services. + +Health check configuration is controlled via the `health_check_config` variable, which behaves in a similar way to other LB modules in this repository. + +```hcl +module "int-tcp-proxy" { + source = "./fabric/modules/net-lb-proxy-int" + name = "int-tcp-proxy" + project_id = var.project_id + region = "europe-west1" + backend_service_config = { + backends = [{ + group = "projects/myprj/zones/europe-west1-a/instanceGroups/my-ig" + }] + } + health_check_config = { + tcp = { port = 80 } + } + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + } +} +# tftest modules=1 resources=4 +``` + +To leverage an existing health check without having the module create them, simply pass its self link: + +```hcl +module "int-tcp-proxy" { + source = "./fabric/modules/net-lb-proxy-int" + name = "int-tcp-proxy" + project_id = var.project_id + region = "europe-west1" + backend_service_config = { + backends = [{ + group = "projects/myprj/zones/europe-west1-a/instanceGroups/my-ig" + }] + } + health_check = "projects/myprj/global/healthChecks/custom" + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + } +} +# tftest modules=1 resources=4 +``` + +### Instance Groups + +The module can optionally create unmanaged instance groups, which can then be referred in backends via their key: + +```hcl +module "int-tcp-proxy" { + source = "./fabric/modules/net-lb-proxy-int" + name = "int-tcp-proxy" + project_id = var.project_id + region = "europe-west1" + backend_service_config = { + port_name = "http" + backends = [ + { group = "default" } + ] + } + group_configs = { + default = { + zone = "europe-west1-b" + instances = [ + "projects/myprj/zones/europe-west1-b/instances/vm-a" + ] + named_ports = { http = 80 } + } + } + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + } +} +# tftest modules=1 resources=5 +``` + +### Network Endpoint Groups (NEGs) + +Network Endpoint Groups (NEGs) can be used as backends, by passing their id as the backend group: + +```hcl +module "int-tcp-proxy" { + source = "./fabric/modules/net-lb-proxy-int" + name = "int-tcp-proxy" + project_id = var.project_id + region = "europe-west1" + backend_service_config = { + backends = [{ + group = "projects/myprj/zones/europe-west1-a/networkEndpointGroups/my-neg" + }] + } + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + } +} +# tftest modules=1 resources=4 +``` + +Similarly to instance groups, NEGs can also be managed by this module which supports GCE, hybrid and Private Service Connect NEGs: + +#### Zonal NEG creation + +```hcl +resource "google_compute_address" "test" { + name = "neg-test" + subnetwork = var.subnet.self_link + address_type = "INTERNAL" + address = "10.0.0.10" + region = "europe-west1" +} + +module "int-tcp-proxy" { + source = "./fabric/modules/net-lb-proxy-int" + name = "int-tcp-proxy" + project_id = var.project_id + region = "europe-west1" + backend_service_config = { + backends = [{ + group = "my-neg" + balancing_mode = "CONNECTION" + max_connections = { + per_endpoint = 10 + } + }] + } + neg_configs = { + my-neg = { + gce = { + zone = "europe-west1-b" + endpoints = { + e-0 = { + instance = "test-1" + ip_address = google_compute_address.test.address + # ip_address = "10.0.0.10" + port = 80 + } + } + } + } + } + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + } +} +# tftest modules=1 resources=7 +``` + +#### Hybrid NEG creation + +```hcl +module "int-tcp-proxy" { + source = "./fabric/modules/net-lb-proxy-int" + name = "int-tcp-proxy" + project_id = var.project_id + region = "europe-west1" + backend_service_config = { + backends = [{ + group = "my-neg" + balancing_mode = "CONNECTION" + max_connections = { + per_endpoint = 10 + } + }] + } + neg_configs = { + my-neg = { + hybrid = { + zone = "europe-west1-b" + endpoints = { + e-0 = { + ip_address = "10.0.0.10" + port = 80 + } + } + } + } + } + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + } +} +# tftest modules=1 resources=6 +``` + +#### Private Service Connect NEG creation + +```hcl +module "int-tcp-proxy" { + source = "./fabric/modules/net-lb-proxy-int" + name = "int-tcp-proxy" + project_id = var.project_id + region = "europe-west1" + backend_service_config = { + backends = [{ + group = "my-neg" + balancing_mode = "CONNECTION" + max_connections = { + per_endpoint = 10 + } + }] + } + neg_configs = { + my-neg = { + psc = { + region = "europe-west1" + target_service = "europe-west1-cloudkms.googleapis.com" + } + } + } + vpc_config = { + network = var.vpc.self_link + subnetwork = var.subnet.self_link + } +} +# tftest modules=1 resources=5 +``` + + + +## Files + +| name | description | resources | +|---|---|---| +| [backend-service.tf](./backend-service.tf) | Backend service resources. | google_compute_region_backend_service | +| [groups.tf](./groups.tf) | None | google_compute_instance_group | +| [health-check.tf](./health-check.tf) | Health check resource. | google_compute_region_health_check | +| [main.tf](./main.tf) | Module-level locals and resources. | google_compute_forwarding_rule · google_compute_network_endpoint · google_compute_network_endpoint_group · google_compute_region_network_endpoint_group · google_compute_region_target_tcp_proxy | +| [outputs.tf](./outputs.tf) | Module outputs. | | +| [variables.tf](./variables.tf) | Module variables. | | +| [versions.tf](./versions.tf) | Version pins. | | + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [name](variables.tf#L198) | Load balancer name. | string | ✓ | | +| [project_id](variables.tf#L256) | Project id. | string | ✓ | | +| [region](variables.tf#L261) | The region where to allocate the ILB resources. | string | ✓ | | +| [vpc_config](variables.tf#L266) | VPC-level configuration. | object({…}) | ✓ | | +| [address](variables.tf#L17) | Optional IP address used for the forwarding rule. | string | | null | +| [backend_service_config](variables.tf#L23) | Backend service level configuration. | object({…}) | | {} | +| [description](variables.tf#L75) | Optional description used for resources. | string | | "Terraform managed." | +| [global_access](variables.tf#L82) | Allow client access from all regions. | bool | | null | +| [group_configs](variables.tf#L88) | Optional unmanaged groups to create. Can be referenced in backends via key or outputs. | map(object({…})) | | {} | +| [health_check](variables.tf#L100) | Name of existing health check to use, disables auto-created health check. | string | | null | +| [health_check_config](variables.tf#L106) | Optional auto-created health check configurations, use the output self-link to set it in the auto healing policy. Refer to examples for usage. | object({…}) | | {…} | +| [labels](variables.tf#L192) | Labels set on resources. | map(string) | | {} | +| [neg_configs](variables.tf#L203) | Optional network endpoint groups to create. Can be referenced in backends via key or outputs. | map(object({…})) | | {} | +| [port](variables.tf#L250) | Port. | number | | 80 | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [backend_service](outputs.tf#L17) | Backend resource. | | +| [backend_service_id](outputs.tf#L22) | Backend id. | | +| [backend_service_self_link](outputs.tf#L27) | Backend self link. | | +| [forwarding_rule](outputs.tf#L32) | Forwarding rule resource. | | +| [group_self_links](outputs.tf#L37) | Optional unmanaged instance group self links. | | +| [groups](outputs.tf#L44) | Optional unmanaged instance group resources. | | +| [health_check](outputs.tf#L49) | Auto-created health-check resource. | | +| [health_check_self_id](outputs.tf#L54) | Auto-created health-check self id. | | +| [health_check_self_link](outputs.tf#L59) | Auto-created health-check self link. | | +| [id](outputs.tf#L64) | Fully qualified forwarding rule id. | | +| [neg_ids](outputs.tf#L69) | Autogenerated network endpoint group ids. | | + diff --git a/assets/modules-fabric/v26/net-lb-proxy-int/backend-service.tf b/assets/modules-fabric/v26/net-lb-proxy-int/backend-service.tf new file mode 100644 index 0000000..f5cf6e1 --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-proxy-int/backend-service.tf @@ -0,0 +1,102 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Backend service resources. + +locals { + group_ids = merge( + { + for k, v in google_compute_instance_group.default : k => v.id + }, + { + for k, v in google_compute_network_endpoint_group.default : k => v.id + }, + { + for k, v in google_compute_region_network_endpoint_group.psc : k => v.id + } + ) + hc_ids = { + for k, v in google_compute_region_health_check.default : k => v.id + } +} + +resource "google_compute_region_backend_service" "default" { + provider = google-beta + project = var.project_id + region = var.region + name = var.name + description = var.description + affinity_cookie_ttl_sec = var.backend_service_config.affinity_cookie_ttl_sec + connection_draining_timeout_sec = var.backend_service_config.connection_draining_timeout_sec + health_checks = [local.health_check] + load_balancing_scheme = "INTERNAL_MANAGED" + port_name = var.backend_service_config.port_name # defaults to http, not for NEGs + protocol = "TCP" + session_affinity = var.backend_service_config.session_affinity + timeout_sec = var.backend_service_config.timeout_sec + + dynamic "backend" { + for_each = { for b in coalesce(var.backend_service_config.backends, []) : b.group => b } + content { + group = lookup(local.group_ids, backend.key, backend.key) + balancing_mode = backend.value.balancing_mode + capacity_scaler = backend.value.capacity_scaler + description = backend.value.description + failover = backend.value.failover + max_connections = try( + backend.value.max_connections.per_group, null + ) + max_connections_per_endpoint = try( + backend.value.max_connections.per_endpoint, null + ) + max_connections_per_instance = try( + backend.value.max_connections.per_instance, null + ) + max_utilization = backend.value.max_utilization + } + } + + dynamic "connection_tracking_policy" { + for_each = var.backend_service_config.connection_tracking == null ? [] : [""] + content { + connection_persistence_on_unhealthy_backends = ( + ar.backend_service_config.connection_tracking.persist_conn_on_unhealthy != null + ? ar.backend_service_config.connection_tracking.persist_conn_on_unhealthy + : null + ) + idle_timeout_sec = var.backend_service_config.connection_tracking.idle_timeout_sec + tracking_mode = try(local.bs_conntrack.track_per_session ? "PER_SESSION" : "PER_CONNECTION", null) + } + } + + dynamic "failover_policy" { + for_each = var.backend_service_config.failover_config == null ? [] : [""] + content { + disable_connection_drain_on_failover = var.backend_service_config.failover_config.disable_conn_drain + drop_traffic_if_unhealthy = var.backend_service_config.failover_config.drop_traffic_if_unhealthy + failover_ratio = var.backend_service_config.failover_config.ratio + } + } + + dynamic "log_config" { + for_each = var.backend_service_config.log_sample_rate == null ? [] : [""] + content { + enable = true + sample_rate = var.backend_service_config.log_sample_rate + } + } + +} diff --git a/assets/modules-fabric/v26/net-lb-proxy-int/groups.tf b/assets/modules-fabric/v26/net-lb-proxy-int/groups.tf new file mode 100644 index 0000000..53ba6b2 --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-proxy-int/groups.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_compute_instance_group" "default" { + for_each = var.group_configs + project = ( + each.value.project_id == null + ? var.project_id + : each.value.project_id + ) + zone = each.value.zone + name = "${var.name}-${each.key}" + description = var.description + instances = each.value.instances + + dynamic "named_port" { + for_each = each.value.named_ports + content { + name = named_port.key + port = named_port.value + } + } +} + diff --git a/assets/modules-fabric/v26/net-lb-proxy-int/health-check.tf b/assets/modules-fabric/v26/net-lb-proxy-int/health-check.tf new file mode 100644 index 0000000..8c0ff4f --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-proxy-int/health-check.tf @@ -0,0 +1,120 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Health check resource. + +locals { + hc = var.health_check_config + hc_grpc = try(local.hc.grpc, null) != null + hc_http = try(local.hc.http, null) != null + hc_http2 = try(local.hc.http2, null) != null + hc_https = try(local.hc.https, null) != null + hc_ssl = try(local.hc.ssl, null) != null + hc_tcp = try(local.hc.tcp, null) != null +} + +resource "google_compute_region_health_check" "default" { + provider = google-beta + count = local.hc != null ? 1 : 0 + project = var.project_id + name = var.name + region = var.region + description = local.hc.description + check_interval_sec = local.hc.check_interval_sec + healthy_threshold = local.hc.healthy_threshold + timeout_sec = local.hc.timeout_sec + unhealthy_threshold = local.hc.unhealthy_threshold + + dynamic "grpc_health_check" { + for_each = local.hc_grpc ? [""] : [] + content { + port = local.hc.grpc.port + port_name = local.hc.grpc.port_name + port_specification = local.hc.grpc.port_specification + grpc_service_name = local.hc.grpc.service_name + } + } + + dynamic "http_health_check" { + for_each = local.hc_http ? [""] : [] + content { + host = local.hc.http.host + port = local.hc.http.port + port_name = local.hc.http.port_name + port_specification = local.hc.http.port_specification + proxy_header = local.hc.http.proxy_header + request_path = local.hc.http.request_path + response = local.hc.http.response + } + } + + dynamic "http2_health_check" { + for_each = local.hc_http2 ? [""] : [] + content { + host = local.hc.http2.host + port = local.hc.http2.port + port_name = local.hc.http2.port_name + port_specification = local.hc.http2.port_specification + proxy_header = local.hc.http2.proxy_header + request_path = local.hc.http2.request_path + response = local.hc.http2.response + } + } + + dynamic "https_health_check" { + for_each = local.hc_https ? [""] : [] + content { + host = local.hc.https.host + port = local.hc.https.port + port_name = local.hc.https.port_name + port_specification = local.hc.https.port_specification + proxy_header = local.hc.https.proxy_header + request_path = local.hc.https.request_path + response = local.hc.https.response + } + } + + dynamic "ssl_health_check" { + for_each = local.hc_ssl ? [""] : [] + content { + port = local.hc.ssl.port + port_name = local.hc.ssl.port_name + port_specification = local.hc.ssl.port_specification + proxy_header = local.hc.ssl.proxy_header + request = local.hc.ssl.request + response = local.hc.ssl.response + } + } + + dynamic "tcp_health_check" { + for_each = local.hc_tcp ? [""] : [] + content { + port = local.hc.tcp.port + port_name = local.hc.tcp.port_name + port_specification = local.hc.tcp.port_specification + proxy_header = local.hc.tcp.proxy_header + request = local.hc.tcp.request + response = local.hc.tcp.response + } + } + + dynamic "log_config" { + for_each = try(local.hc.enable_logging, null) == true ? [""] : [] + content { + enable = true + } + } +} diff --git a/assets/modules-fabric/v26/net-lb-proxy-int/main.tf b/assets/modules-fabric/v26/net-lb-proxy-int/main.tf new file mode 100644 index 0000000..f90d86f --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-proxy-int/main.tf @@ -0,0 +1,124 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # we need keys in the endpoint type to address issue #1055 + _neg_endpoints = flatten([ + for k, v in local.neg_zonal : [ + for kk, vv in v.endpoints : merge(vv, { + key = "${k}-${kk}", neg = k, zone = v.zone + }) + ] + ]) + neg_endpoints = { + for v in local._neg_endpoints : (v.key) => v + } + neg_zonal = { + # we need to rebuild new objects as we cannot merge different types + for k, v in var.neg_configs : k => { + endpoints = v.gce != null ? v.gce.endpoints : v.hybrid.endpoints + network = v.gce != null ? v.gce.network : v.hybrid.network + project_id = v.project_id + subnetwork = v.gce != null ? v.gce.subnetwork : null + type = v.gce != null ? "GCE_VM_IP_PORT" : "NON_GCP_PRIVATE_IP_PORT" + zone = v.gce != null ? v.gce.zone : v.hybrid.zone + } if v.gce != null || v.hybrid != null + } + neg_regional_psc = { + for k, v in var.neg_configs : + k => v if v.psc != null + } + health_check = ( + var.health_check != null + ? var.health_check + : google_compute_region_health_check.default.0.self_link + ) +} + +resource "google_compute_forwarding_rule" "default" { + provider = google-beta + project = var.project_id + region = var.region + name = var.name + description = var.description + ip_address = var.address + ip_protocol = "TCP" + load_balancing_scheme = "INTERNAL_MANAGED" + network = var.vpc_config.network + port_range = var.port + subnetwork = var.vpc_config.subnetwork + labels = var.labels + target = google_compute_region_target_tcp_proxy.default.id + # during the preview phase you cannot change this attribute on an existing rule + allow_global_access = var.global_access +} + +resource "google_compute_region_target_tcp_proxy" "default" { + project = var.project_id + name = var.name + description = var.description + region = var.region + backend_service = google_compute_region_backend_service.default.self_link +} + +resource "google_compute_network_endpoint_group" "default" { + for_each = local.neg_zonal + project = ( + each.value.project_id == null + ? var.project_id + : each.value.project_id + ) + zone = each.value.zone + name = "${var.name}-${each.key}" + # re-enable once provider properly supports this + # default_port = each.value.default_port + description = var.description + network_endpoint_type = each.value.type + network = ( + each.value.network != null ? each.value.network : var.vpc_config.network + ) + subnetwork = ( + each.value.type == "NON_GCP_PRIVATE_IP_PORT" + ? null + : try(each.value.subnetwork, var.vpc_config.subnetwork) + ) +} + +resource "google_compute_network_endpoint" "default" { + for_each = local.neg_endpoints + project = ( + google_compute_network_endpoint_group.default[each.value.neg].project + ) + network_endpoint_group = ( + google_compute_network_endpoint_group.default[each.value.neg].name + ) + instance = try(each.value.instance, null) + ip_address = each.value.ip_address + port = each.value.port + zone = each.value.zone +} + +resource "google_compute_region_network_endpoint_group" "psc" { + for_each = local.neg_regional_psc + project = var.project_id + region = each.value.psc.region + name = "${var.name}-${each.key}" + //description = coalesce(each.value.description, var.description) + network_endpoint_type = "PRIVATE_SERVICE_CONNECT" + psc_target_service = each.value.psc.target_service + network = each.value.psc.network + subnetwork = each.value.psc.subnetwork +} diff --git a/assets/modules-fabric/v26/net-lb-proxy-int/outputs.tf b/assets/modules-fabric/v26/net-lb-proxy-int/outputs.tf new file mode 100644 index 0000000..2bd35ee --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-proxy-int/outputs.tf @@ -0,0 +1,74 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "backend_service" { + description = "Backend resource." + value = google_compute_region_backend_service.default +} + +output "backend_service_id" { + description = "Backend id." + value = google_compute_region_backend_service.default.id +} + +output "backend_service_self_link" { + description = "Backend self link." + value = google_compute_region_backend_service.default.self_link +} + +output "forwarding_rule" { + description = "Forwarding rule resource." + value = google_compute_forwarding_rule.default +} + +output "group_self_links" { + description = "Optional unmanaged instance group self links." + value = { + for k, v in google_compute_instance_group.default : k => v.self_link + } +} + +output "groups" { + description = "Optional unmanaged instance group resources." + value = google_compute_instance_group.default +} + +output "health_check" { + description = "Auto-created health-check resource." + value = try(google_compute_region_health_check.default.0, null) +} + +output "health_check_self_id" { + description = "Auto-created health-check self id." + value = try(google_compute_region_health_check.default.0.id, null) +} + +output "health_check_self_link" { + description = "Auto-created health-check self link." + value = try(google_compute_region_health_check.default.0.self_link, null) +} + +output "id" { + description = "Fully qualified forwarding rule id." + value = google_compute_forwarding_rule.default.id +} + +output "neg_ids" { + description = "Autogenerated network endpoint group ids." + value = { + for k, v in google_compute_network_endpoint_group.default : k => v.id + } +} diff --git a/assets/modules-fabric/v26/net-lb-proxy-int/variables.tf b/assets/modules-fabric/v26/net-lb-proxy-int/variables.tf new file mode 100644 index 0000000..70a725a --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-proxy-int/variables.tf @@ -0,0 +1,273 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "address" { + description = "Optional IP address used for the forwarding rule." + type = string + default = null +} + +variable "backend_service_config" { + description = "Backend service level configuration." + type = object({ + affinity_cookie_ttl_sec = optional(number) + connection_draining_timeout_sec = optional(number) + health_checks = optional(list(string), ["default"]) + log_sample_rate = optional(number) + port_name = optional(string) + project_id = optional(string) + session_affinity = optional(string, "NONE") + timeout_sec = optional(number) + backends = optional(list(object({ + group = string + balancing_mode = optional(string, "UTILIZATION") + capacity_scaler = optional(number, 1) + description = optional(string, "Terraform managed.") + failover = optional(bool, false) + max_connections = optional(object({ + per_endpoint = optional(number) + per_group = optional(number) + per_instance = optional(number) + })) + max_utilization = optional(number) + }))) + connection_tracking = optional(object({ + idle_timeout_sec = optional(number) + persist_conn_on_unhealthy = optional(string) + track_per_session = optional(bool) + })) + failover_config = optional(object({ + disable_conn_drain = optional(bool) + drop_traffic_if_unhealthy = optional(bool) + ratio = optional(number) + })) + }) + default = {} + nullable = false + validation { + condition = (var.backend_service_config == null || contains(["NONE", "CLIENT_IP"], + var.backend_service_config.session_affinity + )) + error_message = "Invalid session affinity value." + } + validation { + condition = alltrue([ + for b in var.backend_service_config.backends : contains( + ["CONNECTION", "UTILIZATION"], coalesce(b.balancing_mode, "CONNECTION") + )]) + error_message = "When specified balancing mode needs to be 'CONNECTION' or 'UTILIZATION'." + } +} + +variable "description" { + description = "Optional description used for resources." + type = string + default = "Terraform managed." +} + +# during the preview phase you cannot change this attribute on an existing rule +variable "global_access" { + description = "Allow client access from all regions." + type = bool + default = null +} + +variable "group_configs" { + description = "Optional unmanaged groups to create. Can be referenced in backends via key or outputs." + type = map(object({ + zone = string + instances = optional(list(string)) + named_ports = optional(map(number), {}) + project_id = optional(string) + })) + default = {} + nullable = false +} + +variable "health_check" { + description = "Name of existing health check to use, disables auto-created health check." + type = string + default = null +} + +variable "health_check_config" { + description = "Optional auto-created health check configurations, use the output self-link to set it in the auto healing policy. Refer to examples for usage." + type = object({ + check_interval_sec = optional(number) + description = optional(string, "Terraform managed.") + enable_logging = optional(bool, false) + healthy_threshold = optional(number) + project_id = optional(string) + timeout_sec = optional(number) + unhealthy_threshold = optional(number) + grpc = optional(object({ + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + service_name = optional(string) + })) + http = optional(object({ + host = optional(string) + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request_path = optional(string) + response = optional(string) + })) + http2 = optional(object({ + host = optional(string) + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request_path = optional(string) + response = optional(string) + })) + https = optional(object({ + host = optional(string) + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request_path = optional(string) + response = optional(string) + })) + tcp = optional(object({ + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request = optional(string) + response = optional(string) + })) + ssl = optional(object({ + port = optional(number) + port_name = optional(string) + port_specification = optional(string) # USE_FIXED_PORT USE_NAMED_PORT USE_SERVING_PORT + proxy_header = optional(string) + request = optional(string) + response = optional(string) + })) + }) + default = { + tcp = { + port_specification = "USE_SERVING_PORT" + } + } + validation { + condition = ( + (try(var.health_check_config.grpc, null) == null ? 0 : 1) + + (try(var.health_check_config.http, null) == null ? 0 : 1) + + (try(var.health_check_config.http2, null) == null ? 0 : 1) + + (try(var.health_check_config.https, null) == null ? 0 : 1) + + (try(var.health_check_config.tcp, null) == null ? 0 : 1) + + (try(var.health_check_config.ssl, null) == null ? 0 : 1) <= 1 + ) + error_message = "Only one health check type can be configured at a time." + } + validation { + condition = alltrue([ + for k, v in var.health_check_config : contains([ + "-", "USE_FIXED_PORT", "USE_NAMED_PORT", "USE_SERVING_PORT" + ], coalesce(try(v.port_specification, null), "-")) + ]) + error_message = "Invalid 'port_specification' value. Supported values are 'USE_FIXED_PORT', 'USE_NAMED_PORT', 'USE_SERVING_PORT'." + } +} + +variable "labels" { + description = "Labels set on resources." + type = map(string) + default = {} +} + +variable "name" { + description = "Load balancer name." + type = string +} + +variable "neg_configs" { + description = "Optional network endpoint groups to create. Can be referenced in backends via key or outputs." + type = map(object({ + project_id = optional(string) + gce = optional(object({ + zone = string + # default_port = optional(number) + network = optional(string) + subnetwork = optional(string) + endpoints = optional(map(object({ + instance = string + ip_address = string + port = number + }))) + + })) + hybrid = optional(object({ + zone = string + network = optional(string) + # re-enable once provider properly support this + # default_port = optional(number) + endpoints = optional(map(object({ + ip_address = string + port = number + }))) + })) + psc = optional(object({ + region = string + target_service = string + network = optional(string) + subnetwork = optional(string) + })) + })) + default = {} + nullable = false + validation { + condition = alltrue([ + for k, v in var.neg_configs : ( + (try(v.gce, null) == null ? 0 : 1) + + (try(v.hybrid, null) == null ? 0 : 1) + + (try(v.psc, null) == null ? 0 : 1) == 1 + ) + ]) + error_message = "Only one type of neg can be configured at a time." + } +} + +variable "port" { + description = "Port." + type = number + default = 80 +} + +variable "project_id" { + description = "Project id." + type = string +} + +variable "region" { + description = "The region where to allocate the ILB resources." + type = string +} + +variable "vpc_config" { + description = "VPC-level configuration." + type = object({ + network = string + subnetwork = string + }) + nullable = false +} diff --git a/assets/modules-fabric/v26/net-lb-proxy-int/versions.tf b/assets/modules-fabric/v26/net-lb-proxy-int/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/net-lb-proxy-int/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/net-swp/README.md b/assets/modules-fabric/v26/net-swp/README.md new file mode 100644 index 0000000..03fe246 --- /dev/null +++ b/assets/modules-fabric/v26/net-swp/README.md @@ -0,0 +1,195 @@ +# Google Cloud Secure Web Proxy + +This module allows creation and management of [Secure Web Proxy](https://cloud.google.com/secure-web-proxy/docs/overview) alongside with its security +policies: + +- Secure tag based rules via the `policy_rules.secure_tags` variable +- Url list rules via the `policy_rules.url_lists` variable +- Custom rules via the `policy_rules.custom` + +## Examples + +### Minimal Secure Web Proxy + +(Note that this will not allow any request to pass.) + +```hcl +module "secure-web-proxy" { + source = "./fabric/modules/net-swp" + + project_id = "my-project" + region = "europe-west4" + name = "secure-web-proxy" + network = "projects/my-project/global/networks/my-network" + subnetwork = "projects/my-project/regions/europe-west4/subnetworks/my-subnetwork" + addresses = ["10.142.68.3"] + certificates = ["projects/my-project/locations/europe-west4/certificates/secure-web-proxy-cert"] + labels = { + example = "value" + } +} +# tftest modules=1 resources=2 inventory=basic.yaml +``` + +### Secure Web Proxy with rules + +```hcl +module "secure-web-proxy" { + source = "./fabric/modules/net-swp" + + project_id = "my-project" + region = "europe-west4" + name = "secure-web-proxy" + network = "projects/my-project/global/networks/my-network" + subnetwork = "projects/my-project/regions/europe-west4/subnetworks/my-subnetwork" + addresses = ["10.142.68.3"] + certificates = ["projects/my-project/locations/europe-west4/certificates/secure-web-proxy-cert"] + ports = [80, 443] + policy_rules = { + secure_tags = { + secure-tag-1 = { + tag = "tagValues/281484836404786" + priority = 1000 + } + secure-tag-2 = { + tag = "tagValues/281484836404786" + session_matcher = "host() != 'google.com'" + priority = 1001 + } + } + url_lists = { + url-list-1 = { + url_list = "my-url-list" + values = ["www.google.com", "google.com"] + priority = 1002 + } + url-list-2 = { + url_list = "projects/my-project/locations/europe-west4/urlLists/my-url-list" + session_matcher = "source.matchServiceAccount('my-sa@my-project.iam.gserviceaccount.com')" + enabled = false + priority = 1003 + } + } + custom = { + custom-rule-1 = { + priority = 1004 + session_matcher = "host() == 'google.com'" + action = "DENY" + } + } + } +} +# tftest modules=1 resources=8 inventory=rules.yaml +``` + +### Secure Web Proxy with TLS inspection + +```hcl +resource "google_privateca_ca_pool" "pool" { + name = "secure-web-proxy-capool" + location = "europe-west4" + project = "my-project" + + tier = "DEVOPS" +} + +resource "google_privateca_certificate_authority" "ca" { + pool = google_privateca_ca_pool.pool.name + certificate_authority_id = "secure-web-proxy-ca" + location = "europe-west4" + project = "my-project" + + deletion_protection = "false" + + config { + subject_config { + subject { + organization = "Cloud Foundation Fabric" + common_name = "fabric" + } + } + x509_config { + ca_options { + is_ca = true + } + key_usage { + base_key_usage { + cert_sign = true + crl_sign = true + } + extended_key_usage { + server_auth = true + } + } + } + } + lifetime = "1209600s" + key_spec { + algorithm = "EC_P256_SHA256" + } +} + +resource "google_privateca_ca_pool_iam_member" "member" { + ca_pool = google_privateca_ca_pool.pool.id + role = "roles/privateca.certificateManager" + member = "serviceAccount:service-123456789@gcp-sa-networksecurity.iam.gserviceaccount.com" +} + +module "secure-web-proxy" { + source = "./fabric/modules/net-swp" + + project_id = "my-project" + region = "europe-west4" + name = "secure-web-proxy" + network = "projects/my-project/global/networks/my-network" + subnetwork = "projects/my-project/regions/europe-west4/subnetworks/my-subnetwork" + addresses = ["10.142.68.3"] + certificates = ["projects/my-project/locations/europe-west4/certificates/secure-web-proxy-cert"] + ports = [443] + policy_rules = { + custom = { + custom-rule-1 = { + priority = 1000 + session_matcher = "host() == 'google.com'" + application_matcher = "request.path.contains('generate_204')" + action = "ALLOW" + tls_inspection_enabled = true + } + } + } + tls_inspection_config = { + ca_pool = google_privateca_ca_pool.pool.id + } +} +# tftest modules=1 resources=7 inventory=tls.yaml +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [addresses](variables.tf#L19) | One or more IP addresses to be used for Secure Web Proxy. | | ✓ | | +| [certificates](variables.tf#L27) | List of certificates to be used for Secure Web Proxy. | list(string) | ✓ | | +| [name](variables.tf#L50) | Name of the Secure Web Proxy resource. | string | ✓ | | +| [network](variables.tf#L55) | Name of the network the Secure Web Proxy is deployed into. | string | ✓ | | +| [project_id](variables.tf#L119) | Project id of the project that holds the network. | string | ✓ | | +| [region](variables.tf#L124) | Region where resources will be created. | string | ✓ | | +| [subnetwork](variables.tf#L135) | Name of the subnetwork the Secure Web Proxy is deployed into. | string | ✓ | | +| [delete_swg_autogen_router_on_destroy](variables.tf#L32) | Delete automatically provisioned Cloud Router on destroy. | bool | | true | +| [description](variables.tf#L38) | Optional description for the created resources. | string | | "Managed by Terraform." | +| [labels](variables.tf#L44) | Resource labels. | map(string) | | {} | +| [policy_rules](variables.tf#L60) | List of policy rule definitions, default to allow action. Available keys: secure_tags, url_lists, custom. URL lists that only have values set will be created. | object({…}) | | {} | +| [ports](variables.tf#L113) | Ports to use for Secure Web Proxy. | list(number) | | [443] | +| [scope](variables.tf#L129) | Scope determines how configuration across multiple Gateway instances are merged. | string | | null | +| [tls_inspection_config](variables.tf#L140) | TLS inspection configuration. | object({…}) | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [gateway](outputs.tf#L17) | The gateway resource. | | +| [gateway_security_policy](outputs.tf#L22) | The gateway security policy resource. | | +| [id](outputs.tf#L27) | ID of the gateway resource. | | + + diff --git a/assets/modules-fabric/v26/net-swp/main.tf b/assets/modules-fabric/v26/net-swp/main.tf new file mode 100644 index 0000000..bf19517 --- /dev/null +++ b/assets/modules-fabric/v26/net-swp/main.tf @@ -0,0 +1,126 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + create_url_lists = { for k, v in var.policy_rules.url_lists : v.url_list => v if v.values != null } +} + +resource "google_network_security_gateway_security_policy" "policy" { + provider = google-beta + project = var.project_id + name = var.name + location = var.region + description = var.description + tls_inspection_policy = var.tls_inspection_config != null ? google_network_security_tls_inspection_policy.tls-policy.0.id : null +} + +resource "google_network_security_tls_inspection_policy" "tls-policy" { + count = var.tls_inspection_config != null ? 1 : 0 + provider = google-beta + project = var.project_id + name = var.name + location = var.region + description = coalesce(var.tls_inspection_config.description, var.description) + ca_pool = var.tls_inspection_config.ca_pool + exclude_public_ca_set = var.tls_inspection_config.exclude_public_ca_set +} + +resource "google_network_security_gateway_security_policy_rule" "secure_tag_rules" { + for_each = var.policy_rules.secure_tags + provider = google-beta + project = var.project_id + name = each.key + location = var.region + description = coalesce(each.value.description, var.description) + gateway_security_policy = google_network_security_gateway_security_policy.policy.name + enabled = each.value.enabled + priority = each.value.priority + session_matcher = trimspace(<<-EOT + source.matchTag('${each.value.tag}')%{if each.value.session_matcher != null} && (${each.value.session_matcher})%{endif~} + EOT + ) + application_matcher = each.value.application_matcher + tls_inspection_enabled = each.value.tls_inspection_enabled + basic_profile = each.value.action +} + +resource "google_network_security_url_lists" "url_lists" { + for_each = local.create_url_lists + provider = google-beta + project = var.project_id + name = each.key + location = var.region + description = coalesce(each.value.description, var.description) + values = each.value.values +} + +resource "google_network_security_gateway_security_policy_rule" "url_list_rules" { + for_each = var.policy_rules.url_lists + provider = google-beta + project = var.project_id + name = each.key + location = var.region + description = coalesce(each.value.description, var.description) + gateway_security_policy = google_network_security_gateway_security_policy.policy.name + enabled = each.value.enabled + priority = each.value.priority + session_matcher = trimspace(<<-EOT + inUrlList(host(), '%{~if each.value.values != null~} + ${~google_network_security_url_lists.url_lists[each.value.url_list].id~} + %{~else~} + ${~each.value.url_list~} + %{~endif~}') %{~if each.value.session_matcher != null} && (${each.value.session_matcher})%{~endif~} + EOT + ) + application_matcher = each.value.application_matcher + tls_inspection_enabled = each.value.tls_inspection_enabled + basic_profile = each.value.action +} + +resource "google_network_security_gateway_security_policy_rule" "custom_rules" { + for_each = var.policy_rules.custom + project = var.project_id + provider = google-beta + name = each.key + location = var.region + description = coalesce(each.value.description, var.description) + gateway_security_policy = google_network_security_gateway_security_policy.policy.name + enabled = each.value.enabled + priority = each.value.priority + session_matcher = each.value.session_matcher + application_matcher = each.value.application_matcher + tls_inspection_enabled = each.value.tls_inspection_enabled + basic_profile = each.value.action +} + +resource "google_network_services_gateway" "gateway" { + provider = google-beta + project = var.project_id + name = var.name + location = var.region + description = var.description + labels = var.labels + addresses = var.addresses != null ? var.addresses : [] + type = "SECURE_WEB_GATEWAY" + ports = var.ports + scope = var.scope != null ? var.scope : "" + certificate_urls = var.certificates + gateway_security_policy = google_network_security_gateway_security_policy.policy.id + network = var.network + subnetwork = var.subnetwork + delete_swg_autogen_router_on_destroy = var.delete_swg_autogen_router_on_destroy +} + \ No newline at end of file diff --git a/assets/modules-fabric/v26/net-swp/outputs.tf b/assets/modules-fabric/v26/net-swp/outputs.tf new file mode 100644 index 0000000..aeb1371 --- /dev/null +++ b/assets/modules-fabric/v26/net-swp/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "gateway" { + description = "The gateway resource." + value = google_network_services_gateway.gateway +} + +output "gateway_security_policy" { + description = "The gateway security policy resource." + value = google_network_services_gateway.gateway.gateway_security_policy +} + +output "id" { + description = "ID of the gateway resource." + value = google_network_services_gateway.gateway.id +} diff --git a/assets/modules-fabric/v26/net-swp/variables.tf b/assets/modules-fabric/v26/net-swp/variables.tf new file mode 100644 index 0000000..17d9061 --- /dev/null +++ b/assets/modules-fabric/v26/net-swp/variables.tf @@ -0,0 +1,148 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +variable "addresses" { + description = "One or more IP addresses to be used for Secure Web Proxy." + validation { + condition = length(var.addresses) > 0 + error_message = "Must specify at least one IP address." + } +} + +variable "certificates" { + description = "List of certificates to be used for Secure Web Proxy." + type = list(string) +} + +variable "delete_swg_autogen_router_on_destroy" { + description = "Delete automatically provisioned Cloud Router on destroy." + type = bool + default = true +} + +variable "description" { + description = "Optional description for the created resources." + type = string + default = "Managed by Terraform." +} + +variable "labels" { + description = "Resource labels." + type = map(string) + default = {} +} + +variable "name" { + description = "Name of the Secure Web Proxy resource." + type = string +} + +variable "network" { + description = "Name of the network the Secure Web Proxy is deployed into." + type = string +} + +variable "policy_rules" { + description = "List of policy rule definitions, default to allow action. Available keys: secure_tags, url_lists, custom. URL lists that only have values set will be created." + type = object({ + secure_tags = optional(map(object({ + tag = string + session_matcher = optional(string) + application_matcher = optional(string) + priority = number + action = optional(string, "ALLOW") + enabled = optional(bool, true) + tls_inspection_enabled = optional(bool, false) + description = optional(string) + })), {}) + + url_lists = optional(map(object({ + url_list = string + values = optional(list(string)) + session_matcher = optional(string) + application_matcher = optional(string) + priority = number + action = optional(string, "ALLOW") + enabled = optional(bool, true) + tls_inspection_enabled = optional(bool, false) + description = optional(string) + })), {}) + + custom = optional(map(object({ + session_matcher = optional(string) + application_matcher = optional(string) + priority = number + action = optional(string, "ALLOW") + enabled = optional(bool, true) + tls_inspection_enabled = optional(bool, false) + description = optional(string) + })), {}) + }) + validation { + condition = ( + length(concat( + [for k, v in var.policy_rules.secure_tags : v.priority], + [for k, v in var.policy_rules.url_lists : v.priority], + [for k, v in var.policy_rules.custom : v.priority])) == + length(distinct(concat( + [for k, v in var.policy_rules.secure_tags : v.priority], + [for k, v in var.policy_rules.url_lists : v.priority], + [for k, v in var.policy_rules.custom : v.priority]))) + ) + error_message = "Each rule must have unique priority." + } + default = {} + nullable = false +} + +variable "ports" { + description = "Ports to use for Secure Web Proxy." + type = list(number) + default = [443] +} + +variable "project_id" { + description = "Project id of the project that holds the network." + type = string +} + +variable "region" { + description = "Region where resources will be created." + type = string +} + +variable "scope" { + description = "Scope determines how configuration across multiple Gateway instances are merged." + type = string + default = null +} + +variable "subnetwork" { + description = "Name of the subnetwork the Secure Web Proxy is deployed into." + type = string +} + +variable "tls_inspection_config" { + description = "TLS inspection configuration." + type = object({ + ca_pool = optional(string, null) + exclude_public_ca_set = optional(bool, false) + description = optional(string) + }) + default = null +} diff --git a/assets/modules-fabric/v26/net-swp/versions.tf b/assets/modules-fabric/v26/net-swp/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/net-swp/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/net-vlan-attachment/README.md b/assets/modules-fabric/v26/net-vlan-attachment/README.md new file mode 100644 index 0000000..a171170 --- /dev/null +++ b/assets/modules-fabric/v26/net-vlan-attachment/README.md @@ -0,0 +1,571 @@ +# VLAN Attachment module + +This module allows for the provisioning of VLAN Attachments for [Dedicated Interconnect](https://cloud.google.com/network-connectivity/docs/interconnect/how-to/dedicated/creating-vlan-attachments) or [Partner Interconnect](https://cloud.google.com/network-connectivity/docs/interconnect/how-to/partner/creating-vlan-attachments). + +## Examples + +### Dedicated Interconnect - Single VLAN Attachment (No SLA) + +```hcl +resource "google_compute_router" "interconnect-router" { + name = "interconnect-router" + network = "mynet" + project = "myproject" + region = "europe-west8" + bgp { + advertise_mode = "CUSTOM" + asn = 64514 + advertised_groups = ["ALL_SUBNETS"] + advertised_ip_ranges { + range = "10.255.255.0/24" + } + advertised_ip_ranges { + range = "192.168.255.0/24" + } + } +} + +module "example-va" { + source = "./fabric/modules/net-vlan-attachment" + network = "mynet" + project_id = "myproject" + region = "europe-west8" + name = "vlan-attachment" + description = "Example vlan attachment" + peer_asn = "65000" + router_config = { + create = false + name = google_compute_router.interconnect-router.name + } + dedicated_interconnect_config = { + bandwidth = "BPS_10G" + bgp_range = "169.254.0.0/30" + interconnect = "interconnect-a" + vlan_tag = 12345 + } +} +# tftest modules=1 resources=4 +``` + +### Partner Interconnect - Single VLAN Attachment (No SLA) + +```hcl +resource "google_compute_router" "interconnect-router" { + name = "interconnect-router" + network = "mynet" + project = "myproject" + region = "europe-west8" + bgp { + advertise_mode = "CUSTOM" + asn = 16550 + advertised_groups = ["ALL_SUBNETS"] + advertised_ip_ranges { + range = "10.255.255.0/24" + } + advertised_ip_ranges { + range = "192.168.255.0/24" + } + } +} + +module "example-va" { + source = "./fabric/modules/net-vlan-attachment" + network = "mynet" + project_id = "myproject" + region = "europe-west8" + name = "vlan-attachment" + description = "Example vlan attachment" + peer_asn = "65000" + router_config = { + create = false + name = google_compute_router.interconnect-router.name + } +} +# tftest modules=1 resources=2 +``` + +### Dedicated Interconnect - Two VLAN Attachments on a single region (99.9% SLA) + +```hcl +resource "google_compute_router" "interconnect-router" { + name = "interconnect-router" + network = "mynet" + project = "myproject" + region = "europe-west8" + bgp { + asn = 64514 + advertise_mode = "CUSTOM" + advertised_groups = ["ALL_SUBNETS"] + advertised_ip_ranges { + range = "10.255.255.0/24" + } + advertised_ip_ranges { + range = "192.168.255.0/24" + } + } +} + +module "example-va-a" { + source = "./fabric/modules/net-vlan-attachment" + network = "mynet" + project_id = "myproject" + region = "europe-west8" + name = "vlan-attachment-a" + description = "interconnect-a vlan attachment 0" + peer_asn = "65000" + router_config = { + create = false + name = google_compute_router.interconnect-router.name + } + dedicated_interconnect_config = { + bandwidth = "BPS_10G" + bgp_range = "169.254.0.0/30" + interconnect = "interconnect-a" + vlan_tag = 1001 + } +} + +module "example-va-b" { + source = "./fabric/modules/net-vlan-attachment" + network = "mynet" + project_id = "myproject" + region = "europe-west8" + name = "vlan-attachment-b" + description = "interconnect-b vlan attachment 0" + peer_asn = "65000" + router_config = { + create = false + name = google_compute_router.interconnect-router.name + } + dedicated_interconnect_config = { + bandwidth = "BPS_10G" + bgp_range = "169.254.0.4/30" + interconnect = "interconnect-b" + vlan_tag = 1002 + } +} +# tftest modules=2 resources=7 +``` + +### Partner Interconnect - Two VLAN Attachments on a single region (99.9% SLA) + +```hcl +resource "google_compute_router" "interconnect-router" { + name = "interconnect-router" + network = "mynet" + project = "myproject" + region = "europe-west8" + bgp { + asn = 16550 + advertise_mode = "CUSTOM" + advertised_groups = ["ALL_SUBNETS"] + advertised_ip_ranges { + range = "10.255.255.0/24" + } + advertised_ip_ranges { + range = "192.168.255.0/24" + } + } +} + +module "example-va-a" { + source = "./fabric/modules/net-vlan-attachment" + network = "mynet" + project_id = "myproject" + region = "europe-west8" + name = "vlan-attachment-a" + description = "interconnect-a vlan attachment 0" + peer_asn = "65000" + router_config = { + create = false + name = google_compute_router.interconnect-router.name + } + partner_interconnect_config = { + edge_availability_domain = "AVAILABILITY_DOMAIN_1" + } +} + +module "example-va-b" { + source = "./fabric/modules/net-vlan-attachment" + network = "mynet" + project_id = "myproject" + region = "europe-west8" + name = "vlan-attachment-b" + description = "interconnect-b vlan attachment 0" + peer_asn = "65000" + router_config = { + create = false + name = google_compute_router.interconnect-router.name + } + partner_interconnect_config = { + edge_availability_domain = "AVAILABILITY_DOMAIN_2" + } +} +# tftest modules=2 resources=3 +``` + +### Dedicated Interconnect - Four VLAN Attachments on two regions (99.99% SLA) + +```hcl +resource "google_compute_router" "interconnect-router-ew8" { + name = "interconnect-router-ew8" + network = "mynet" + project = "myproject" + region = "europe-west8" + bgp { + asn = 64514 + advertise_mode = "CUSTOM" + advertised_groups = ["ALL_SUBNETS"] + advertised_ip_ranges { + range = "10.255.255.0/24" + } + advertised_ip_ranges { + range = "192.168.255.0/24" + } + } +} + +resource "google_compute_router" "interconnect-router-ew12" { + name = "interconnect-router-ew12" + network = "mynet" + project = "myproject" + region = "europe-west12" + bgp { + asn = 64514 + advertise_mode = "CUSTOM" + advertised_groups = ["ALL_SUBNETS"] + advertised_ip_ranges { + range = "10.255.255.0/24" + } + advertised_ip_ranges { + range = "192.168.255.0/24" + } + } +} + +module "example-va-a-ew8" { + source = "./fabric/modules/net-vlan-attachment" + network = "mynet" + project_id = "myproject" + region = "europe-west8" + name = "vlan-attachment-a-ew8" + description = "interconnect-a-ew8 vlan attachment 0" + peer_asn = "65000" + router_config = { + create = false + name = google_compute_router.interconnect-router-ew8.name + } + dedicated_interconnect_config = { + bandwidth = "BPS_10G" + bgp_range = "169.254.0.0/30" + interconnect = "interconnect-a-ew8" + vlan_tag = 1001 + } +} + +module "example-va-b-ew8" { + source = "./fabric/modules/net-vlan-attachment" + network = "mynet" + project_id = "myproject" + region = "europe-west8" + name = "vlan-attachment-b-ew8" + description = "interconnect-b-ew8 vlan attachment 0" + peer_asn = "65000" + router_config = { + create = false + name = google_compute_router.interconnect-router-ew8.name + } + dedicated_interconnect_config = { + bandwidth = "BPS_10G" + bgp_range = "169.254.0.4/30" + interconnect = "interconnect-b-ew8" + vlan_tag = 1002 + } +} + +module "example-va-a-ew12" { + source = "./fabric/modules/net-vlan-attachment" + network = "mynet" + project_id = "myproject" + region = "europe-west12" + name = "vlan-attachment-a-ew12" + description = "interconnect-a-ew12 vlan attachment 0" + peer_asn = "65000" + router_config = { + create = false + name = google_compute_router.interconnect-router-ew12.name + } + dedicated_interconnect_config = { + bandwidth = "BPS_10G" + bgp_range = "169.254.1.0/30" + interconnect = "interconnect-a-ew12" + vlan_tag = 1003 + } +} + +module "example-va-b-ew12" { + source = "./fabric/modules/net-vlan-attachment" + network = "mynet" + project_id = "myproject" + region = "europe-west12" + name = "vlan-attachment-b-ew12" + description = "interconnect-b-ew12 vlan attachment 0" + peer_asn = "65000" + router_config = { + create = false + name = google_compute_router.interconnect-router-ew12.name + } + dedicated_interconnect_config = { + bandwidth = "BPS_10G" + bgp_range = "169.254.1.4/30" + interconnect = "interconnect-b-ew12" + vlan_tag = 1004 + } +} +# tftest modules=4 resources=14 +``` + +### Partner Interconnect - Four VLAN Attachments on two regions (99.99% SLA) + +```hcl +resource "google_compute_router" "interconnect-router-ew8" { + name = "interconnect-router-ew8" + network = "mynet" + project = "myproject" + region = "europe-west8" + bgp { + asn = 16550 + advertise_mode = "CUSTOM" + advertised_groups = ["ALL_SUBNETS"] + advertised_ip_ranges { + range = "10.255.255.0/24" + } + advertised_ip_ranges { + range = "192.168.255.0/24" + } + } +} + +resource "google_compute_router" "interconnect-router-ew12" { + name = "interconnect-router-ew12" + network = "mynet" + project = "myproject" + region = "europe-west12" + bgp { + asn = 16550 + advertise_mode = "CUSTOM" + advertised_groups = ["ALL_SUBNETS"] + advertised_ip_ranges { + range = "10.255.255.0/24" + } + advertised_ip_ranges { + range = "192.168.255.0/24" + } + } +} + +module "example-va-a-ew8" { + source = "./fabric/modules/net-vlan-attachment" + network = "mynet" + project_id = "myproject" + region = "europe-west8" + name = "vlan-attachment-a-ew8" + description = "interconnect-a-ew8 vlan attachment 0" + peer_asn = "65000" + router_config = { + create = false + name = google_compute_router.interconnect-router-ew8.name + } + partner_interconnect_config = { + edge_availability_domain = "AVAILABILITY_DOMAIN_1" + } +} + +module "example-va-b-ew8" { + source = "./fabric/modules/net-vlan-attachment" + network = "mynet" + project_id = "myproject" + region = "europe-west8" + name = "vlan-attachment-b-ew8" + description = "interconnect-b-ew8 vlan attachment 0" + peer_asn = "65000" + router_config = { + create = false + name = google_compute_router.interconnect-router-ew8.name + } + partner_interconnect_config = { + edge_availability_domain = "AVAILABILITY_DOMAIN_2" + } +} + +module "example-va-a-ew12" { + source = "./fabric/modules/net-vlan-attachment" + network = "mynet" + project_id = "myproject" + region = "europe-west12" + name = "vlan-attachment-a-ew12" + description = "interconnect-a-ew12 vlan attachment 0" + peer_asn = "65000" + router_config = { + create = false + name = google_compute_router.interconnect-router-ew12.name + } + partner_interconnect_config = { + edge_availability_domain = "AVAILABILITY_DOMAIN_1" + } +} + +module "example-va-b-ew12" { + source = "./fabric/modules/net-vlan-attachment" + network = "mynet" + project_id = "myproject" + region = "europe-west12" + name = "vlan-attachment-b-ew12" + description = "interconnect-b-ew12 vlan attachment 0" + peer_asn = "65000" + router_config = { + create = false + name = google_compute_router.interconnect-router-ew12.name + } + partner_interconnect_config = { + edge_availability_domain = "AVAILABILITY_DOMAIN_2" + } +} +# tftest modules=4 resources=6 +``` + +### IPSec for Dedicated Interconnect + +Refer to the [HA VPN over Interconnect Blueprint](../../blueprints/networking/ha-vpn-over-interconnect/) for an all-encompassing example. + +```hcl +resource "google_compute_router" "encrypted-interconnect-underlay-router-ew8" { + name = "encrypted-interconnect-underlay-router-ew8" + project = "myproject" + network = "mynet" + region = "europe-west8" + encrypted_interconnect_router = true + bgp { + advertise_mode = "DEFAULT" + asn = 64514 + } +} + +module "example-va-a" { + source = "./fabric/modules/net-vlan-attachment" + project_id = "myproject" + network = "mynet" + region = "europe-west8" + name = "encrypted-vlan-attachment-a" + description = "example-va-a vlan attachment" + peer_asn = "65001" + router_config = { + create = false + name = google_compute_router.encrypted-interconnect-underlay-router-ew8.name + } + dedicated_interconnect_config = { + bandwidth = "BPS_10G" + bgp_range = "169.254.0.0/30" + interconnect = "interconnect-a" + vlan_tag = 1001 + } + vpn_gateways_ip_range = "10.255.255.0/29" # Allows for up to 8 tunnels +} + +module "example-va-b" { + source = "./fabric/modules/net-vlan-attachment" + project_id = "myproject" + network = "mynet" + region = "europe-west8" + name = "encrypted-vlan-attachment-b" + description = "example-va-b vlan attachment" + peer_asn = "65001" + router_config = { + create = false + name = google_compute_router.encrypted-interconnect-underlay-router-ew8.name + } + dedicated_interconnect_config = { + bandwidth = "BPS_10G" + bgp_range = "169.254.0.4/30" + interconnect = "interconnect-b" + vlan_tag = 1002 + } + vpn_gateways_ip_range = "10.255.255.8/29" # Allows for up to 8 tunnels +} +# tftest modules=2 resources=9 +``` + +### IPSec for Partner Interconnect + +```hcl +module "example-va-a" { + source = "./fabric/modules/net-vlan-attachment" + project_id = "myproject" + network = "mynet" + region = "europe-west8" + name = "encrypted-vlan-attachment-a" + description = "example-va-a vlan attachment" + peer_asn = "65001" + router_config = { + create = true + } + partner_interconnect_config = { + edge_availability_domain = "AVAILABILITY_DOMAIN_1" + } + vpn_gateways_ip_range = "10.255.255.0/29" # Allows for up to 8 tunnels +} + +module "example-va-b" { + source = "./fabric/modules/net-vlan-attachment" + project_id = "myproject" + network = "mynet" + region = "europe-west8" + name = "encrypted-vlan-attachment-b" + description = "example-va-b vlan attachment" + peer_asn = "65001" + router_config = { + create = true + } + partner_interconnect_config = { + edge_availability_domain = "AVAILABILITY_DOMAIN_2" + } + vpn_gateways_ip_range = "10.255.255.8/29" # Allows for up to 8 tunnels +} +# tftest modules=2 resources=6 +``` + + + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [description](variables.tf#L35) | VLAN attachment description. | string | ✓ | | +| [name](variables.tf#L52) | The common resources name, used after resource type prefix and suffix. | string | ✓ | | +| [network](variables.tf#L57) | The VPC name to which resources are associated to. | string | ✓ | | +| [peer_asn](variables.tf#L74) | The on-premises underlay router ASN. | string | ✓ | | +| [project_id](variables.tf#L79) | The project id where resources are created. | string | ✓ | | +| [region](variables.tf#L84) | The region where resources are created. | string | ✓ | | +| [router_config](variables.tf#L89) | Cloud Router configuration for the VPN. If you want to reuse an existing router, set create to false and use name to specify the desired router. | object({…}) | ✓ | | +| [admin_enabled](variables.tf#L17) | Whether the VLAN attachment is enabled. | bool | | true | +| [dedicated_interconnect_config](variables.tf#L23) | Partner interconnect configuration. | object({…}) | | null | +| [ipsec_gateway_ip_ranges](variables.tf#L40) | IPSec Gateway IP Ranges. | map(string) | | {} | +| [mtu](variables.tf#L46) | The MTU associated to the VLAN attachment (1440 / 1500). | number | | 1500 | +| [partner_interconnect_config](variables.tf#L62) | Partner interconnect configuration. | object({…}) | | null | +| [vlan_tag](variables.tf#L110) | The VLAN id to be used for this VLAN attachment. | number | | null | +| [vpn_gateways_ip_range](variables.tf#L116) | The IP range (cidr notation) to be used for the GCP VPN gateways. If null IPSec over Interconnect is not enabled. | string | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [attachment](outputs.tf#L17) | VLAN Attachment resource. | | +| [id](outputs.tf#L22) | Fully qualified VLAN attachment id. | | +| [name](outputs.tf#L27) | The name of the VLAN attachment created. | | +| [pairing_key](outputs.tf#L32) | Opaque identifier of an PARTNER attachment used to initiate provisioning with a selected partner. | | +| [router](outputs.tf#L37) | Router resource (only if auto-created). | | +| [router_interface](outputs.tf#L42) | Router interface created for the VLAN attachment. | | +| [router_name](outputs.tf#L47) | Router name. | | + + diff --git a/assets/modules-fabric/v26/net-vlan-attachment/main.tf b/assets/modules-fabric/v26/net-vlan-attachment/main.tf new file mode 100644 index 0000000..c5e3334 --- /dev/null +++ b/assets/modules-fabric/v26/net-vlan-attachment/main.tf @@ -0,0 +1,149 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + ipsec_enabled = var.vpn_gateways_ip_range == null ? false : true + router = ( + var.router_config.create + ? local.ipsec_enabled ? try(google_compute_router.encrypted[0].name, null) : try(google_compute_router.unencrypted[0].name, null) + : var.router_config.name + ) +} + +resource "google_compute_address" "default" { + count = local.ipsec_enabled ? 1 : 0 + project = var.project_id + network = var.network + region = var.region + name = "pool-${var.name}" + address_type = "INTERNAL" + purpose = "IPSEC_INTERCONNECT" + address = split("/", var.vpn_gateways_ip_range)[0] + prefix_length = split("/", var.vpn_gateways_ip_range)[1] +} + +resource "google_compute_interconnect_attachment" "default" { + project = var.project_id + region = var.region + router = local.router + name = var.name + description = var.description + interconnect = try(var.dedicated_interconnect_config.interconnect, null) + bandwidth = try(var.dedicated_interconnect_config.bandwidth, null) + mtu = local.ipsec_enabled ? null : var.mtu + candidate_subnets = var.dedicated_interconnect_config != null ? [var.dedicated_interconnect_config.bgp_range] : null + vlan_tag8021q = try(var.dedicated_interconnect_config.vlan_tag, null) + admin_enabled = var.admin_enabled + encryption = local.ipsec_enabled ? "IPSEC" : null + type = var.dedicated_interconnect_config == null ? "PARTNER" : "DEDICATED" + edge_availability_domain = try(var.partner_interconnect_config.edge_availability_domain, null) + ipsec_internal_addresses = local.ipsec_enabled ? [google_compute_address.default[0].self_link] : null +} + +resource "google_compute_router" "encrypted" { + count = var.router_config.create && local.ipsec_enabled ? 1 : 0 + name = "${var.name}-underlay" + network = var.network + project = var.project_id + region = var.region + encrypted_interconnect_router = true + bgp { + asn = var.router_config.asn + advertise_mode = var.dedicated_interconnect_config == null ? "DEFAULT" : "CUSTOM" + dynamic "advertised_ip_ranges" { + for_each = var.dedicated_interconnect_config == null ? var.ipsec_gateway_ip_ranges : {} + content { + description = advertised_ip_ranges.key + range = advertised_ip_ranges.value + } + } + } +} + +resource "google_compute_router" "unencrypted" { + count = var.router_config.create && !local.ipsec_enabled ? 1 : 0 + name = coalesce(var.router_config.name, "underlay-${var.name}") + project = var.project_id + region = var.region + network = var.network + bgp { + advertise_mode = ( + var.router_config.custom_advertise != null + ? "CUSTOM" + : "DEFAULT" + ) + advertised_groups = ( + try(var.router_config.custom_advertise.all_subnets, false) + ? ["ALL_SUBNETS"] + : [] + ) + dynamic "advertised_ip_ranges" { + for_each = try(var.router_config.custom_advertise.ip_ranges, {}) + iterator = range + content { + range = range.key + description = range.value + } + } + keepalive_interval = try(var.router_config.keepalive, null) + asn = var.router_config.asn + } +} + +resource "google_compute_router_interface" "default" { + count = var.dedicated_interconnect_config != null ? 1 : 0 + project = var.project_id + region = var.region + name = "${var.name}-intf" + router = local.router + ip_range = google_compute_interconnect_attachment.default.cloud_router_ip_address + interconnect_attachment = google_compute_interconnect_attachment.default.self_link +} + +resource "google_compute_router_peer" "default" { + count = var.dedicated_interconnect_config != null ? 1 : 0 + name = "${var.name}-peer" + project = var.project_id + router = local.router + region = var.region + peer_ip_address = split("/", google_compute_interconnect_attachment.default.customer_router_ip_address)[0] + peer_asn = var.peer_asn + interface = google_compute_router_interface.default[0].name + advertised_route_priority = 100 + advertise_mode = "CUSTOM" + + dynamic "advertised_ip_ranges" { + for_each = var.ipsec_gateway_ip_ranges + content { + description = advertised_ip_ranges.key + range = advertised_ip_ranges.value + } + } + + dynamic "bfd" { + for_each = var.router_config.bfd != null ? toset([var.router_config.bfd]) : [] + content { + session_initialization_mode = bfd.session_initialization_mode + min_receive_interval = bfd.min_receive_interval + min_transmit_interval = bfd.min_transmit_interval + multiplier = bfd.multiplier + } + } + + depends_on = [ + google_compute_router_interface.default + ] +} diff --git a/assets/modules-fabric/v26/net-vlan-attachment/outputs.tf b/assets/modules-fabric/v26/net-vlan-attachment/outputs.tf new file mode 100644 index 0000000..19abbc8 --- /dev/null +++ b/assets/modules-fabric/v26/net-vlan-attachment/outputs.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "attachment" { + description = "VLAN Attachment resource." + value = google_compute_interconnect_attachment.default +} + +output "id" { + description = "Fully qualified VLAN attachment id." + value = google_compute_interconnect_attachment.default.id +} + +output "name" { + description = "The name of the VLAN attachment created." + value = google_compute_interconnect_attachment.default.name +} + +output "pairing_key" { + description = "Opaque identifier of an PARTNER attachment used to initiate provisioning with a selected partner." + value = google_compute_interconnect_attachment.default.pairing_key +} + +output "router" { + description = "Router resource (only if auto-created)." + value = local.ipsec_enabled ? one(google_compute_router.encrypted[*]) : one(google_compute_router.unencrypted[*]) +} + +output "router_interface" { + description = "Router interface created for the VLAN attachment." + value = google_compute_router_interface.default +} + +output "router_name" { + description = "Router name." + value = local.router +} diff --git a/assets/modules-fabric/v26/net-vlan-attachment/variables.tf b/assets/modules-fabric/v26/net-vlan-attachment/variables.tf new file mode 100644 index 0000000..8ec2dcf --- /dev/null +++ b/assets/modules-fabric/v26/net-vlan-attachment/variables.tf @@ -0,0 +1,120 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "admin_enabled" { + description = "Whether the VLAN attachment is enabled." + type = bool + default = true +} + +variable "dedicated_interconnect_config" { + description = "Partner interconnect configuration." + type = object({ + # Possible values @ https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_interconnect_attachment#bandwidth + bandwidth = optional(string, "BPS_10G") + bgp_range = optional(string, "169.254.128.0/29") + interconnect = string + vlan_tag = string + }) + default = null +} + +variable "description" { + description = "VLAN attachment description." + type = string +} + +variable "ipsec_gateway_ip_ranges" { + description = "IPSec Gateway IP Ranges." + type = map(string) + default = {} +} + +variable "mtu" { + description = "The MTU associated to the VLAN attachment (1440 / 1500)." + type = number + default = "1500" +} + +variable "name" { + description = "The common resources name, used after resource type prefix and suffix." + type = string +} + +variable "network" { + description = "The VPC name to which resources are associated to." + type = string +} + +variable "partner_interconnect_config" { + description = "Partner interconnect configuration." + type = object({ + edge_availability_domain = string + }) + validation { + condition = var.partner_interconnect_config == null ? true : contains(["AVAILABILITY_DOMAIN_1", "AVAILABILITY_DOMAIN_2", "AVAILABILITY_DOMAIN_ANY"], var.partner_interconnect_config.edge_availability_domain) + error_message = "The edge_availability_domain must have one of these values: AVAILABILITY_DOMAIN_1, AVAILABILITY_DOMAIN_2, AVAILABILITY_DOMAIN_ANY." + } + default = null +} + +variable "peer_asn" { + description = "The on-premises underlay router ASN." + type = string +} + +variable "project_id" { + description = "The project id where resources are created." + type = string +} + +variable "region" { + description = "The region where resources are created." + type = string +} + +variable "router_config" { + description = "Cloud Router configuration for the VPN. If you want to reuse an existing router, set create to false and use name to specify the desired router." + type = object({ + create = optional(bool, true) + asn = optional(number, 65001) + name = optional(string, "router") + keepalive = optional(number) + custom_advertise = optional(object({ + all_subnets = bool + ip_ranges = map(string) + })) + bfd = optional(object({ + session_initialization_mode = optional(string, "ACTIVE") + min_receive_interval = optional(number) + min_transmit_interval = optional(number) + multiplier = optional(number) + })) + }) + nullable = false +} + +variable "vlan_tag" { + description = "The VLAN id to be used for this VLAN attachment." + type = number + default = null +} + +variable "vpn_gateways_ip_range" { + description = "The IP range (cidr notation) to be used for the GCP VPN gateways. If null IPSec over Interconnect is not enabled." + type = string + default = null +} diff --git a/assets/modules-fabric/v26/net-vlan-attachment/versions.tf b/assets/modules-fabric/v26/net-vlan-attachment/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/net-vlan-attachment/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/net-vpc-firewall/README.md b/assets/modules-fabric/v26/net-vpc-firewall/README.md new file mode 100644 index 0000000..47a696d --- /dev/null +++ b/assets/modules-fabric/v26/net-vpc-firewall/README.md @@ -0,0 +1,250 @@ +# Google Cloud VPC Firewall + +This module allows creation and management of different types of firewall rules for a single VPC network: + +- custom rules via the `egress_rules` and `ingress_rules` variables +- optional predefined rules that simplify prototyping via the `default_rules_config` variable + +The predefined rules are enabled by default and set to the ranges of the GCP health checkers for HTTP/HTTPS, and the IAP forwarders for SSH. See the relevant section below on how to configure or disable them. + +## Examples + +### Minimal open firewall + +This is often useful for prototyping or testing infrastructure, allowing open ingress from the private range, enabling SSH to private addresses from IAP, and HTTP/HTTPS from the health checkers. + +```hcl +module "firewall" { + source = "./fabric/modules/net-vpc-firewall" + project_id = "my-project" + network = "my-network" + default_rules_config = { + admin_ranges = ["10.0.0.0/8"] + } +} +# tftest modules=1 resources=4 inventory=basic.yaml +``` + +### Custom rules + +This is an example of how to define custom rules, with a sample rule allowing open ingress for the NTP protocol to instances with the `ntp-svc` tag. + +Some implicit defaults are used in the rules variable types and can be controlled by explicitly setting specific attributes: + +- action is controlled via the `deny` attribute which defaults to `true` for egress and `false` for ingress +- priority defaults to `1000` +- destination ranges (for egress) and source ranges (for ingress) default to `["0.0.0.0/0"]` if not explicitly set or set to `null`, to disable the behaviour set ranges to the empty list (`[]`) +- rules default to all protocols if not set + +```hcl +module "firewall" { + source = "./fabric/modules/net-vpc-firewall" + project_id = "my-project" + network = "my-network" + default_rules_config = { + admin_ranges = ["10.0.0.0/8"] + } + egress_rules = { + # implicit deny action + allow-egress-rfc1918 = { + deny = false + description = "Allow egress to RFC 1918 ranges." + destination_ranges = [ + "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16" + ] + } + allow-egress-tag = { + deny = false + description = "Allow egress from a specific tag to 0/0." + targets = ["target-tag"] + } + deny-egress-all = { + description = "Block egress." + } + } + ingress_rules = { + # implicit allow action + allow-ingress-ntp = { + description = "Allow NTP service based on tag." + targets = ["ntp-svc"] + rules = [{ protocol = "udp", ports = [123] }] + } + allow-ingress-tag = { + description = "Allow ingress from a specific tag." + source_ranges = [] + sources = ["client-tag"] + targets = ["target-tag"] + } + } +} +# tftest modules=1 resources=9 inventory=custom-rules.yaml +``` + +### Controlling or turning off default rules + +Predefined rules can be controlled or turned off via the `default_rules_config` variable. + +#### Overriding default tags and ranges + +Each protocol rule has a default set of tags and ranges: + +- the health check range and the `http-server`/`https-server` tag for HTTP/HTTPS, matching tags set via GCP console flags on GCE instances +- the IAP forwarders range and `ssh` tag for SSH + +Default tags and ranges can be overridden for each protocol, like shown here for SSH: + +```hcl +module "firewall" { + source = "./fabric/modules/net-vpc-firewall" + project_id = "my-project" + network = "my-network" + default_rules_config = { + ssh_ranges = ["10.0.0.0/8"] + ssh_tags = ["ssh-default"] + } +} +# tftest modules=1 resources=3 inventory=custom-ssh-default-rule.yaml +``` + +#### Disabling predefined rules + +Default rules can be disabled individually by specifying an empty set of ranges: + +```hcl +module "firewall" { + source = "./fabric/modules/net-vpc-firewall" + project_id = "my-project" + network = "my-network" + default_rules_config = { + ssh_ranges = [] + } +} +# tftest modules=1 resources=2 inventory=no-ssh-default-rules.yaml +``` + +Or the entire set of rules can be disabled via the `disabled` attribute: + +```hcl +module "firewall" { + source = "./fabric/modules/net-vpc-firewall" + project_id = "my-project" + network = "my-network" + default_rules_config = { + disabled = true + } +} +# tftest modules=0 resources=0 inventory=no-default-rules.yaml +``` + +### Including source & destination ranges + +Custom rules now support including both source & destination ranges in ingress and egress rules: + +```hcl +module "firewall" { + source = "./fabric/modules/net-vpc-firewall" + project_id = "my-project" + network = "my-network" + default_rules_config = { + disabled = true + } + egress_rules = { + deny-egress-source-destination-ranges = { + description = "Deny egress using source and destination ranges" + source_ranges = ["10.132.0.0/20", "10.138.0.0/20"] + destination_ranges = ["172.16.0.0/12"] + } + } + ingress_rules = { + allow-ingress-source-destination-ranges = { + description = "Allow ingress using source and destination ranges" + source_ranges = ["172.16.0.0/12"] + destination_ranges = ["10.132.0.0/20", "10.138.0.0/20"] + } + } +} +# tftest modules=1 resources=2 inventory=local-ranges.yaml +``` + +### Rules Factory + +The module includes a rules factory (see [Resource Factories](../../blueprints/factories/)) for the massive creation of rules leveraging YaML configuration files. Each configuration file can optionally contain more than one rule which a structure that reflects the `custom_rules` variable. + +```hcl +module "firewall" { + source = "./fabric/modules/net-vpc-firewall" + project_id = "my-project" + network = "my-network" + factories_config = { + rules_folder = "configs/firewall/rules" + cidr_tpl_file = "configs/firewall/cidrs.yaml" + } + default_rules_config = { disabled = true } +} +# tftest modules=1 resources=3 files=lbs,cidrs inventory=factory.yaml +``` + +```yaml +# tftest-file id=lbs path=configs/firewall/rules/load_balancers.yaml +ingress: + allow-healthchecks: + description: Allow ingress from healthchecks. + source_ranges: + - healthchecks + targets: ["lb-backends"] + rules: + - protocol: tcp + ports: + - 80 + - 443 + allow-service-1-to-service-2: + description: Allow ingress from service-1 SA + targets: ["service-2"] + use_service_accounts: true + sources: + - service-1@my-project.iam.gserviceaccount.com + rules: + - protocol: tcp + ports: + - 80 + - 443 +egress: + block-telnet: + description: block outbound telnet + deny: true + rules: + - protocol: tcp + ports: + - 23 +``` + +```yaml +# tftest-file id=cidrs path=configs/firewall/cidrs.yaml +healthchecks: + - 35.191.0.0/16 + - 130.211.0.0/22 + - 209.85.152.0/22 + - 209.85.204.0/22 +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [network](variables.tf#L110) | Name of the network this set of firewall rules applies to. | string | ✓ | | +| [project_id](variables.tf#L115) | Project id of the project that holds the network. | string | ✓ | | +| [default_rules_config](variables.tf#L17) | Optionally created convenience rules. Set the 'disabled' attribute to true, or individual rule attributes to empty lists to disable. | object({…}) | | {} | +| [egress_rules](variables.tf#L37) | List of egress rule definitions, default to deny action. Null destination ranges will be replaced with 0/0. | map(object({…})) | | {} | +| [factories_config](variables.tf#L60) | Paths to data files and folders that enable factory functionality. | object({…}) | | null | +| [ingress_rules](variables.tf#L69) | List of ingress rule definitions, default to allow action. Null source ranges will be replaced with 0/0. | map(object({…})) | | {} | +| [named_ranges](variables.tf#L93) | Define mapping of names to ranges that can be used in custom rules. | map(list(string)) | | {…} | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [default_rules](outputs.tf#L17) | Default rule resources. | | +| [rules](outputs.tf#L27) | Custom rule resources. | | + + diff --git a/assets/modules-fabric/v26/net-vpc-firewall/default-rules.tf b/assets/modules-fabric/v26/net-vpc-firewall/default-rules.tf new file mode 100644 index 0000000..bbca6dd --- /dev/null +++ b/assets/modules-fabric/v26/net-vpc-firewall/default-rules.tf @@ -0,0 +1,77 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Optional default rule resources. + +locals { + default_rules = { + for k, v in var.default_rules_config : + k => var.default_rules_config.disabled == true || v == null ? [] : v + if k != "disabled" + } +} + +resource "google_compute_firewall" "allow-admins" { + count = length(local.default_rules.admin_ranges) > 0 ? 1 : 0 + name = "${var.network}-ingress-admins" + description = "Access from the admin subnet to all subnets." + network = var.network + project = var.project_id + source_ranges = local.default_rules.admin_ranges + allow { protocol = "all" } +} + +resource "google_compute_firewall" "allow-tag-http" { + count = length(local.default_rules.http_ranges) > 0 ? 1 : 0 + name = "${var.network}-ingress-tag-http" + description = "Allow http to machines with matching tags." + network = var.network + project = var.project_id + source_ranges = local.default_rules.http_ranges + target_tags = local.default_rules.http_tags + allow { + protocol = "tcp" + ports = ["80"] + } +} + +resource "google_compute_firewall" "allow-tag-https" { + count = length(local.default_rules.https_ranges) > 0 ? 1 : 0 + name = "${var.network}-ingress-tag-https" + description = "Allow http to machines with matching tags." + network = var.network + project = var.project_id + source_ranges = local.default_rules.https_ranges + target_tags = local.default_rules.https_tags + allow { + protocol = "tcp" + ports = ["443"] + } +} + +resource "google_compute_firewall" "allow-tag-ssh" { + count = length(local.default_rules.ssh_ranges) > 0 ? 1 : 0 + name = "${var.network}-ingress-tag-ssh" + description = "Allow SSH to machines with matching tags." + network = var.network + project = var.project_id + source_ranges = local.default_rules.ssh_ranges + target_tags = local.default_rules.ssh_tags + allow { + protocol = "tcp" + ports = ["22"] + } +} diff --git a/assets/modules-fabric/v26/net-vpc-firewall/main.tf b/assets/modules-fabric/v26/net-vpc-firewall/main.tf new file mode 100644 index 0000000..bd528b0 --- /dev/null +++ b/assets/modules-fabric/v26/net-vpc-firewall/main.tf @@ -0,0 +1,164 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # define list of rule files + _factory_rule_files = [ + for f in try(fileset(var.factories_config.rules_folder, "**/*.yaml"), []) : + "${var.factories_config.rules_folder}/${f}" + ] + # decode rule files and account for optional attributes + _factory_rule_list = flatten([ + for f in local._factory_rule_files : [ + for direction, ruleset in yamldecode(file(f)) : [ + for name, rule in ruleset : { + name = name + deny = try(rule.deny, false) + rules = try(rule.rules, [{ protocol = "all" }]) + description = try(rule.description, null) + destination_ranges = try(rule.destination_ranges, null) + direction = upper(direction) + disabled = try(rule.disabled, null) + enable_logging = try(rule.enable_logging, null) + priority = try(rule.priority, 1000) + source_ranges = try(rule.source_ranges, null) + sources = try(rule.sources, null) + targets = try(rule.targets, null) + use_service_accounts = try(rule.use_service_accounts, false) + } + ] + ] + ]) + _factory_rules = { + for r in local._factory_rule_list : r.name => r + if contains(["EGRESS", "INGRESS"], r.direction) + } + _named_ranges = merge( + try(yamldecode(file(var.factories_config.cidr_tpl_file)), {}), + var.named_ranges + ) + _rules = merge( + local._factory_rules, local._rules_egress, local._rules_ingress + ) + _rules_egress = { + for name, rule in merge(var.egress_rules) : + name => merge(rule, { direction = "EGRESS" }) + } + _rules_ingress = { + for name, rule in merge(var.ingress_rules) : + name => merge(rule, { direction = "INGRESS" }) + } + # convert rules data to resource format and replace range template variables + rules = { + for name, rule in local._rules : + name => merge(rule, { + action = rule.deny == true ? "DENY" : "ALLOW" + destination_ranges = ( + try(rule.destination_ranges, null) == null + ? null + : flatten([ + for range in rule.destination_ranges : + try(local._named_ranges[range], range) + ]) + ) + rules = { for k, v in rule.rules : k => v } + source_ranges = ( + try(rule.source_ranges, null) == null + ? null + : flatten([ + for range in rule.source_ranges : + try(local._named_ranges[range], range) + ]) + ) + }) + } +} + +resource "google_compute_firewall" "custom-rules" { + for_each = local.rules + project = var.project_id + network = var.network + name = each.key + description = each.value.description + direction = each.value.direction + source_ranges = ( + each.value.direction == "INGRESS" + ? ( + each.value.source_ranges == null + ? ["0.0.0.0/0"] + : each.value.source_ranges + ) + #for egress, we will include the source_ranges when provided. Previously, null was forced + : each.value.source_ranges + ) + destination_ranges = ( + each.value.direction == "EGRESS" + ? ( + each.value.destination_ranges == null + ? ["0.0.0.0/0"] + : each.value.destination_ranges + ) + #for ingress, we will include the destination_ranges when provided. Previously, null was forced + : each.value.destination_ranges + ) + source_tags = ( + each.value.use_service_accounts || each.value.direction == "EGRESS" + ? null + : each.value.sources + ) + source_service_accounts = ( + each.value.use_service_accounts && each.value.direction == "INGRESS" + ? each.value.sources + : null + ) + target_tags = ( + each.value.use_service_accounts ? null : each.value.targets + ) + target_service_accounts = ( + each.value.use_service_accounts ? each.value.targets : null + ) + disabled = each.value.disabled == true + priority = each.value.priority + + dynamic "log_config" { + for_each = each.value.enable_logging == null ? [] : [""] + content { + metadata = ( + try(each.value.enable_logging.include_metadata, null) == true + ? "INCLUDE_ALL_METADATA" + : "EXCLUDE_ALL_METADATA" + ) + } + } + + dynamic "deny" { + for_each = each.value.action == "DENY" ? each.value.rules : {} + iterator = rule + content { + protocol = rule.value.protocol + ports = rule.value.ports + } + } + + dynamic "allow" { + for_each = each.value.action == "ALLOW" ? each.value.rules : {} + iterator = rule + content { + protocol = rule.value.protocol + ports = rule.value.ports + } + } +} diff --git a/assets/modules-fabric/v26/net-vpc-firewall/outputs.tf b/assets/modules-fabric/v26/net-vpc-firewall/outputs.tf new file mode 100644 index 0000000..9206ab5 --- /dev/null +++ b/assets/modules-fabric/v26/net-vpc-firewall/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "default_rules" { + description = "Default rule resources." + value = { + admin = try(google_compute_firewall.allow-admins, null) + http = try(google_compute_firewall.allow-tag-http, null) + https = try(google_compute_firewall.allow-tag-https, null) + ssh = try(google_compute_firewall.allow-tag-ssh, null) + } +} + +output "rules" { + description = "Custom rule resources." + value = google_compute_firewall.custom-rules +} diff --git a/assets/modules-fabric/v26/net-vpc-firewall/variables.tf b/assets/modules-fabric/v26/net-vpc-firewall/variables.tf new file mode 100644 index 0000000..132f00e --- /dev/null +++ b/assets/modules-fabric/v26/net-vpc-firewall/variables.tf @@ -0,0 +1,118 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "default_rules_config" { + description = "Optionally created convenience rules. Set the 'disabled' attribute to true, or individual rule attributes to empty lists to disable." + type = object({ + admin_ranges = optional(list(string)) + disabled = optional(bool, false) + http_ranges = optional(list(string), [ + "35.191.0.0/16", "130.211.0.0/22", "209.85.152.0/22", "209.85.204.0/22"] + ) + http_tags = optional(list(string), ["http-server"]) + https_ranges = optional(list(string), [ + "35.191.0.0/16", "130.211.0.0/22", "209.85.152.0/22", "209.85.204.0/22"] + ) + https_tags = optional(list(string), ["https-server"]) + ssh_ranges = optional(list(string), ["35.235.240.0/20"]) + ssh_tags = optional(list(string), ["ssh"]) + }) + default = {} + nullable = false +} + +variable "egress_rules" { + description = "List of egress rule definitions, default to deny action. Null destination ranges will be replaced with 0/0." + type = map(object({ + deny = optional(bool, true) + description = optional(string) + destination_ranges = optional(list(string)) + disabled = optional(bool, false) + enable_logging = optional(object({ + include_metadata = optional(bool) + })) + priority = optional(number, 1000) + source_ranges = optional(list(string)) + targets = optional(list(string)) + use_service_accounts = optional(bool, false) + rules = optional(list(object({ + protocol = string + ports = optional(list(string)) + })), [{ protocol = "all" }]) + })) + default = {} + nullable = false +} + +variable "factories_config" { + description = "Paths to data files and folders that enable factory functionality." + type = object({ + cidr_tpl_file = optional(string) + rules_folder = string + }) + default = null +} + +variable "ingress_rules" { + description = "List of ingress rule definitions, default to allow action. Null source ranges will be replaced with 0/0." + type = map(object({ + deny = optional(bool, false) + description = optional(string) + destination_ranges = optional(list(string), []) # empty list is needed as default to allow deletion after initial creation with a value. See https://github.com/hashicorp/terraform-provider-google/issues/14270 + disabled = optional(bool, false) + enable_logging = optional(object({ + include_metadata = optional(bool) + })) + priority = optional(number, 1000) + source_ranges = optional(list(string)) + sources = optional(list(string)) + targets = optional(list(string)) + use_service_accounts = optional(bool, false) + rules = optional(list(object({ + protocol = string + ports = optional(list(string)) + })), [{ protocol = "all" }]) + })) + default = {} + nullable = false +} + +variable "named_ranges" { + description = "Define mapping of names to ranges that can be used in custom rules." + type = map(list(string)) + default = { + any = ["0.0.0.0/0"] + dns-forwarders = ["35.199.192.0/19"] + health-checkers = [ + "35.191.0.0/16", "130.211.0.0/22", "209.85.152.0/22", "209.85.204.0/22" + ] + iap-forwarders = ["35.235.240.0/20"] + private-googleapis = ["199.36.153.8/30"] + restricted-googleapis = ["199.36.153.4/30"] + rfc1918 = ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"] + } + nullable = false +} + +variable "network" { + description = "Name of the network this set of firewall rules applies to." + type = string +} + +variable "project_id" { + description = "Project id of the project that holds the network." + type = string +} diff --git a/assets/modules-fabric/v26/net-vpc-firewall/versions.tf b/assets/modules-fabric/v26/net-vpc-firewall/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/net-vpc-firewall/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/net-vpc-peering/README.md b/assets/modules-fabric/v26/net-vpc-peering/README.md new file mode 100644 index 0000000..0998d7b --- /dev/null +++ b/assets/modules-fabric/v26/net-vpc-peering/README.md @@ -0,0 +1,92 @@ +# Google Network Peering + +This module allows creation of a [VPC Network Peering](https://cloud.google.com/vpc/docs/vpc-peering) between two networks. + +The resources created/managed by this module are: + +- one network peering from `local network` to `peer network` +- one network peering from `peer network` to `local network` + + +- [Examples](#examples) + - [Basic Usage](#basic-usage) + - [Multiple Peerings](#multiple-peerings) + - [Route Configuration](#route-configuration) +- [Variables](#variables) +- [Outputs](#outputs) + + +## Examples + +### Basic Usage + +Basic usage of this module is as follows: + +```hcl +module "peering" { + source = "./fabric/modules/net-vpc-peering" + prefix = "name-prefix" + local_network = "projects/project-1/global/networks/vpc-1" + peer_network = "projects/project-1/global/networks/vpc-2" +} +# tftest modules=1 resources=2 +``` + +### Multiple Peerings + +If you need to create more than one peering for the same VPC Network `(A -> B, A -> C)` you use a `depends_on` for second one to keep order of peering creation (It is not currently possible to create more than one peering connection for a VPC Network at the same time). + +```hcl +module "peering-a-b" { + source = "./fabric/modules/net-vpc-peering" + prefix = "name-prefix" + local_network = "projects/project-a/global/networks/vpc-a" + peer_network = "projects/project-b/global/networks/vpc-b" +} + +module "peering-a-c" { + source = "./fabric/modules/net-vpc-peering" + prefix = "name-prefix" + local_network = "projects/project-a/global/networks/vpc-a" + peer_network = "projects/project-c/global/networks/vpc-c" + depends_on = [module.peering-a-b] +} +# tftest modules=2 resources=4 +``` + +### Route Configuration + +You can control export/import of routes in both the local and peer via the `routes_config` variable. Defaults are to import and export from both sides, when the peer side only configured if the peering is managed by the module via `peer_create_peering`. + +```hcl +module "peering" { + source = "./fabric/modules/net-vpc-peering" + prefix = "name-prefix" + local_network = "projects/project-1/global/networks/vpc-1" + peer_network = "projects/project-1/global/networks/vpc-2" + routes_config = { + local = { + import = false + } + } +} +# tftest modules=1 resources=2 inventory=route-config.yaml +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [local_network](variables.tf#L17) | Resource link of the network to add a peering to. | string | ✓ | | +| [peer_network](variables.tf#L28) | Resource link of the peer network. | string | ✓ | | +| [peer_create_peering](variables.tf#L22) | Create the peering on the remote side. If false, only the peering from this network to the remote network is created. | bool | | true | +| [prefix](variables.tf#L33) | Optional name prefix for the network peerings. | string | | null | +| [routes_config](variables.tf#L43) | Control import/export for local and remote peer. Remote configuration is only used when creating remote peering. | object({…}) | | {} | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [local_network_peering](outputs.tf#L17) | Network peering resource. | | +| [peer_network_peering](outputs.tf#L22) | Peer network peering resource. | | + diff --git a/assets/modules-fabric/v26/net-vpc-peering/main.tf b/assets/modules-fabric/v26/net-vpc-peering/main.tf new file mode 100644 index 0000000..2d88a0b --- /dev/null +++ b/assets/modules-fabric/v26/net-vpc-peering/main.tf @@ -0,0 +1,59 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + local_network_name = element(reverse(split("/", var.local_network)), 0) + peer_network_name = element(reverse(split("/", var.peer_network)), 0) + prefix = var.prefix == null ? "" : "${var.prefix}-" +} + +resource "google_compute_network_peering" "local_network_peering" { + name = "${local.prefix}${local.local_network_name}-${local.peer_network_name}" + network = var.local_network + peer_network = var.peer_network + export_custom_routes = try( + var.routes_config.local.export, null + ) + import_custom_routes = try( + var.routes_config.local.import, null + ) + export_subnet_routes_with_public_ip = try( + var.routes_config.local.public_export, null + ) + import_subnet_routes_with_public_ip = try( + var.routes_config.local.public_import, null + ) +} + +resource "google_compute_network_peering" "peer_network_peering" { + count = var.peer_create_peering ? 1 : 0 + name = "${local.prefix}${local.peer_network_name}-${local.local_network_name}" + network = var.peer_network + peer_network = var.local_network + export_custom_routes = try( + var.routes_config.peer.export, null + ) + import_custom_routes = try( + var.routes_config.peer.import, null + ) + export_subnet_routes_with_public_ip = try( + var.routes_config.peer.public_export, null + ) + import_subnet_routes_with_public_ip = try( + var.routes_config.peer.public_import, null + ) + depends_on = [google_compute_network_peering.local_network_peering] +} diff --git a/assets/modules-fabric/v26/net-vpc-peering/outputs.tf b/assets/modules-fabric/v26/net-vpc-peering/outputs.tf new file mode 100644 index 0000000..558c4ba --- /dev/null +++ b/assets/modules-fabric/v26/net-vpc-peering/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "local_network_peering" { + description = "Network peering resource." + value = google_compute_network_peering.local_network_peering +} + +output "peer_network_peering" { + description = "Peer network peering resource." + value = google_compute_network_peering.peer_network_peering +} diff --git a/assets/modules-fabric/v26/net-vpc-peering/variables.tf b/assets/modules-fabric/v26/net-vpc-peering/variables.tf new file mode 100644 index 0000000..ade8074 --- /dev/null +++ b/assets/modules-fabric/v26/net-vpc-peering/variables.tf @@ -0,0 +1,61 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "local_network" { + description = "Resource link of the network to add a peering to." + type = string +} + +variable "peer_create_peering" { + description = "Create the peering on the remote side. If false, only the peering from this network to the remote network is created." + type = bool + default = true +} + +variable "peer_network" { + description = "Resource link of the peer network." + type = string +} + +variable "prefix" { + description = "Optional name prefix for the network peerings." + type = string + default = null + validation { + condition = var.prefix != "" + error_message = "Prefix cannot be empty, please use null instead." + } +} + +variable "routes_config" { + description = "Control import/export for local and remote peer. Remote configuration is only used when creating remote peering." + type = object({ + local = optional(object({ + export = optional(bool, true) + import = optional(bool, true) + public_export = optional(bool) + public_import = optional(bool) + }), {}) + peer = optional(object({ + export = optional(bool, true) + import = optional(bool, true) + public_export = optional(bool) + public_import = optional(bool) + }), {}) + }) + nullable = false + default = {} +} diff --git a/assets/modules-fabric/v26/net-vpc-peering/versions.tf b/assets/modules-fabric/v26/net-vpc-peering/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/net-vpc-peering/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/net-vpc/README.md b/assets/modules-fabric/v26/net-vpc/README.md new file mode 100644 index 0000000..ea86930 --- /dev/null +++ b/assets/modules-fabric/v26/net-vpc/README.md @@ -0,0 +1,581 @@ +# VPC module + +This module allows creation and management of VPC networks including subnetworks and subnetwork IAM bindings, and most features and options related to VPCs and subnets. + +## Examples + + +- [Examples](#examples) + - [Simple VPC](#simple-vpc) + - [Subnet Options](#subnet-options) + - [Subnet IAM](#subnet-iam) + - [Peering](#peering) + - [Shared VPC](#shared-vpc) + - [Private Service Networking](#private-service-networking) + - [Private Service Networking with peering routes](#private-service-networking-with-peering-routes) + - [Subnets for Private Service Connect, Proxy-only subnets](#subnets-for-private-service-connect-proxy-only-subnets) + - [DNS Policies](#dns-policies) + - [Subnet Factory](#subnet-factory) + - [Custom Routes](#custom-routes) + - [Private Google Access routes](#private-google-access-routes) + - [Allow Firewall Policy to be evaluated before Firewall Rules](#allow-firewall-policy-to-be-evaluated-before-firewall-rules) + - [IPv6](#ipv6) +- [Variables](#variables) +- [Outputs](#outputs) + + +### Simple VPC + +```hcl +module "vpc" { + source = "./fabric/modules/net-vpc" + project_id = "my-project" + name = "my-network" + subnets = [ + { + ip_cidr_range = "10.0.0.0/24" + name = "production" + region = "europe-west1" + secondary_ip_ranges = { + pods = "172.16.0.0/20" + services = "192.168.0.0/24" + } + }, + { + ip_cidr_range = "10.0.16.0/24" + name = "production" + region = "europe-west2" + } + ] +} +# tftest modules=1 resources=5 inventory=simple.yaml +``` + +### Subnet Options + +```hcl +module "vpc" { + source = "./fabric/modules/net-vpc" + project_id = "my-project" + name = "my-network" + subnets = [ + # simple subnet + { + name = "simple" + region = "europe-west1" + ip_cidr_range = "10.0.0.0/24" + }, + # custom description and PGA disabled + { + name = "no-pga" + region = "europe-west1" + ip_cidr_range = "10.0.1.0/24", + description = "Subnet b" + enable_private_access = false + }, + # secondary ranges + { + name = "with-secondary-ranges" + region = "europe-west1" + ip_cidr_range = "10.0.2.0/24" + secondary_ip_ranges = { + a = "192.168.0.0/24" + b = "192.168.1.0/24" + } + }, + # enable flow logs + { + name = "with-flow-logs" + region = "europe-west1" + ip_cidr_range = "10.0.3.0/24" + flow_logs_config = { + flow_sampling = 0.5 + aggregation_interval = "INTERVAL_10_MIN" + } + } + ] +} +# tftest modules=1 resources=7 inventory=subnet-options.yaml +``` + +### Subnet IAM + +Subnet IAM variables follow our general interface, with extra keys/members for the subnet to which each binding will be applied. + +```hcl +module "vpc" { + source = "./fabric/modules/net-vpc" + project_id = "my-project" + name = "my-network" + subnets = [ + { + name = "subnet-1" + region = "europe-west1" + ip_cidr_range = "10.0.1.0/24" + iam = { + "roles/compute.networkUser" = [ + "user:user1@example.com", "group:group1@example.com" + ] + } + iam_bindings = { + subnet-1-iam = { + members = ["group:group2@example.com"] + role = "roles/compute.networkUser" + condition = { + expression = "resource.matchTag('123456789012/env', 'prod')" + title = "test_condition" + } + } + } + }, + { + name = "subnet-2" + region = "europe-west1" + ip_cidr_range = "10.0.1.0/24" + iam_bindings_additive = { + subnet-2-iam = { + member = "user:am1@example.com" + role = "roles/compute.networkUser" + subnet = "europe-west1/subnet-2" + } + } + } + ] +} +# tftest modules=1 resources=8 inventory=subnet-iam.yaml +``` + +### Peering + +A single peering can be configured for the VPC, so as to allow management of simple scenarios, and more complex configurations like hub and spoke by defining the peering configuration on the spoke VPCs. Care must be taken so as a single peering is created/changed/destroyed at a time, due to the specific behaviour of the peering API calls. + +If you only want to create the "local" side of the peering, use `peering_create_remote_end` to `false`. This is useful if you don't have permissions on the remote project/VPC to create peerings. + +```hcl +module "vpc-hub" { + source = "./fabric/modules/net-vpc" + project_id = "hub" + name = "vpc-hub" + subnets = [{ + ip_cidr_range = "10.0.0.0/24" + name = "subnet-1" + region = "europe-west1" + }] +} + +module "vpc-spoke-1" { + source = "./fabric/modules/net-vpc" + project_id = "spoke1" + name = "vpc-spoke1" + subnets = [{ + ip_cidr_range = "10.0.1.0/24" + name = "subnet-2" + region = "europe-west1" + }] + peering_config = { + peer_vpc_self_link = module.vpc-hub.self_link + import_routes = true + } +} +# tftest modules=2 resources=10 inventory=peering.yaml +``` + +### Shared VPC + +[Shared VPC](https://cloud.google.com/vpc/docs/shared-vpc) is a project-level functionality which enables a project to share its VPCs with other projects. The `shared_vpc_host` variable is here to help with rapid prototyping, we recommend leveraging the project module for production usage. + +```hcl +locals { + service_project_1 = { + project_id = "project1" + gke_service_account = "serviceAccount:gke" + cloud_services_service_account = "serviceAccount:cloudsvc" + } + service_project_2 = { + project_id = "project2" + } +} + +module "vpc-host" { + source = "./fabric/modules/net-vpc" + project_id = "my-project" + name = "my-host-network" + subnets = [ + { + ip_cidr_range = "10.0.0.0/24" + name = "subnet-1" + region = "europe-west1" + secondary_ip_ranges = { + pods = "172.16.0.0/20" + services = "192.168.0.0/24" + } + iam = { + "roles/compute.networkUser" = [ + local.service_project_1.cloud_services_service_account, + local.service_project_1.gke_service_account + ] + "roles/compute.securityAdmin" = [ + local.service_project_1.gke_service_account + ] + } + } + ] + shared_vpc_host = true + shared_vpc_service_projects = [ + local.service_project_1.project_id, + local.service_project_2.project_id + ] +} +# tftest modules=1 resources=9 inventory=shared-vpc.yaml +``` + +### Private Service Networking + +```hcl +module "vpc" { + source = "./fabric/modules/net-vpc" + project_id = "my-project" + name = "my-network" + subnets = [ + { + ip_cidr_range = "10.0.0.0/24" + name = "production" + region = "europe-west1" + } + ] + psa_config = { + ranges = { myrange = "10.0.1.0/24" } + } +} +# tftest modules=1 resources=7 inventory=psc.yaml +``` + +### Private Service Networking with peering routes + +Custom routes can be optionally exported/imported through the peering formed with the Google managed PSA VPC. + +```hcl +module "vpc" { + source = "./fabric/modules/net-vpc" + project_id = "my-project" + name = "my-network" + subnets = [ + { + ip_cidr_range = "10.0.0.0/24" + name = "production" + region = "europe-west1" + } + ] + psa_config = { + ranges = { myrange = "10.0.1.0/24" } + export_routes = true + import_routes = true + } +} +# tftest modules=1 resources=7 inventory=psc-routes.yaml +``` + +### Subnets for Private Service Connect, Proxy-only subnets + +Along with common private subnets module supports creation more service specific subnets for the following purposes: + +- [Proxy-only subnets](https://cloud.google.com/load-balancing/docs/proxy-only-subnets) for Regional HTTPS Internal HTTPS Load Balancers +- [Private Service Connect](https://cloud.google.com/vpc/docs/private-service-connect#psc-subnets) subnets + +```hcl +module "vpc" { + source = "./fabric/modules/net-vpc" + project_id = "my-project" + name = "my-network" + + subnets_proxy_only = [ + { + ip_cidr_range = "10.0.1.0/24" + name = "regional-proxy" + region = "europe-west1" + active = true + }, + { + ip_cidr_range = "10.0.4.0/24" + name = "global-proxy" + region = "australia-southeast2" + active = true + global = true + } + ] + subnets_psc = [ + { + ip_cidr_range = "10.0.3.0/24" + name = "psc" + region = "europe-west1" + } + ] +} +# tftest modules=1 resources=6 inventory=proxy-only-subnets.yaml +``` + +### DNS Policies + +```hcl +module "vpc" { + source = "./fabric/modules/net-vpc" + project_id = "my-project" + name = "my-network" + dns_policy = { + inbound = true + outbound = { + private_ns = ["10.0.0.1"] + public_ns = ["8.8.8.8"] + } + } + subnets = [ + { + ip_cidr_range = "10.0.0.0/24" + name = "production" + region = "europe-west1" + } + ] +} +# tftest modules=1 resources=5 inventory=dns-policies.yaml +``` + +### Subnet Factory + +The `net-vpc` module includes a subnet factory (see [Resource Factories](../../blueprints/factories/)) for the massive creation of subnets leveraging one configuration file per subnet. The factory also supports proxy-only and PSC subnets via the `purpose` attribute. The `name` attribute is optional and defaults to the file name, allowing to use the same name for subnets in different regions. + +```hcl +module "vpc" { + source = "./fabric/modules/net-vpc" + project_id = "my-project" + name = "my-network" + factories_config = { + subnets_folder = "config/subnets" + } +} +# tftest modules=1 resources=10 files=subnet-simple,subnet-simple-2,subnet-detailed,subnet-proxy,subnet-proxy-global,subnet-psc inventory=factory.yaml +``` + +```yaml +# tftest-file id=subnet-simple path=config/subnets/subnet-simple.yaml +name: simple +region: europe-west4 +ip_cidr_range: 10.0.1.0/24 +``` + +```yaml +# tftest-file id=subnet-simple-2 path=config/subnets/subnet-simple-2.yaml +name: simple +region: europe-west8 +ip_cidr_range: 10.0.2.0/24 +``` + +```yaml +# tftest-file id=subnet-detailed path=config/subnets/subnet-detailed.yaml +region: europe-west1 +description: Sample description +ip_cidr_range: 10.0.0.0/24 +# optional attributes +enable_private_access: false # defaults to true +iam: + roles/compute.networkUser: + - group:lorem@example.com + - serviceAccount:fbz@prj.iam.gserviceaccount.com + - user:foobar@example.com +secondary_ip_ranges: # map of secondary ip ranges + secondary-range-a: 192.168.0.0/24 +flow_logs_config: # enable, set to empty map to use defaults + aggregation_interval: "INTERVAL_5_SEC" + flow_sampling: 0.5 + metadata: "INCLUDE_ALL_METADATA" +``` + +```yaml +# tftest-file id=subnet-proxy path=config/subnets/subnet-proxy.yaml +region: europe-west4 +ip_cidr_range: 10.1.0.0/24 +proxy_only: true +``` + +```yaml +# tftest-file id=subnet-proxy-global path=config/subnets/subnet-proxy-global.yaml +region: australia-southeast2 +ip_cidr_range: 10.4.0.0/24 +proxy_only: true +global: true +``` + +```yaml +# tftest-file id=subnet-psc path=config/subnets/subnet-psc.yaml +region: europe-west4 +ip_cidr_range: 10.2.0.0/24 +psc: true +``` + +### Custom Routes + +VPC routes can be configured through the `routes` variable. + +```hcl +locals { + route_types = { + gateway = "global/gateways/default-internet-gateway" + instance = "zones/europe-west1-b/test" + ip = "192.168.0.128" + ilb = "regions/europe-west1/forwardingRules/test" + vpn_tunnel = "regions/europe-west1/vpnTunnels/foo" + } +} + +module "vpc" { + source = "./fabric/modules/net-vpc" + for_each = local.route_types + project_id = "my-project" + name = "my-network-with-route-${replace(each.key, "_", "-")}" + routes = { + next-hop = { + description = "Route to internal range." + dest_range = "192.168.128.0/24" + tags = null + next_hop_type = each.key + next_hop = each.value + } + gateway = { + dest_range = "0.0.0.0/0", + priority = 100 + tags = ["tag-a"] + next_hop_type = "gateway", + next_hop = "global/gateways/default-internet-gateway" + } + } + create_googleapis_routes = null +} +# tftest modules=5 resources=15 inventory=routes.yaml +``` + +### Private Google Access routes + +By default the VPC module creates IPv4 routes for the [Private Google Access ranges](https://cloud.google.com/vpc/docs/configure-private-google-access#config-routing). This behavior can be controlled through the `create_googleapis_routes` variable: + +```hcl +module "vpc" { + source = "./fabric/modules/net-vpc" + project_id = "my-project" + name = "my-vpc" + create_googleapis_routes = { + restricted = false + restricted-6 = true + private = false + private-6 = true + } +} +# tftest modules=1 resources=3 inventory=googleapis.yaml +``` + +### Allow Firewall Policy to be evaluated before Firewall Rules + +```hcl +module "vpc" { + source = "./fabric/modules/net-vpc" + project_id = "my-project" + name = "my-network" + firewall_policy_enforcement_order = "BEFORE_CLASSIC_FIREWALL" + subnets = [ + { + ip_cidr_range = "10.0.0.0/24" + name = "production" + region = "europe-west1" + secondary_ip_ranges = { + pods = "172.16.0.0/20" + services = "192.168.0.0/24" + } + }, + { + ip_cidr_range = "10.0.16.0/24" + name = "production" + region = "europe-west2" + } + ] +} +# tftest modules=1 resources=5 inventory=firewall_policy_enforcement_order.yaml +``` + +### IPv6 + +A non-overlapping private IPv6 address space can be configured for the VPC via the `ipv6_config` variable. If an internal range is not specified, a unique /48 ULA prefix from the `fd20::/20` range is assigned. + +```hcl +module "vpc" { + source = "./fabric/modules/net-vpc" + project_id = "my-project" + name = "my-network" + ipv6_config = { + # internal_range is optional + enable_ula_internal = true + internal_range = "fd20:6b2:27e5:0:0:0:0:0/48" + } + subnets = [ + { + ip_cidr_range = "10.0.0.0/24" + name = "test" + region = "europe-west1" + ipv6 = {} + }, + { + ip_cidr_range = "10.0.1.0/24" + name = "test" + region = "europe-west3" + ipv6 = { + access_type = "EXTERNAL" + } + } + ] +} +# tftest modules=1 resources=5 inventory=ipv6.yaml +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [name](variables.tf#L95) | The name of the network being created. | string | ✓ | | +| [project_id](variables.tf#L111) | The ID of the project where this VPC will be created. | string | ✓ | | +| [auto_create_subnetworks](variables.tf#L17) | Set to true to create an auto mode subnet, defaults to custom mode. | bool | | false | +| [create_googleapis_routes](variables.tf#L23) | Toggle creation of googleapis private/restricted routes. Disabled when vpc creation is turned off, or when set to null. | object({…}) | | {} | +| [delete_default_routes_on_create](variables.tf#L34) | Set to true to delete the default routes at creation time. | bool | | false | +| [description](variables.tf#L40) | An optional description of this resource (triggers recreation on change). | string | | "Terraform-managed." | +| [dns_policy](variables.tf#L46) | DNS policy setup for the VPC. | object({…}) | | null | +| [factories_config](variables.tf#L59) | Paths to data files and folders that enable factory functionality. | object({…}) | | null | +| [firewall_policy_enforcement_order](variables.tf#L67) | Order that Firewall Rules and Firewall Policies are evaluated. Can be either 'BEFORE_CLASSIC_FIREWALL' or 'AFTER_CLASSIC_FIREWALL'. | string | | "AFTER_CLASSIC_FIREWALL" | +| [ipv6_config](variables.tf#L79) | Optional IPv6 configuration for this network. | object({…}) | | {} | +| [mtu](variables.tf#L89) | Maximum Transmission Unit in bytes. The minimum value for this field is 1460 (the default) and the maximum value is 1500 bytes. | number | | null | +| [peering_config](variables.tf#L100) | VPC peering configuration. | object({…}) | | null | +| [psa_config](variables.tf#L116) | The Private Service Access configuration for Service Networking. | object({…}) | | null | +| [routes](variables.tf#L126) | Network routes, keyed by name. | map(object({…})) | | {} | +| [routing_mode](variables.tf#L147) | The network routing mode (default 'GLOBAL'). | string | | "GLOBAL" | +| [shared_vpc_host](variables.tf#L157) | Enable shared VPC for this project. | bool | | false | +| [shared_vpc_service_projects](variables.tf#L163) | Shared VPC service projects to register with this host. | list(string) | | [] | +| [subnets](variables.tf#L169) | Subnet configuration. | list(object({…})) | | [] | +| [subnets_proxy_only](variables.tf#L216) | List of proxy-only subnets for Regional HTTPS or Internal HTTPS load balancers. Note: Only one proxy-only subnet for each VPC network in each region can be active. | list(object({…})) | | [] | +| [subnets_psc](variables.tf#L250) | List of subnets for Private Service Connect service producers. | list(object({…})) | | [] | +| [vpc_create](variables.tf#L282) | Create VPC. When set to false, uses a data source to reference existing VPC. | bool | | true | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [id](outputs.tf#L17) | Fully qualified network id. | | +| [internal_ipv6_range](outputs.tf#L29) | ULA range. | | +| [name](outputs.tf#L34) | Network name. | | +| [network](outputs.tf#L46) | Network resource. | | +| [project_id](outputs.tf#L58) | Project ID containing the network. Use this when you need to create resources *after* the VPC is fully set up (e.g. subnets created, shared VPC service projects attached, Private Service Networking configured). | | +| [self_link](outputs.tf#L71) | Network self link. | | +| [subnet_ids](outputs.tf#L83) | Map of subnet IDs keyed by name. | | +| [subnet_ips](outputs.tf#L88) | Map of subnet address ranges keyed by name. | | +| [subnet_ipv6_external_prefixes](outputs.tf#L95) | Map of subnet external IPv6 prefixes keyed by name. | | +| [subnet_regions](outputs.tf#L103) | Map of subnet regions keyed by name. | | +| [subnet_secondary_ranges](outputs.tf#L110) | Map of subnet secondary ranges keyed by name. | | +| [subnet_self_links](outputs.tf#L121) | Map of subnet self links keyed by name. | | +| [subnets](outputs.tf#L126) | Subnet resources. | | +| [subnets_proxy_only](outputs.tf#L131) | L7 ILB or L7 Regional LB subnet resources. | | +| [subnets_psc](outputs.tf#L136) | Private Service Connect subnet resources. | | + diff --git a/assets/modules-fabric/v26/net-vpc/main.tf b/assets/modules-fabric/v26/net-vpc/main.tf new file mode 100644 index 0000000..7c7ee56 --- /dev/null +++ b/assets/modules-fabric/v26/net-vpc/main.tf @@ -0,0 +1,132 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + network = ( + var.vpc_create + ? try(google_compute_network.network.0, null) + : try(data.google_compute_network.network.0, null) + ) + peer_network = ( + var.peering_config == null + ? null + : element(reverse(split("/", var.peering_config.peer_vpc_self_link)), 0) + ) +} + +data "google_compute_network" "network" { + count = var.vpc_create ? 0 : 1 + project = var.project_id + name = var.name +} + +resource "google_compute_network" "network" { + count = var.vpc_create ? 1 : 0 + project = var.project_id + name = var.name + description = var.description + auto_create_subnetworks = var.auto_create_subnetworks + delete_default_routes_on_create = var.delete_default_routes_on_create + mtu = var.mtu + routing_mode = var.routing_mode + network_firewall_policy_enforcement_order = var.firewall_policy_enforcement_order + enable_ula_internal_ipv6 = var.ipv6_config.enable_ula_internal + internal_ipv6_range = var.ipv6_config.internal_range +} + +resource "google_compute_network_peering" "local" { + provider = google-beta + count = var.peering_config == null ? 0 : 1 + name = "${var.name}-${local.peer_network}" + network = local.network.self_link + peer_network = var.peering_config.peer_vpc_self_link + export_custom_routes = var.peering_config.export_routes + import_custom_routes = var.peering_config.import_routes +} + +resource "google_compute_network_peering" "remote" { + provider = google-beta + count = ( + var.peering_config != null && try(var.peering_config.create_remote_peer, true) + ? 1 + : 0 + ) + name = "${local.peer_network}-${var.name}" + network = var.peering_config.peer_vpc_self_link + peer_network = local.network.self_link + export_custom_routes = var.peering_config.import_routes + import_custom_routes = var.peering_config.export_routes + depends_on = [google_compute_network_peering.local] +} + +resource "google_compute_shared_vpc_host_project" "shared_vpc_host" { + provider = google-beta + count = var.shared_vpc_host ? 1 : 0 + project = var.project_id + depends_on = [local.network] +} + +resource "google_compute_shared_vpc_service_project" "service_projects" { + provider = google-beta + for_each = toset( + var.shared_vpc_host && var.shared_vpc_service_projects != null + ? var.shared_vpc_service_projects + : [] + ) + host_project = var.project_id + service_project = each.value + depends_on = [google_compute_shared_vpc_host_project.shared_vpc_host] +} + +resource "google_dns_policy" "default" { + count = var.dns_policy == null ? 0 : 1 + project = var.project_id + name = var.name + enable_inbound_forwarding = try(var.dns_policy.inbound, null) + enable_logging = try(var.dns_policy.logging, null) + networks { + network_url = local.network.id + } + + dynamic "alternative_name_server_config" { + for_each = var.dns_policy.outbound != null ? [""] : [] + content { + dynamic "target_name_servers" { + for_each = ( + var.dns_policy.outbound.private_ns != null + ? var.dns_policy.outbound.private_ns + : [] + ) + iterator = ns + content { + ipv4_address = ns.value + forwarding_path = "private" + } + } + dynamic "target_name_servers" { + for_each = ( + var.dns_policy.outbound.public_ns != null + ? var.dns_policy.outbound.public_ns + : [] + ) + iterator = ns + content { + ipv4_address = ns.value + } + } + } + } +} diff --git a/assets/modules-fabric/v26/net-vpc/outputs.tf b/assets/modules-fabric/v26/net-vpc/outputs.tf new file mode 100644 index 0000000..503923d --- /dev/null +++ b/assets/modules-fabric/v26/net-vpc/outputs.tf @@ -0,0 +1,139 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "id" { + description = "Fully qualified network id." + value = local.network.id + depends_on = [ + google_compute_network_peering.local, + google_compute_network_peering.remote, + google_compute_shared_vpc_host_project.shared_vpc_host, + google_compute_shared_vpc_service_project.service_projects, + google_service_networking_connection.psa_connection + ] +} + +output "internal_ipv6_range" { + description = "ULA range." + value = try(local.network.internal_ipv6_range, null) +} + +output "name" { + description = "Network name." + value = local.network.name + depends_on = [ + google_compute_network_peering.local, + google_compute_network_peering.remote, + google_compute_shared_vpc_host_project.shared_vpc_host, + google_compute_shared_vpc_service_project.service_projects, + google_service_networking_connection.psa_connection + ] +} + +output "network" { + description = "Network resource." + value = local.network + depends_on = [ + google_compute_network_peering.local, + google_compute_network_peering.remote, + google_compute_shared_vpc_host_project.shared_vpc_host, + google_compute_shared_vpc_service_project.service_projects, + google_service_networking_connection.psa_connection + ] +} + +output "project_id" { + description = "Project ID containing the network. Use this when you need to create resources *after* the VPC is fully set up (e.g. subnets created, shared VPC service projects attached, Private Service Networking configured)." + value = var.project_id + depends_on = [ + google_compute_subnetwork.subnetwork, + google_compute_network_peering.local, + google_compute_network_peering.remote, + google_compute_shared_vpc_host_project.shared_vpc_host, + google_compute_shared_vpc_service_project.service_projects, + google_service_networking_connection.psa_connection + ] +} + +output "self_link" { + description = "Network self link." + value = local.network.self_link + depends_on = [ + google_compute_network_peering.local, + google_compute_network_peering.remote, + google_compute_shared_vpc_host_project.shared_vpc_host, + google_compute_shared_vpc_service_project.service_projects, + google_service_networking_connection.psa_connection + ] +} + +output "subnet_ids" { + description = "Map of subnet IDs keyed by name." + value = { for k, v in google_compute_subnetwork.subnetwork : k => v.id } +} + +output "subnet_ips" { + description = "Map of subnet address ranges keyed by name." + value = { + for k, v in google_compute_subnetwork.subnetwork : k => v.ip_cidr_range + } +} + +output "subnet_ipv6_external_prefixes" { + description = "Map of subnet external IPv6 prefixes keyed by name." + value = { + for k, v in google_compute_subnetwork.subnetwork : + k => try(v.external_ipv6_prefix, null) + } +} + +output "subnet_regions" { + description = "Map of subnet regions keyed by name." + value = { + for k, v in google_compute_subnetwork.subnetwork : k => v.region + } +} + +output "subnet_secondary_ranges" { + description = "Map of subnet secondary ranges keyed by name." + value = { + for k, v in google_compute_subnetwork.subnetwork : + k => { + for range in v.secondary_ip_range : + range.range_name => range.ip_cidr_range + } + } +} + +output "subnet_self_links" { + description = "Map of subnet self links keyed by name." + value = { for k, v in google_compute_subnetwork.subnetwork : k => v.self_link } +} + +output "subnets" { + description = "Subnet resources." + value = { for k, v in google_compute_subnetwork.subnetwork : k => v } +} + +output "subnets_proxy_only" { + description = "L7 ILB or L7 Regional LB subnet resources." + value = { for k, v in google_compute_subnetwork.proxy_only : k => v } +} + +output "subnets_psc" { + description = "Private Service Connect subnet resources." + value = { for k, v in google_compute_subnetwork.psc : k => v } +} \ No newline at end of file diff --git a/assets/modules-fabric/v26/net-vpc/psa.tf b/assets/modules-fabric/v26/net-vpc/psa.tf new file mode 100644 index 0000000..19c47d4 --- /dev/null +++ b/assets/modules-fabric/v26/net-vpc/psa.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Private Service Access resources. + +locals { + psa_config_ranges = try(var.psa_config.ranges, {}) +} + +resource "google_compute_global_address" "psa_ranges" { + for_each = local.psa_config_ranges + project = var.project_id + name = each.key + purpose = "VPC_PEERING" + address_type = "INTERNAL" + address = split("/", each.value)[0] + prefix_length = split("/", each.value)[1] + network = local.network.id +} + +resource "google_service_networking_connection" "psa_connection" { + for_each = var.psa_config != null ? { 1 = 1 } : {} + network = local.network.id + service = "servicenetworking.googleapis.com" + reserved_peering_ranges = [ + for k, v in google_compute_global_address.psa_ranges : v.name + ] +} + +resource "google_compute_network_peering_routes_config" "psa_routes" { + for_each = var.psa_config != null ? { 1 = 1 } : {} + project = var.project_id + peering = google_service_networking_connection.psa_connection["1"].peering + network = local.network.name + export_custom_routes = var.psa_config.export_routes + import_custom_routes = var.psa_config.import_routes +} diff --git a/assets/modules-fabric/v26/net-vpc/routes.tf b/assets/modules-fabric/v26/net-vpc/routes.tf new file mode 100644 index 0000000..e6904e8 --- /dev/null +++ b/assets/modules-fabric/v26/net-vpc/routes.tf @@ -0,0 +1,110 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Route resources. + +locals { + _googleapis_ranges = { + private = "199.36.153.8/30" + private-6 = "2600:2d00:0002:2000::/64" + restricted = "199.36.153.4/30" + restricted-6 = "2600:2d00:0002:1000::/64" + } + _googleapis_routes = { + for k, v in local._googleapis_ranges : "${k}-googleapis" => { + description = "Terraform-managed." + dest_range = v + next_hop = "default-internet-gateway" + next_hop_type = "gateway" + priority = 1000 + tags = null + } + if( + var.vpc_create && + lookup(coalesce(var.create_googleapis_routes, {}), k, false) + ) + } + _routes = merge(local._googleapis_routes, coalesce(var.routes, {})) + routes = { + gateway = { for k, v in local._routes : k => v if v.next_hop_type == "gateway" } + ilb = { for k, v in local._routes : k => v if v.next_hop_type == "ilb" } + instance = { for k, v in local._routes : k => v if v.next_hop_type == "instance" } + ip = { for k, v in local._routes : k => v if v.next_hop_type == "ip" } + vpn_tunnel = { for k, v in local._routes : k => v if v.next_hop_type == "vpn_tunnel" } + } +} + +resource "google_compute_route" "gateway" { + for_each = local.routes.gateway + project = var.project_id + network = local.network.name + name = "${var.name}-${each.key}" + description = each.value.description + dest_range = each.value.dest_range + priority = each.value.priority + tags = each.value.tags + next_hop_gateway = each.value.next_hop +} + +resource "google_compute_route" "ilb" { + for_each = local.routes.ilb + project = var.project_id + network = local.network.name + name = "${var.name}-${each.key}" + description = each.value.description + dest_range = each.value.dest_range + priority = each.value.priority + tags = each.value.tags + next_hop_ilb = each.value.next_hop +} + +resource "google_compute_route" "instance" { + for_each = local.routes.instance + project = var.project_id + network = local.network.name + name = "${var.name}-${each.key}" + description = each.value.description + dest_range = each.value.dest_range + priority = each.value.priority + tags = each.value.tags + next_hop_instance = each.value.next_hop + # not setting the instance zone will trigger a refresh + next_hop_instance_zone = regex("zones/([^/]+)/", each.value.next_hop)[0] +} + +resource "google_compute_route" "ip" { + for_each = local.routes.ip + project = var.project_id + network = local.network.name + name = "${var.name}-${each.key}" + description = each.value.description + dest_range = each.value.dest_range + priority = each.value.priority + tags = each.value.tags + next_hop_ip = each.value.next_hop +} + +resource "google_compute_route" "vpn_tunnel" { + for_each = local.routes.vpn_tunnel + project = var.project_id + network = local.network.name + name = "${var.name}-${each.key}" + description = each.value.description + dest_range = each.value.dest_range + priority = each.value.priority + tags = each.value.tags + next_hop_vpn_tunnel = each.value.next_hop +} diff --git a/assets/modules-fabric/v26/net-vpc/subnets.tf b/assets/modules-fabric/v26/net-vpc/subnets.tf new file mode 100644 index 0000000..fe5abea --- /dev/null +++ b/assets/modules-fabric/v26/net-vpc/subnets.tf @@ -0,0 +1,246 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Subnet resources. + +locals { + _factory_data = { + for f in try(fileset(var.factories_config.subnets_folder, "**/*.yaml"), []) : + trimsuffix(basename(f), ".yaml") => yamldecode(file("${var.factories_config.subnets_folder}/${f}")) + } + _factory_subnets = { + for k, v in local._factory_data : + "${v.region}/${try(v.name, k)}" => { + active = try(v.active, true) + description = try(v.description, null) + enable_private_access = try(v.enable_private_access, true) + flow_logs_config = can(v.flow_logs_config) ? { + aggregation_interval = try(v.flow_logs_config.aggregation_interval, null) + filter_expression = try(v.flow_logs_config.filter_expression, null) + flow_sampling = try(v.flow_logs_config.flow_sampling, null) + metadata = try(v.flow_logs_config.metadata, null) + metadata_fields = try(v.flow_logs_config.metadata_fields, null) + } : null + global = try(v.global, false) + ip_cidr_range = v.ip_cidr_range + ipv6 = !can(v.ipv6) ? null : { + access_type = try(v.ipv6.access_type, "INTERNAL") + } + name = try(v.name, k) + region = v.region + secondary_ip_ranges = try(v.secondary_ip_ranges, null) + iam = try(v.iam, {}) + iam_bindings = !can(v.iam_bindings) ? {} : { + for k2, v2 in v.iam_bindings : + k2 => { + role = v2.role + members = v2.members + condition = !can(v2.condition) ? null : { + expression = v2.condition.expression + title = v2.condition.title + description = try(v2.condition.description, null) + } + } + } + iam_bindings_additive = !can(v.iam_bindings_additive) ? {} : { + for k2, v2 in v.iam_bindings_additive : + k2 => { + member = v2.member + role = v2.role + condition = !can(v2.condition) ? null : { + expression = v2.condition.expression + title = v2.condition.title + description = try(v2.condition.description, null) + } + } + } + _is_regular = !try(v.psc == true, false) && !try(v.proxy_only == true, false) + _is_psc = try(v.psc == true, false) + _is_proxy_only = try(v.proxy_only == true, false) + } + } + + all_subnets = merge( + { for k, v in google_compute_subnetwork.subnetwork : k => v }, + { for k, v in google_compute_subnetwork.proxy_only : k => v }, + { for k, v in google_compute_subnetwork.psc : k => v } + ) + subnet_iam = flatten(concat( + [ + for s in concat(var.subnets, var.subnets_psc, var.subnets_proxy_only, values(local._factory_subnets)) : [ + for role, members in s.iam : + { + role = role + members = members + subnet = "${s.region}/${s.name}" + } + ] + ], + )) + subnet_iam_bindings = merge([ + for s in concat(var.subnets, var.subnets_psc, var.subnets_proxy_only, values(local._factory_subnets)) : { + for key, data in s.iam_bindings : + key => { + role = data.role + subnet = "${s.region}/${s.name}" + members = data.members + condition = data.condition + } + } + ]...) + # note: all additive bindings share a single namespace for the key. + # In other words, if you have multiple additive bindings with the + # same name, only one will be used + subnet_iam_bindings_additive = merge([ + for s in concat(var.subnets, var.subnets_psc, var.subnets_proxy_only, values(local._factory_subnets)) : { + for key, data in s.iam_bindings_additive : + key => { + role = data.role + subnet = "${s.region}/${s.name}" + member = data.member + condition = data.condition + } + } + ]...) + subnets = merge( + { for s in var.subnets : "${s.region}/${s.name}" => s }, + { for k, v in local._factory_subnets : k => v if v._is_regular } + ) + subnets_proxy_only = merge( + { for s in var.subnets_proxy_only : "${s.region}/${s.name}" => s }, + { for k, v in local._factory_subnets : k => v if v._is_proxy_only }, + ) + subnets_psc = merge( + { for s in var.subnets_psc : "${s.region}/${s.name}" => s }, + { for k, v in local._factory_subnets : k => v if v._is_psc } + ) +} + +resource "google_compute_subnetwork" "subnetwork" { + for_each = local.subnets + project = var.project_id + network = local.network.name + name = each.value.name + region = each.value.region + ip_cidr_range = each.value.ip_cidr_range + description = ( + each.value.description == null + ? "Terraform-managed." + : each.value.description + ) + private_ip_google_access = each.value.enable_private_access + secondary_ip_range = each.value.secondary_ip_ranges == null ? [] : [ + for name, range in each.value.secondary_ip_ranges : + { range_name = name, ip_cidr_range = range } + ] + stack_type = ( + try(each.value.ipv6, null) != null ? "IPV4_IPV6" : null + ) + ipv6_access_type = ( + try(each.value.ipv6, null) != null ? each.value.ipv6.access_type : null + ) + # private_ipv6_google_access = try(each.value.ipv6.enable_private_access, null) + dynamic "log_config" { + for_each = each.value.flow_logs_config != null ? [""] : [] + content { + aggregation_interval = each.value.flow_logs_config.aggregation_interval + filter_expr = each.value.flow_logs_config.filter_expression + flow_sampling = each.value.flow_logs_config.flow_sampling + metadata = each.value.flow_logs_config.metadata + metadata_fields = ( + each.value.flow_logs_config.metadata == "CUSTOM_METADATA" + ? each.value.flow_logs_config.metadata_fields + : null + ) + } + } +} + +resource "google_compute_subnetwork" "proxy_only" { + for_each = local.subnets_proxy_only + project = var.project_id + network = local.network.name + name = each.value.name + region = each.value.region + ip_cidr_range = each.value.ip_cidr_range + description = coalesce( + each.value.description, + "Terraform-managed proxy-only subnet for Regional HTTPS, Internal HTTPS or Cross-Regional HTTPS Internal LB." + ) + purpose = each.value.global ? "GLOBAL_MANAGED_PROXY" : "REGIONAL_MANAGED_PROXY" + role = each.value.active ? "ACTIVE" : "BACKUP" +} + +resource "google_compute_subnetwork" "psc" { + for_each = local.subnets_psc + project = var.project_id + network = local.network.name + name = each.value.name + region = each.value.region + ip_cidr_range = each.value.ip_cidr_range + description = coalesce( + each.value.description, + "Terraform-managed subnet for Private Service Connect (PSC NAT)." + ) + purpose = "PRIVATE_SERVICE_CONNECT" +} + + +resource "google_compute_subnetwork_iam_binding" "authoritative" { + for_each = { + for binding in local.subnet_iam : + "${binding.subnet}.${binding.role}" => binding + } + project = var.project_id + subnetwork = local.all_subnets[each.value.subnet].name + region = local.all_subnets[each.value.subnet].region + role = each.value.role + members = each.value.members +} + +resource "google_compute_subnetwork_iam_binding" "bindings" { + for_each = local.subnet_iam_bindings + project = var.project_id + subnetwork = local.all_subnets[each.value.subnet].name + region = local.all_subnets[each.value.subnet].region + role = each.value.role + members = each.value.members + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +resource "google_compute_subnetwork_iam_member" "bindings" { + for_each = local.subnet_iam_bindings_additive + project = var.project_id + subnetwork = local.all_subnets[each.value.subnet].name + region = local.all_subnets[each.value.subnet].region + role = each.value.role + member = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} diff --git a/assets/modules-fabric/v26/net-vpc/variables.tf b/assets/modules-fabric/v26/net-vpc/variables.tf new file mode 100644 index 0000000..5c4cc69 --- /dev/null +++ b/assets/modules-fabric/v26/net-vpc/variables.tf @@ -0,0 +1,286 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "auto_create_subnetworks" { + description = "Set to true to create an auto mode subnet, defaults to custom mode." + type = bool + default = false +} + +variable "create_googleapis_routes" { + description = "Toggle creation of googleapis private/restricted routes. Disabled when vpc creation is turned off, or when set to null." + type = object({ + private = optional(bool, true) + private-6 = optional(bool, false) + restricted = optional(bool, true) + restricted-6 = optional(bool, false) + }) + default = {} +} + +variable "delete_default_routes_on_create" { + description = "Set to true to delete the default routes at creation time." + type = bool + default = false +} + +variable "description" { + description = "An optional description of this resource (triggers recreation on change)." + type = string + default = "Terraform-managed." +} + +variable "dns_policy" { + description = "DNS policy setup for the VPC." + type = object({ + inbound = optional(bool) + logging = optional(bool) + outbound = optional(object({ + private_ns = list(string) + public_ns = list(string) + })) + }) + default = null +} + +variable "factories_config" { + description = "Paths to data files and folders that enable factory functionality." + type = object({ + subnets_folder = string + }) + default = null +} + +variable "firewall_policy_enforcement_order" { + description = "Order that Firewall Rules and Firewall Policies are evaluated. Can be either 'BEFORE_CLASSIC_FIREWALL' or 'AFTER_CLASSIC_FIREWALL'." + type = string + nullable = false + default = "AFTER_CLASSIC_FIREWALL" + + validation { + condition = var.firewall_policy_enforcement_order == "BEFORE_CLASSIC_FIREWALL" || var.firewall_policy_enforcement_order == "AFTER_CLASSIC_FIREWALL" + error_message = "Enforcement order must be BEFORE_CLASSIC_FIREWALL or AFTER_CLASSIC_FIREWALL." + } +} + +variable "ipv6_config" { + description = "Optional IPv6 configuration for this network." + type = object({ + enable_ula_internal = optional(bool) + internal_range = optional(string) + }) + nullable = false + default = {} +} + +variable "mtu" { + description = "Maximum Transmission Unit in bytes. The minimum value for this field is 1460 (the default) and the maximum value is 1500 bytes." + type = number + default = null +} + +variable "name" { + description = "The name of the network being created." + type = string +} + +variable "peering_config" { + description = "VPC peering configuration." + type = object({ + peer_vpc_self_link = string + create_remote_peer = optional(bool, true) + export_routes = optional(bool) + import_routes = optional(bool) + }) + default = null +} + +variable "project_id" { + description = "The ID of the project where this VPC will be created." + type = string +} + +variable "psa_config" { + description = "The Private Service Access configuration for Service Networking." + type = object({ + ranges = map(string) + export_routes = optional(bool, false) + import_routes = optional(bool, false) + }) + default = null +} + +variable "routes" { + description = "Network routes, keyed by name." + type = map(object({ + description = optional(string, "Terraform-managed.") + dest_range = string + next_hop_type = string # gateway, instance, ip, vpn_tunnel, ilb + next_hop = string + priority = optional(number) + tags = optional(list(string)) + })) + default = {} + nullable = false + validation { + condition = alltrue([ + for r in var.routes : + contains(["gateway", "instance", "ip", "vpn_tunnel", "ilb"], r.next_hop_type) + ]) + error_message = "Unsupported next hop type for route." + } +} + +variable "routing_mode" { + description = "The network routing mode (default 'GLOBAL')." + type = string + default = "GLOBAL" + validation { + condition = var.routing_mode == "GLOBAL" || var.routing_mode == "REGIONAL" + error_message = "Routing type must be GLOBAL or REGIONAL." + } +} + +variable "shared_vpc_host" { + description = "Enable shared VPC for this project." + type = bool + default = false +} + +variable "shared_vpc_service_projects" { + description = "Shared VPC service projects to register with this host." + type = list(string) + default = [] +} + +variable "subnets" { + description = "Subnet configuration." + type = list(object({ + name = string + ip_cidr_range = string + region = string + description = optional(string) + enable_private_access = optional(bool, true) + flow_logs_config = optional(object({ + aggregation_interval = optional(string) + filter_expression = optional(string) + flow_sampling = optional(number) + metadata = optional(string) + # only if metadata == "CUSTOM_METADATA" + metadata_fields = optional(list(string)) + })) + ipv6 = optional(object({ + access_type = optional(string, "INTERNAL") + # this field is marked for internal use in the API documentation + # enable_private_access = optional(string) + })) + secondary_ip_ranges = optional(map(string)) + + iam = optional(map(list(string)), {}) + iam_bindings = optional(map(object({ + role = string + members = list(string) + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })), {}) + iam_bindings_additive = optional(map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })), {}) + })) + default = [] + nullable = false +} + +variable "subnets_proxy_only" { + description = "List of proxy-only subnets for Regional HTTPS or Internal HTTPS load balancers. Note: Only one proxy-only subnet for each VPC network in each region can be active." + type = list(object({ + name = string + ip_cidr_range = string + region = string + description = optional(string) + active = optional(bool, true) + global = optional(bool, false) + + iam = optional(map(list(string)), {}) + iam_bindings = optional(map(object({ + role = string + members = list(string) + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })), {}) + iam_bindings_additive = optional(map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })), {}) + })) + default = [] + nullable = false +} + +variable "subnets_psc" { + description = "List of subnets for Private Service Connect service producers." + type = list(object({ + name = string + ip_cidr_range = string + region = string + description = optional(string) + + iam = optional(map(list(string)), {}) + iam_bindings = optional(map(object({ + role = string + members = list(string) + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })), {}) + iam_bindings_additive = optional(map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })), {}) + })) + default = [] + nullable = false +} + +variable "vpc_create" { + description = "Create VPC. When set to false, uses a data source to reference existing VPC." + type = bool + default = true +} diff --git a/assets/modules-fabric/v26/net-vpc/versions.tf b/assets/modules-fabric/v26/net-vpc/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/net-vpc/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/net-vpn-dynamic/README.md b/assets/modules-fabric/v26/net-vpn-dynamic/README.md new file mode 100644 index 0000000..5f79ffa --- /dev/null +++ b/assets/modules-fabric/v26/net-vpn-dynamic/README.md @@ -0,0 +1,87 @@ +# Cloud VPN Dynamic Module + +## Example + +This example shows how to configure a single VPN tunnel using a couple of extra features + +- custom advertisement on the tunnel's BGP session; if custom advertisement is not needed, simply set the `bgp_peer_options` attribute to `null` +- internally generated shared secret, which can be fetched from the module's `random_secret` output for reuse; a predefined secret can be used instead by assigning it to the `shared_secret` attribute + +```hcl +module "vm" { + source = "./fabric/modules/compute-vm" + project_id = "my-project" + zone = "europe-west1-b" + name = "my-vm" + network_interfaces = [{ + nat = true + network = var.vpc.self_link + subnetwork = var.subnet.self_link + }] + service_account = { + auto_create = true + } +} + +module "vpn-dynamic" { + source = "./fabric/modules/net-vpn-dynamic" + project_id = "my-project" + region = "europe-west1" + network = var.vpc.name + name = "gateway-1" + router_config = { + asn = 64514 + } + tunnels = { + remote-1 = { + bgp_peer = { + address = "169.254.139.134" + asn = 64513 + custom_advertise = { + all_subnets = true + all_vpc_subnets = false + all_peer_vpc_subnets = false + ip_ranges = { + "192.168.0.0/24" = "Advertised range description" + } + } + } + bgp_session_range = "169.254.139.133/30" + peer_ip = module.vm.external_ip + } + } +} +# tftest modules=2 resources=12 +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [name](variables.tf#L29) | VPN gateway name, and prefix used for dependent resources. | string | ✓ | | +| [network](variables.tf#L34) | VPC used for the gateway and routes. | string | ✓ | | +| [project_id](variables.tf#L39) | Project where resources will be created. | string | ✓ | | +| [region](variables.tf#L44) | Region used for resources. | string | ✓ | | +| [router_config](variables.tf#L49) | Cloud Router configuration for the VPN. If you want to reuse an existing router, set create to false and use name to specify the desired router. | object({…}) | ✓ | | +| [gateway_address](variables.tf#L17) | Optional address assigned to the VPN gateway. Ignored unless gateway_address_create is set to false. | string | | null | +| [gateway_address_create](variables.tf#L23) | Create external address assigned to the VPN gateway. Needs to be explicitly set to false to use address in gateway_address variable. | bool | | true | +| [tunnels](variables.tf#L64) | VPN tunnel configurations. | map(object({…})) | | {} | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [address](outputs.tf#L17) | VPN gateway address. | | +| [gateway](outputs.tf#L22) | VPN gateway resource. | | +| [id](outputs.tf#L27) | Fully qualified VPN gateway id. | | +| [name](outputs.tf#L32) | VPN gateway name. | | +| [random_secret](outputs.tf#L37) | Generated secret. | | +| [router](outputs.tf#L43) | Router resource (only if auto-created). | | +| [router_name](outputs.tf#L48) | Router name. | | +| [self_link](outputs.tf#L53) | VPN gateway self link. | | +| [tunnel_names](outputs.tf#L58) | VPN tunnel names. | | +| [tunnel_self_links](outputs.tf#L66) | VPN tunnel self links. | | +| [tunnels](outputs.tf#L74) | VPN tunnel resources. | | + + diff --git a/assets/modules-fabric/v26/net-vpn-dynamic/main.tf b/assets/modules-fabric/v26/net-vpn-dynamic/main.tf new file mode 100644 index 0000000..fcf2693 --- /dev/null +++ b/assets/modules-fabric/v26/net-vpn-dynamic/main.tf @@ -0,0 +1,160 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + gateway_address = ( + var.gateway_address_create + ? google_compute_address.gateway[0].address + : var.gateway_address + ) + router = ( + var.router_config.create + ? try(google_compute_router.router[0].name, null) + : var.router_config.name + ) + secret = random_id.secret.b64_url +} + +resource "google_compute_address" "gateway" { + count = var.gateway_address_create ? 1 : 0 + name = "vpn-${var.name}" + project = var.project_id + region = var.region +} + +resource "google_compute_forwarding_rule" "esp" { + name = "vpn-${var.name}-esp" + project = var.project_id + region = var.region + target = google_compute_vpn_gateway.gateway.self_link + ip_address = local.gateway_address + ip_protocol = "ESP" +} + +resource "google_compute_forwarding_rule" "udp-500" { + name = "vpn-${var.name}-udp-500" + project = var.project_id + region = var.region + target = google_compute_vpn_gateway.gateway.self_link + ip_address = local.gateway_address + ip_protocol = "UDP" + port_range = "500" +} + +resource "google_compute_forwarding_rule" "udp-4500" { + name = "vpn-${var.name}-udp-4500" + project = var.project_id + region = var.region + target = google_compute_vpn_gateway.gateway.self_link + ip_address = local.gateway_address + ip_protocol = "UDP" + port_range = "4500" +} + +resource "google_compute_router" "router" { + count = var.router_config.create ? 1 : 0 + name = coalesce(var.router_config.name, "vpn-${var.name}") + project = var.project_id + region = var.region + network = var.network + bgp { + advertise_mode = ( + var.router_config.custom_advertise != null + ? "CUSTOM" + : "DEFAULT" + ) + advertised_groups = ( + try(var.router_config.custom_advertise.all_subnets, false) + ? ["ALL_SUBNETS"] + : [] + ) + dynamic "advertised_ip_ranges" { + for_each = try(var.router_config.custom_advertise.ip_ranges, {}) + iterator = range + content { + range = range.key + description = range.value + } + } + keepalive_interval = try(var.router_config.keepalive, null) + asn = var.router_config.asn + } +} + +resource "google_compute_router_peer" "bgp_peer" { + for_each = var.tunnels + region = var.region + project = var.project_id + name = "${var.name}-${each.key}" + router = coalesce(each.value.router, local.router) + peer_ip_address = each.value.bgp_peer.address + peer_asn = each.value.bgp_peer.asn + advertised_route_priority = each.value.bgp_peer.route_priority + advertise_mode = ( + try(each.value.bgp_peer.custom_advertise, null) != null + ? "CUSTOM" + : "DEFAULT" + ) + advertised_groups = concat( + try(each.value.bgp_peer.custom_advertise.all_subnets, false) ? ["ALL_SUBNETS"] : [], + try(each.value.bgp_peer.custom_advertise.all_vpc_subnets, false) ? ["ALL_VPC_SUBNETS"] : [], + try(each.value.bgp_peer.custom_advertise.all_peer_vpc_subnets, false) ? ["ALL_PEER_VPC_SUBNETS"] : [] + ) + dynamic "advertised_ip_ranges" { + for_each = try(each.value.bgp_peer.custom_advertise.ip_ranges, {}) + iterator = range + content { + range = range.key + description = range.value + } + } + interface = google_compute_router_interface.router_interface[each.key].name +} + +resource "google_compute_router_interface" "router_interface" { + for_each = var.tunnels + project = var.project_id + region = var.region + name = "${var.name}-${each.key}" + router = coalesce(each.value.router, local.router) + # FIXME: can bgp_session_range be null? + ip_range = each.value.bgp_session_range == "" ? null : each.value.bgp_session_range + vpn_tunnel = google_compute_vpn_tunnel.tunnels[each.key].name +} + +resource "google_compute_vpn_gateway" "gateway" { + name = var.name + project = var.project_id + region = var.region + network = var.network +} + +resource "google_compute_vpn_tunnel" "tunnels" { + for_each = var.tunnels + project = var.project_id + region = var.region + name = "${var.name}-${each.key}" + router = coalesce(each.value.router, local.router) + peer_ip = each.value.peer_ip + ike_version = each.value.ike_version + shared_secret = coalesce(each.value.shared_secret, local.secret) + target_vpn_gateway = google_compute_vpn_gateway.gateway.self_link + depends_on = [google_compute_forwarding_rule.esp] +} + +resource "random_id" "secret" { + byte_length = 8 +} diff --git a/assets/modules-fabric/v26/net-vpn-dynamic/outputs.tf b/assets/modules-fabric/v26/net-vpn-dynamic/outputs.tf new file mode 100644 index 0000000..2595a8f --- /dev/null +++ b/assets/modules-fabric/v26/net-vpn-dynamic/outputs.tf @@ -0,0 +1,80 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "address" { + description = "VPN gateway address." + value = local.gateway_address +} + +output "gateway" { + description = "VPN gateway resource." + value = google_compute_vpn_gateway.gateway +} + +output "id" { + description = "Fully qualified VPN gateway id." + value = google_compute_vpn_gateway.gateway.id +} + +output "name" { + description = "VPN gateway name." + value = google_compute_vpn_gateway.gateway.name +} + +output "random_secret" { + description = "Generated secret." + value = local.secret +} + + +output "router" { + description = "Router resource (only if auto-created)." + value = one(google_compute_router.router[*]) +} + +output "router_name" { + description = "Router name." + value = local.router +} + +output "self_link" { + description = "VPN gateway self link." + value = google_compute_vpn_gateway.gateway.self_link +} + +output "tunnel_names" { + description = "VPN tunnel names." + value = { + for name in keys(var.tunnels) : + name => try(google_compute_vpn_tunnel.tunnels[name].name, null) + } +} + +output "tunnel_self_links" { + description = "VPN tunnel self links." + value = { + for name in keys(var.tunnels) : + name => try(google_compute_vpn_tunnel.tunnels[name].self_link, null) + } +} + +output "tunnels" { + description = "VPN tunnel resources." + value = { + for name in keys(var.tunnels) : + name => try(google_compute_vpn_tunnel.tunnels[name], null) + } +} diff --git a/assets/modules-fabric/v26/net-vpn-dynamic/variables.tf b/assets/modules-fabric/v26/net-vpn-dynamic/variables.tf new file mode 100644 index 0000000..33d23a0 --- /dev/null +++ b/assets/modules-fabric/v26/net-vpn-dynamic/variables.tf @@ -0,0 +1,88 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "gateway_address" { + description = "Optional address assigned to the VPN gateway. Ignored unless gateway_address_create is set to false." + type = string + default = null +} + +variable "gateway_address_create" { + description = "Create external address assigned to the VPN gateway. Needs to be explicitly set to false to use address in gateway_address variable." + type = bool + default = true +} + +variable "name" { + description = "VPN gateway name, and prefix used for dependent resources." + type = string +} + +variable "network" { + description = "VPC used for the gateway and routes." + type = string +} + +variable "project_id" { + description = "Project where resources will be created." + type = string +} + +variable "region" { + description = "Region used for resources." + type = string +} + +variable "router_config" { + description = "Cloud Router configuration for the VPN. If you want to reuse an existing router, set create to false and use name to specify the desired router." + type = object({ + create = optional(bool, true) + asn = number + name = optional(string) + keepalive = optional(number) + custom_advertise = optional(object({ + all_subnets = bool + ip_ranges = map(string) + })) + }) + nullable = false +} + +variable "tunnels" { + description = "VPN tunnel configurations." + type = map(object({ + bgp_peer = object({ + address = string + asn = number + route_priority = optional(number, 1000) + custom_advertise = optional(object({ + all_subnets = bool + all_vpc_subnets = bool + all_peer_vpc_subnets = bool + ip_ranges = map(string) + })) + }) + # each BGP session on the same Cloud Router must use a unique /30 CIDR + # from the 169.254.0.0/16 block. + bgp_session_range = string + ike_version = optional(number, 2) + peer_ip = string + router = optional(string) + shared_secret = optional(string) + })) + default = {} + nullable = false +} diff --git a/assets/modules-fabric/v26/net-vpn-dynamic/versions.tf b/assets/modules-fabric/v26/net-vpn-dynamic/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/net-vpn-dynamic/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/net-vpn-ha/README.md b/assets/modules-fabric/v26/net-vpn-ha/README.md new file mode 100644 index 0000000..d2769cd --- /dev/null +++ b/assets/modules-fabric/v26/net-vpn-ha/README.md @@ -0,0 +1,158 @@ +# Cloud HA VPN Module + +This module makes it easy to deploy either GCP-to-GCP or GCP-to-On-prem [Cloud HA VPN](https://cloud.google.com/network-connectivity/docs/vpn/concepts/overview#ha-vpn). + +## Examples + +### GCP to GCP + +```hcl +module "vpn-1" { + source = "./fabric/modules/net-vpn-ha" + project_id = var.project_id + region = "europe-west4" + network = var.vpc1.self_link + name = "net1-to-net-2" + peer_gateways = { + default = { gcp = module.vpn-2.self_link } + } + router_config = { + asn = 64514 + custom_advertise = { + all_subnets = true + ip_ranges = { + "10.0.0.0/8" = "default" + } + } + } + tunnels = { + remote-0 = { + bgp_peer = { + address = "169.254.1.1" + asn = 64513 + } + bgp_session_range = "169.254.1.2/30" + vpn_gateway_interface = 0 + } + remote-1 = { + bgp_peer = { + address = "169.254.2.1" + asn = 64513 + } + bgp_session_range = "169.254.2.2/30" + vpn_gateway_interface = 1 + } + } +} + +module "vpn-2" { + source = "./fabric/modules/net-vpn-ha" + project_id = var.project_id + region = "europe-west4" + network = var.vpc2.self_link + name = "net2-to-net1" + router_config = { asn = 64513 } + peer_gateways = { + default = { gcp = module.vpn-1.self_link } + } + tunnels = { + remote-0 = { + bgp_peer = { + address = "169.254.1.2" + asn = 64514 + } + bgp_session_range = "169.254.1.1/30" + shared_secret = module.vpn-1.random_secret + vpn_gateway_interface = 0 + } + remote-1 = { + bgp_peer = { + address = "169.254.2.2" + asn = 64514 + } + bgp_session_range = "169.254.2.1/30" + shared_secret = module.vpn-1.random_secret + vpn_gateway_interface = 1 + } + } +} +# tftest modules=2 resources=18 +``` + +Note: When using the `for_each` meta-argument you might experience a Cycle Error due to the multiple `net-vpn-ha` modules referencing each other. To fix this you can create the [google_compute_ha_vpn_gateway](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_ha_vpn_gateway) resources separately and reference them in the `net-vpn-ha` module via the `vpn_gateway` and `peer_gcp_gateway` variables. + +### GCP to on-prem + +```hcl +module "vpn_ha" { + source = "./fabric/modules/net-vpn-ha" + project_id = var.project_id + region = var.region + network = var.vpc.self_link + name = "mynet-to-onprem" + peer_gateways = { + default = { + external = { + redundancy_type = "SINGLE_IP_INTERNALLY_REDUNDANT" + interfaces = ["8.8.8.8"] # on-prem router ip address + } + } + } + router_config = { asn = 64514 } + tunnels = { + remote-0 = { + bgp_peer = { + address = "169.254.1.1" + asn = 64513 + } + bgp_session_range = "169.254.1.2/30" + peer_external_gateway_interface = 0 + shared_secret = "mySecret" + vpn_gateway_interface = 0 + } + remote-1 = { + bgp_peer = { + address = "169.254.2.1" + asn = 64513 + } + bgp_session_range = "169.254.2.2/30" + peer_external_gateway_interface = 0 + shared_secret = "mySecret" + vpn_gateway_interface = 1 + } + } +} +# tftest modules=1 resources=10 +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [name](variables.tf#L17) | VPN Gateway name (if an existing VPN Gateway is not used), and prefix used for dependent resources. | string | ✓ | | +| [network](variables.tf#L22) | VPC used for the gateway and routes. | string | ✓ | | +| [project_id](variables.tf#L47) | Project where resources will be created. | string | ✓ | | +| [region](variables.tf#L52) | Region used for resources. | string | ✓ | | +| [router_config](variables.tf#L57) | Cloud Router configuration for the VPN. If you want to reuse an existing router, set create to false and use name to specify the desired router. | object({…}) | ✓ | | +| [peer_gateways](variables.tf#L27) | Configuration of the (external or GCP) peer gateway. | map(object({…})) | | {} | +| [tunnels](variables.tf#L72) | VPN tunnel configurations. | map(object({…})) | | {} | +| [vpn_gateway](variables.tf#L100) | HA VPN Gateway Self Link for using an existing HA VPN Gateway. Ignored if `vpn_gateway_create` is set to `true`. | string | | null | +| [vpn_gateway_create](variables.tf#L106) | Create HA VPN Gateway. Set to null to avoid creation. | object({…}) | | {} | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [bgp_peers](outputs.tf#L18) | BGP peer resources. | | +| [external_gateway](outputs.tf#L25) | External VPN gateway resource. | | +| [gateway](outputs.tf#L30) | VPN gateway resource (only if auto-created). | | +| [id](outputs.tf#L35) | Fully qualified VPN gateway id. | | +| [name](outputs.tf#L42) | VPN gateway name (only if auto-created). . | | +| [random_secret](outputs.tf#L47) | Generated secret. | | +| [router](outputs.tf#L52) | Router resource (only if auto-created). | | +| [router_name](outputs.tf#L57) | Router name. | | +| [self_link](outputs.tf#L62) | HA VPN gateway self link. | | +| [tunnel_names](outputs.tf#L67) | VPN tunnel names. | | +| [tunnel_self_links](outputs.tf#L75) | VPN tunnel self links. | | +| [tunnels](outputs.tf#L83) | VPN tunnel resources. | | + diff --git a/assets/modules-fabric/v26/net-vpn-ha/main.tf b/assets/modules-fabric/v26/net-vpn-ha/main.tf new file mode 100644 index 0000000..d1a555e --- /dev/null +++ b/assets/modules-fabric/v26/net-vpn-ha/main.tf @@ -0,0 +1,154 @@ + +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + peer_gateways_external = { + for k, v in var.peer_gateways : k => v.external if v.external != null + } + peer_gateways_gcp = { + for k, v in var.peer_gateways : k => v.gcp if v.gcp != null + } + router = ( + var.router_config.create + ? try(google_compute_router.router[0].name, null) + : var.router_config.name + ) + vpn_gateway = ( + var.vpn_gateway_create != null + ? try(google_compute_ha_vpn_gateway.ha_gateway[0].self_link, null) + : var.vpn_gateway + ) + secret = random_id.secret.b64_url +} + +resource "google_compute_ha_vpn_gateway" "ha_gateway" { + count = var.vpn_gateway_create != null ? 1 : 0 + name = var.name + project = var.project_id + region = var.region + network = var.network +} + +resource "google_compute_external_vpn_gateway" "external_gateway" { + for_each = local.peer_gateways_external + name = "${var.name}-${each.key}" + project = var.project_id + redundancy_type = each.value.redundancy_type + description = each.value.description + dynamic "interface" { + for_each = each.value.interfaces + content { + id = interface.key + ip_address = interface.value + } + } +} + +resource "google_compute_router" "router" { + count = var.router_config.create ? 1 : 0 + name = coalesce(var.router_config.name, "vpn-${var.name}") + project = var.project_id + region = var.region + network = var.network + bgp { + advertise_mode = ( + var.router_config.custom_advertise != null + ? "CUSTOM" + : "DEFAULT" + ) + advertised_groups = ( + try(var.router_config.custom_advertise.all_subnets, false) + ? ["ALL_SUBNETS"] + : [] + ) + dynamic "advertised_ip_ranges" { + for_each = try(var.router_config.custom_advertise.ip_ranges, {}) + iterator = range + content { + range = range.key + description = range.value + } + } + keepalive_interval = try(var.router_config.keepalive, null) + asn = var.router_config.asn + } +} + +resource "google_compute_router_peer" "bgp_peer" { + for_each = var.tunnels + region = var.region + project = var.project_id + name = "${var.name}-${each.key}" + router = coalesce(each.value.router, local.router) + peer_ip_address = each.value.bgp_peer.address + peer_asn = each.value.bgp_peer.asn + advertised_route_priority = each.value.bgp_peer.route_priority + advertise_mode = ( + try(each.value.bgp_peer.custom_advertise, null) != null + ? "CUSTOM" + : "DEFAULT" + ) + advertised_groups = concat( + try(each.value.bgp_peer.custom_advertise.all_subnets, false) ? ["ALL_SUBNETS"] : [], + try(each.value.bgp_peer.custom_advertise.all_vpc_subnets, false) ? ["ALL_VPC_SUBNETS"] : [], + try(each.value.bgp_peer.custom_advertise.all_peer_vpc_subnets, false) ? ["ALL_PEER_VPC_SUBNETS"] : [] + ) + dynamic "advertised_ip_ranges" { + for_each = try(each.value.bgp_peer.custom_advertise.ip_ranges, {}) + iterator = range + content { + range = range.key + description = range.value + } + } + interface = google_compute_router_interface.router_interface[each.key].name +} + +resource "google_compute_router_interface" "router_interface" { + for_each = var.tunnels + project = var.project_id + region = var.region + name = "${var.name}-${each.key}" + router = local.router + # FIXME: can bgp_session_range be null? + ip_range = each.value.bgp_session_range == "" ? null : each.value.bgp_session_range + vpn_tunnel = google_compute_vpn_tunnel.tunnels[each.key].name +} + +resource "google_compute_vpn_tunnel" "tunnels" { + for_each = var.tunnels + project = var.project_id + region = var.region + name = "${var.name}-${each.key}" + router = local.router + peer_external_gateway = try( + google_compute_external_vpn_gateway.external_gateway[each.value.peer_gateway].id, + null + ) + peer_external_gateway_interface = each.value.peer_external_gateway_interface + peer_gcp_gateway = lookup( + local.peer_gateways_gcp, each.value.peer_gateway, null + ) + vpn_gateway_interface = each.value.vpn_gateway_interface + ike_version = each.value.ike_version + shared_secret = coalesce(each.value.shared_secret, local.secret) + vpn_gateway = local.vpn_gateway +} + +resource "random_id" "secret" { + byte_length = 8 +} diff --git a/assets/modules-fabric/v26/net-vpn-ha/outputs.tf b/assets/modules-fabric/v26/net-vpn-ha/outputs.tf new file mode 100644 index 0000000..2655eea --- /dev/null +++ b/assets/modules-fabric/v26/net-vpn-ha/outputs.tf @@ -0,0 +1,89 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +output "bgp_peers" { + description = "BGP peer resources." + value = { + for k, v in google_compute_router_peer.bgp_peer : k => v + } +} + +output "external_gateway" { + description = "External VPN gateway resource." + value = one(google_compute_external_vpn_gateway.external_gateway[*]) +} + +output "gateway" { + description = "VPN gateway resource (only if auto-created)." + value = one(google_compute_ha_vpn_gateway.ha_gateway[*]) +} + +output "id" { + description = "Fully qualified VPN gateway id." + value = ( + "projects/${var.project_id}/regions/${var.region}/vpnGateways/${var.name}" + ) +} + +output "name" { + description = "VPN gateway name (only if auto-created). ." + value = one(google_compute_ha_vpn_gateway.ha_gateway[*].name) +} + +output "random_secret" { + description = "Generated secret." + value = local.secret +} + +output "router" { + description = "Router resource (only if auto-created)." + value = one(google_compute_router.router[*]) +} + +output "router_name" { + description = "Router name." + value = local.router +} + +output "self_link" { + description = "HA VPN gateway self link." + value = local.vpn_gateway +} + +output "tunnel_names" { + description = "VPN tunnel names." + value = { + for name in keys(var.tunnels) : + name => try(google_compute_vpn_tunnel.tunnels[name].name, null) + } +} + +output "tunnel_self_links" { + description = "VPN tunnel self links." + value = { + for name in keys(var.tunnels) : + name => try(google_compute_vpn_tunnel.tunnels[name].self_link, null) + } +} + +output "tunnels" { + description = "VPN tunnel resources." + value = { + for name in keys(var.tunnels) : + name => try(google_compute_vpn_tunnel.tunnels[name], null) + } +} diff --git a/assets/modules-fabric/v26/net-vpn-ha/variables.tf b/assets/modules-fabric/v26/net-vpn-ha/variables.tf new file mode 100644 index 0000000..50a123a --- /dev/null +++ b/assets/modules-fabric/v26/net-vpn-ha/variables.tf @@ -0,0 +1,112 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "name" { + description = "VPN Gateway name (if an existing VPN Gateway is not used), and prefix used for dependent resources." + type = string +} + +variable "network" { + description = "VPC used for the gateway and routes." + type = string +} + +variable "peer_gateways" { + description = "Configuration of the (external or GCP) peer gateway." + type = map(object({ + external = optional(object({ + redundancy_type = string + interfaces = list(string) + description = optional(string, "Terraform managed external VPN gateway") + })) + gcp = optional(string) + })) + nullable = false + default = {} + validation { + condition = alltrue([ + for k, v in var.peer_gateways : (v.external != null) != (v.gcp != null) + ]) + error_message = "Peer gateway configuration must define exactly one between `external` and `gcp`." + } +} + +variable "project_id" { + description = "Project where resources will be created." + type = string +} + +variable "region" { + description = "Region used for resources." + type = string +} + +variable "router_config" { + description = "Cloud Router configuration for the VPN. If you want to reuse an existing router, set create to false and use name to specify the desired router." + type = object({ + create = optional(bool, true) + asn = number + name = optional(string) + keepalive = optional(number) + custom_advertise = optional(object({ + all_subnets = bool + ip_ranges = map(string) + })) + }) + nullable = false +} + +variable "tunnels" { + description = "VPN tunnel configurations." + type = map(object({ + bgp_peer = object({ + address = string + asn = number + route_priority = optional(number, 1000) + custom_advertise = optional(object({ + all_subnets = bool + all_vpc_subnets = bool + all_peer_vpc_subnets = bool + ip_ranges = map(string) + })) + }) + # each BGP session on the same Cloud Router must use a unique /30 CIDR + # from the 169.254.0.0/16 block. + bgp_session_range = string + ike_version = optional(number, 2) + peer_external_gateway_interface = optional(number) + peer_gateway = optional(string, "default") + router = optional(string) + shared_secret = optional(string) + vpn_gateway_interface = number + })) + default = {} + nullable = false +} + +variable "vpn_gateway" { + description = "HA VPN Gateway Self Link for using an existing HA VPN Gateway. Ignored if `vpn_gateway_create` is set to `true`." + type = string + default = null +} + +variable "vpn_gateway_create" { + description = "Create HA VPN Gateway. Set to null to avoid creation." + type = object({ + description = optional(string, "Terraform managed external VPN gateway") + }) + default = {} +} diff --git a/assets/modules-fabric/v26/net-vpn-ha/versions.tf b/assets/modules-fabric/v26/net-vpn-ha/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/net-vpn-ha/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/net-vpn-static/README.md b/assets/modules-fabric/v26/net-vpn-static/README.md new file mode 100644 index 0000000..902fc1e --- /dev/null +++ b/assets/modules-fabric/v26/net-vpn-static/README.md @@ -0,0 +1,63 @@ +# Cloud VPN Route-based Module + +## Example + +```hcl +module "addresses" { + source = "./fabric/modules/net-address" + project_id = var.project_id + external_addresses = { + vpn = { region = "europe-west1" } + } +} + +module "vpn" { + source = "./fabric/modules/net-vpn-static" + project_id = var.project_id + region = var.region + network = var.vpc.self_link + name = "remote" + gateway_address_create = false + gateway_address = module.addresses.external_addresses["vpn"].address + remote_ranges = ["10.10.0.0/24"] + tunnels = { + remote-0 = { + peer_ip = "1.1.1.1" + shared_secret = "mysecret" + traffic_selectors = { local = ["0.0.0.0/0"], remote = ["0.0.0.0/0"] } + } + } +} +# tftest modules=2 resources=8 +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [name](variables.tf#L29) | VPN gateway name, and prefix used for dependent resources. | string | ✓ | | +| [network](variables.tf#L34) | VPC used for the gateway and routes. | string | ✓ | | +| [project_id](variables.tf#L39) | Project where resources will be created. | string | ✓ | | +| [region](variables.tf#L44) | Region used for resources. | string | ✓ | | +| [gateway_address](variables.tf#L17) | Optional address assigned to the VPN gateway. Ignored unless gateway_address_create is set to false. | string | | null | +| [gateway_address_create](variables.tf#L23) | Create external address assigned to the VPN gateway. Needs to be explicitly set to false to use address in gateway_address variable. | bool | | true | +| [remote_ranges](variables.tf#L49) | Remote IP CIDR ranges. | list(string) | | [] | +| [route_priority](variables.tf#L56) | Route priority, defaults to 1000. | number | | 1000 | +| [tunnels](variables.tf#L62) | VPN tunnel configurations. | map(object({…})) | | {} | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [address](outputs.tf#L17) | VPN gateway address. | | +| [gateway](outputs.tf#L22) | VPN gateway resource. | | +| [id](outputs.tf#L27) | Fully qualified VPN gateway id. | | +| [name](outputs.tf#L32) | VPN gateway name. | | +| [random_secret](outputs.tf#L37) | Generated secret. | | +| [self_link](outputs.tf#L42) | VPN gateway self link. | | +| [tunnel_names](outputs.tf#L47) | VPN tunnel names. | | +| [tunnel_self_links](outputs.tf#L55) | VPN tunnel self links. | | +| [tunnels](outputs.tf#L63) | VPN tunnel resources. | | + + diff --git a/assets/modules-fabric/v26/net-vpn-static/main.tf b/assets/modules-fabric/v26/net-vpn-static/main.tf new file mode 100644 index 0000000..f05771c --- /dev/null +++ b/assets/modules-fabric/v26/net-vpn-static/main.tf @@ -0,0 +1,101 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + gateway_address = ( + var.gateway_address_create + ? google_compute_address.gateway[0].address + : var.gateway_address + ) + route_pairs = { + for pair in setproduct(keys(var.tunnels), var.remote_ranges) : + "${pair[0]}-${join("-", regexall("[0-9]+", pair[1]))}" => { + tunnel = pair[0], range = pair[1] + } + } + secret = random_id.secret.b64_url +} + +resource "google_compute_address" "gateway" { + count = var.gateway_address_create ? 1 : 0 + name = "vpn-${var.name}" + project = var.project_id + region = var.region +} + +resource "google_compute_forwarding_rule" "esp" { + name = "vpn-${var.name}-esp" + project = var.project_id + region = var.region + target = google_compute_vpn_gateway.gateway.self_link + ip_address = local.gateway_address + ip_protocol = "ESP" +} + +resource "google_compute_forwarding_rule" "udp-500" { + name = "vpn-${var.name}-udp-500" + project = var.project_id + region = var.region + target = google_compute_vpn_gateway.gateway.self_link + ip_address = local.gateway_address + ip_protocol = "UDP" + port_range = "500" +} + +resource "google_compute_forwarding_rule" "udp-4500" { + name = "vpn-${var.name}-udp-4500" + project = var.project_id + region = var.region + target = google_compute_vpn_gateway.gateway.self_link + ip_address = local.gateway_address + ip_protocol = "UDP" + port_range = "4500" +} + +resource "google_compute_route" "route" { + for_each = local.route_pairs + name = "vpn-${var.name}-${each.key}" + project = var.project_id + network = var.network + dest_range = each.value.range + priority = var.route_priority + next_hop_vpn_tunnel = google_compute_vpn_tunnel.tunnels[each.value.tunnel].self_link +} + +resource "google_compute_vpn_gateway" "gateway" { + name = var.name + project = var.project_id + region = var.region + network = var.network +} + +resource "google_compute_vpn_tunnel" "tunnels" { + for_each = var.tunnels + name = "${var.name}-${each.key}" + project = var.project_id + region = var.region + peer_ip = each.value.peer_ip + local_traffic_selector = each.value.traffic_selectors.local + remote_traffic_selector = each.value.traffic_selectors.remote + ike_version = each.value.ike_version + shared_secret = coalesce(each.value.shared_secret, local.secret) + target_vpn_gateway = google_compute_vpn_gateway.gateway.self_link + depends_on = [google_compute_forwarding_rule.esp] +} + +resource "random_id" "secret" { + byte_length = 8 +} diff --git a/assets/modules-fabric/v26/net-vpn-static/outputs.tf b/assets/modules-fabric/v26/net-vpn-static/outputs.tf new file mode 100644 index 0000000..946063f --- /dev/null +++ b/assets/modules-fabric/v26/net-vpn-static/outputs.tf @@ -0,0 +1,69 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "address" { + description = "VPN gateway address." + value = local.gateway_address +} + +output "gateway" { + description = "VPN gateway resource." + value = google_compute_vpn_gateway.gateway +} + +output "id" { + description = "Fully qualified VPN gateway id." + value = google_compute_vpn_gateway.gateway.id +} + +output "name" { + description = "VPN gateway name." + value = google_compute_vpn_gateway.gateway.name +} + +output "random_secret" { + description = "Generated secret." + value = local.secret +} + +output "self_link" { + description = "VPN gateway self link." + value = google_compute_vpn_gateway.gateway.self_link +} + +output "tunnel_names" { + description = "VPN tunnel names." + value = { + for name in keys(var.tunnels) : + name => google_compute_vpn_tunnel.tunnels[name].name + } +} + +output "tunnel_self_links" { + description = "VPN tunnel self links." + value = { + for name in keys(var.tunnels) : + name => google_compute_vpn_tunnel.tunnels[name].self_link + } +} + +output "tunnels" { + description = "VPN tunnel resources." + value = { + for name in keys(var.tunnels) : + name => google_compute_vpn_tunnel.tunnels[name] + } +} diff --git a/assets/modules-fabric/v26/net-vpn-static/variables.tf b/assets/modules-fabric/v26/net-vpn-static/variables.tf new file mode 100644 index 0000000..935c543 --- /dev/null +++ b/assets/modules-fabric/v26/net-vpn-static/variables.tf @@ -0,0 +1,75 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "gateway_address" { + description = "Optional address assigned to the VPN gateway. Ignored unless gateway_address_create is set to false." + type = string + default = null +} + +variable "gateway_address_create" { + description = "Create external address assigned to the VPN gateway. Needs to be explicitly set to false to use address in gateway_address variable." + type = bool + default = true +} + +variable "name" { + description = "VPN gateway name, and prefix used for dependent resources." + type = string +} + +variable "network" { + description = "VPC used for the gateway and routes." + type = string +} + +variable "project_id" { + description = "Project where resources will be created." + type = string +} + +variable "region" { + description = "Region used for resources." + type = string +} + +variable "remote_ranges" { + description = "Remote IP CIDR ranges." + type = list(string) + default = [] + nullable = false +} + +variable "route_priority" { + description = "Route priority, defaults to 1000." + type = number + default = 1000 +} + +variable "tunnels" { + description = "VPN tunnel configurations." + type = map(object({ + ike_version = optional(number, 2) + peer_ip = string + shared_secret = optional(string) + traffic_selectors = object({ + local = list(string) + remote = list(string) + }) + })) + default = {} + nullable = false +} diff --git a/assets/modules-fabric/v26/net-vpn-static/versions.tf b/assets/modules-fabric/v26/net-vpn-static/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/net-vpn-static/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/organization/README.md b/assets/modules-fabric/v26/organization/README.md new file mode 100644 index 0000000..fd9ca09 --- /dev/null +++ b/assets/modules-fabric/v26/organization/README.md @@ -0,0 +1,482 @@ +# Organization Module + +This module allows managing several organization properties: + +- IAM bindings, both authoritative and additive +- custom IAM roles +- audit logging configuration for services +- organization policies +- organization policy custom constraints + +To manage organization policies, the `orgpolicy.googleapis.com` service should be enabled in the quota project. + +## TOC + + +- [TOC](#toc) +- [Example](#example) +- [IAM](#iam) +- [Organization Policies](#organization-policies) + - [Organization Policy Factory](#organization-policy-factory) + - [Organization Policy Custom Constraints](#organization-policy-custom-constraints) + - [Organization Policy Custom Constraints Factory](#organization-policy-custom-constraints-factory) +- [Hierarchical Firewall Policy Attachments](#hierarchical-firewall-policy-attachments) +- [Log Sinks](#log-sinks) +- [Data Access Logs](#data-access-logs) +- [Custom Roles](#custom-roles) +- [Tags](#tags) +- [Files](#files) +- [Variables](#variables) +- [Outputs](#outputs) + + +## Example + +```hcl +module "org" { + source = "./fabric/modules/organization" + organization_id = "organizations/1234567890" + group_iam = { + "cloud-owners@example.org" = ["roles/owner", "roles/projectCreator"] + } + iam = { + "roles/resourcemanager.projectCreator" = ["group:cloud-admins@example.org"] + } + iam_bindings_additive = { + am1-storage-admin = { + member = "user:am1@example.org" + role = "roles/storage.admin" + } + } + tags = { + allowexternal = { + description = "Allow external identities." + values = { + true = {}, false = {} + } + } + } + org_policies = { + "custom.gkeEnableAutoUpgrade" = { + rules = [{ enforce = true }] + } + "compute.disableGuestAttributesAccess" = { + rules = [{ enforce = true }] + } + "compute.skipDefaultNetworkCreation" = { + rules = [{ enforce = true }] + } + "iam.disableServiceAccountKeyCreation" = { + rules = [{ enforce = true }] + } + "iam.disableServiceAccountKeyUpload" = { + rules = [ + { + condition = { + expression = "resource.matchTagId('tagKeys/1234', 'tagValues/1234')" + title = "condition" + description = "test condition" + location = "somewhere" + } + enforce = true + }, + { + enforce = false + } + ] + } + "iam.allowedPolicyMemberDomains" = { + rules = [ + { + allow = { all = true } + condition = { + expression = "resource.matchTag('1234567890/allowexternal', 'true')" + title = "Allow external identities" + description = "Allow external identities when resource has the `allowexternal` tag set to true." + } + }, + { + allow = { values = ["C0xxxxxxx", "C0yyyyyyy"] } + condition = { + expression = "!resource.matchTag('1234567890/allowexternal', 'true')" + title = "" + description = "For any resource without allowexternal=true, only allow identities from restricted domains." + } + } + ] + } + + "compute.trustedImageProjects" = { + rules = [{ + allow = { + values = ["projects/my-project"] + } + }] + } + "compute.vmExternalIpAccess" = { + rules = [{ deny = { all = true } }] + } + } +} +# tftest modules=1 resources=15 inventory=basic.yaml +``` + +## IAM + +IAM is managed via several variables that implement different features and levels of control: + +- `iam` and `group_iam` configure authoritative bindings that manage individual roles exclusively, and are internally merged +- `iam_bindings` configure authoritative bindings with optional support for conditions, and are not internally merged with the previous two variables +- `iam_bindings_additive` configure additive bindings via individual role/member pairs with optional support conditions + +The authoritative and additive approaches can be used together, provided different roles are managed by each. Some care must also be taken with the `groups_iam` variable to ensure that variable keys are static values, so that Terraform is able to compute the dependency graph. + +Refer to the [project module](../project/README.md#iam) for examples of the IAM interface. + +## Organization Policies + +### Organization Policy Factory + +See the [organization policy factory in the project module](../project#organization-policy-factory). + +### Organization Policy Custom Constraints + +Refer to the [Creating and managing custom constraints](https://cloud.google.com/resource-manager/docs/organization-policy/creating-managing-custom-constraints) documentation for details on usage. +To manage organization policy custom constraints, the `orgpolicy.googleapis.com` service should be enabled in the quota project. + +```hcl +module "org" { + source = "./fabric/modules/organization" + organization_id = var.organization_id + + org_policy_custom_constraints = { + "custom.gkeEnableAutoUpgrade" = { + resource_types = ["container.googleapis.com/NodePool"] + method_types = ["CREATE"] + condition = "resource.management.autoUpgrade == true" + action_type = "ALLOW" + display_name = "Enable node auto-upgrade" + description = "All node pools must have node auto-upgrade enabled." + } + } + + # not necessarily to enforce on the org level, policy may be applied on folder/project levels + org_policies = { + "custom.gkeEnableAutoUpgrade" = { + rules = [{ enforce = true }] + } + } +} +# tftest modules=1 resources=2 inventory=custom-constraints.yaml +``` + +You can use the `id` or `custom_constraint_ids` outputs to prevent race conditions between the creation of a custom constraint and an organization policy using that constraint. Both of these outputs depend on the actual constraint, which would make any resource referring to them to wait for the creation of the constraint. + +### Organization Policy Custom Constraints Factory + +Org policy custom constraints can be loaded from a directory containing YAML files where each file defines one or more custom constraints. The structure of the YAML files is exactly the same as the `org_policy_custom_constraints` variable. + +The example below deploys a few org policy custom constraints split between two YAML files. + +```hcl +module "org" { + source = "./fabric/modules/organization" + organization_id = var.organization_id + org_policy_custom_constraints_data_path = "configs/custom-constraints" + org_policies = { + "custom.gkeEnableAutoUpgrade" = { + rules = [{ enforce = true }] + } + } +} +# tftest modules=1 resources=3 files=gke inventory=custom-constraints.yaml +``` + +```yaml +# tftest-file id=gke path=configs/custom-constraints/gke.yaml +custom.gkeEnableLogging: + resource_types: + - container.googleapis.com/Cluster + method_types: + - CREATE + - UPDATE + condition: resource.loggingService == "none" + action_type: DENY + display_name: Do not disable Cloud Logging +custom.gkeEnableAutoUpgrade: + resource_types: + - container.googleapis.com/NodePool + method_types: + - CREATE + condition: resource.management.autoUpgrade == true + action_type: ALLOW + display_name: Enable node auto-upgrade + description: All node pools must have node auto-upgrade enabled. +``` + +```yaml +# tftest-file id=dataproc path=configs/custom-constraints/dataproc.yaml +custom.dataprocNoMoreThan10Workers: + resource_types: + - dataproc.googleapis.com/Cluster + method_types: + - CREATE + - UPDATE + condition: resource.config.workerConfig.numInstances + resource.config.secondaryWorkerConfig.numInstances > 10 + action_type: DENY + display_name: Total number of worker instances cannot be larger than 10 + description: Cluster cannot have more than 10 workers, including primary and secondary workers. +``` + +## Hierarchical Firewall Policy Attachments + +Hierarchical firewall policies can be managed via the [`net-firewall-policy`](../net-firewall-policy/) module, including support for factories. Once a policy is available, attaching it to the organization can be done either in the firewall policy module itself, or here: + +```hcl +module "firewall-policy" { + source = "./fabric/modules/net-firewall-policy" + name = "test-1" + parent_id = var.organization_id + # attachment via the firewall policy module + # attachments = { + # org = var.organization_id + # } +} + +module "org" { + source = "./fabric/modules/organization" + organization_id = var.organization_id + # attachment via the organization module + firewall_policy = { + name = "test-1" + policy = module.firewall-policy.id + } +} +# tftest modules=2 resources=2 +``` + +## Log Sinks + +The following example shows how to define organization-level log sinks: + +```hcl +module "gcs" { + source = "./fabric/modules/gcs" + project_id = var.project_id + name = "gcs_sink" + force_destroy = true +} + +module "dataset" { + source = "./fabric/modules/bigquery-dataset" + project_id = var.project_id + id = "bq_sink" +} + +module "pubsub" { + source = "./fabric/modules/pubsub" + project_id = var.project_id + name = "pubsub_sink" +} + +module "bucket" { + source = "./fabric/modules/logging-bucket" + parent_type = "project" + parent = "my-project" + id = "bucket" +} + +module "org" { + source = "./fabric/modules/organization" + organization_id = var.organization_id + + logging_sinks = { + warnings = { + destination = module.gcs.id + filter = "severity=WARNING" + type = "storage" + } + info = { + bq_partitioned_table = true + destination = module.dataset.id + filter = "severity=INFO" + type = "bigquery" + } + notice = { + destination = module.pubsub.id + filter = "severity=NOTICE" + type = "pubsub" + } + debug = { + destination = module.bucket.id + filter = "severity=DEBUG" + exclusions = { + no-compute = "logName:compute" + } + type = "logging" + } + } + logging_exclusions = { + no-gce-instances = "resource.type=gce_instance" + } +} +# tftest modules=5 resources=13 inventory=logging.yaml +``` + +## Data Access Logs + +Activation of data access logs can be controlled via the `logging_data_access` variable. + +```hcl +module "org" { + source = "./fabric/modules/organization" + organization_id = var.organization_id + logging_data_access = { + allServices = { + # logs for principals listed here will be excluded + ADMIN_READ = ["group:organization-admins@example.org"] + } + "storage.googleapis.com" = { + DATA_READ = [] + DATA_WRITE = [] + } + } +} +# tftest modules=1 resources=2 inventory=logging-data-access.yaml +``` + +## Custom Roles + +Custom roles can be defined via the `custom_roles` variable, and referenced via the `custom_role_id` output: + +```hcl +module "org" { + source = "./fabric/modules/organization" + organization_id = var.organization_id + custom_roles = { + "myRole" = [ + "compute.instances.list", + ] + } + iam = { + (module.org.custom_role_id.myRole) = ["user:me@example.com"] + } +} +# tftest modules=1 resources=2 inventory=roles.yaml +``` + +## Tags + +Refer to the [Creating and managing tags](https://cloud.google.com/resource-manager/docs/tags/tags-creating-and-managing) documentation for details on usage. + +```hcl +module "org" { + source = "./fabric/modules/organization" + organization_id = var.organization_id + tags = { + environment = { + description = "Environment specification." + iam = { + "roles/resourcemanager.tagAdmin" = ["group:admins@example.com"] + } + values = { + dev = {} + prod = { + description = "Environment: production." + iam = { + "roles/resourcemanager.tagViewer" = ["user:user1@example.com"] + } + } + } + } + } + tag_bindings = { + env-prod = module.org.tag_values["environment/prod"].id + foo = "tagValues/12345678" + } +} +# tftest modules=1 resources=7 inventory=tags.yaml +``` + +You can also define network tags, through a dedicated variable *network_tags*: + +```hcl +module "org" { + source = "./fabric/modules/organization" + organization_id = var.organization_id + network_tags = { + net-environment = { + description = "This is a network tag." + network = "my_project/my_vpc" + iam = { + "roles/resourcemanager.tagAdmin" = ["group:admins@example.com"] + } + values = { + dev = null + prod = { + description = "Environment: production." + iam = { + "roles/resourcemanager.tagUser" = ["user:user1@example.com"] + } + } + } + } + } +} +# tftest modules=1 resources=5 inventory=network-tags.yaml +``` + + + +## Files + +| name | description | resources | +|---|---|---| +| [iam.tf](./iam.tf) | IAM bindings, roles and audit logging resources. | google_organization_iam_binding · google_organization_iam_custom_role · google_organization_iam_member | +| [logging.tf](./logging.tf) | Log sinks and data access logs. | google_bigquery_dataset_iam_member · google_logging_organization_exclusion · google_logging_organization_sink · google_organization_iam_audit_config · google_project_iam_member · google_pubsub_topic_iam_member · google_storage_bucket_iam_member | +| [main.tf](./main.tf) | Module-level locals and resources. | google_compute_firewall_policy_association · google_essential_contacts_contact | +| [org-policy-custom-constraints.tf](./org-policy-custom-constraints.tf) | None | google_org_policy_custom_constraint | +| [organization-policies.tf](./organization-policies.tf) | Organization-level organization policies. | google_org_policy_policy | +| [outputs.tf](./outputs.tf) | Module outputs. | | +| [tags.tf](./tags.tf) | None | google_tags_tag_binding · google_tags_tag_key · google_tags_tag_key_iam_binding · google_tags_tag_value · google_tags_tag_value_iam_binding | +| [variables.tf](./variables.tf) | Module variables. | | +| [versions.tf](./versions.tf) | Version pins. | | + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [organization_id](variables.tf#L211) | Organization id in organizations/nnnnnn format. | string | ✓ | | +| [contacts](variables.tf#L17) | List of essential contacts for this resource. Must be in the form EMAIL -> [NOTIFICATION_TYPES]. Valid notification types are ALL, SUSPENSION, SECURITY, TECHNICAL, BILLING, LEGAL, PRODUCT_UPDATES. | map(list(string)) | | {} | +| [custom_roles](variables.tf#L24) | Map of role name => list of permissions to create in this project. | map(list(string)) | | {} | +| [firewall_policy](variables.tf#L31) | Hierarchical firewall policies to associate to the organization. | object({…}) | | null | +| [group_iam](variables.tf#L40) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | +| [iam](variables.tf#L47) | IAM bindings, in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [iam_bindings](variables.tf#L54) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | map(object({…})) | | {} | +| [iam_bindings_additive](variables.tf#L69) | Individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | +| [logging_data_access](variables.tf#L84) | Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services. | map(map(list(string))) | | {} | +| [logging_exclusions](variables.tf#L99) | Logging exclusions for this organization in the form {NAME -> FILTER}. | map(string) | | {} | +| [logging_sinks](variables.tf#L106) | Logging sinks to create for the organization. | map(object({…})) | | {} | +| [network_tags](variables.tf#L136) | Network tags by key name. If `id` is provided, key creation is skipped. The `iam` attribute behaves like the similarly named one at module level. | map(object({…})) | | {} | +| [org_policies](variables.tf#L158) | Organization policies applied to this organization keyed by policy name. | map(object({…})) | | {} | +| [org_policies_data_path](variables.tf#L185) | Path containing org policies in YAML format. | string | | null | +| [org_policy_custom_constraints](variables.tf#L191) | Organization policy custom constraints keyed by constraint name. | map(object({…})) | | {} | +| [org_policy_custom_constraints_data_path](variables.tf#L205) | Path containing org policy custom constraints in YAML format. | string | | null | +| [tag_bindings](variables.tf#L220) | Tag bindings for this organization, in key => tag value id format. | map(string) | | null | +| [tags](variables.tf#L226) | Tags by key name. If `id` is provided, key or value creation is skipped. The `iam` attribute behaves like the similarly named one at module level. | map(object({…})) | | {} | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [custom_constraint_ids](outputs.tf#L17) | Map of CUSTOM_CONSTRAINTS => ID in the organization. | | +| [custom_role_id](outputs.tf#L22) | Map of custom role IDs created in the organization. | | +| [custom_roles](outputs.tf#L35) | Map of custom roles resources created in the organization. | | +| [id](outputs.tf#L40) | Fully qualified organization id. | | +| [network_tag_keys](outputs.tf#L57) | Tag key resources. | | +| [network_tag_values](outputs.tf#L66) | Tag value resources. | | +| [organization_id](outputs.tf#L76) | Organization id dependent on module resources. | | +| [sink_writer_identities](outputs.tf#L93) | Writer identities created for each sink. | | +| [tag_keys](outputs.tf#L101) | Tag key resources. | | +| [tag_values](outputs.tf#L110) | Tag value resources. | | + diff --git a/assets/modules-fabric/v26/organization/iam.tf b/assets/modules-fabric/v26/organization/iam.tf new file mode 100644 index 0000000..81a8d2b --- /dev/null +++ b/assets/modules-fabric/v26/organization/iam.tf @@ -0,0 +1,79 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description IAM bindings, roles and audit logging resources. + +locals { + _group_iam_roles = distinct(flatten(values(var.group_iam))) + _group_iam = { + for r in local._group_iam_roles : r => [ + for k, v in var.group_iam : "group:${k}" if try(index(v, r), null) != null + ] + } + iam = { + for role in distinct(concat(keys(var.iam), keys(local._group_iam))) : + role => concat( + try(var.iam[role], []), + try(local._group_iam[role], []) + ) + } +} + +resource "google_organization_iam_custom_role" "roles" { + for_each = var.custom_roles + org_id = local.organization_id_numeric + role_id = each.key + title = "Custom role ${each.key}" + description = "Terraform-managed." + permissions = each.value +} + +resource "google_organization_iam_binding" "authoritative" { + for_each = local.iam + org_id = local.organization_id_numeric + role = each.key + members = each.value +} + +resource "google_organization_iam_binding" "bindings" { + for_each = var.iam_bindings + org_id = local.organization_id_numeric + role = each.value.role + members = each.value.members + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +resource "google_organization_iam_member" "bindings" { + for_each = var.iam_bindings_additive + org_id = local.organization_id_numeric + role = each.value.role + member = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} diff --git a/assets/modules-fabric/v26/organization/logging.tf b/assets/modules-fabric/v26/organization/logging.tf new file mode 100644 index 0000000..7719c0f --- /dev/null +++ b/assets/modules-fabric/v26/organization/logging.tf @@ -0,0 +1,117 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Log sinks and data access logs. + +locals { + sink_bindings = { + for type in ["bigquery", "logging", "pubsub", "storage"] : + type => { + for name, sink in var.logging_sinks : + name => sink if sink.type == type + } + } +} + +resource "google_organization_iam_audit_config" "default" { + for_each = var.logging_data_access + org_id = local.organization_id_numeric + service = each.key + dynamic "audit_log_config" { + for_each = each.value + iterator = config + content { + log_type = config.key + exempted_members = config.value + } + } +} + +resource "google_logging_organization_sink" "sink" { + for_each = var.logging_sinks + name = each.key + description = coalesce(each.value.description, "${each.key} (Terraform-managed).") + org_id = local.organization_id_numeric + destination = "${each.value.type}.googleapis.com/${each.value.destination}" + filter = each.value.filter + include_children = each.value.include_children + disabled = each.value.disabled + + dynamic "bigquery_options" { + for_each = each.value.type == "biquery" && each.value.bq_partitioned_table != null ? [""] : [] + content { + use_partitioned_tables = each.value.bq_partitioned_table + } + } + + dynamic "exclusions" { + for_each = each.value.exclusions + iterator = exclusion + content { + name = exclusion.key + filter = exclusion.value + } + } + depends_on = [ + google_organization_iam_binding.authoritative, + google_organization_iam_binding.bindings, + google_organization_iam_member.bindings + ] +} + +resource "google_storage_bucket_iam_member" "storage-sinks-binding" { + for_each = local.sink_bindings["storage"] + bucket = each.value.destination + role = "roles/storage.objectCreator" + member = google_logging_organization_sink.sink[each.key].writer_identity +} + +resource "google_bigquery_dataset_iam_member" "bq-sinks-binding" { + for_each = local.sink_bindings["bigquery"] + project = split("/", each.value.destination)[1] + dataset_id = split("/", each.value.destination)[3] + role = "roles/bigquery.dataEditor" + member = google_logging_organization_sink.sink[each.key].writer_identity +} + +resource "google_pubsub_topic_iam_member" "pubsub-sinks-binding" { + for_each = local.sink_bindings["pubsub"] + project = split("/", each.value.destination)[1] + topic = split("/", each.value.destination)[3] + role = "roles/pubsub.publisher" + member = google_logging_organization_sink.sink[each.key].writer_identity +} + +resource "google_project_iam_member" "bucket-sinks-binding" { + for_each = local.sink_bindings["logging"] + project = split("/", each.value.destination)[1] + role = "roles/logging.bucketWriter" + member = google_logging_organization_sink.sink[each.key].writer_identity + + condition { + title = "${each.key} bucket writer" + description = "Grants bucketWriter to ${google_logging_organization_sink.sink[each.key].writer_identity} used by log sink ${each.key} on ${var.organization_id}" + expression = "resource.name.endsWith('${each.value.destination}')" + } +} + +resource "google_logging_organization_exclusion" "logging-exclusion" { + for_each = var.logging_exclusions + name = each.key + org_id = local.organization_id_numeric + description = "${each.key} (Terraform-managed)." + filter = each.value +} diff --git a/assets/modules-fabric/v26/organization/main.tf b/assets/modules-fabric/v26/organization/main.tf new file mode 100644 index 0000000..5b448df --- /dev/null +++ b/assets/modules-fabric/v26/organization/main.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + organization_id_numeric = split("/", var.organization_id)[1] +} + +resource "google_essential_contacts_contact" "contact" { + provider = google-beta + for_each = var.contacts + parent = var.organization_id + email = each.key + language_tag = "en" + notification_category_subscriptions = each.value +} + + +resource "google_compute_firewall_policy_association" "default" { + count = var.firewall_policy == null ? 0 : 1 + attachment_target = var.organization_id + name = var.firewall_policy.name + firewall_policy = var.firewall_policy.policy +} diff --git a/assets/modules-fabric/v26/organization/org-policy-custom-constraints.tf b/assets/modules-fabric/v26/organization/org-policy-custom-constraints.tf new file mode 100644 index 0000000..6a8cf5e --- /dev/null +++ b/assets/modules-fabric/v26/organization/org-policy-custom-constraints.tf @@ -0,0 +1,59 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + _custom_constraints_factory_data_raw = merge([ + for f in try(fileset(var.org_policy_custom_constraints_data_path, "*.yaml"), []) : + yamldecode(file("${var.org_policy_custom_constraints_data_path}/${f}")) + ]...) + + + _custom_constraints_factory_data = { + for k, v in local._custom_constraints_factory_data_raw : + k => { + display_name = try(v.display_name, null) + description = try(v.description, null) + action_type = v.action_type + condition = v.condition + method_types = v.method_types + resource_types = v.resource_types + } + } + + _custom_constraints = merge(local._custom_constraints_factory_data, var.org_policy_custom_constraints) + + custom_constraints = { + for k, v in local._custom_constraints : + k => merge(v, { + name = k + parent = var.organization_id + }) + } +} + +resource "google_org_policy_custom_constraint" "constraint" { + provider = google-beta + + for_each = local.custom_constraints + name = each.value.name + parent = each.value.parent + display_name = each.value.display_name + description = each.value.description + action_type = each.value.action_type + condition = each.value.condition + method_types = each.value.method_types + resource_types = each.value.resource_types +} diff --git a/assets/modules-fabric/v26/organization/organization-policies.tf b/assets/modules-fabric/v26/organization/organization-policies.tf new file mode 100644 index 0000000..8d867f6 --- /dev/null +++ b/assets/modules-fabric/v26/organization/organization-policies.tf @@ -0,0 +1,126 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Organization-level organization policies. + +locals { + _factory_data_raw = merge([ + for f in try(fileset(var.org_policies_data_path, "*.yaml"), []) : + yamldecode(file("${var.org_policies_data_path}/${f}")) + ]...) + + # simulate applying defaults to data coming from yaml files + _factory_data = { + for k, v in local._factory_data_raw : + k => { + inherit_from_parent = try(v.inherit_from_parent, null) + reset = try(v.reset, null) + rules = [ + for r in try(v.rules, []) : { + allow = can(r.allow) ? { + all = try(r.allow.all, null) + values = try(r.allow.values, null) + } : null + deny = can(r.deny) ? { + all = try(r.deny.all, null) + values = try(r.deny.values, null) + } : null + enforce = try(r.enforce, null) + condition = { + description = try(r.condition.description, null) + expression = try(r.condition.expression, null) + location = try(r.condition.location, null) + title = try(r.condition.title, null) + } + } + ] + } + } + + _org_policies = merge(local._factory_data, var.org_policies) + + org_policies = { + for k, v in local._org_policies : + k => merge(v, { + name = "${var.organization_id}/policies/${k}" + parent = var.organization_id + is_boolean_policy = ( + alltrue([for r in v.rules : r.allow == null && r.deny == null]) + ) + has_values = ( + length(coalesce(try(v.allow.values, []), [])) > 0 || + length(coalesce(try(v.deny.values, []), [])) > 0 + ) + rules = [ + for r in v.rules : + merge(r, { + has_values = ( + length(coalesce(try(r.allow.values, []), [])) > 0 || + length(coalesce(try(r.deny.values, []), [])) > 0 + ) + }) + ] + }) + } +} + +resource "google_org_policy_policy" "default" { + for_each = local.org_policies + name = each.value.name + parent = each.value.parent + spec { + inherit_from_parent = each.value.inherit_from_parent + reset = each.value.reset + dynamic "rules" { + for_each = each.value.rules + iterator = rule + content { + allow_all = try(rule.value.allow.all, false) == true ? "TRUE" : null + deny_all = try(rule.value.deny.all, false) == true ? "TRUE" : null + enforce = ( + each.value.is_boolean_policy && rule.value.enforce != null + ? upper(tostring(rule.value.enforce)) + : null + ) + dynamic "condition" { + for_each = rule.value.condition.expression != null ? [1] : [] + content { + description = rule.value.condition.description + expression = rule.value.condition.expression + location = rule.value.condition.location + title = rule.value.condition.title + } + } + dynamic "values" { + for_each = rule.value.has_values ? [1] : [] + content { + allowed_values = try(rule.value.allow.values, null) + denied_values = try(rule.value.deny.values, null) + } + } + } + } + } + depends_on = [ + google_organization_iam_binding.authoritative, + google_organization_iam_binding.bindings, + google_organization_iam_member.bindings, + google_organization_iam_custom_role.roles, + google_org_policy_custom_constraint.constraint, + google_tags_tag_key.default, + google_tags_tag_value.default, + ] +} diff --git a/assets/modules-fabric/v26/organization/outputs.tf b/assets/modules-fabric/v26/organization/outputs.tf new file mode 100644 index 0000000..12c133e --- /dev/null +++ b/assets/modules-fabric/v26/organization/outputs.tf @@ -0,0 +1,116 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "custom_constraint_ids" { + description = "Map of CUSTOM_CONSTRAINTS => ID in the organization." + value = { for k, v in google_org_policy_custom_constraint.constraint : k => v.id } +} + +output "custom_role_id" { + description = "Map of custom role IDs created in the organization." + value = { + for role_id, role in google_organization_iam_custom_role.roles : + # build the string manually so that role IDs can be used as map + # keys (useful for folder/organization/project-level iam bindings) + (role_id) => "${var.organization_id}/roles/${role_id}" + } + depends_on = [ + google_organization_iam_custom_role.roles + ] +} + +output "custom_roles" { + description = "Map of custom roles resources created in the organization." + value = google_organization_iam_custom_role.roles +} + +output "id" { + description = "Fully qualified organization id." + value = var.organization_id + depends_on = [ + google_org_policy_custom_constraint.constraint, + google_org_policy_policy.default, + google_organization_iam_binding.authoritative, + google_organization_iam_binding.bindings, + google_organization_iam_member.bindings, + google_organization_iam_custom_role.roles, + google_tags_tag_key.default, + google_tags_tag_key_iam_binding.default, + google_tags_tag_value.default, + google_tags_tag_value_iam_binding.default, + ] +} + +output "network_tag_keys" { + description = "Tag key resources." + value = { + for k, v in google_tags_tag_key.default : k => v if( + v.purpose != null && v.purpose != "" + ) + } +} + +output "network_tag_values" { + description = "Tag value resources." + value = { + for k, v in google_tags_tag_value.default : + k => v if local.tag_values[k].tag_network + } +} + +# TODO: deprecate in favor of id + +output "organization_id" { + description = "Organization id dependent on module resources." + value = var.organization_id + depends_on = [ + google_org_policy_custom_constraint.constraint, + google_org_policy_policy.default, + google_organization_iam_binding.authoritative, + google_organization_iam_binding.bindings, + google_organization_iam_member.bindings, + google_organization_iam_custom_role.roles, + google_tags_tag_key.default, + google_tags_tag_key_iam_binding.default, + google_tags_tag_value.default, + google_tags_tag_value_iam_binding.default, + ] +} + +output "sink_writer_identities" { + description = "Writer identities created for each sink." + value = { + for name, sink in google_logging_organization_sink.sink : + name => sink.writer_identity + } +} + +output "tag_keys" { + description = "Tag key resources." + value = { + for k, v in google_tags_tag_key.default : k => v if( + v.purpose == null || v.purpose == "" + ) + } +} + +output "tag_values" { + description = "Tag value resources." + value = { + for k, v in google_tags_tag_value.default : + k => v if !local.tag_values[k].tag_network + } +} diff --git a/assets/modules-fabric/v26/organization/tags.tf b/assets/modules-fabric/v26/organization/tags.tf new file mode 100644 index 0000000..7fb1c06 --- /dev/null +++ b/assets/modules-fabric/v26/organization/tags.tf @@ -0,0 +1,135 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + _tag_values = flatten([ + for tag, attrs in local.tags : [ + for value, value_attrs in coalesce(attrs.values, {}) : { + description = coalesce( + value_attrs == null ? null : value_attrs.description, + "Managed by the Terraform organization module." + ) + key = "${tag}/${value}" + id = try(value_attrs.id, null) + name = value + roles = keys(coalesce( + value_attrs == null ? null : value_attrs.iam, {} + )) + tag = tag + tag_id = attrs.id + tag_network = try(attrs.network, null) != null + } + ] + ]) + _tag_values_iam = flatten([ + for key, value_attrs in local.tag_values : [ + for role in value_attrs.roles : { + id = value_attrs.id + key = value_attrs.key + name = value_attrs.name + role = role + tag = value_attrs.tag + } + ] + ]) + _tags_iam = flatten([ + for tag, attrs in local.tags : [ + for role in keys(coalesce(attrs.iam, {})) : { + role = role + tag = tag + tag_id = attrs.id + } + ] + ]) + tag_values = { + for t in local._tag_values : t.key => t + } + tag_values_iam = { + for t in local._tag_values_iam : "${t.key}:${t.role}" => t + } + tags = merge(var.tags, var.network_tags) + tags_iam = { + for t in local._tags_iam : "${t.tag}:${t.role}" => t + } +} + +# keys + +resource "google_tags_tag_key" "default" { + for_each = { for k, v in local.tags : k => v if v.id == null } + parent = var.organization_id + purpose = ( + lookup(each.value, "network", null) == null ? null : "GCE_FIREWALL" + ) + purpose_data = ( + lookup(each.value, "network", null) == null ? null : { network = each.value.network } + ) + short_name = each.key + description = each.value.description + depends_on = [ + google_organization_iam_binding.authoritative, + google_organization_iam_binding.bindings, + google_organization_iam_member.bindings + ] +} + +resource "google_tags_tag_key_iam_binding" "default" { + for_each = local.tags_iam + tag_key = ( + each.value.tag_id == null + ? google_tags_tag_key.default[each.value.tag].id + : each.value.tag_id + ) + role = each.value.role + members = coalesce( + local.tags[each.value.tag]["iam"][each.value.role], [] + ) +} + +# values + +resource "google_tags_tag_value" "default" { + for_each = { for k, v in local.tag_values : k => v if v.id == null } + parent = ( + each.value.tag_id == null + ? google_tags_tag_key.default[each.value.tag].id + : each.value.tag_id + ) + short_name = each.value.name + description = each.value.description +} + +resource "google_tags_tag_value_iam_binding" "default" { + for_each = local.tag_values_iam + tag_value = ( + each.value.id == null + ? google_tags_tag_value.default[each.value.key].id + : each.value.id + ) + role = each.value.role + members = coalesce( + local.tags[each.value.tag]["values"][each.value.name]["iam"][each.value.role], + [] + ) +} + +# bindings + +resource "google_tags_tag_binding" "binding" { + for_each = coalesce(var.tag_bindings, {}) + parent = "//cloudresourcemanager.googleapis.com/${var.organization_id}" + tag_value = each.value +} diff --git a/assets/modules-fabric/v26/organization/variables.tf b/assets/modules-fabric/v26/organization/variables.tf new file mode 100644 index 0000000..c9899e2 --- /dev/null +++ b/assets/modules-fabric/v26/organization/variables.tf @@ -0,0 +1,246 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "contacts" { + description = "List of essential contacts for this resource. Must be in the form EMAIL -> [NOTIFICATION_TYPES]. Valid notification types are ALL, SUSPENSION, SECURITY, TECHNICAL, BILLING, LEGAL, PRODUCT_UPDATES." + type = map(list(string)) + default = {} + nullable = false +} + +variable "custom_roles" { + description = "Map of role name => list of permissions to create in this project." + type = map(list(string)) + default = {} + nullable = false +} + +variable "firewall_policy" { + description = "Hierarchical firewall policies to associate to the organization." + type = object({ + name = string + policy = string + }) + default = null +} + +variable "group_iam" { + description = "Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable." + type = map(list(string)) + default = {} + nullable = false +} + +variable "iam" { + description = "IAM bindings, in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} + nullable = false +} + +variable "iam_bindings" { + description = "Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary." + type = map(object({ + members = list(string) + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} +} + +variable "iam_bindings_additive" { + description = "Individual additive IAM bindings. Keys are arbitrary." + type = map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} +} + +variable "logging_data_access" { + description = "Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services." + type = map(map(list(string))) + nullable = false + default = {} + validation { + condition = alltrue(flatten([ + for k, v in var.logging_data_access : [ + for kk, vv in v : contains(["DATA_READ", "DATA_WRITE", "ADMIN_READ"], kk) + ] + ])) + error_message = "Log type keys for each service can only be one of 'DATA_READ', 'DATA_WRITE', 'ADMIN_READ'." + } +} + +variable "logging_exclusions" { + description = "Logging exclusions for this organization in the form {NAME -> FILTER}." + type = map(string) + default = {} + nullable = false +} + +variable "logging_sinks" { + description = "Logging sinks to create for the organization." + type = map(object({ + bq_partitioned_table = optional(bool) + description = optional(string) + destination = string + disabled = optional(bool, false) + exclusions = optional(map(string), {}) + filter = string + include_children = optional(bool, true) + type = string + })) + default = {} + nullable = false + validation { + condition = alltrue([ + for k, v in var.logging_sinks : + contains(["bigquery", "logging", "pubsub", "storage"], v.type) + ]) + error_message = "Type must be one of 'bigquery', 'logging', 'pubsub', 'storage'." + } + validation { + condition = alltrue([ + for k, v in var.logging_sinks : + v.bq_partitioned_table != true || v.type == "bigquery" + ]) + error_message = "Can only set bq_partitioned_table when type is `bigquery`." + } +} + +variable "network_tags" { + description = "Network tags by key name. If `id` is provided, key creation is skipped. The `iam` attribute behaves like the similarly named one at module level." + type = map(object({ + description = optional(string, "Managed by the Terraform organization module.") + iam = optional(map(list(string)), {}) + id = optional(string) + network = string # project_id/vpc_name + values = optional(map(object({ + description = optional(string, "Managed by the Terraform organization module.") + iam = optional(map(list(string)), {}) + })), {}) + })) + nullable = false + default = {} + validation { + condition = alltrue([ + for k, v in var.network_tags : v != null + ]) + error_message = "Use an empty map instead of null as value." + } +} + +variable "org_policies" { + description = "Organization policies applied to this organization keyed by policy name." + type = map(object({ + inherit_from_parent = optional(bool) # for list policies only. + reset = optional(bool) + rules = optional(list(object({ + allow = optional(object({ + all = optional(bool) + values = optional(list(string)) + })) + deny = optional(object({ + all = optional(bool) + values = optional(list(string)) + })) + enforce = optional(bool) # for boolean policies only. + condition = optional(object({ + description = optional(string) + expression = optional(string) + location = optional(string) + title = optional(string) + }), {}) + })), []) + })) + default = {} + nullable = false +} + +variable "org_policies_data_path" { + description = "Path containing org policies in YAML format." + type = string + default = null +} + +variable "org_policy_custom_constraints" { + description = "Organization policy custom constraints keyed by constraint name." + type = map(object({ + display_name = optional(string) + description = optional(string) + action_type = string + condition = string + method_types = list(string) + resource_types = list(string) + })) + default = {} + nullable = false +} + +variable "org_policy_custom_constraints_data_path" { + description = "Path containing org policy custom constraints in YAML format." + type = string + default = null +} + +variable "organization_id" { + description = "Organization id in organizations/nnnnnn format." + type = string + validation { + condition = can(regex("^organizations/[0-9]+", var.organization_id)) + error_message = "The organization_id must in the form organizations/nnn." + } +} + +variable "tag_bindings" { + description = "Tag bindings for this organization, in key => tag value id format." + type = map(string) + default = null +} + +variable "tags" { + description = "Tags by key name. If `id` is provided, key or value creation is skipped. The `iam` attribute behaves like the similarly named one at module level." + type = map(object({ + description = optional(string, "Managed by the Terraform organization module.") + iam = optional(map(list(string)), {}) + id = optional(string) + values = optional(map(object({ + description = optional(string, "Managed by the Terraform organization module.") + iam = optional(map(list(string)), {}) + id = optional(string) + })), {}) + })) + nullable = false + default = {} + validation { + condition = alltrue([ + for k, v in var.tags : v != null + ]) + error_message = "Use an empty map instead of null as value." + } +} diff --git a/assets/modules-fabric/v26/organization/versions.tf b/assets/modules-fabric/v26/organization/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/organization/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/project/README.md b/assets/modules-fabric/v26/project/README.md new file mode 100644 index 0000000..3fddf95 --- /dev/null +++ b/assets/modules-fabric/v26/project/README.md @@ -0,0 +1,638 @@ +# Project Module + +This module implements the creation and management of one GCP project including IAM, organization policies, Shared VPC host or service attachment, service API activation, and tag attachment. It also offers a convenient way to refer to managed service identities (aka robot service accounts) for APIs. + +## TOC + + +- [TOC](#toc) +- [Basic Project Creation](#basic-project-creation) +- [IAM](#iam) + - [Authoritative IAM](#authoritative-iam) + - [Additive IAM](#additive-iam) + - [Service Identities and Authoritative IAM](#service-identities-and-authoritative-iam) + - [Service Identities Requiring Manual Iam Grants](#service-identities-requiring-manual-iam-grants) +- [Shared VPC](#shared-vpc) +- [Organization Policies](#organization-policies) + - [Organization Policy Factory](#organization-policy-factory) +- [Log Sinks](#log-sinks) +- [Data Access Logs](#data-access-logs) +- [Cloud Kms Encryption Keys](#cloud-kms-encryption-keys) +- [Tags](#tags) +- [Outputs](#outputs) +- [Files](#files) +- [Variables](#variables) +- [Outputs](#outputs) + + +## Basic Project Creation + +```hcl +module "project" { + source = "./fabric/modules/project" + billing_account = "123456-123456-123456" + name = "myproject" + parent = "folders/1234567890" + prefix = "foo" + services = [ + "container.googleapis.com", + "stackdriver.googleapis.com" + ] +} +# tftest modules=1 resources=3 inventory=basic.yaml +``` + +## IAM + +IAM is managed via several variables that implement different features and levels of control: + +- `iam` and `group_iam` configure authoritative bindings that manage individual roles exclusively, and are internally merged +- `iam_bindings` configure authoritative bindings with optional support for conditions, and are not internally merged with the previous two variables +- `iam_bindings_additive` configure additive bindings via individual role/member pairs with optional support conditions + +The authoritative and additive approaches can be used together, provided different roles are managed by each. Some care must also be taken with the `groups_iam` variable to ensure that variable keys are static values, so that Terraform is able to compute the dependency graph. + +Be mindful about service identity roles when using authoritative IAM, as you might inadvertently remove a role from a [service identity](https://cloud.google.com/iam/docs/service-account-types#google-managed) or default service account. For example, using `roles/editor` with `iam` or `group_iam` will remove the default permissions for the Cloud Services identity. A simple workaround for these scenarios is described below. + +### Authoritative IAM + +The `iam` variable is based on role keys and is typically used for service accounts, or where member values can be dynamic and would create potential problems in the underlying `for_each` cycle. + +```hcl +locals { + gke_service_account = "my_gke_service_account" +} + +module "project" { + source = "./fabric/modules/project" + billing_account = "123456-123456-123456" + name = "project-example" + parent = "folders/1234567890" + prefix = "foo" + services = [ + "container.googleapis.com", + "stackdriver.googleapis.com" + ] + iam = { + "roles/container.hostServiceAgentUser" = [ + "serviceAccount:${local.gke_service_account}" + ] + } +} +# tftest modules=1 resources=4 inventory=iam-authoritative.yaml +``` + +The `group_iam` variable uses group email addresses as keys and is a convenient way to assign roles to humans following Google's best practices. The end result is readable code that also serves as documentation. + +```hcl +module "project" { + source = "./fabric/modules/project" + billing_account = "123456-123456-123456" + name = "project-example" + parent = "folders/1234567890" + prefix = "foo" + group_iam = { + "gcp-security-admins@example.com" = [ + "roles/cloudasset.owner", + "roles/cloudsupport.techSupportEditor", + "roles/iam.securityReviewer", + "roles/logging.admin", + ] + } +} +# tftest modules=1 resources=5 inventory=iam-group.yaml +``` + +The `iam_bindings` variable behaves like a more verbose version of `iam`, and allows setting binding-level IAM conditions. + +```hcl +module "project" { + source = "./fabric/modules/project" + billing_account = "123456-123456-123456" + name = "project-example" + parent = "folders/1234567890" + prefix = "foo" + services = [ + "container.googleapis.com", + "stackdriver.googleapis.com" + ] + iam_bindings = { + iam_admin_conditional = { + members = [ + "group:test-admins@example.org" + ] + role = "roles/resourcemanager.projectIamAdmin" + condition = { + title = "delegated_network_user_one" + expression = <<-END + api.getAttribute( + 'iam.googleapis.com/modifiedGrantsByRole', [] + ).hasOnly([ + 'roles/compute.networkAdmin' + ]) + END + } + } + } +} +# tftest modules=1 resources=4 inventory=iam-bindings.yaml +``` + +### Additive IAM + +Additive IAM is typically used where bindings for specific roles are controlled by different modules or in different Terraform stages. One common example is a host project managed by the networking team, and a project factory that manages service projects and needs to assign `roles/networkUser` on the host project. + +The `iam_bindings_additive` variable allows setting individual role/principal binding pairs. Support for IAM conditions is implemented like for `iam_bindings` above. + +```hcl +module "project" { + source = "./fabric/modules/project" + name = "project-1" + services = [ + "compute.googleapis.com" + ] + iam_bindings_additive = { + group-owner = { + member = "group:p1-owners@example.org" + role = "roles/owner" + } + } +} +# tftest modules=1 resources=3 inventory=iam-bindings-additive.yaml +``` + +### Service Identities and Authoritative IAM + +As mentioned above, there are cases where authoritative management of specific IAM roles results in removal of default bindings from service identities. One example is outlined below, with a simple workaround leveraging the `service_accounts` output to identify the service identity. A full list of service identities and their roles can be found [here](https://cloud.google.com/iam/docs/service-agents). + +```hcl +module "project" { + source = "./fabric/modules/project" + name = "project-example" + group_iam = { + "foo@example.com" = [ + "roles/editor" + ] + } + iam = { + "roles/editor" = [ + "serviceAccount:${module.project.service_accounts.cloud_services}" + ] + } +} +# tftest modules=1 resources=2 +``` + +### Service Identities Requiring Manual Iam Grants + +The module will create service identities at project creation instead of creating of them at the time of first use. This allows granting these service identities roles in other projects, something which is usually necessary in a Shared VPC context. + +You can grant roles to service identities using the following construct: + +```hcl +module "project" { + source = "./fabric/modules/project" + name = "project-example" + iam = { + "roles/apigee.serviceAgent" = [ + "serviceAccount:${module.project.service_accounts.robots.apigee}" + ] + } +} +# tftest modules=1 resources=2 +``` + +This table lists all affected services and roles that you need to grant to service identities + +| service | service identity | role | +|---|---|---| +| apigee.googleapis.com | apigee | roles/apigee.serviceAgent | +| artifactregistry.googleapis.com | artifactregistry | roles/artifactregistry.serviceAgent | +| cloudasset.googleapis.com | cloudasset | roles/cloudasset.serviceAgent | +| cloudbuild.googleapis.com | cloudbuild | roles/cloudbuild.builds.builder | +| dataplex.googleapis.com | dataplex | roles/dataplex.serviceAgent | +| gkehub.googleapis.com | fleet | roles/gkehub.serviceAgent | +| meshconfig.googleapis.com | servicemesh | roles/anthosservicemesh.serviceAgent | +| multiclusteringress.googleapis.com | multicluster-ingress | roles/multiclusteringress.serviceAgent | +| pubsub.googleapis.com | pubsub | roles/pubsub.serviceAgent | +| sqladmin.googleapis.com | sqladmin | roles/cloudsql.serviceAgent | + +## Shared VPC + +The module allows managing Shared VPC status for both hosts and service projects, and includes a simple way of assigning Shared VPC roles to service identities. + +You can enable Shared VPC Host at the project level and manage project service association independently. + +```hcl +module "host-project" { + source = "./fabric/modules/project" + name = "my-host-project" + shared_vpc_host_config = { + enabled = true + } +} + +module "service-project" { + source = "./fabric/modules/project" + name = "my-service-project" + shared_vpc_service_config = { + host_project = module.host-project.project_id + service_identity_iam = { + "roles/compute.networkUser" = [ + "cloudservices", "container-engine" + ] + "roles/vpcaccess.user" = [ + "cloudrun" + ] + "roles/container.hostServiceAgentUser" = [ + "container-engine" + ] + } + } +} +# tftest modules=2 resources=8 inventory=shared-vpc.yaml +``` + +The module allows also granting necessary permissions in host project to service identities by specifying which services will be used in service project in `grant_iam_for_services`. + +```hcl +module "host-project" { + source = "./fabric/modules/project" + name = "my-host-project" + shared_vpc_host_config = { + enabled = true + } +} + +module "service-project" { + source = "./fabric/modules/project" + name = "my-service-project" + services = [ + "container.googleapis.com", + ] + shared_vpc_service_config = { + host_project = module.host-project.project_id + service_iam_grants = module.service-project.services + } +} +# tftest modules=2 resources=9 inventory=shared-vpc-auto-grants.yaml +``` + +## Organization Policies + +To manage organization policies, the `orgpolicy.googleapis.com` service should be enabled in the quota project. + +```hcl +module "project" { + source = "./fabric/modules/project" + billing_account = "123456-123456-123456" + name = "project-example" + parent = "folders/1234567890" + prefix = "foo" + org_policies = { + "compute.disableGuestAttributesAccess" = { + rules = [{ enforce = true }] + } + "compute.skipDefaultNetworkCreation" = { + rules = [{ enforce = true }] + } + "iam.disableServiceAccountKeyCreation" = { + rules = [{ enforce = true }] + } + "iam.disableServiceAccountKeyUpload" = { + rules = [ + { + condition = { + expression = "resource.matchTagId('tagKeys/1234', 'tagValues/1234')" + title = "condition" + description = "test condition" + location = "somewhere" + } + enforce = true + }, + { + enforce = false + } + ] + } + "iam.allowedPolicyMemberDomains" = { + rules = [{ + allow = { + values = ["C0xxxxxxx", "C0yyyyyyy"] + } + }] + } + "compute.trustedImageProjects" = { + rules = [{ + allow = { + values = ["projects/my-project"] + } + }] + } + "compute.vmExternalIpAccess" = { + rules = [{ deny = { all = true } }] + } + } +} +# tftest modules=1 resources=8 inventory=org-policies.yaml +``` + +### Organization Policy Factory + +Organization policies can be loaded from a directory containing YAML files where each file defines one or more constraints. The structure of the YAML files is exactly the same as the `org_policies` variable. + +Note that constraints defined via `org_policies` take precedence over those in `org_policies_data_path`. In other words, if you specify the same constraint in a YAML file *and* in the `org_policies` variable, the latter will take priority. + +The example below deploys a few organization policies split between two YAML files. + +```hcl +module "project" { + source = "./fabric/modules/project" + billing_account = "123456-123456-123456" + name = "project-example" + parent = "folders/1234567890" + prefix = "foo" + org_policies_data_path = "configs/org-policies/" +} +# tftest modules=1 resources=8 files=boolean,list inventory=org-policies.yaml +``` + +```yaml +# tftest-file id=boolean path=configs/org-policies/boolean.yaml +compute.disableGuestAttributesAccess: + rules: + - enforce: true +compute.skipDefaultNetworkCreation: + rules: + - enforce: true +iam.disableServiceAccountKeyCreation: + rules: + - enforce: true +iam.disableServiceAccountKeyUpload: + rules: + - condition: + description: test condition + expression: resource.matchTagId('tagKeys/1234', 'tagValues/1234') + location: somewhere + title: condition + enforce: true + - enforce: false +``` + +```yaml +# tftest-file id=list path=configs/org-policies/list.yaml +compute.trustedImageProjects: + rules: + - allow: + values: + - projects/my-project +compute.vmExternalIpAccess: + rules: + - deny: + all: true +iam.allowedPolicyMemberDomains: + rules: + - allow: + values: + - C0xxxxxxx + - C0yyyyyyy +``` + +## Log Sinks + +```hcl +module "gcs" { + source = "./fabric/modules/gcs" + project_id = var.project_id + name = "gcs_sink" + force_destroy = true +} + +module "dataset" { + source = "./fabric/modules/bigquery-dataset" + project_id = var.project_id + id = "bq_sink" +} + +module "pubsub" { + source = "./fabric/modules/pubsub" + project_id = var.project_id + name = "pubsub_sink" +} + +module "bucket" { + source = "./fabric/modules/logging-bucket" + parent_type = "project" + parent = "my-project" + id = "bucket" +} + +module "project-host" { + source = "./fabric/modules/project" + name = "my-project" + billing_account = "123456-123456-123456" + parent = "folders/1234567890" + logging_sinks = { + warnings = { + destination = module.gcs.id + filter = "severity=WARNING" + type = "storage" + } + info = { + destination = module.dataset.id + filter = "severity=INFO" + type = "bigquery" + } + notice = { + destination = module.pubsub.id + filter = "severity=NOTICE" + type = "pubsub" + } + debug = { + destination = module.bucket.id + filter = "severity=DEBUG" + exclusions = { + no-compute = "logName:compute" + } + type = "logging" + } + } + logging_exclusions = { + no-gce-instances = "resource.type=gce_instance" + } +} +# tftest modules=5 resources=14 inventory=logging.yaml +``` + +## Data Access Logs + +Activation of data access logs can be controlled via the `logging_data_access` variable. If the `iam_bindings_authoritative` variable is used to set a resource-level IAM policy, the data access log configuration will also be authoritative as part of the policy. + +This example shows how to set a non-authoritative access log configuration: + +```hcl +module "project" { + source = "./fabric/modules/project" + name = "my-project" + billing_account = "123456-123456-123456" + parent = "folders/1234567890" + logging_data_access = { + allServices = { + # logs for principals listed here will be excluded + ADMIN_READ = ["group:organization-admins@example.org"] + } + "storage.googleapis.com" = { + DATA_READ = [] + DATA_WRITE = [] + } + } +} +# tftest modules=1 resources=3 inventory=logging-data-access.yaml +``` + +## Cloud Kms Encryption Keys + +The module offers a simple, centralized way to assign `roles/cloudkms.cryptoKeyEncrypterDecrypter` to service identities. + +```hcl +module "project" { + source = "./fabric/modules/project" + name = "my-project" + prefix = "foo" + services = [ + "compute.googleapis.com", + "storage.googleapis.com" + ] + service_encryption_key_ids = { + compute = [ + "projects/kms-central-prj/locations/europe-west3/keyRings/my-keyring/cryptoKeys/europe3-gce", + "projects/kms-central-prj/locations/europe-west4/keyRings/my-keyring/cryptoKeys/europe4-gce" + ] + storage = [ + "projects/kms-central-prj/locations/europe/keyRings/my-keyring/cryptoKeys/europe-gcs" + ] + } +} +# tftest modules=1 resources=7 +``` + +## Tags + +Refer to the [Creating and managing tags](https://cloud.google.com/resource-manager/docs/tags/tags-creating-and-managing) documentation for details on usage. + +```hcl +module "org" { + source = "./fabric/modules/organization" + organization_id = var.organization_id + tags = { + environment = { + description = "Environment specification." + iam = null + values = { + dev = null + prod = null + } + } + } +} + +module "project" { + source = "./fabric/modules/project" + name = "test-project" + tag_bindings = { + env-prod = module.org.tag_values["environment/prod"].id + foo = "tagValues/12345678" + } +} +# tftest modules=2 resources=6 +``` + +## Outputs + +Most of this module's outputs depend on its resources, to allow Terraform to compute all dependencies required for the project to be correctly configured. This allows you to reference outputs like `project_id` in other modules or resources without having to worry about setting `depends_on` blocks manually. + +One non-obvious output is `service_accounts`, which offers a simple way to discover service identities and default service accounts, and guarantees that service identities that require an API call to trigger creation (like GCS or BigQuery) exist before use. + +```hcl +module "project" { + source = "./fabric/modules/project" + name = "project-example" + services = [ + "compute.googleapis.com" + ] +} + +output "compute_robot" { + value = module.project.service_accounts.robots.compute +} +# tftest modules=1 resources=2 inventory:outputs.yaml +``` + + + +## Files + +| name | description | resources | +|---|---|---| +| [iam.tf](./iam.tf) | Generic and OSLogin-specific IAM bindings and roles. | google_project_iam_binding · google_project_iam_custom_role · google_project_iam_member | +| [logging.tf](./logging.tf) | Log sinks and supporting resources. | google_bigquery_dataset_iam_member · google_logging_project_exclusion · google_logging_project_sink · google_project_iam_audit_config · google_project_iam_member · google_pubsub_topic_iam_member · google_storage_bucket_iam_member | +| [main.tf](./main.tf) | Module-level locals and resources. | google_compute_project_metadata_item · google_essential_contacts_contact · google_monitoring_monitored_project · google_project · google_project_service · google_resource_manager_lien | +| [organization-policies.tf](./organization-policies.tf) | Project-level organization policies. | google_org_policy_policy | +| [outputs.tf](./outputs.tf) | Module outputs. | | +| [service-accounts.tf](./service-accounts.tf) | Service identities and supporting resources. | google_kms_crypto_key_iam_member · google_project_default_service_accounts · google_project_iam_member · google_project_service_identity | +| [shared-vpc.tf](./shared-vpc.tf) | Shared VPC project-level configuration. | google_compute_shared_vpc_host_project · google_compute_shared_vpc_service_project · google_project_iam_member | +| [tags.tf](./tags.tf) | None | google_tags_tag_binding | +| [variables.tf](./variables.tf) | Module variables. | | +| [versions.tf](./versions.tf) | Version pins. | | +| [vpc-sc.tf](./vpc-sc.tf) | VPC-SC project-level perimeter configuration. | google_access_context_manager_service_perimeter_resource | + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [name](variables.tf#L186) | Project name and id suffix. | string | ✓ | | +| [auto_create_network](variables.tf#L17) | Whether to create the default network for the project. | bool | | false | +| [billing_account](variables.tf#L23) | Billing account id. | string | | null | +| [compute_metadata](variables.tf#L29) | Optional compute metadata key/values. Only usable if compute API has been enabled. | map(string) | | {} | +| [contacts](variables.tf#L36) | List of essential contacts for this resource. Must be in the form EMAIL -> [NOTIFICATION_TYPES]. Valid notification types are ALL, SUSPENSION, SECURITY, TECHNICAL, BILLING, LEGAL, PRODUCT_UPDATES. | map(list(string)) | | {} | +| [custom_roles](variables.tf#L43) | Map of role name => list of permissions to create in this project. | map(list(string)) | | {} | +| [default_service_account](variables.tf#L50) | Project default service account setting: can be one of `delete`, `deprivilege`, `disable`, or `keep`. | string | | "keep" | +| [descriptive_name](variables.tf#L63) | Name of the project name. Used for project name instead of `name` variable. | string | | null | +| [group_iam](variables.tf#L69) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | +| [iam](variables.tf#L76) | Authoritative IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [iam_bindings](variables.tf#L83) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | map(object({…})) | | {} | +| [iam_bindings_additive](variables.tf#L98) | Individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | +| [labels](variables.tf#L113) | Resource labels. | map(string) | | {} | +| [lien_reason](variables.tf#L120) | If non-empty, creates a project lien with this description. | string | | null | +| [logging_data_access](variables.tf#L126) | Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services. | map(map(list(string))) | | {} | +| [logging_exclusions](variables.tf#L141) | Logging exclusions for this project in the form {NAME -> FILTER}. | map(string) | | {} | +| [logging_sinks](variables.tf#L148) | Logging sinks to create for this project. | map(object({…})) | | {} | +| [metric_scopes](variables.tf#L179) | List of projects that will act as metric scopes for this project. | list(string) | | [] | +| [org_policies](variables.tf#L191) | Organization policies applied to this project keyed by policy name. | map(object({…})) | | {} | +| [org_policies_data_path](variables.tf#L218) | Path containing org policies in YAML format. | string | | null | +| [parent](variables.tf#L224) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | string | | null | +| [prefix](variables.tf#L234) | Optional prefix used to generate project id and name. | string | | null | +| [project_create](variables.tf#L244) | Create project. When set to false, uses a data source to reference existing project. | bool | | true | +| [service_config](variables.tf#L250) | Configure service API activation. | object({…}) | | {…} | +| [service_encryption_key_ids](variables.tf#L262) | Cloud KMS encryption key in {SERVICE => [KEY_URL]} format. | map(list(string)) | | {} | +| [service_perimeter_bridges](variables.tf#L269) | Name of VPC-SC Bridge perimeters to add project into. See comment in the variables file for format. | list(string) | | null | +| [service_perimeter_standard](variables.tf#L276) | Name of VPC-SC Standard perimeter to add project into. See comment in the variables file for format. | string | | null | +| [services](variables.tf#L282) | Service APIs to enable. | list(string) | | [] | +| [shared_vpc_host_config](variables.tf#L288) | Configures this project as a Shared VPC host project (mutually exclusive with shared_vpc_service_project). | object({…}) | | null | +| [shared_vpc_service_config](variables.tf#L297) | Configures this project as a Shared VPC service project (mutually exclusive with shared_vpc_host_config). | object({…}) | | {…} | +| [skip_delete](variables.tf#L319) | Allows the underlying resources to be destroyed without destroying the project itself. | bool | | false | +| [tag_bindings](variables.tf#L325) | Tag bindings for this project, in key => tag value id format. | map(string) | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [custom_roles](outputs.tf#L17) | Ids of the created custom roles. | | +| [id](outputs.tf#L25) | Project id. | | +| [name](outputs.tf#L44) | Project name. | | +| [number](outputs.tf#L56) | Project number. | | +| [project_id](outputs.tf#L75) | Project id. | | +| [service_accounts](outputs.tf#L94) | Product robot service accounts in project. | | +| [services](outputs.tf#L110) | Service APIs to enabled in the project. | | +| [sink_writer_identities](outputs.tf#L119) | Writer identities created for each sink. | | + diff --git a/assets/modules-fabric/v26/project/iam.tf b/assets/modules-fabric/v26/project/iam.tf new file mode 100644 index 0000000..0f00f28 --- /dev/null +++ b/assets/modules-fabric/v26/project/iam.tf @@ -0,0 +1,94 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Generic and OSLogin-specific IAM bindings and roles. + +# IAM notes: +# - external users need to have accepted the invitation email to join + +locals { + _group_iam_roles = distinct(flatten(values(var.group_iam))) + _group_iam = { + for r in local._group_iam_roles : r => [ + for k, v in var.group_iam : "group:${k}" if try(index(v, r), null) != null + ] + } + iam = { + for role in distinct(concat(keys(var.iam), keys(local._group_iam))) : + role => concat( + try(var.iam[role], []), + try(local._group_iam[role], []) + ) + } +} + +resource "google_project_iam_custom_role" "roles" { + for_each = var.custom_roles + project = local.project.project_id + role_id = each.key + title = "Custom role ${each.key}" + description = "Terraform-managed." + permissions = each.value +} + +resource "google_project_iam_binding" "authoritative" { + for_each = local.iam + project = local.project.project_id + role = each.key + members = each.value + depends_on = [ + google_project_service.project_services, + google_project_iam_custom_role.roles + ] +} + +resource "google_project_iam_binding" "bindings" { + for_each = var.iam_bindings + project = local.project.project_id + role = each.value.role + members = each.value.members + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } + depends_on = [ + google_project_service.project_services, + google_project_iam_custom_role.roles + ] +} + +resource "google_project_iam_member" "bindings" { + for_each = var.iam_bindings_additive + project = local.project.project_id + role = each.value.role + member = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } + depends_on = [ + google_project_service.project_services, + google_project_iam_custom_role.roles + ] +} diff --git a/assets/modules-fabric/v26/project/logging.tf b/assets/modules-fabric/v26/project/logging.tf new file mode 100644 index 0000000..0181f04 --- /dev/null +++ b/assets/modules-fabric/v26/project/logging.tf @@ -0,0 +1,118 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Log sinks and supporting resources. + +locals { + sink_bindings = { + for type in ["bigquery", "pubsub", "logging", "storage"] : + type => { + for name, sink in var.logging_sinks : + name => sink if sink.iam && sink.type == type + } + } +} + +resource "google_project_iam_audit_config" "default" { + for_each = var.logging_data_access + project = local.project.project_id + service = each.key + dynamic "audit_log_config" { + for_each = each.value + iterator = config + content { + log_type = config.key + exempted_members = config.value + } + } +} + +resource "google_logging_project_sink" "sink" { + for_each = var.logging_sinks + name = each.key + description = coalesce(each.value.description, "${each.key} (Terraform-managed).") + project = local.project.project_id + destination = "${each.value.type}.googleapis.com/${each.value.destination}" + filter = each.value.filter + unique_writer_identity = each.value.unique_writer + disabled = each.value.disabled + + dynamic "bigquery_options" { + for_each = each.value.type == "biquery" && each.value.bq_partitioned_table != null ? [""] : [] + content { + use_partitioned_tables = each.value.bq_partitioned_table + } + } + + dynamic "exclusions" { + for_each = each.value.exclusions + iterator = exclusion + content { + name = exclusion.key + filter = exclusion.value + } + } + + depends_on = [ + google_project_iam_binding.authoritative, + google_project_iam_binding.bindings, + google_project_iam_member.bindings + ] +} + +resource "google_storage_bucket_iam_member" "gcs-sinks-binding" { + for_each = local.sink_bindings["storage"] + bucket = each.value.destination + role = "roles/storage.objectCreator" + member = google_logging_project_sink.sink[each.key].writer_identity +} + +resource "google_bigquery_dataset_iam_member" "bq-sinks-binding" { + for_each = local.sink_bindings["bigquery"] + project = split("/", each.value.destination)[1] + dataset_id = split("/", each.value.destination)[3] + role = "roles/bigquery.dataEditor" + member = google_logging_project_sink.sink[each.key].writer_identity +} + +resource "google_pubsub_topic_iam_member" "pubsub-sinks-binding" { + for_each = local.sink_bindings["pubsub"] + project = split("/", each.value.destination)[1] + topic = split("/", each.value.destination)[3] + role = "roles/pubsub.publisher" + member = google_logging_project_sink.sink[each.key].writer_identity +} + +resource "google_project_iam_member" "bucket-sinks-binding" { + for_each = local.sink_bindings["logging"] + project = split("/", each.value.destination)[1] + role = "roles/logging.bucketWriter" + member = google_logging_project_sink.sink[each.key].writer_identity + + condition { + title = "${each.key} bucket writer" + description = "Grants bucketWriter to ${google_logging_project_sink.sink[each.key].writer_identity} used by log sink ${each.key} on ${local.project.project_id}" + expression = "resource.name.endsWith('${each.value.destination}')" + } +} + +resource "google_logging_project_exclusion" "logging-exclusion" { + for_each = var.logging_exclusions + name = each.key + project = local.project.project_id + description = "${each.key} (Terraform-managed)." + filter = each.value +} diff --git a/assets/modules-fabric/v26/project/main.tf b/assets/modules-fabric/v26/project/main.tf new file mode 100644 index 0000000..547f1aa --- /dev/null +++ b/assets/modules-fabric/v26/project/main.tf @@ -0,0 +1,96 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + descriptive_name = ( + var.descriptive_name != null ? var.descriptive_name : "${local.prefix}${var.name}" + ) + parent_type = var.parent == null ? null : split("/", var.parent)[0] + parent_id = var.parent == null ? null : split("/", var.parent)[1] + prefix = var.prefix == null ? "" : "${var.prefix}-" + project = ( + var.project_create ? + { + project_id = try(google_project.project.0.project_id, null) + number = try(google_project.project.0.number, null) + name = try(google_project.project.0.name, null) + } + : { + project_id = "${local.prefix}${var.name}" + number = try(data.google_project.project.0.number, null) + name = try(data.google_project.project.0.name, null) + } + ) +} + +data "google_project" "project" { + count = var.project_create ? 0 : 1 + project_id = "${local.prefix}${var.name}" +} + +resource "google_project" "project" { + count = var.project_create ? 1 : 0 + org_id = local.parent_type == "organizations" ? local.parent_id : null + folder_id = local.parent_type == "folders" ? local.parent_id : null + project_id = "${local.prefix}${var.name}" + name = local.descriptive_name + billing_account = var.billing_account + auto_create_network = var.auto_create_network + labels = var.labels + skip_delete = var.skip_delete +} + +resource "google_project_service" "project_services" { + for_each = toset(var.services) + project = local.project.project_id + service = each.value + disable_on_destroy = var.service_config.disable_on_destroy + disable_dependent_services = var.service_config.disable_dependent_services +} + +resource "google_compute_project_metadata_item" "default" { + for_each = ( + contains(var.services, "compute.googleapis.com") ? var.compute_metadata : {} + ) + project = local.project.project_id + key = each.key + value = each.value + depends_on = [google_project_service.project_services] +} + +resource "google_resource_manager_lien" "lien" { + count = var.lien_reason != null ? 1 : 0 + parent = "projects/${local.project.number}" + restrictions = ["resourcemanager.projects.delete"] + origin = "created-by-terraform" + reason = var.lien_reason +} + +resource "google_essential_contacts_contact" "contact" { + provider = google-beta + for_each = var.contacts + parent = "projects/${local.project.project_id}" + email = each.key + language_tag = "en" + notification_category_subscriptions = each.value +} + +resource "google_monitoring_monitored_project" "primary" { + provider = google-beta + for_each = toset(var.metric_scopes) + metrics_scope = each.value + name = local.project.project_id +} diff --git a/assets/modules-fabric/v26/project/organization-policies.tf b/assets/modules-fabric/v26/project/organization-policies.tf new file mode 100644 index 0000000..37e6f25 --- /dev/null +++ b/assets/modules-fabric/v26/project/organization-policies.tf @@ -0,0 +1,117 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Project-level organization policies. + +locals { + _factory_data_raw = merge([ + for f in try(fileset(var.org_policies_data_path, "*.yaml"), []) : + yamldecode(file("${var.org_policies_data_path}/${f}")) + ]...) + + # simulate applying defaults to data coming from yaml files + _factory_data = { + for k, v in local._factory_data_raw : + k => { + inherit_from_parent = try(v.inherit_from_parent, null) + reset = try(v.reset, null) + rules = [ + for r in try(v.rules, []) : { + allow = can(r.allow) ? { + all = try(r.allow.all, null) + values = try(r.allow.values, null) + } : null + deny = can(r.deny) ? { + all = try(r.deny.all, null) + values = try(r.deny.values, null) + } : null + enforce = try(r.enforce, null) + condition = { + description = try(r.condition.description, null) + expression = try(r.condition.expression, null) + location = try(r.condition.location, null) + title = try(r.condition.title, null) + } + } + ] + } + } + + _org_policies = merge(local._factory_data, var.org_policies) + + org_policies = { + for k, v in local._org_policies : + k => merge(v, { + name = "projects/${local.project.project_id}/policies/${k}" + parent = "projects/${local.project.project_id}" + is_boolean_policy = ( + alltrue([for r in v.rules : r.allow == null && r.deny == null]) + ) + has_values = ( + length(coalesce(try(v.allow.values, []), [])) > 0 || + length(coalesce(try(v.deny.values, []), [])) > 0 + ) + rules = [ + for r in v.rules : + merge(r, { + has_values = ( + length(coalesce(try(r.allow.values, []), [])) > 0 || + length(coalesce(try(r.deny.values, []), [])) > 0 + ) + }) + ] + }) + } +} + +resource "google_org_policy_policy" "default" { + for_each = local.org_policies + name = each.value.name + parent = each.value.parent + spec { + inherit_from_parent = each.value.inherit_from_parent + reset = each.value.reset + dynamic "rules" { + for_each = each.value.rules + iterator = rule + content { + allow_all = try(rule.value.allow.all, false) == true ? "TRUE" : null + deny_all = try(rule.value.deny.all, false) == true ? "TRUE" : null + enforce = ( + each.value.is_boolean_policy && rule.value.enforce != null + ? upper(tostring(rule.value.enforce)) + : null + ) + dynamic "condition" { + for_each = rule.value.condition.expression != null ? [1] : [] + content { + description = rule.value.condition.description + expression = rule.value.condition.expression + location = rule.value.condition.location + title = rule.value.condition.title + } + } + dynamic "values" { + for_each = rule.value.has_values ? [1] : [] + content { + allowed_values = try(rule.value.allow.values, null) + denied_values = try(rule.value.deny.values, null) + } + } + } + } + } +} diff --git a/assets/modules-fabric/v26/project/outputs.tf b/assets/modules-fabric/v26/project/outputs.tf new file mode 100644 index 0000000..ae7bbc6 --- /dev/null +++ b/assets/modules-fabric/v26/project/outputs.tf @@ -0,0 +1,124 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "custom_roles" { + description = "Ids of the created custom roles." + value = { + for name, role in google_project_iam_custom_role.roles : + name => role.id + } +} + +output "id" { + description = "Project id." + value = "${local.prefix}${var.name}" + depends_on = [ + google_project.project, + data.google_project.project, + google_org_policy_policy.default, + google_project_service.project_services, + google_compute_shared_vpc_host_project.shared_vpc_host, + google_compute_shared_vpc_service_project.shared_vpc_service, + google_compute_shared_vpc_service_project.service_projects, + google_project_iam_member.shared_vpc_host_robots, + google_kms_crypto_key_iam_member.service_identity_cmek, + google_project_service_identity.jit_si, + google_project_service_identity.servicenetworking, + google_project_iam_member.servicenetworking + ] +} + +output "name" { + description = "Project name." + value = local.project.name + depends_on = [ + google_org_policy_policy.default, + google_project_service.project_services, + google_compute_shared_vpc_service_project.service_projects, + google_project_iam_member.shared_vpc_host_robots, + google_kms_crypto_key_iam_member.service_identity_cmek + ] +} + +output "number" { + description = "Project number." + value = local.project.number + depends_on = [ + google_org_policy_policy.default, + google_project_service.project_services, + google_compute_shared_vpc_host_project.shared_vpc_host, + google_compute_shared_vpc_service_project.shared_vpc_service, + google_compute_shared_vpc_service_project.service_projects, + google_project_iam_member.shared_vpc_host_robots, + google_kms_crypto_key_iam_member.service_identity_cmek, + google_project_service_identity.jit_si, + google_project_service_identity.servicenetworking, + google_project_iam_member.servicenetworking + ] +} + +# TODO: deprecate in favor of id + +output "project_id" { + description = "Project id." + value = "${local.prefix}${var.name}" + depends_on = [ + google_project.project, + data.google_project.project, + google_org_policy_policy.default, + google_project_service.project_services, + google_compute_shared_vpc_host_project.shared_vpc_host, + google_compute_shared_vpc_service_project.shared_vpc_service, + google_compute_shared_vpc_service_project.service_projects, + google_project_iam_member.shared_vpc_host_robots, + google_kms_crypto_key_iam_member.service_identity_cmek, + google_project_service_identity.jit_si, + google_project_service_identity.servicenetworking, + google_project_iam_member.servicenetworking + ] +} + +output "service_accounts" { + description = "Product robot service accounts in project." + value = { + cloud_services = local.service_account_cloud_services + default = local.service_accounts_default + robots = local.service_accounts_robots + } + depends_on = [ + google_project_service.project_services, + google_kms_crypto_key_iam_member.service_identity_cmek, + google_project_service_identity.jit_si, + data.google_bigquery_default_service_account.bq_sa, + data.google_storage_project_service_account.gcs_sa + ] +} + +output "services" { + description = "Service APIs to enabled in the project." + value = var.services + depends_on = [ + google_project_service.project_services, + google_project_service_identity.jit_si, + ] +} + +output "sink_writer_identities" { + description = "Writer identities created for each sink." + value = { + for name, sink in google_logging_project_sink.sink : name => sink.writer_identity + } +} diff --git a/assets/modules-fabric/v26/project/service-accounts.tf b/assets/modules-fabric/v26/project/service-accounts.tf new file mode 100644 index 0000000..5e06efb --- /dev/null +++ b/assets/modules-fabric/v26/project/service-accounts.tf @@ -0,0 +1,129 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Service identities and supporting resources. + +locals { + _service_accounts_cmek_service_dependencies = { + "composer" : [ + "composer", + "artifactregistry", "container-engine", "compute", "pubsub", "storage" + ] + "dataflow" : ["dataflow", "compute"] + } + _service_agents_data = yamldecode(file("${path.module}/service-agents.yaml")) + service_accounts_default = { + cloudbuild = "${local.project.number}@cloudbuild.gserviceaccount.com" + compute = "${local.project.number}-compute@developer.gserviceaccount.com" + gae = "${local.project.project_id}@appspot.gserviceaccount.com" + workstations = "service-${local.project.number}@gcp-sa-workstationsvm.iam.gserviceaccount.com" + } + service_account_cloud_services = ( + "${local.project.number}@cloudservices.gserviceaccount.com" + ) + service_accounts_robots = merge( + { + for agent in local._service_agents_data : + agent.name => format(agent.service_agent, local.project.number) + }, + { + for agent in local._service_agents_data : + agent.alias => format(agent.service_agent, local.project.number) + if lookup(agent, "alias", null) != null + }, + { + gke-mcs-importer = "${local.project.project_id}.svc.id.goog[gke-mcs/gke-mcs-importer]" + } + ) + service_accounts_jit_services = [ + for agent in local._service_agents_data : + "${agent.name}.googleapis.com" + if lookup(agent, "jit", false) + ] + service_accounts_cmek_service_keys = distinct(flatten([ + for s in keys(var.service_encryption_key_ids) : [ + for ss in try(local._service_accounts_cmek_service_dependencies[s], [s]) : [ + for key in var.service_encryption_key_ids[s] : { + service = ss + key = key + } if key != null + ] + ] + ])) +} + +data "google_storage_project_service_account" "gcs_sa" { + count = contains(var.services, "storage.googleapis.com") ? 1 : 0 + project = local.project.project_id + depends_on = [google_project_service.project_services] +} + +data "google_bigquery_default_service_account" "bq_sa" { + count = contains(var.services, "bigquery.googleapis.com") ? 1 : 0 + project = local.project.project_id + depends_on = [google_project_service.project_services] +} + +resource "google_project_service_identity" "servicenetworking" { + provider = google-beta + count = contains(var.services, "servicenetworking.googleapis.com") ? 1 : 0 + project = local.project.project_id + service = "servicenetworking.googleapis.com" + depends_on = [google_project_service.project_services] +} + +resource "google_project_iam_member" "servicenetworking" { + count = contains(var.services, "servicenetworking.googleapis.com") ? 1 : 0 + project = local.project.project_id + role = "roles/servicenetworking.serviceAgent" + member = "serviceAccount:${google_project_service_identity.servicenetworking.0.email}" +} + +# Secret Manager SA created just in time, we need to trigger the creation. +resource "google_project_service_identity" "jit_si" { + for_each = setintersection(var.services, local.service_accounts_jit_services) + provider = google-beta + project = local.project.project_id + service = each.value + depends_on = [google_project_service.project_services] +} + +resource "google_kms_crypto_key_iam_member" "service_identity_cmek" { + for_each = { + for service_key in local.service_accounts_cmek_service_keys : + "${service_key.service}.${service_key.key}" => service_key + if service_key != service_key.key + } + crypto_key_id = each.value.key + role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" + member = "serviceAccount:${local.service_accounts_robots[each.value.service]}" + depends_on = [ + google_project.project, + google_project_service.project_services, + google_project_service_identity.jit_si, + data.google_bigquery_default_service_account.bq_sa, + data.google_project.project, + data.google_storage_project_service_account.gcs_sa, + ] +} + +resource "google_project_default_service_accounts" "default_service_accounts" { + count = upper(var.default_service_account) == "KEEP" ? 0 : 1 + action = upper(var.default_service_account) + project = local.project.project_id + restore_policy = "REVERT_AND_IGNORE_FAILURE" + depends_on = [google_project_service.project_services] +} diff --git a/assets/modules-fabric/v26/project/service-agents.yaml b/assets/modules-fabric/v26/project/service-agents.yaml new file mode 100644 index 0000000..c8eff2d --- /dev/null +++ b/assets/modules-fabric/v26/project/service-agents.yaml @@ -0,0 +1,399 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: "accessapproval" + service_agent: "service-p%s@gcp-sa-accessapproval.iam.gserviceaccount.com" +- name: "adsdatahub" + service_agent: "service-%s@gcp-sa-adsdatahub.iam.gserviceaccount.com" +- name: "aiplatform" + service_agent: "service-%s@gcp-sa-aiplatform.iam.gserviceaccount.com" + jit: true +- name: "aiplatform-cc" + service_agent: "service-%s@gcp-sa-aiplatform-cc.iam.gserviceaccount.com" +- name: "alloydb" + service_agent: "service-%s@gcp-sa-alloydb.iam.gserviceaccount.com" +- name: "anthos" + service_agent: "service-%s@gcp-sa-anthos.iam.gserviceaccount.com" +- name: "anthosaudit" + service_agent: "service-%s@gcp-sa-anthosaudit.iam.gserviceaccount.com" +- name: "anthosconfigmanagement" + service_agent: "service-%s@gcp-sa-anthosconfigmanagement.iam.gserviceaccount.com" +- name: "anthosidentityservice" + service_agent: "service-%s@gcp-sa-anthosidentityservice.iam.gserviceaccount.com" +- name: "apigateway" + service_agent: "service-%s@gcp-sa-apigateway.iam.gserviceaccount.com" +- name: "apigateway-mgmt" + service_agent: "service-%s@gcp-sa-apigateway-mgmt.iam.gserviceaccount.com" +- name: "apigee" + service_agent: "service-%s@gcp-sa-apigee.iam.gserviceaccount.com" + jit: true #roles/apigee.serviceAgent +- name: "apigeeregistry" + service_agent: "service-%s@gcp-sa-apigeeregistry.iam.gserviceaccount.com" +- name: "appdevelopmentexperience" + service_agent: "service-%s@gcp-sa-appdevexperience.iam.gserviceaccount.com" +- name: "appengineflex" + alias: "gae-flex" + service_agent: "service-%s@gae-api-prod.google.com.iam.gserviceaccount.com" +- name: "appenginestandard" + service_agent: "service-%s@gcp-gae-service.iam.gserviceaccount.com" +- name: "artifactregistry" + service_agent: "service-%s@gcp-sa-artifactregistry.iam.gserviceaccount.com" + jit: true # roles/artifactregistry.serviceAgent +- name: "assuredworkloads" + service_agent: "service-%s@gcp-sa-assuredworkloads.iam.gserviceaccount.com" +- name: "automl" + service_agent: "service-%s@gcp-sa-automl.iam.gserviceaccount.com" +- name: "backupdr" + service_agent: "service-%s@gcp-sa-backupdr.iam.gserviceaccount.com" +- name: "backupdr-run" + service_agent: "service-%s@gcp-sa-backupdr-run.iam.gserviceaccount.com" +- name: "baremetalsolution" + service_agent: "service-%s@gcp-sa-bms.iam.gserviceaccount.com" +- name: "batch" + service_agent: "service-%s@gcp-sa-cloudbatch.iam.gserviceaccount.com" +- name: "bigquery" + alias: "bq" + service_agent: "bq-%s@bigquery-encryption.iam.gserviceaccount.com" +- name: "bigquery-omni" + service_agent: "service-%s@gcp-sa-prod-bigqueryomni.iam.gserviceaccount.com" +- name: "bigquery-ri" + service_agent: "service-%s@gcp-sa-bigqueryri.iam.gserviceaccount.com" +- name: "bigquerydatatransfer" + service_agent: "service-%s@gcp-sa-bigquerydatatransfer.iam.gserviceaccount.com" +- name: "bigtableadmin" + service_agent: "service-%s@gcp-sa-bigtable.iam.gserviceaccount.com" +- name: "binaryauthorization" + service_agent: "service-%s@gcp-sa-binaryauthorization.iam.gserviceaccount.com" +- name: "certificatemanager" + service_agent: "service-%s@gcp-sa-certificatemanager.iam.gserviceaccount.com" +- name: "chronicle" + service_agent: "service-%s@gcp-sa-chronicle.iam.gserviceaccount.com" +- name: "cloudasset" + service_agent: "service-%s@gcp-sa-cloudasset.iam.gserviceaccount.com" + jit: true # roles/cloudasset.serviceAgent +- name: "cloudbuild" + service_agent: "service-%s@gcp-sa-cloudbuild.iam.gserviceaccount.com" + jit: true # roles/cloudbuild.builds.builder +- name: "cloudbuild-builder" + service_agent: "%s@cloudbuild.gserviceaccount.com.iam.gserviceaccount.com" +- name: "cloudbuild-logging" + service_agent: "service-%s@gcp-sa-log-cloudbuild.iam.gserviceaccount.com" +- name: "clouddeploy" + service_agent: "service-%s@gcp-sa-clouddeploy.iam.gserviceaccount.com" +- name: "cloudfunctions" + alias: "gcf" + service_agent: "service-%s@gcf-admin-robot.iam.gserviceaccount.com" +- name: "cloudiot" + service_agent: "service-%s@gcp-sa-cloudiot.iam.gserviceaccount.com" +- name: "cloudkms" + service_agent: "service-%s@gcp-sa-cloudkms.iam.gserviceaccount.com" +- name: "cloudkms-ekms" + service_agent: "service-%s@gcp-sa-ekms.iam.gserviceaccount.com" +- name: "cloudoptimization" + service_agent: "service-%s@gcp-sa-cloudoptim.iam.gserviceaccount.com" +- name: "cloudscheduler" + service_agent: "service-%s@gcp-sa-cloudscheduler.iam.gserviceaccount.com" +- name: "cloudtasks" + service_agent: "service-%s@gcp-sa-cloudtasks.iam.gserviceaccount.com" +- name: "cloudtrace" + service_agent: "service-%s@gcp-sa-cloud-trace.iam.gserviceaccount.com" +- name: "composer" + service_agent: "service-%s@cloudcomposer-accounts.iam.gserviceaccount.com" +- name: "compute" + service_agent: "service-%s@compute-system.iam.gserviceaccount.com" +- name: "compute-usage" + service_agent: "service-%s@gcp-sa-compute-usage.iam.gserviceaccount.com" +- name: "config" + service_agent: "service-%s@gcp-sa-config.iam.gserviceaccount.com" +- name: "connectgateway" + service_agent: "service-%s@gcp-sa-anthossupport.iam.gserviceaccount.com" +- name: "connectors" + service_agent: "service-%s@gcp-sa-connectors.iam.gserviceaccount.com" +- name: "contactcenteraiplatform" + service_agent: "service-%s@gcp-sa-ccaip.iam.gserviceaccount.com" +- name: "contactcenterinsights" + service_agent: "service-%s@gcp-sa-contactcenterinsights.iam.gserviceaccount.com" +- name: "container" + alias: "container-engine" + service_agent: "service-%s@container-engine-robot.iam.gserviceaccount.com" +- name: "container-gkenode" + service_agent: "service-%s@gcp-sa-gkenode.iam.gserviceaccount.com" +- name: "containeranalysis" + service_agent: "service-%s@container-analysis.iam.gserviceaccount.com" +- name: "containerregistry" + service_agent: "service-%s@containerregistry.iam.gserviceaccount.com" +- name: "containerscanning" + service_agent: "service-%s@gcp-sa-containerscanning.iam.gserviceaccount.com" +- name: "containerthreatdetection" + service_agent: "service-%s@gcp-sa-ktd-control.iam.gserviceaccount.com" +- name: "contentwarehouse" + service_agent: "service-%s@gcp-sa-cloud-cw.iam.gserviceaccount.com" +- name: "dataconnectors" + service_agent: "service-%s@gcp-sa-dataconnectors.iam.gserviceaccount.com" +- name: "dataflow" + service_agent: "service-%s@dataflow-service-producer-prod.iam.gserviceaccount.com" +- name: "dataform" + service_agent: "service-%s@gcp-sa-dataform.iam.gserviceaccount.com" +- name: "datafusion" + service_agent: "service-%s@gcp-sa-datafusion.iam.gserviceaccount.com" +- name: "datalabeling" + service_agent: "service-%s@gcp-sa-datalabeling.iam.gserviceaccount.com" +- name: "datamigration" + service_agent: "service-%s@gcp-sa-datamigration.iam.gserviceaccount.com" +- name: "datapipelines" + service_agent: "service-%s@gcp-sa-datapipelines.iam.gserviceaccount.com" +- name: "dataplex" + service_agent: "service-%s@gcp-sa-dataplex.iam.gserviceaccount.com" + jit: true # roles/dataplex.serviceAgent +- name: "dataproc" + service_agent: "service-%s@dataproc-accounts.iam.gserviceaccount.com" +- name: "datastream" + service_agent: "service-%s@gcp-sa-datastream.iam.gserviceaccount.com" +- name: "datastudio" + service_agent: "service-%s@gcp-sa-datastudio.iam.gserviceaccount.com" +- name: "dialogflow" + service_agent: "service-%s@gcp-sa-dialogflow.iam.gserviceaccount.com" +- name: "discoveryengine" + service_agent: "service-%s@gcp-sa-discoveryengine.iam.gserviceaccount.com" + # dlp ="organizations-ORGANIZATION_NUMBER@gcp-sa-riskmanager" +- name: "dlp" + service_agent: "service-%s@dlp-api.iam.gserviceaccount.com" +- name: "documentai" + service_agent: "service-%s@gcp-sa-prod-dai-core.iam.gserviceaccount.com" +- name: "edgecontainer" + service_agent: "service-%s@gcp-sa-edgecontainer.iam.gserviceaccount.com" +- name: "edgecontainer-cluster" + service_agent: "service-%s@gcp-sa-edgecontainercluster.iam.gserviceaccount.com" +- name: "endpoints" + service_agent: "service-%s@gcp-sa-endpoints.iam.gserviceaccount.com" +- name: "endpointsportal" + service_agent: "service-%s@endpoints-portal.iam.gserviceaccount.com" +- name: "enterpriseknowledgegraph" + service_agent: "service-%s@gcp-sa-cloud-ekg.iam.gserviceaccount.com" +- name: "eventarc" + service_agent: "service-%s@gcp-sa-eventarc.iam.gserviceaccount.com" +- name: "file" + service_agent: "service-%s@cloud-filer.iam.gserviceaccount.com" +- name: "firebase" + service_agent: "service-%s@gcp-sa-firebase.iam.gserviceaccount.com" +- name: "firebaseappcheck" + service_agent: "service-%s@gcp-sa-firebaseappcheck.iam.gserviceaccount.com" +- name: "firebasedatabase" + service_agent: "service-%s@gcp-sa-firebasedatabase.iam.gserviceaccount.com" +- name: "firebaseextensions" + service_agent: "service-%s@gcp-sa-firebasemods.iam.gserviceaccount.com" +- name: "firebaserules" + service_agent: "service-%s@firebase-rules.iam.gserviceaccount.com" +- name: "firebasestorage" + service_agent: "service-%s@gcp-sa-firebasestorage.iam.gserviceaccount.com" +- name: "firestore" + service_agent: "service-%s@gcp-sa-firestore.iam.gserviceaccount.com" +- name: "firewallinsights" + service_agent: "service-%s@gcp-sa-firewallinsights.iam.gserviceaccount.com" +- name: "gameservices" + service_agent: "service-%s@gcp-sa-gameservices.iam.gserviceaccount.com" +- name: "genomics" + service_agent: "service-%s@genomics-api.google.com.iam.gserviceaccount.com" +- name: "gkebackup" + service_agent: "service-%s@gcp-sa-gkebackup.iam.gserviceaccount.com" +- name: "gkehub" + alias: "fleet" + service_agent: "service-%s@gcp-sa-gkehub.iam.gserviceaccount.com" + jit: true # roles/gkehub.serviceAgent +- name: "gkemulticloud" + service_agent: "service-%s@gcp-sa-gkemulticloud.iam.gserviceaccount.com" +- name: "gkeonprem" + service_agent: "service-%s@gcp-sa-gkeonprem.iam.gserviceaccount.com" +- name: "gsuiteaddons" + service_agent: "service-%s@gcp-sa-gsuiteaddons.iam.gserviceaccount.com" +- name: "healthcare" + service_agent: "service-%s@gcp-sa-healthcare.iam.gserviceaccount.com" +- name: "iap" + service_agent: "service-%s@gcp-sa-iap.iam.gserviceaccount.com" + jit: true +- name: "identitytoolkit" + service_agent: "service-%s@gcp-sa-identitytoolkit.iam.gserviceaccount.com" +- name: "ids" + service_agent: "service-%s@gcp-sa-cloud-ids.iam.gserviceaccount.com" +- name: "integrations" + service_agent: "service-%s@gcp-sa-integrations.iam.gserviceaccount.com" +- name: "krmapihosting" + service_agent: "service-%s@gcp-sa-krmapihosting.iam.gserviceaccount.com" +- name: "krmapihosting-dataplane" + service_agent: "service-%s@gcp-sa-krmapihosting-dataplane.iam.gserviceaccount.com" +- name: "lifesciences" + service_agent: "service-%s@gcp-sa-lifesciences.iam.gserviceaccount.com" +- name: "livestream" + service_agent: "service-%s@gcp-sa-livestream.iam.gserviceaccount.com" +- name: "logging" + service_agent: "service-%s@gcp-sa-logging.iam.gserviceaccount.com" +- name: "managedidentities" + service_agent: "service-%s@gcp-sa-mi.iam.gserviceaccount.com" +- name: "memcache" + service_agent: "service-%s@cloud-memcache-sa.iam.gserviceaccount.com" +- name: "meshconfig" + service_agent: "service-%s@gcp-sa-meshconfig.iam.gserviceaccount.com" + jit: true # roles/anthosservicemesh.serviceAgent +- name: "meshconfig-servicemesh" + alias: "servicemesh" + service_agent: "service-%s@gcp-sa-servicemesh.iam.gserviceaccount.com" +- name: "meshconfig-controlplane" + service_agent: "service-%s@gcp-sa-meshcontrolplane.iam.gserviceaccount.com" +- name: "meshconfig-dataplane" + service_agent: "service-%s@gcp-sa-meshdataplane.iam.gserviceaccount.com" +- name: "metastore" + service_agent: "service-%s@gcp-sa-metastore.iam.gserviceaccount.com" +- name: "migrationcenter" + service_agent: "service-%s@gcp-sa-migcenter.iam.gserviceaccount.com" +- name: "ml" + service_agent: "service-%s@cloud-ml.google.com.iam.gserviceaccount.com" +- name: "monitoring-deprecated" + service_agent: "service-%s@gcp-sa-monitoring.iam.gserviceaccount.com" +- name: "monitoring" + alias: "monitoring-notifications" + service_agent: "service-%s@gcp-sa-monitoring-notification.iam.gserviceaccount.com" +- name: "multiclusteringress" + alias: "multicluster-ingress" + service_agent: "service-%s@gcp-sa-multiclusteringress.iam.gserviceaccount.com" + jit: true # roles/multiclusteringress.serviceAgent +- name: "multiclustermetering" + service_agent: "service-%s@gcp-sa-mcmetering.iam.gserviceaccount.com" +- name: "multiclusterservicediscovery" + alias: "gke-mcs" + service_agent: "service-%s@gcp-sa-mcsd.iam.gserviceaccount.com" +- name: "networkconnectivity" + service_agent: "service-%s@gcp-sa-networkconnectivity.iam.gserviceaccount.com" +- name: "networkmanagement" + service_agent: "service-%s@gcp-sa-networkmanagement.iam.gserviceaccount.com" +- name: "networksecurity" + service_agent: "service-%s@gcp-sa-networksecurity.iam.gserviceaccount.com" + jit: true +- name: "networkservices" + service_agent: "service-%s@gcp-sa-networkactions.iam.gserviceaccount.com" +- name: "notebooks" + service_agent: "service-%s@gcp-sa-notebooks.iam.gserviceaccount.com" + jit: true +- name: "ondemandscanning" + service_agent: "service-%s@gcp-sa-ondemandscanning.iam.gserviceaccount.com" +- name: "osconfig" + service_agent: "service-%s@gcp-sa-osconfig.iam.gserviceaccount.com" +- name: "privateca" + service_agent: "service-%s@gcp-sa-privateca.iam.gserviceaccount.com" +- name: "pubsub" + service_agent: "service-%s@gcp-sa-pubsub.iam.gserviceaccount.com" + jit: true # roles/pubsub.serviceAgent +- name: "pubsublite" + service_agent: "service-%s@gcp-sa-pubsublite.iam.gserviceaccount.com" +- name: "rapidmigrationassessment" + service_agent: "service-%s@gcp-sa-rma.iam.gserviceaccount.com" +- name: "recommendationengine" + service_agent: "service-%s@gcp-sa-recommendationengine.iam.gserviceaccount.com" +- name: "redis" + service_agent: "service-%s@cloud-redis.iam.gserviceaccount.com" + #remotebuildexecution ="service-%s@gcp-sa-rbe" + #remotebuildexecution ="service-%s@remotebuildexecution" +- name: "retail" + service_agent: "service-%s@gcp-sa-retail.iam.gserviceaccount.com" +- name: "run" + alias: "cloudrun" + service_agent: "service-%s@serverless-robot-prod.iam.gserviceaccount.com" +- name: "runapps" + service_agent: "service-%s@gcp-sa-runapps.iam.gserviceaccount.com" +- name: "sasportal" + service_agent: "service-%s@gcp-sa-spectrumsas.iam.gserviceaccount.com" +- name: "secretmanager" + service_agent: "service-%s@gcp-sa-secretmanager.iam.gserviceaccount.com" + jit: true +- name: "securedlandingzone" + service_agent: "service-%s@gcp-sa-slz.iam.gserviceaccount.com" +- name: "securitycenter-notification" + service_agent: "service-%s@gcp-sa-scc-notification.iam.gserviceaccount.com" +- name: "securitycenter-vmtd" + service_agent: "service-%s@gcp-sa-scc-vmtd.iam.gserviceaccount.com" + # securitycenter ="service-org-ORGANIZATION_NUMBER@security-center-api" +- name: "serviceconsumermanagement" + service_agent: "service-%s@service-consumer-management.iam.gserviceaccount.com" +- name: "servicedirectory" + service_agent: "service-%s@gcp-sa-servicedirectory.iam.gserviceaccount.com" +- name: "servicenetworking" + service_agent: "service-%s@service-networking.iam.gserviceaccount.com" +- name: "sourcerepo" + service_agent: "service-%s@sourcerepo-service-accounts.iam.gserviceaccount.com" +- name: "spanner" + service_agent: "service-%s@gcp-sa-spanner.iam.gserviceaccount.com" +- name: "speech" + service_agent: "service-%s@gcp-sa-speech.iam.gserviceaccount.com" +- name: "sqladmin" + alias: "sql" + service_agent: "service-%s@gcp-sa-cloud-sql.iam.gserviceaccount.com" + jit: true # roles/cloudsql.serviceAgent +- name: "storage" + service_agent: "service-%s@gs-project-accounts.iam.gserviceaccount.com" +- name: "storagetransfer" + service_agent: "project-%s@storage-transfer-service.iam.gserviceaccount.com" +- name: "stream" + service_agent: "service-%s@gcp-sa-stream.iam.gserviceaccount.com" +- name: "tpu" + service_agent: "service-%s@cloud-tpu.iam.gserviceaccount.com" +- name: "tpu-v2" + service_agent: "service-%s@gcp-sa-tpu.iam.gserviceaccount.com" +- name: "transcoder" + service_agent: "service-%s@gcp-sa-transcoder.iam.gserviceaccount.com" +- name: "transferappliance" + service_agent: "service-%s@gcp-sa-transferappliance.iam.gserviceaccount.com" +- name: "translate" + service_agent: "service-%s@gcp-sa-translation.iam.gserviceaccount.com" +- name: "visionai" + service_agent: "service-%s@gcp-sa-visionai.iam.gserviceaccount.com" +- name: "vmmigration" + service_agent: "service-%s@gcp-sa-vmmigration.iam.gserviceaccount.com" +- name: "vmwareengine" + service_agent: "service-%s@gcp-sa-vmwareengine.iam.gserviceaccount.com" +- name: "vpcaccess" + service_agent: "service-%s@gcp-sa-vpcaccess.iam.gserviceaccount.com" +- name: "websecurityscanner" + service_agent: "service-%s@gcp-sa-websecurityscanner.iam.gserviceaccount.com" +- name: "workflows" + service_agent: "service-%s@gcp-sa-workflows.iam.gserviceaccount.com" +- name: "workloadcertificate" + service_agent: "service-%s@gcp-sa-workloadcert.iam.gserviceaccount.com" +- name: "workloadmanager" + service_agent: "service-%s@gcp-sa-workloadmanager.iam.gserviceaccount.com" +- name: "workstations" + service_agent: "service-%s@gcp-sa-workstations.iam.gserviceaccount.com" + + + # "accessapproval.googleapis.com. + # For the project: service-p%s@gcp-sa-accessapproval + # For the folder: service-fFOLDER_NUMBER@gcp-sa-accessapproval + # For the organization: service-oORGANIZATION_NUMBER@gcp-sa-accessapproval" + + # "bigqueryconnection.googleapis.com. + # bqcx-PROJECT_NUMBER-IDENTIFIER@gcp-sa-bigquery-condel + # connection-PROJECT_NUMBER-IDENTIFIER@gcp-sa-bigquery-condel" + + # sqladmin.googleapis.com. + # For the project:pPROJECT_NUMBER-IDENTIFIER@gcp-sa-cloud-sql + # For the folder:fFOLDER_NUMBER-IDENTIFIER@gcp-sa-cloud-sql + # For the organization:oORGANIZATION_NUMBER-IDENTIFIER@gcp-sa-cloud-sql + + # logging.googleapis.com. + # For the project:pPROJECT_NUMBER-IDENTIFIER@gcp-sa-logging + # For the folder:fFOLDER_NUMBER-IDENTIFIER@gcp-sa-logging + # For the organization:oORGANIZATION_NUMBER-IDENTIFIER@gcp-sa-logging + + # integrations.googleapis.com. + # For the project:pPROJECT_NUMBER-IDENTIFIER@gcp-sa-playbooks + # For the folder:fFOLDER_NUMBER-IDENTIFIER@gcp-sa-playbooks + # For the organization:oORGANIZATION_NUMBER-IDENTIFIER@gcp-sa-playbooks diff --git a/assets/modules-fabric/v26/project/shared-vpc.tf b/assets/modules-fabric/v26/project/shared-vpc.tf new file mode 100644 index 0000000..d728f42 --- /dev/null +++ b/assets/modules-fabric/v26/project/shared-vpc.tf @@ -0,0 +1,92 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Shared VPC project-level configuration. + +locals { + _shared_vpc_agent_config = yamldecode(file("${path.module}/sharedvpc-agent-iam.yaml")) + _shared_vpc_agent_config_filtered = [ + for config in local._shared_vpc_agent_config : config + if contains(var.shared_vpc_service_config.service_iam_grants, config.service) + ] + _shared_vpc_agent_grants = flatten(flatten([ + for api in local._shared_vpc_agent_config_filtered : [ + for service, roles in api.agents : [ + for role in roles : { role = role, service = service } + ] + ] + ])) + + # compute the host project IAM bindings for this project's service identities + _svpc_service_iam = flatten([ + for role, services in var.shared_vpc_service_config.service_identity_iam : [ + for service in services : { role = role, service = service } + ] + ]) + svpc_host_config = { + enabled = coalesce( + try(var.shared_vpc_host_config.enabled, null), false + ) + service_projects = coalesce( + try(var.shared_vpc_host_config.service_projects, null), [] + ) + } + + svpc_service_iam = { + for b in setunion(local._svpc_service_iam, local._shared_vpc_agent_grants) : "${b.role}:${b.service}" => b + } +} + +resource "google_compute_shared_vpc_host_project" "shared_vpc_host" { + provider = google-beta + count = local.svpc_host_config.enabled ? 1 : 0 + project = local.project.project_id + depends_on = [google_project_service.project_services] +} + +resource "google_compute_shared_vpc_service_project" "service_projects" { + provider = google-beta + for_each = toset(local.svpc_host_config.service_projects) + host_project = local.project.project_id + service_project = each.value + depends_on = [google_compute_shared_vpc_host_project.shared_vpc_host] +} + +resource "google_compute_shared_vpc_service_project" "shared_vpc_service" { + provider = google-beta + count = var.shared_vpc_service_config.host_project != null ? 1 : 0 + host_project = var.shared_vpc_service_config.host_project + service_project = local.project.project_id +} + +resource "google_project_iam_member" "shared_vpc_host_robots" { + for_each = local.svpc_service_iam + project = var.shared_vpc_service_config.host_project + role = each.value.role + member = ( + each.value.service == "cloudservices" + ? "serviceAccount:${local.service_account_cloud_services}" + : "serviceAccount:${local.service_accounts_robots[each.value.service]}" + ) + depends_on = [ + google_project_service.project_services, + google_project_service_identity.servicenetworking, + google_project_service_identity.jit_si, + google_project_default_service_accounts.default_service_accounts, + data.google_bigquery_default_service_account.bq_sa, + data.google_storage_project_service_account.gcs_sa, + ] +} diff --git a/assets/modules-fabric/v26/project/sharedvpc-agent-iam.yaml b/assets/modules-fabric/v26/project/sharedvpc-agent-iam.yaml new file mode 100644 index 0000000..3cb8ee3 --- /dev/null +++ b/assets/modules-fabric/v26/project/sharedvpc-agent-iam.yaml @@ -0,0 +1,97 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Cloud Composer +# https://cloud.google.com/composer/docs/how-to/managing/configuring-shared-vpc#edit_permissions_for_the_composer_agent_service_account +- service: composer.googleapis.com + agents: + composer: + - roles/compute.networkUser + - roles/composer.sharedVpcAgent + +# Compute Engine +# TODO: identify docs +- service: compute.googleapis.com + agents: + cloudservices: + - roles/compute.networkUser + +# Google Kubernetes Engine +# https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-shared-vpc#enabling_and_granting_roles +- service: container.googleapis.com + agents: + container: + - roles/compute.networkUser + - roles/container.hostServiceAgentUser + - roles/compute.securityAdmin # to manage firewall rules + cloudservices: + - roles/compute.networkUser + +# Dataflow +# https://cloud.google.com/dataflow/docs/guides/specifying-networks#shared +- service: dataflow.googleapis.com + agents: + dataflow: + - roles/compute.networkUser + +# Cloud Data Fusion +# https://cloud.google.com/data-fusion/docs/how-to/create-private-ip#shared-vpc-network_1 +- service: datafusion.googleapis.com + agents: + datafusion: + - roles/compute.networkUser + dataproc: + - roles/compute.networkUser + +# Dataproc +# https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/network#create_a_cluster_that_uses_a_network_in_another_project +- service: dataproc.googleapis.com + agents: + dataproc: + - roles/compute.networkUser + cloudservices: + - roles/compute.networkUser + +# Change Data Capture | Datastream +# https://cloud.google.com/datastream/docs/create-a-private-connectivity-configuration +- service: datastream.googleapis.com + agents: + datastream: + - roles/compute.networkAdmin + +# Cloud Functions +# For shared connectors in host project +# https://cloud.google.com/functions/docs/networking/shared-vpc-host-project +- service: cloudfunctions.googleapis.com + agents: + cloudfunctions: + - roles/vpcaccess.user + +# Cloud Run +# For shared connectors in host project +# https://cloud.google.com/run/docs/configuring/shared-vpc-host-project +- service: run.googleapis.com + agents: + run: + - roles/vpcaccess.user + +# Cloud Run / Cloud Functions +# For connectors in service project +# https://cloud.google.com/functions/docs/networking/shared-vpc-service-projects#grant-permissions +- service: vpcaccess.googleapis.com + agents: + vpcaccess: + - roles/compute.networkUser + cloudservices: + - roles/compute.networkUser diff --git a/assets/modules-fabric/v26/project/tags.tf b/assets/modules-fabric/v26/project/tags.tf new file mode 100644 index 0000000..683143b --- /dev/null +++ b/assets/modules-fabric/v26/project/tags.tf @@ -0,0 +1,21 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_tags_tag_binding" "binding" { + for_each = coalesce(var.tag_bindings, {}) + parent = "//cloudresourcemanager.googleapis.com/projects/${local.project.number}" + tag_value = each.value +} diff --git a/assets/modules-fabric/v26/project/variables.tf b/assets/modules-fabric/v26/project/variables.tf new file mode 100644 index 0000000..68f8b6c --- /dev/null +++ b/assets/modules-fabric/v26/project/variables.tf @@ -0,0 +1,329 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "auto_create_network" { + description = "Whether to create the default network for the project." + type = bool + default = false +} + +variable "billing_account" { + description = "Billing account id." + type = string + default = null +} + +variable "compute_metadata" { + description = "Optional compute metadata key/values. Only usable if compute API has been enabled." + type = map(string) + nullable = false + default = {} +} + +variable "contacts" { + description = "List of essential contacts for this resource. Must be in the form EMAIL -> [NOTIFICATION_TYPES]. Valid notification types are ALL, SUSPENSION, SECURITY, TECHNICAL, BILLING, LEGAL, PRODUCT_UPDATES." + type = map(list(string)) + default = {} + nullable = false +} + +variable "custom_roles" { + description = "Map of role name => list of permissions to create in this project." + type = map(list(string)) + default = {} + nullable = false +} + +variable "default_service_account" { + description = "Project default service account setting: can be one of `delete`, `deprivilege`, `disable`, or `keep`." + default = "keep" + type = string + validation { + condition = ( + var.default_service_account == null || + contains(["delete", "deprivilege", "disable", "keep"], var.default_service_account) + ) + error_message = "Only `delete`, `deprivilege`, `disable`, or `keep` are supported." + } +} + +variable "descriptive_name" { + description = "Name of the project name. Used for project name instead of `name` variable." + type = string + default = null +} + +variable "group_iam" { + description = "Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable." + type = map(list(string)) + default = {} + nullable = false +} + +variable "iam" { + description = "Authoritative IAM bindings in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} + nullable = false +} + +variable "iam_bindings" { + description = "Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary." + type = map(object({ + members = list(string) + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} +} + +variable "iam_bindings_additive" { + description = "Individual additive IAM bindings. Keys are arbitrary." + type = map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} +} + +variable "labels" { + description = "Resource labels." + type = map(string) + default = {} + nullable = false +} + +variable "lien_reason" { + description = "If non-empty, creates a project lien with this description." + type = string + default = null +} + +variable "logging_data_access" { + description = "Control activation of data access logs. Format is service => { log type => [exempted members]}. The special 'allServices' key denotes configuration for all services." + type = map(map(list(string))) + nullable = false + default = {} + validation { + condition = alltrue(flatten([ + for k, v in var.logging_data_access : [ + for kk, vv in v : contains(["DATA_READ", "DATA_WRITE", "ADMIN_READ"], kk) + ] + ])) + error_message = "Log type keys for each service can only be one of 'DATA_READ', 'DATA_WRITE', 'ADMIN_READ'." + } +} + +variable "logging_exclusions" { + description = "Logging exclusions for this project in the form {NAME -> FILTER}." + type = map(string) + default = {} + nullable = false +} + +variable "logging_sinks" { + description = "Logging sinks to create for this project." + type = map(object({ + bq_partitioned_table = optional(bool) + description = optional(string) + destination = string + disabled = optional(bool, false) + exclusions = optional(map(string), {}) + filter = string + iam = optional(bool, true) + type = string + unique_writer = optional(bool) + })) + default = {} + nullable = false + validation { + condition = alltrue([ + for k, v in var.logging_sinks : + contains(["bigquery", "logging", "pubsub", "storage"], v.type) + ]) + error_message = "Type must be one of 'bigquery', 'logging', 'pubsub', 'storage'." + } + validation { + condition = alltrue([ + for k, v in var.logging_sinks : + v.bq_partitioned_table != true || v.type == "bigquery" + ]) + error_message = "Can only set bq_partitioned_table when type is `bigquery`." + } +} + +variable "metric_scopes" { + description = "List of projects that will act as metric scopes for this project." + type = list(string) + default = [] + nullable = false +} + +variable "name" { + description = "Project name and id suffix." + type = string +} + +variable "org_policies" { + description = "Organization policies applied to this project keyed by policy name." + type = map(object({ + inherit_from_parent = optional(bool) # for list policies only. + reset = optional(bool) + rules = optional(list(object({ + allow = optional(object({ + all = optional(bool) + values = optional(list(string)) + })) + deny = optional(object({ + all = optional(bool) + values = optional(list(string)) + })) + enforce = optional(bool) # for boolean policies only. + condition = optional(object({ + description = optional(string) + expression = optional(string) + location = optional(string) + title = optional(string) + }), {}) + })), []) + })) + default = {} + nullable = false +} + +variable "org_policies_data_path" { + description = "Path containing org policies in YAML format." + type = string + default = null +} + +variable "parent" { + description = "Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format." + type = string + default = null + validation { + condition = var.parent == null || can(regex("(organizations|folders)/[0-9]+", var.parent)) + error_message = "Parent must be of the form folders/folder_id or organizations/organization_id." + } +} + +variable "prefix" { + description = "Optional prefix used to generate project id and name." + type = string + default = null + validation { + condition = var.prefix != "" + error_message = "Prefix cannot be empty, please use null instead." + } +} + +variable "project_create" { + description = "Create project. When set to false, uses a data source to reference existing project." + type = bool + default = true +} + +variable "service_config" { + description = "Configure service API activation." + type = object({ + disable_on_destroy = bool + disable_dependent_services = bool + }) + default = { + disable_on_destroy = false + disable_dependent_services = false + } +} + +variable "service_encryption_key_ids" { + description = "Cloud KMS encryption key in {SERVICE => [KEY_URL]} format." + type = map(list(string)) + default = {} +} + +# accessPolicies/ACCESS_POLICY_NAME/servicePerimeters/PERIMETER_NAME +variable "service_perimeter_bridges" { + description = "Name of VPC-SC Bridge perimeters to add project into. See comment in the variables file for format." + type = list(string) + default = null +} + +# accessPolicies/ACCESS_POLICY_NAME/servicePerimeters/PERIMETER_NAME +variable "service_perimeter_standard" { + description = "Name of VPC-SC Standard perimeter to add project into. See comment in the variables file for format." + type = string + default = null +} + +variable "services" { + description = "Service APIs to enable." + type = list(string) + default = [] +} + +variable "shared_vpc_host_config" { + description = "Configures this project as a Shared VPC host project (mutually exclusive with shared_vpc_service_project)." + type = object({ + enabled = bool + service_projects = optional(list(string), []) + }) + default = null +} + +variable "shared_vpc_service_config" { + description = "Configures this project as a Shared VPC service project (mutually exclusive with shared_vpc_host_config)." + # the list of valid service identities is in service-agents.yaml + type = object({ + host_project = string + service_identity_iam = optional(map(list(string)), {}) + service_iam_grants = optional(list(string), []) + }) + default = { + host_project = null + } + nullable = false + validation { + condition = var.shared_vpc_service_config.host_project != null || ( + var.shared_vpc_service_config.host_project == null && + length(var.shared_vpc_service_config.service_iam_grants) == 0 && + length(var.shared_vpc_service_config.service_iam_grants) == 0 + ) + error_message = "You need to provide host_project when providing service_identity_iam or service_iam_grants" + } +} + +variable "skip_delete" { + description = "Allows the underlying resources to be destroyed without destroying the project itself." + type = bool + default = false +} + +variable "tag_bindings" { + description = "Tag bindings for this project, in key => tag value id format." + type = map(string) + default = null +} diff --git a/assets/modules-fabric/v26/project/versions.tf b/assets/modules-fabric/v26/project/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/project/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/project/vpc-sc.tf b/assets/modules-fabric/v26/project/vpc-sc.tf new file mode 100644 index 0000000..edaa203 --- /dev/null +++ b/assets/modules-fabric/v26/project/vpc-sc.tf @@ -0,0 +1,45 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description VPC-SC project-level perimeter configuration. + +moved { + from = google_access_context_manager_service_perimeter_resource.service-perimeter-resource-standard + to = google_access_context_manager_service_perimeter_resource.standard +} + +resource "google_access_context_manager_service_perimeter_resource" "standard" { + count = var.service_perimeter_standard != null ? 1 : 0 + # this needs an additional lifecycle block in the vpc module on the + # google_access_context_manager_service_perimeter resource + perimeter_name = var.service_perimeter_standard + resource = "projects/${local.project.number}" +} + +moved { + from = google_access_context_manager_service_perimeter_resource.service-perimeter-resource-bridges + to = google_access_context_manager_service_perimeter_resource.bridge +} + +resource "google_access_context_manager_service_perimeter_resource" "bridge" { + for_each = toset( + var.service_perimeter_bridges != null ? var.service_perimeter_bridges : [] + ) + # this needs an additional lifecycle block in the vpc module on the + # google_access_context_manager_service_perimeter resource + perimeter_name = each.value + resource = "projects/${local.project.number}" +} diff --git a/assets/modules-fabric/v26/projects-data-source/README.md b/assets/modules-fabric/v26/projects-data-source/README.md new file mode 100644 index 0000000..93dd67f --- /dev/null +++ b/assets/modules-fabric/v26/projects-data-source/README.md @@ -0,0 +1,95 @@ +# Projects Data Source Module + +This module extends functionality of [google_projects](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/projects) data source by retrieving all the projects under a specific `parent` recursively with only one API call against [Cloud Asset Inventory](https://cloud.google.com/asset-inventory) service. + +A good usage pattern would be when we want all the projects under a specific folder (including nested subfolders) to be included into [VPC Service Controls](../vpc-sc/). Instead of manually maintaining the list of project numbers as an input to the `vpc-sc` module we can use that module to retrieve all the project numbers dynamically. + +### IAM Permissions required + +- `roles/cloudasset.viewer` on the `parent` level or above + + +## Examples + +### All projects in my org + +```hcl +module "my-org" { + source = "./fabric/modules/projects-data-source" + parent = "organizations/123456789" +} + +output "project_numbers" { + value = module.my-org.project_numbers +} + +# tftest skip (uses data sources) +``` + +### My dev projects based on parent and label + +```hcl +module "my-dev" { + source = "./fabric/modules/projects-data-source" + parent = "folders/123456789" + query = "labels.env:DEV state:ACTIVE" +} + +output "dev-projects" { + value = module.my-dev.projects +} + +# tftest skip (uses data sources) +``` + +### Projects under org with folder/project exclusions +```hcl +module "my-filtered" { + source = "./fabric/modules/projects-data-source" + parent = "organizations/123456789" + ignore_projects = [ + "sandbox-*", # wildcard ignore + "project-full-id", # specific project id + "0123456789" # specific project number + ] + + include_projects = [ + "sandbox-114", # include specific project which was excluded by wildcard + "415216609246" # include specific project which was excluded by wildcard (by project number) + ] + + ignore_folders = [ # subfolders are ingoner as well + "343991594985", + "437102807785", + "345245235245" + ] + query = "state:ACTIVE" +} + +output "filtered-projects" { + value = module.my-filtered.projects +} + +# tftest skip (uses data sources) + +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [parent](variables.tf#L55) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | string | ✓ | | +| [ignore_folders](variables.tf#L17) | A list of folder IDs or numbers to be excluded from the output, all the subfolders and projects are excluded from the output regardless of the include_projects variable. | list(string) | | [] | +| [ignore_projects](variables.tf#L28) | A list of project IDs, numbers or prefixes to exclude matching projects from the module output. | list(string) | | [] | +| [include_projects](variables.tf#L41) | A list of project IDs/numbers to include to the output if some of them are excluded by `ignore_projects` wildcard entries. | list(string) | | [] | +| [query](variables.tf#L64) | A string query as defined in the [Query Syntax](https://cloud.google.com/asset-inventory/docs/query-syntax). | string | | "state:ACTIVE" | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [project_numbers](outputs.tf#L17) | List of project numbers. | | +| [projects](outputs.tf#L22) | List of projects in [StandardResourceMetadata](https://cloud.google.com/asset-inventory/docs/reference/rest/v1p1beta1/resources/searchAll#StandardResourceMetadata) format. | | + + diff --git a/assets/modules-fabric/v26/projects-data-source/main.tf b/assets/modules-fabric/v26/projects-data-source/main.tf new file mode 100644 index 0000000..6bd5631 --- /dev/null +++ b/assets/modules-fabric/v26/projects-data-source/main.tf @@ -0,0 +1,41 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + _ignore_folder_numbers = [for folder_id in var.ignore_folders : trimprefix(folder_id, "folders/")] + _ignore_folders_query = join(" AND NOT folders:", concat([""], local._ignore_folder_numbers)) + query = var.query != "" ? ( + format("%s%s", var.query, local._ignore_folders_query) + ) : ( + format("%s%s", var.query, trimprefix(local._ignore_folders_query, " AND ")) + ) + + ignore_patterns = [for item in var.ignore_projects : "^${replace(item, "*", ".*")}$"] + ignore_regexp = length(local.ignore_patterns) > 0 ? join("|", local.ignore_patterns) : "^NO_PROJECTS_TO_IGNORE$" + projects_after_ignore = [for item in data.google_cloud_asset_resources_search_all.projects.results : item if( + length(concat(try(regexall(local.ignore_regexp, trimprefix(item.project, "projects/")), []), try(regexall(local.ignore_regexp, trimprefix(item.name, "//cloudresourcemanager.googleapis.com/projects/")), []))) == 0 + ) || contains(var.include_projects, trimprefix(item.name, "//cloudresourcemanager.googleapis.com/projects/")) || contains(var.include_projects, trimprefix(item.project, "projects/")) + ] +} + +data "google_cloud_asset_resources_search_all" "projects" { + provider = google-beta + scope = var.parent + asset_types = [ + "cloudresourcemanager.googleapis.com/Project" + ] + query = local.query +} diff --git a/assets/modules-fabric/v26/projects-data-source/outputs.tf b/assets/modules-fabric/v26/projects-data-source/outputs.tf new file mode 100644 index 0000000..b1710fa --- /dev/null +++ b/assets/modules-fabric/v26/projects-data-source/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_numbers" { + description = "List of project numbers." + value = [for item in local.projects_after_ignore : trimprefix(item.project, "projects/")] +} + +output "projects" { + description = "List of projects in [StandardResourceMetadata](https://cloud.google.com/asset-inventory/docs/reference/rest/v1p1beta1/resources/searchAll#StandardResourceMetadata) format." + value = local.projects_after_ignore +} diff --git a/assets/modules-fabric/v26/projects-data-source/variables.tf b/assets/modules-fabric/v26/projects-data-source/variables.tf new file mode 100644 index 0000000..888cab2 --- /dev/null +++ b/assets/modules-fabric/v26/projects-data-source/variables.tf @@ -0,0 +1,68 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "ignore_folders" { + description = "A list of folder IDs or numbers to be excluded from the output, all the subfolders and projects are excluded from the output regardless of the include_projects variable." + type = list(string) + default = [] + # example exlusing a folder + # ignore_folders = [ + # "folders/0123456789", + # "2345678901" + # ] +} + +variable "ignore_projects" { + description = "A list of project IDs, numbers or prefixes to exclude matching projects from the module output." + type = list(string) + default = [] + # example + #ignore_projects = [ + # "dev-proj-1", + # "uat-proj-2", + # "0123456789", + # "prd-proj-*" + #] +} + +variable "include_projects" { + description = "A list of project IDs/numbers to include to the output if some of them are excluded by `ignore_projects` wildcard entries." + type = list(string) + default = [] + # example excluding all the projects starting with "prf-" except "prd-123457" + #ignore_projects = [ + # "prd-*" + #] + #include_projects = [ + # "prd-123457", + # "0123456789" + #] +} + +variable "parent" { + description = "Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format." + type = string + validation { + condition = can(regex("(organizations|folders)/[0-9]+", var.parent)) + error_message = "Parent must be of the form folders/folder_id or organizations/organization_id." + } +} + +variable "query" { + description = "A string query as defined in the [Query Syntax](https://cloud.google.com/asset-inventory/docs/query-syntax)." + type = string + default = "state:ACTIVE" +} diff --git a/assets/modules-fabric/v26/projects-data-source/versions.tf b/assets/modules-fabric/v26/projects-data-source/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/projects-data-source/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/pubsub/README.md b/assets/modules-fabric/v26/pubsub/README.md new file mode 100644 index 0000000..69a18db --- /dev/null +++ b/assets/modules-fabric/v26/pubsub/README.md @@ -0,0 +1,187 @@ +# Google Cloud Pub/Sub Module + +This module allows managing a single Pub/Sub topic, including multiple subscriptions and IAM bindings at the topic and subscriptions levels, as well as schemas. + +## Examples + +### Simple topic with IAM + +```hcl +module "pubsub" { + source = "./fabric/modules/pubsub" + project_id = "my-project" + name = "my-topic" + iam = { + "roles/pubsub.viewer" = ["group:foo@example.com"] + "roles/pubsub.subscriber" = ["user:user1@example.com"] + } +} +# tftest modules=1 resources=3 inventory=simple.yaml +``` + +### Topic with schema + +```hcl +module "topic_with_schema" { + source = "./fabric/modules/pubsub" + project_id = "my-project" + name = "my-topic" + schema = { + msg_encoding = "JSON" + schema_type = "AVRO" + definition = jsonencode({ + "type" = "record", + "name" = "Avro", + "fields" = [{ + "name" = "StringField", + "type" = "string" + }, + { + "name" = "FloatField", + "type" = "float" + }, + { + "name" = "BooleanField", + "type" = "boolean" + }, + ] + }) + } +} +# tftest modules=1 resources=2 inventory=schema.yaml +``` + +### Subscriptions + +Subscriptions are defined with the `subscriptions` variable, allowing optional configuration of per-subscription defaults. Push subscriptions need extra configuration, shown in the following example. + +```hcl +module "pubsub" { + source = "./fabric/modules/pubsub" + project_id = "my-project" + name = "my-topic" + subscriptions = { + test-pull = {} + test-pull-override = { + labels = { test = "override" } + retain_acked_messages = true + } + } +} +# tftest modules=1 resources=3 inventory=subscriptions.yaml +``` + +### Push subscriptions + +Push subscriptions need extra configuration in the `push_configs` variable. + +```hcl +module "pubsub" { + source = "./fabric/modules/pubsub" + project_id = "my-project" + name = "my-topic" + subscriptions = { + test-push = { + push = { + endpoint = "https://example.com/foo" + } + } + } +} +# tftest modules=1 resources=2 +``` + +### BigQuery subscriptions + +BigQuery subscriptions need extra configuration in the `bigquery_subscription_configs` variable. + +```hcl +module "pubsub" { + source = "./fabric/modules/pubsub" + project_id = "my-project" + name = "my-topic" + subscriptions = { + test-bigquery = { + bigquery = { + table = "my_project_id:my_dataset.my_table" + use_topic_schema = true + write_metadata = false + drop_unknown_fields = true + } + } + } +} +# tftest modules=1 resources=2 +``` + +### Cloud Storage subscriptions + +Cloud Storage subscriptions need extra configuration in the `cloud_storage_subscription_configs` variable. + +```hcl +module "pubsub" { + source = "./fabric/modules/pubsub" + project_id = "my-project" + name = "my-topic" + subscriptions = { + test-cloudstorage = { + cloud_storage = { + bucket = "my-bucket" + filename_prefix = "test_prefix" + filename_suffix = "test_suffix" + max_duration = "100s" + max_bytes = 1000 + avro_config = { + write_metadata = true + } + } + } + } +} +# tftest modules=1 resources=2 +``` +### Subscriptions with IAM + +```hcl +module "pubsub" { + source = "./fabric/modules/pubsub" + project_id = "my-project" + name = "my-topic" + subscriptions = { + test-1 = { + iam = { + "roles/pubsub.subscriber" = ["user:user1@example.com"] + } + } + } +} +# tftest modules=1 resources=3 +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [name](variables.tf#L73) | PubSub topic name. | string | ✓ | | +| [project_id](variables.tf#L78) | Project used for resources. | string | ✓ | | +| [iam](variables.tf#L17) | IAM bindings for topic in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [iam_bindings](variables.tf#L24) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | map(object({…})) | | {} | +| [iam_bindings_additive](variables.tf#L39) | Keyring individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | +| [kms_key](variables.tf#L54) | KMS customer managed encryption key. | string | | null | +| [labels](variables.tf#L60) | Labels. | map(string) | | {} | +| [message_retention_duration](variables.tf#L67) | Minimum duration to retain a message after it is published to the topic. | string | | null | +| [regions](variables.tf#L83) | List of regions used to set persistence policy. | list(string) | | [] | +| [schema](variables.tf#L90) | Topic schema. If set, all messages in this topic should follow this schema. | object({…}) | | null | +| [subscriptions](variables.tf#L100) | Topic subscriptions. Also define push configs for push subscriptions. If options is set to null subscription defaults will be used. Labels default to topic labels if set to null. | map(object({…})) | | {} | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [id](outputs.tf#L17) | Fully qualified topic id. | | +| [schema](outputs.tf#L27) | Schema resource. | | +| [schema_id](outputs.tf#L32) | Schema resource id. | | +| [subscription_id](outputs.tf#L37) | Subscription ids. | | +| [subscriptions](outputs.tf#L48) | Subscription resources. | | +| [topic](outputs.tf#L57) | Topic resource. | | + diff --git a/assets/modules-fabric/v26/pubsub/iam.tf b/assets/modules-fabric/v26/pubsub/iam.tf new file mode 100644 index 0000000..4e39b43 --- /dev/null +++ b/assets/modules-fabric/v26/pubsub/iam.tf @@ -0,0 +1,140 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the authoritative. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + subscription_iam = flatten([ + for k, v in var.subscriptions : [ + for role, members in v.iam : { + subscription = k + role = role + members = members + } + ] + ]) + subscription_iam_bindings = merge([ + for k, v in var.subscriptions : { + for binding_key, data in v.iam_bindings : + binding_key => { + subscription = k + role = data.role + members = data.members + condition = data.condition + } + } + ]...) + subscription_iam_bindings_additive = merge([ + for k, v in var.subscriptions : { + for binding_key, data in v.iam_bindings_additive : + binding_key => { + subscription = k + role = data.role + member = data.member + condition = data.condition + } + } + ]...) +} + +moved { + from = google_pubsub_topic_iam_binding.default + to = google_pubsub_topic_iam_binding.authoritative +} + +resource "google_pubsub_topic_iam_binding" "authoritative" { + for_each = var.iam + project = var.project_id + topic = google_pubsub_topic.default.name + role = each.key + members = each.value +} + +resource "google_pubsub_topic_iam_binding" "bindings" { + for_each = var.iam_bindings + topic = google_pubsub_topic.default.name + role = each.value.role + members = each.value.members + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +resource "google_pubsub_topic_iam_member" "bindings" { + for_each = var.iam_bindings_additive + topic = google_pubsub_topic.default.name + role = each.value.role + member = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +moved { + from = google_pubsub_subscription_iam_binding.default + to = google_pubsub_subscription_iam_binding.authoritative +} + +resource "google_pubsub_subscription_iam_binding" "authoritative" { + for_each = { + for binding in local.subscription_iam : + "${binding.subscription}.${binding.role}" => binding + } + project = var.project_id + subscription = google_pubsub_subscription.default[each.value.subscription].name + role = each.value.role + members = each.value.members +} + +resource "google_pubsub_subscription_iam_binding" "bindings" { + for_each = local.subscription_iam_bindings + project = var.project_id + subscription = google_pubsub_subscription.default[each.value.subscription].name + role = each.value.role + members = each.value.members + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +resource "google_pubsub_subscription_iam_member" "members" { + for_each = local.subscription_iam_bindings_additive + project = var.project_id + subscription = google_pubsub_subscription.default[each.value.subscription].name + role = each.value.role + member = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} diff --git a/assets/modules-fabric/v26/pubsub/main.tf b/assets/modules-fabric/v26/pubsub/main.tf new file mode 100644 index 0000000..de06502 --- /dev/null +++ b/assets/modules-fabric/v26/pubsub/main.tf @@ -0,0 +1,121 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + topic_id_static = "projects/${var.project_id}/topics/${var.name}" +} + +resource "google_pubsub_schema" "default" { + count = var.schema == null ? 0 : 1 + name = "${var.name}-schema" + type = var.schema.schema_type + definition = var.schema.definition + project = var.project_id +} + +resource "google_pubsub_topic" "default" { + project = var.project_id + name = var.name + kms_key_name = var.kms_key + labels = var.labels + message_retention_duration = var.message_retention_duration + + dynamic "message_storage_policy" { + for_each = length(var.regions) > 0 ? [var.regions] : [] + content { + allowed_persistence_regions = var.regions + } + } + + dynamic "schema_settings" { + for_each = var.schema == null ? [] : [""] + content { + schema = google_pubsub_schema.default[0].id + encoding = var.schema.msg_encoding + } + } +} + +resource "google_pubsub_subscription" "default" { + for_each = var.subscriptions + project = var.project_id + name = each.key + topic = google_pubsub_topic.default.name + labels = each.value.labels + ack_deadline_seconds = each.value.ack_deadline_seconds + message_retention_duration = each.value.message_retention_duration + retain_acked_messages = each.value.retain_acked_messages + filter = each.value.filter + enable_message_ordering = each.value.enable_message_ordering + enable_exactly_once_delivery = each.value.enable_exactly_once_delivery + + dynamic "expiration_policy" { + for_each = each.value.expiration_policy_ttl == null ? [] : [""] + content { + ttl = each.value.expiration_policy_ttl + } + } + + dynamic "dead_letter_policy" { + for_each = each.value.dead_letter_policy == null ? [] : [""] + content { + dead_letter_topic = each.value.dead_letter_policy.topic + max_delivery_attempts = each.value.dead_letter_policy.max_delivery_attempts + } + } + + dynamic "push_config" { + for_each = each.value.push == null ? [] : [""] + content { + push_endpoint = each.value.push.endpoint + attributes = each.value.push.attributes + dynamic "oidc_token" { + for_each = each.value.push.oidc_token == null ? [] : [""] + content { + service_account_email = each.value.push.oidc_token.service_account_email + audience = each.value.push.oidc_token.audience + } + } + } + } + + dynamic "bigquery_config" { + for_each = each.value.bigquery == null ? [] : [""] + content { + table = each.value.bigquery.table + use_topic_schema = each.value.bigquery.use_topic_schema + write_metadata = each.value.bigquery.write_metadata + drop_unknown_fields = each.value.bigquery.drop_unknown_fields + } + } + + dynamic "cloud_storage_config" { + for_each = each.value.cloud_storage == null ? [] : [""] + content { + bucket = each.value.cloud_storage.bucket + filename_prefix = each.value.cloud_storage.filename_prefix + filename_suffix = each.value.cloud_storage.filename_suffix + max_duration = each.value.cloud_storage.max_duration + max_bytes = each.value.cloud_storage.max_bytes + dynamic "avro_config" { + for_each = each.value.cloud_storage.avro_config == null ? [] : [""] + content { + write_metadata = each.value.cloud_storage.avro_config.write_metadata + } + } + } + } +} diff --git a/assets/modules-fabric/v26/pubsub/outputs.tf b/assets/modules-fabric/v26/pubsub/outputs.tf new file mode 100644 index 0000000..8218e2b --- /dev/null +++ b/assets/modules-fabric/v26/pubsub/outputs.tf @@ -0,0 +1,64 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "id" { + description = "Fully qualified topic id." + value = local.topic_id_static + depends_on = [ + google_pubsub_topic.default, + google_pubsub_topic_iam_binding.authoritative, + google_pubsub_topic_iam_binding.bindings + ] +} + +output "schema" { + description = "Schema resource." + value = try(google_pubsub_schema.default[0], null) +} + +output "schema_id" { + description = "Schema resource id." + value = try(google_pubsub_schema.default[0].id, null) +} + +output "subscription_id" { + description = "Subscription ids." + value = { + for k, v in google_pubsub_subscription.default : k => v.id + } + depends_on = [ + google_pubsub_subscription_iam_binding.authoritative, + google_pubsub_subscription_iam_binding.bindings + ] +} + +output "subscriptions" { + description = "Subscription resources." + value = google_pubsub_subscription.default + depends_on = [ + google_pubsub_subscription_iam_binding.authoritative, + google_pubsub_subscription_iam_binding.bindings + ] +} + +output "topic" { + description = "Topic resource." + value = google_pubsub_topic.default + depends_on = [ + google_pubsub_topic_iam_binding.authoritative, + google_pubsub_topic_iam_binding.bindings + ] +} diff --git a/assets/modules-fabric/v26/pubsub/variables.tf b/assets/modules-fabric/v26/pubsub/variables.tf new file mode 100644 index 0000000..370c42f --- /dev/null +++ b/assets/modules-fabric/v26/pubsub/variables.tf @@ -0,0 +1,168 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "iam" { + description = "IAM bindings for topic in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} + nullable = false +} + +variable "iam_bindings" { + description = "Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary." + type = map(object({ + members = list(string) + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} +} + +variable "iam_bindings_additive" { + description = "Keyring individual additive IAM bindings. Keys are arbitrary." + type = map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} +} + +variable "kms_key" { + description = "KMS customer managed encryption key." + type = string + default = null +} + +variable "labels" { + description = "Labels." + type = map(string) + default = {} + nullable = false +} + +variable "message_retention_duration" { + description = "Minimum duration to retain a message after it is published to the topic." + type = string + default = null +} + +variable "name" { + description = "PubSub topic name." + type = string +} + +variable "project_id" { + description = "Project used for resources." + type = string +} + +variable "regions" { + description = "List of regions used to set persistence policy." + type = list(string) + default = [] + nullable = false +} + +variable "schema" { + description = "Topic schema. If set, all messages in this topic should follow this schema." + type = object({ + definition = string + msg_encoding = optional(string, "ENCODING_UNSPECIFIED") + schema_type = string + }) + default = null +} + +variable "subscriptions" { + description = "Topic subscriptions. Also define push configs for push subscriptions. If options is set to null subscription defaults will be used. Labels default to topic labels if set to null." + type = map(object({ + labels = optional(map(string)) + ack_deadline_seconds = optional(number) + message_retention_duration = optional(string) + retain_acked_messages = optional(bool, false) + expiration_policy_ttl = optional(string) + filter = optional(string) + enable_message_ordering = optional(bool, false) + enable_exactly_once_delivery = optional(bool, false) + dead_letter_policy = optional(object({ + topic = string + max_delivery_attempts = optional(number) + })) + retry_policy = optional(object({ + minimum_backoff = optional(number) + maximum_backoff = optional(number) + })) + + bigquery = optional(object({ + table = string + use_topic_schema = optional(bool, false) + write_metadata = optional(bool, false) + drop_unknown_fields = optional(bool, false) + })) + cloud_storage = optional(object({ + bucket = string + filename_prefix = optional(string) + filename_suffix = optional(string) + max_duration = optional(string) + max_bytes = optional(number) + avro_config = optional(object({ + write_metadata = optional(bool, false) + })) + })) + push = optional(object({ + endpoint = string + attributes = optional(map(string)) + no_wrapper = optional(bool, false) + oidc_token = optional(object({ + audience = optional(string) + service_account_email = string + })) + })) + + iam = optional(map(list(string)), {}) + iam_bindings = optional(map(object({ + members = list(string) + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })), {}) + iam_bindings_additive = optional(map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })), {}) + })) + default = {} + nullable = false +} diff --git a/assets/modules-fabric/v26/pubsub/versions.tf b/assets/modules-fabric/v26/pubsub/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/pubsub/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/secret-manager/README.md b/assets/modules-fabric/v26/secret-manager/README.md new file mode 100644 index 0000000..dc09298 --- /dev/null +++ b/assets/modules-fabric/v26/secret-manager/README.md @@ -0,0 +1,133 @@ +# Google Secret Manager Module + +Simple Secret Manager module that allows managing one or more secrets, their versions, and IAM bindings. + +Secret Manager locations are available via the `gcloud secrets locations list` command. + +**Warning:** managing versions will persist their data (the actual secret you want to protect) in the Terraform state in unencrypted form, accessible to any identity able to read or pull the state file. + +## Examples + +### Secrets + +The secret replication policy is automatically managed if no location is set, or manually managed if a list of locations is passed to the secret. + +```hcl +module "secret-manager" { + source = "./fabric/modules/secret-manager" + project_id = "my-project" + secrets = { + test-auto = null + test-manual = ["europe-west1", "europe-west4"] + } +} +# tftest modules=1 resources=2 +``` + +### Secret IAM bindings + +IAM bindings can be set per secret in the same way as for most other modules supporting IAM, using the `iam` variable. + +```hcl +module "secret-manager" { + source = "./fabric/modules/secret-manager" + project_id = "my-project" + secrets = { + test-auto = null + test-manual = ["europe-west1", "europe-west4"] + } + iam = { + test-auto = { + "roles/secretmanager.secretAccessor" = ["group:auto-readers@example.com"] + } + test-manual = { + "roles/secretmanager.secretAccessor" = ["group:manual-readers@example.com"] + } + } +} +# tftest modules=1 resources=4 inventory=iam.yaml +``` + +### Secret versions + +As mentioned above, please be aware that **version data will be stored in state in unencrypted form**. + +```hcl +module "secret-manager" { + source = "./fabric/modules/secret-manager" + project_id = "my-project" + secrets = { + test-auto = null + test-manual = ["europe-west1", "europe-west4"] + } + versions = { + test-auto = { + v1 = { enabled = false, data = "auto foo bar baz" } + v2 = { enabled = true, data = "auto foo bar spam" } + }, + test-manual = { + v1 = { enabled = true, data = "manual foo bar spam" } + } + } +} +# tftest modules=1 resources=5 inventory=versions.yaml +``` + +### Secret with customer managed encryption key + +Secrets will be used if an encryption key is set in the `encryption_key` variable for the secret region. + +```hcl +module "secret-manager" { + source = "./fabric/modules/secret-manager" + project_id = "my-project" + secrets = { + test-encryption = ["europe-west1", "europe-west4"] + } + encryption_key = { + europe-west1 = "projects/PROJECT_ID/locations/europe-west1/keyRings/KEYRING/cryptoKeys/KEY" + europe-west4 = "projects/PROJECT_ID/locations/europe-west4/keyRings/KEYRING/cryptoKeys/KEY" + } +} +# tftest modules=1 resources=1 +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [project_id](variables.tf#L35) | Project id where the keyring will be created. | string | ✓ | | +| [encryption_key](variables.tf#L17) | Self link of the KMS keys in {LOCATION => KEY} format. A key must be provided for all replica locations. | map(string) | | null | +| [iam](variables.tf#L23) | IAM bindings in {SECRET => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {} | +| [labels](variables.tf#L29) | Optional labels for each secret. | map(map(string)) | | {} | +| [secrets](variables.tf#L40) | Map of secrets to manage and their locations. If locations is null, automatic management will be set. | map(list(string)) | | {} | +| [versions](variables.tf#L46) | Optional versions to manage for each secret. Version names are only used internally to track individual versions. | map(map(object({…}))) | | {} | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [ids](outputs.tf#L17) | Fully qualified secret ids. | | +| [secrets](outputs.tf#L24) | Secret resources. | | +| [version_ids](outputs.tf#L29) | Version ids keyed by secret name : version name. | | +| [versions](outputs.tf#L36) | Secret versions. | ✓ | + + +## Requirements + +These sections describe requirements for using this module. + +### IAM + +The following roles must be used to provision the resources of this module: + +- Cloud KMS Admin: `roles/cloudkms.admin` or +- Owner: `roles/owner` + +### APIs + +A project with the following APIs enabled must be used to host the +resources of this module: + +- Google Cloud Key Management Service: `cloudkms.googleapis.com` diff --git a/assets/modules-fabric/v26/secret-manager/main.tf b/assets/modules-fabric/v26/secret-manager/main.tf new file mode 100644 index 0000000..73932b5 --- /dev/null +++ b/assets/modules-fabric/v26/secret-manager/main.tf @@ -0,0 +1,90 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + # distinct is needed to make the expanding function argument work + iam = flatten([ + for secret, roles in var.iam : [ + for role, members in roles : { + secret = secret + role = role + members = members + } + ] + ]) + version_pairs = flatten([ + for secret, versions in var.versions : [ + for name, attrs in versions : merge(attrs, { name = name, secret = secret }) + ] + ]) + version_keypairs = { + for pair in local.version_pairs : "${pair.secret}:${pair.name}" => pair + } +} + +resource "google_secret_manager_secret" "default" { + for_each = var.secrets + project = var.project_id + secret_id = each.key + labels = lookup(var.labels, each.key, null) + + dynamic "replication" { + for_each = each.value == null ? [""] : [] + content { + automatic = true + } + } + + dynamic "replication" { + for_each = each.value == null ? [] : [each.value] + iterator = locations + content { + user_managed { + dynamic "replicas" { + for_each = locations.value + iterator = location + content { + location = location.value + dynamic "customer_managed_encryption" { + for_each = try(var.encryption_key[location.value] != null ? [""] : [], []) + content { + kms_key_name = var.encryption_key[location.value] + } + } + } + } + } + } + } +} + +resource "google_secret_manager_secret_version" "default" { + provider = google-beta + for_each = local.version_keypairs + secret = google_secret_manager_secret.default[each.value.secret].id + enabled = each.value.enabled + secret_data = each.value.data +} + +resource "google_secret_manager_secret_iam_binding" "default" { + provider = google-beta + for_each = { + for binding in local.iam : "${binding.secret}.${binding.role}" => binding + } + role = each.value.role + secret_id = google_secret_manager_secret.default[each.value.secret].id + members = each.value.members +} diff --git a/assets/modules-fabric/v26/secret-manager/outputs.tf b/assets/modules-fabric/v26/secret-manager/outputs.tf new file mode 100644 index 0000000..fcd6e1f --- /dev/null +++ b/assets/modules-fabric/v26/secret-manager/outputs.tf @@ -0,0 +1,40 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "ids" { + description = "Fully qualified secret ids." + value = { + for k, v in google_secret_manager_secret.default : v.secret_id => v.id + } +} + +output "secrets" { + description = "Secret resources." + value = google_secret_manager_secret.default +} + +output "version_ids" { + description = "Version ids keyed by secret name : version name." + value = { + for k, v in google_secret_manager_secret_version.default : k => v.id + } +} + +output "versions" { + description = "Secret versions." + value = google_secret_manager_secret_version.default + sensitive = true +} diff --git a/assets/modules-fabric/v26/secret-manager/variables.tf b/assets/modules-fabric/v26/secret-manager/variables.tf new file mode 100644 index 0000000..7d7b528 --- /dev/null +++ b/assets/modules-fabric/v26/secret-manager/variables.tf @@ -0,0 +1,53 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "encryption_key" { + description = "Self link of the KMS keys in {LOCATION => KEY} format. A key must be provided for all replica locations." + type = map(string) + default = null +} + +variable "iam" { + description = "IAM bindings in {SECRET => {ROLE => [MEMBERS]}} format." + type = map(map(list(string))) + default = {} +} + +variable "labels" { + description = "Optional labels for each secret." + type = map(map(string)) + default = {} +} + +variable "project_id" { + description = "Project id where the keyring will be created." + type = string +} + +variable "secrets" { + description = "Map of secrets to manage and their locations. If locations is null, automatic management will be set." + type = map(list(string)) + default = {} +} + +variable "versions" { + description = "Optional versions to manage for each secret. Version names are only used internally to track individual versions." + type = map(map(object({ + enabled = bool + data = string + }))) + default = {} +} diff --git a/assets/modules-fabric/v26/secret-manager/versions.tf b/assets/modules-fabric/v26/secret-manager/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/secret-manager/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/service-directory/README.md b/assets/modules-fabric/v26/service-directory/README.md new file mode 100644 index 0000000..d79c530 --- /dev/null +++ b/assets/modules-fabric/v26/service-directory/README.md @@ -0,0 +1,120 @@ +# Google Cloud Service Directory Module + +This module allows managing a single [Service Directory](https://cloud.google.com/service-directory) namespace, including multiple services, endpoints and IAM bindings at the namespace and service levels. + +It can be used in conjunction with the [DNS](../dns) module to create [service-directory based DNS zones](https://cloud.google.com/service-directory/docs/configuring-service-directory-zone, offloading IAM control of `A` and `SRV` records at the namespace or service level to Service Directory. The last examples shows how to wire the two modules together. + + +## Examples + +### Namespace with IAM + +```hcl +module "service-directory" { + source = "./fabric/modules/service-directory" + project_id = "my-project" + location = "europe-west1" + name = "sd-1" + iam = { + "roles/servicedirectory.editor" = [ + "serviceAccount:namespace-editor@example.com" + ] + } +} +# tftest modules=1 resources=2 inventory=simple.yaml +``` + +### Services with IAM and endpoints + +```hcl +module "service-directory" { + source = "./fabric/modules/service-directory" + project_id = "my-project" + location = "europe-west1" + name = "sd-1" + services = { + one = { + endpoints = ["first", "second"] + metadata = null + } + } + service_iam = { + one = { + "roles/servicedirectory.editor" = [ + "serviceAccount:service-editor.example.com" + ] + } + } + endpoint_config = { + "one/first" = { address = "127.0.0.1", port = 80, metadata = {} } + "one/second" = { address = "127.0.0.2", port = 80, metadata = {} } + } +} +# tftest modules=1 resources=5 inventory=services.yaml +``` + +### DNS based zone + +Wiring a service directory namespace to a private DNS zone allows querying the namespace, and delegating control of DNS records at the namespace or service level. This effectively allows fine grained ACL control of Cloud DNS zones. + +```hcl +module "service-directory" { + source = "./fabric/modules/service-directory" + project_id = "my-project" + location = "europe-west1" + name = "apps" + iam = { + "roles/servicedirectory.editor" = [ + "serviceAccount:namespace-editor@example.com" + ] + } + services = { + app1 = { endpoints = ["one"], metadata = null } + } + endpoint_config = { + "app1/one" = { address = "127.0.0.1", port = 80, metadata = {} } + } +} + +module "dns-sd" { + source = "./fabric/modules/dns" + project_id = "my-project" + name = "apps" + zone_config = { + domain = "apps.example.org." + private = { + client_networks = [var.vpc.self_link] + service_directory_namespace = module.service-directory.id + } + } +} +# tftest modules=2 resources=6 inventory=dns.yaml +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [location](variables.tf#L40) | Namespace location. | string | ✓ | | +| [name](variables.tf#L45) | Namespace name. | string | ✓ | | +| [project_id](variables.tf#L50) | Project used for resources. | string | ✓ | | +| [endpoint_config](variables.tf#L18) | Map of endpoint attributes, keys are in service/endpoint format. | map(object({…})) | | {} | +| [iam](variables.tf#L28) | IAM bindings for namespace, in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [labels](variables.tf#L34) | Labels. | map(string) | | {} | +| [service_iam](variables.tf#L55) | IAM bindings for services, in {SERVICE => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {} | +| [services](variables.tf#L61) | Service configuration, using service names as keys. | map(object({…})) | | {} | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [endpoints](outputs.tf#L17) | Endpoint resources. | | +| [id](outputs.tf#L22) | Fully qualified namespace id. | | +| [name](outputs.tf#L27) | Namespace name. | | +| [namespace](outputs.tf#L32) | Namespace resource. | | +| [service_id](outputs.tf#L40) | Service ids (short names). | | +| [service_names](outputs.tf#L50) | Service ids (long names). | | +| [services](outputs.tf#L60) | Service resources. | | + + diff --git a/assets/modules-fabric/v26/service-directory/main.tf b/assets/modules-fabric/v26/service-directory/main.tf new file mode 100644 index 0000000..781bae6 --- /dev/null +++ b/assets/modules-fabric/v26/service-directory/main.tf @@ -0,0 +1,78 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + endpoint_list = flatten([ + for name, attrs in var.services : [ + for endpoint in attrs.endpoints : { service : name, endpoint : endpoint } + ] + ]) + endpoints = { + for ep in local.endpoint_list : "${ep.service}/${ep.endpoint}" => ep + } + iam_pairs = var.service_iam == null ? [] : flatten([ + for name, bindings in var.service_iam : + [for role in keys(bindings) : { name = name, role = role }] + ]) + iam_keypairs = { + for pair in local.iam_pairs : + "${pair.name}-${pair.role}" => pair + } +} + +resource "google_service_directory_namespace" "default" { + provider = google-beta + project = var.project_id + namespace_id = var.name + location = var.location + labels = var.labels +} + +resource "google_service_directory_namespace_iam_binding" "default" { + provider = google-beta + for_each = var.iam + name = google_service_directory_namespace.default.name + role = each.key + members = each.value +} + +resource "google_service_directory_service" "default" { + provider = google-beta + for_each = var.services + namespace = google_service_directory_namespace.default.id + service_id = each.key + metadata = each.value.metadata +} + +resource "google_service_directory_service_iam_binding" "default" { + provider = google-beta + for_each = local.iam_keypairs + name = google_service_directory_service.default[each.value.name].name + role = each.value.role + members = lookup( + lookup(var.service_iam, each.value.name, {}), each.value.role, [] + ) +} + +resource "google_service_directory_endpoint" "default" { + provider = google-beta + for_each = local.endpoints + endpoint_id = each.value.endpoint + service = google_service_directory_service.default[each.value.service].id + metadata = try(var.endpoint_config[each.key].metadata, null) + address = try(var.endpoint_config[each.key].address, null) + port = try(var.endpoint_config[each.key].port, null) +} diff --git a/assets/modules-fabric/v26/service-directory/outputs.tf b/assets/modules-fabric/v26/service-directory/outputs.tf new file mode 100644 index 0000000..964a213 --- /dev/null +++ b/assets/modules-fabric/v26/service-directory/outputs.tf @@ -0,0 +1,66 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "endpoints" { + description = "Endpoint resources." + value = google_service_directory_endpoint.default +} + +output "id" { + description = "Fully qualified namespace id." + value = google_service_directory_namespace.default.id +} + +output "name" { + description = "Namespace name." + value = google_service_directory_namespace.default.name +} + +output "namespace" { + description = "Namespace resource." + value = google_service_directory_namespace.default + depends_on = [ + google_service_directory_namespace_iam_binding.default + ] +} + +output "service_id" { + description = "Service ids (short names)." + value = { + for k, v in google_service_directory_service.default : k => v.id + } + depends_on = [ + google_service_directory_service_iam_binding.default + ] +} + +output "service_names" { + description = "Service ids (long names)." + value = { + for k, v in google_service_directory_service.default : k => v.name + } + depends_on = [ + google_service_directory_service_iam_binding.default + ] +} + +output "services" { + description = "Service resources." + value = google_service_directory_service.default + depends_on = [ + google_service_directory_service_iam_binding.default + ] +} diff --git a/assets/modules-fabric/v26/service-directory/variables.tf b/assets/modules-fabric/v26/service-directory/variables.tf new file mode 100644 index 0000000..326aeff --- /dev/null +++ b/assets/modules-fabric/v26/service-directory/variables.tf @@ -0,0 +1,68 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# we need a separate variable as address will be dynamic in most cases +variable "endpoint_config" { + description = "Map of endpoint attributes, keys are in service/endpoint format." + type = map(object({ + address = string + port = number + metadata = map(string) + })) + default = {} +} + +variable "iam" { + description = "IAM bindings for namespace, in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} +} + +variable "labels" { + description = "Labels." + type = map(string) + default = {} +} + +variable "location" { + description = "Namespace location." + type = string +} + +variable "name" { + description = "Namespace name." + type = string +} + +variable "project_id" { + description = "Project used for resources." + type = string +} + +variable "service_iam" { + description = "IAM bindings for services, in {SERVICE => {ROLE => [MEMBERS]}} format." + type = map(map(list(string))) + default = {} +} + +variable "services" { + description = "Service configuration, using service names as keys." + type = map(object({ + endpoints = list(string) + metadata = map(string) + })) + default = {} +} diff --git a/assets/modules-fabric/v26/service-directory/versions.tf b/assets/modules-fabric/v26/service-directory/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/service-directory/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/source-repository/README.md b/assets/modules-fabric/v26/source-repository/README.md new file mode 100644 index 0000000..c60ba7e --- /dev/null +++ b/assets/modules-fabric/v26/source-repository/README.md @@ -0,0 +1,93 @@ +# Google Cloud Source Repository Module + +This module allows managing a single Cloud Source Repository, including IAM bindings and basic Cloud Build triggers. + + +- [Examples](#examples) + - [Repository with IAM](#repository-with-iam) + - [Repository with Cloud Build trigger](#repository-with-cloud-build-trigger) +- [Files](#files) +- [Variables](#variables) +- [Outputs](#outputs) + + +## Examples + +### Repository with IAM + +```hcl +module "repo" { + source = "./fabric/modules/source-repository" + project_id = "my-project" + name = "my-repo" + iam = { + "roles/source.reader" = ["user:foo@example.com"] + } + iam_bindings_additive = { + am1-reader = { + member = "user:am1@example.com" + role = "roles/source.reader" + } + } +} +# tftest modules=1 resources=3 inventory=simple.yaml +``` + +### Repository with Cloud Build trigger + +```hcl +module "repo" { + source = "./fabric/modules/source-repository" + project_id = "my-project" + name = "my-repo" + triggers = { + foo = { + filename = "ci/workflow-foo.yaml" + included_files = ["**/*tf"] + service_account = null + substitutions = { + BAR = 1 + } + template = { + branch_name = "main" + project_id = null + tag_name = null + } + } + } +} +# tftest modules=1 resources=2 inventory=trigger.yaml +``` + + + +## Files + +| name | description | resources | +|---|---|---| +| [iam.tf](./iam.tf) | IAM resources. | google_sourcerepo_repository_iam_binding · google_sourcerepo_repository_iam_member | +| [main.tf](./main.tf) | Module-level locals and resources. | google_cloudbuild_trigger · google_sourcerepo_repository | +| [outputs.tf](./outputs.tf) | Module outputs. | | +| [variables.tf](./variables.tf) | Module variables. | | +| [versions.tf](./versions.tf) | Version pins. | | + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [name](variables.tf#L61) | Repository name. | string | ✓ | | +| [project_id](variables.tf#L66) | Project used for resources. | string | ✓ | | +| [group_iam](variables.tf#L17) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | +| [iam](variables.tf#L24) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [iam_bindings](variables.tf#L31) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | map(object({…})) | | {} | +| [iam_bindings_additive](variables.tf#L46) | Individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | +| [triggers](variables.tf#L71) | Cloud Build triggers. | map(object({…})) | | {} | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [id](outputs.tf#L17) | Fully qualified repository id. | | +| [name](outputs.tf#L22) | Repository name. | | +| [url](outputs.tf#L27) | Repository URL. | | + diff --git a/assets/modules-fabric/v26/source-repository/iam.tf b/assets/modules-fabric/v26/source-repository/iam.tf new file mode 100644 index 0000000..1b225d1 --- /dev/null +++ b/assets/modules-fabric/v26/source-repository/iam.tf @@ -0,0 +1,73 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description IAM resources. + +locals { + _group_iam_roles = distinct(flatten(values(var.group_iam))) + _group_iam = { + for r in local._group_iam_roles : r => [ + for k, v in var.group_iam : "group:${k}" if try(index(v, r), null) != null + ] + } + iam = { + for role in distinct(concat(keys(var.iam), keys(local._group_iam))) : + role => concat( + try(var.iam[role], []), + try(local._group_iam[role], []) + ) + } +} + +resource "google_sourcerepo_repository_iam_binding" "authoritative" { + for_each = local.iam + project = var.project_id + repository = google_sourcerepo_repository.default.name + role = each.key + members = each.value +} + +resource "google_sourcerepo_repository_iam_binding" "bindings" { + for_each = var.iam_bindings + project = var.project_id + repository = google_sourcerepo_repository.default.name + role = each.value.role + members = each.value.members + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} + +resource "google_sourcerepo_repository_iam_member" "bindings" { + for_each = var.iam_bindings_additive + project = var.project_id + repository = google_sourcerepo_repository.default.name + role = each.value.role + member = each.value.member + dynamic "condition" { + for_each = each.value.condition == null ? [] : [""] + content { + expression = each.value.condition.expression + title = each.value.condition.title + description = each.value.condition.description + } + } +} diff --git a/assets/modules-fabric/v26/source-repository/main.tf b/assets/modules-fabric/v26/source-repository/main.tf new file mode 100644 index 0000000..d74b7e6 --- /dev/null +++ b/assets/modules-fabric/v26/source-repository/main.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_sourcerepo_repository" "default" { + project = var.project_id + name = var.name +} + +resource "google_cloudbuild_trigger" "default" { + for_each = coalesce(var.triggers, {}) + project = var.project_id + name = each.key + filename = each.value.filename + included_files = each.value.included_files + service_account = each.value.service_account + substitutions = each.value.substitutions + trigger_template { + project_id = try(each.value.template.project_id, var.project_id) + branch_name = try(each.value.template.branch_name, null) + repo_name = google_sourcerepo_repository.default.name + tag_name = try(each.value.template.tag_name, null) + } +} diff --git a/assets/modules-fabric/v26/source-repository/outputs.tf b/assets/modules-fabric/v26/source-repository/outputs.tf new file mode 100644 index 0000000..3b06d30 --- /dev/null +++ b/assets/modules-fabric/v26/source-repository/outputs.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "id" { + description = "Fully qualified repository id." + value = google_sourcerepo_repository.default.id +} + +output "name" { + description = "Repository name." + value = google_sourcerepo_repository.default.name +} + +output "url" { + description = "Repository URL." + value = google_sourcerepo_repository.default.url +} diff --git a/assets/modules-fabric/v26/source-repository/variables.tf b/assets/modules-fabric/v26/source-repository/variables.tf new file mode 100644 index 0000000..23bfa78 --- /dev/null +++ b/assets/modules-fabric/v26/source-repository/variables.tf @@ -0,0 +1,86 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "group_iam" { + description = "Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable." + type = map(list(string)) + default = {} + nullable = false +} + +variable "iam" { + description = "IAM bindings in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} + nullable = false +} + +variable "iam_bindings" { + description = "Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary." + type = map(object({ + members = list(string) + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} +} + +variable "iam_bindings_additive" { + description = "Individual additive IAM bindings. Keys are arbitrary." + type = map(object({ + member = string + role = string + condition = optional(object({ + expression = string + title = string + description = optional(string) + })) + })) + nullable = false + default = {} +} + +variable "name" { + description = "Repository name." + type = string +} + +variable "project_id" { + description = "Project used for resources." + type = string +} + +variable "triggers" { + description = "Cloud Build triggers." + type = map(object({ + filename = string + included_files = list(string) + service_account = string + substitutions = map(string) + template = object({ + branch_name = string + project_id = string + tag_name = string + }) + })) + default = {} + nullable = false +} diff --git a/assets/modules-fabric/v26/source-repository/versions.tf b/assets/modules-fabric/v26/source-repository/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/source-repository/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/modules-fabric/v26/vpc-sc/README.md b/assets/modules-fabric/v26/vpc-sc/README.md new file mode 100644 index 0000000..91d90d4 --- /dev/null +++ b/assets/modules-fabric/v26/vpc-sc/README.md @@ -0,0 +1,223 @@ +# VPC Service Controls + +This module offers a unified interface to manage VPC Service Controls [Access Policy](https://cloud.google.com/access-context-manager/docs/create-access-policy), [Access Levels](https://cloud.google.com/access-context-manager/docs/manage-access-levels), and [Service Perimeters](https://cloud.google.com/vpc-service-controls/docs/service-perimeters). + +Given the complexity of the underlying resources, the module intentionally mimics their interfaces to make it easier to map their documentation onto its variables, and reduce the internal complexity. + +If you are using [Application Default Credentials](https://cloud.google.com/sdk/gcloud/reference/auth/application-default) with Terraform and run into permissions issues, make sure to check out the recommended provider configuration in the [VPC SC resources documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/access_context_manager_access_level). + +## Examples + +### Access policy + +By default, the module is configured to use an existing policy, passed in by name in the `access_policy` variable: + +```hcl +module "test" { + source = "./fabric/modules/vpc-sc" + access_policy = "12345678" +} +# tftest modules=0 resources=0 +``` + +If you need the module to create the policy for you, use the `access_policy_create` variable, and set `access_policy` to `null`: + +```hcl +module "test" { + source = "./fabric/modules/vpc-sc" + access_policy = null + access_policy_create = { + parent = "organizations/123456" + title = "vpcsc-policy" + } +} +# tftest modules=1 resources=1 inventory=access-policy.yaml +``` + +If you need the module to create a scoped policy for you, specify 'scopes' of the policy in the `access_policy_create` variable: + +```hcl +module "test" { + source = "./fabric/modules/vpc-sc" + access_policy = null + access_policy_create = { + parent = "organizations/123456" + title = "vpcsc-policy" + scopes = ["folders/456789"] + } +} +# tftest modules=1 resources=1 inventory=scoped-access-policy.yaml +``` + +### Access levels + +As highlighted above, the `access_levels` type replicates the underlying resource structure. + +```hcl +module "test" { + source = "./fabric/modules/vpc-sc" + access_policy = "12345678" + access_levels = { + a1 = { + conditions = [ + { members = ["user:user1@example.com"] } + ] + } + a2 = { + combining_function = "OR" + conditions = [ + { regions = ["IT", "FR"] }, + { ip_subnetworks = ["101.101.101.0/24"] } + ] + } + } +} +# tftest modules=1 resources=2 inventory=access-levels.yaml +``` + +### Service perimeters + +Bridge and regular service perimeters use two separate variables, as bridge perimeters only accept a limited number of arguments, and can leverage a much simpler interface. + +The regular perimeters variable exposes all the complexity of the underlying resource, use [its documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/access_context_manager_service_perimeter) as a reference about the possible values and configurations. + +If you need to refer to access levels created by the same module in regular service perimeters, you can either use the module's outputs in the provided variables, or the key used to identify the relevant access level. The example below shows how to do this in practice. + +/* +Resources for both perimeters have a `lifecycle` block that ignores changes to `spec` and `status` resources (projects), to allow using the additive resource `google_access_context_manager_service_perimeter_resource` at project creation. If this is not needed, the `lifecycle` blocks can be safely commented in the code. +*/ + +#### Bridge type + +```hcl +module "test" { + source = "./fabric/modules/vpc-sc" + access_policy = "12345678" + service_perimeters_bridge = { + b1 = { + status_resources = ["projects/111110", "projects/111111"] + } + b2 = { + spec_resources = ["projects/222220", "projects/222221"] + use_explicit_dry_run_spec = true + } + } +} +# tftest modules=1 resources=2 inventory=bridge.yaml +``` + +#### Regular type + +```hcl +module "test" { + source = "./fabric/modules/vpc-sc" + access_policy = "12345678" + access_levels = { + a1 = { + conditions = [ + { members = ["user:user1@example.com"] } + ] + } + a2 = { + conditions = [ + { members = ["user:user2@example.com"] } + ] + } + } + egress_policies = { + # allow writing to external GCS bucket from a specific SA + gcs-sa-foo = { + from = { + identities = [ + "serviceAccount:foo@myproject.iam.gserviceaccount.com" + ] + } + to = { + operations = [{ + method_selectors = ["*"] + service_name = "storage.googleapis.com" + }] + resources = ["projects/123456789"] + } + } + } + ingress_policies = { + # allow management from external automation SA + sa-tf-test = { + from = { + identities = [ + "serviceAccount:test-tf@myproject.iam.gserviceaccount.com", + ] + access_levels = ["*"] + } + to = { + operations = [{ service_name = "*" }] + resources = ["*"] + } + } + } + service_perimeters_regular = { + r1 = { + status = { + access_levels = ["a1", "a2"] + resources = ["projects/11111", "projects/111111"] + restricted_services = ["storage.googleapis.com"] + egress_policies = ["gcs-sa-foo"] + ingress_policies = ["sa-tf-test"] + vpc_accessible_services = { + allowed_services = ["storage.googleapis.com"] + enable_restriction = true + } + } + } + } +} +# tftest modules=1 resources=3 inventory=regular.yaml +``` + +## Notes + +- To remove an access level, first remove the binding between perimeter and the access level in `status` and/or `spec` without removing the access level itself. Once you have run `terraform apply`, you'll then be able to remove the access level and run `terraform apply` again. + +## TODO + +- [ ] implement support for the `google_access_context_manager_gcp_user_access_binding` resource + + + +## Files + +| name | description | resources | +|---|---|---| +| [access-levels.tf](./access-levels.tf) | Access level resources. | google_access_context_manager_access_level | +| [main.tf](./main.tf) | Module-level locals and resources. | google_access_context_manager_access_policy | +| [outputs.tf](./outputs.tf) | Module outputs. | | +| [service-perimeters-bridge.tf](./service-perimeters-bridge.tf) | Bridge service perimeter resources. | google_access_context_manager_service_perimeter | +| [service-perimeters-regular.tf](./service-perimeters-regular.tf) | Regular service perimeter resources. | google_access_context_manager_service_perimeter | +| [variables.tf](./variables.tf) | Module variables. | | +| [versions.tf](./versions.tf) | Version pins. | | + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [access_policy](variables.tf#L56) | Access Policy name, set to null if creating one. | string | ✓ | | +| [access_levels](variables.tf#L17) | Access level definitions. | map(object({…})) | | {} | +| [access_policy_create](variables.tf#L61) | Access Policy configuration, fill in to create. Parent is in 'organizations/123456' format, scopes are in 'folders/456789' or 'projects/project_id' format. | object({…}) | | null | +| [egress_policies](variables.tf#L71) | Egress policy definitions that can be referenced in perimeters. | map(object({…})) | | {} | +| [ingress_policies](variables.tf#L102) | Ingress policy definitions that can be referenced in perimeters. | map(object({…})) | | {} | +| [service_perimeters_bridge](variables.tf#L134) | Bridge service perimeters. | map(object({…})) | | {} | +| [service_perimeters_regular](variables.tf#L144) | Regular service perimeters. | map(object({…})) | | {} | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [access_level_names](outputs.tf#L17) | Access level resources. | | +| [access_levels](outputs.tf#L25) | Access level resources. | | +| [access_policy](outputs.tf#L30) | Access policy resource, if autocreated. | | +| [access_policy_name](outputs.tf#L37) | Access policy name. | | +| [id](outputs.tf#L42) | Fully qualified access policy id. | | +| [service_perimeters_bridge](outputs.tf#L47) | Bridge service perimeter resources. | | +| [service_perimeters_regular](outputs.tf#L52) | Regular service perimeter resources. | | + diff --git a/assets/modules-fabric/v26/vpc-sc/access-levels.tf b/assets/modules-fabric/v26/vpc-sc/access-levels.tf new file mode 100644 index 0000000..1eb8534 --- /dev/null +++ b/assets/modules-fabric/v26/vpc-sc/access-levels.tf @@ -0,0 +1,79 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Access level resources. + +# this code implements "additive" access levels, if "authoritative" +# access levels are needed, switch to the +# google_access_context_manager_access_levels resource + +resource "google_access_context_manager_access_level" "basic" { + for_each = var.access_levels + parent = "accessPolicies/${local.access_policy}" + name = "accessPolicies/${local.access_policy}/accessLevels/${each.key}" + title = each.key + description = each.value.description + + basic { + combining_function = each.value.combining_function + + dynamic "conditions" { + for_each = toset(each.value.conditions) + iterator = c + content { + ip_subnetworks = c.value.ip_subnetworks + members = c.value.members + negate = c.value.negate + regions = c.value.regions + required_access_levels = coalesce(c.value.required_access_levels, []) + + dynamic "device_policy" { + for_each = c.value.device_policy == null ? [] : [c.value.device_policy] + iterator = dp + content { + + allowed_device_management_levels = ( + dp.value.allowed_device_management_levels + ) + allowed_encryption_statuses = ( + dp.value.allowed_encryption_statuses + ) + require_admin_approval = dp.value.key.require_admin_approval + require_corp_owned = dp.value.require_corp_owned + require_screen_lock = dp.value.require_screen_lock + + dynamic "os_constraints" { + for_each = toset( + dp.value.os_constraints == null + ? [] + : dp.value.os_constraints + ) + iterator = oc + content { + minimum_version = oc.value.minimum_version + os_type = oc.value.os_type + require_verified_chrome_os = oc.value.require_verified_chrome_os + } + } + + } + } + + } + } + + } +} diff --git a/assets/modules-fabric/v26/vpc-sc/main.tf b/assets/modules-fabric/v26/vpc-sc/main.tf new file mode 100644 index 0000000..7dd5890 --- /dev/null +++ b/assets/modules-fabric/v26/vpc-sc/main.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + access_policy = try( + google_access_context_manager_access_policy.default.0.name, + var.access_policy + ) +} + +resource "google_access_context_manager_access_policy" "default" { + count = var.access_policy_create != null ? 1 : 0 + parent = var.access_policy_create.parent + title = var.access_policy_create.title + scopes = var.access_policy_create.scopes +} diff --git a/assets/modules-fabric/v26/vpc-sc/outputs.tf b/assets/modules-fabric/v26/vpc-sc/outputs.tf new file mode 100644 index 0000000..6a0ce68 --- /dev/null +++ b/assets/modules-fabric/v26/vpc-sc/outputs.tf @@ -0,0 +1,55 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "access_level_names" { + description = "Access level resources." + value = { + for k, v in google_access_context_manager_access_level.basic : + k => v.name + } +} + +output "access_levels" { + description = "Access level resources." + value = google_access_context_manager_access_level.basic +} + +output "access_policy" { + description = "Access policy resource, if autocreated." + value = try(google_access_context_manager_access_policy.default.0, null) +} + +# TODO: deprecate in favor of id + +output "access_policy_name" { + description = "Access policy name." + value = local.access_policy +} + +output "id" { + description = "Fully qualified access policy id." + value = local.access_policy +} + +output "service_perimeters_bridge" { + description = "Bridge service perimeter resources." + value = google_access_context_manager_service_perimeter.bridge +} + +output "service_perimeters_regular" { + description = "Regular service perimeter resources." + value = google_access_context_manager_service_perimeter.regular +} diff --git a/assets/modules-fabric/v26/vpc-sc/service-perimeters-bridge.tf b/assets/modules-fabric/v26/vpc-sc/service-perimeters-bridge.tf new file mode 100644 index 0000000..c3ca5ca --- /dev/null +++ b/assets/modules-fabric/v26/vpc-sc/service-perimeters-bridge.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Bridge service perimeter resources. + +# this code implements "additive" service perimeters, if "authoritative" +# service perimeters are needed, switch to the +# google_access_context_manager_service_perimeters resource + +resource "google_access_context_manager_service_perimeter" "bridge" { + for_each = var.service_perimeters_bridge + parent = "accessPolicies/${local.access_policy}" + name = "accessPolicies/${local.access_policy}/servicePerimeters/${each.key}" + title = each.key + perimeter_type = "PERIMETER_TYPE_BRIDGE" + use_explicit_dry_run_spec = each.value.use_explicit_dry_run_spec + + dynamic "spec" { + for_each = each.value.spec_resources == null ? [] : [""] + content { + resources = each.value.spec_resources + } + } + + status { + resources = each.value.status_resources == null ? [] : each.value.status_resources + } + + # lifecycle { + # ignore_changes = [spec[0].resources, status[0].resources] + # } + + depends_on = [ + google_access_context_manager_access_policy.default, + google_access_context_manager_access_level.basic, + google_access_context_manager_service_perimeter.regular + ] +} diff --git a/assets/modules-fabric/v26/vpc-sc/service-perimeters-regular.tf b/assets/modules-fabric/v26/vpc-sc/service-perimeters-regular.tf new file mode 100644 index 0000000..6742a1c --- /dev/null +++ b/assets/modules-fabric/v26/vpc-sc/service-perimeters-regular.tf @@ -0,0 +1,286 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Regular service perimeter resources. + +# this code implements "additive" service perimeters, if "authoritative" +# service perimeters are needed, switch to the +# google_access_context_manager_service_perimeters resource + +resource "google_access_context_manager_service_perimeter" "regular" { + for_each = var.service_perimeters_regular + parent = "accessPolicies/${local.access_policy}" + name = "accessPolicies/${local.access_policy}/servicePerimeters/${each.key}" + title = each.key + perimeter_type = "PERIMETER_TYPE_REGULAR" + use_explicit_dry_run_spec = each.value.use_explicit_dry_run_spec + dynamic "spec" { + for_each = each.value.spec == null ? [] : [each.value.spec] + iterator = spec + content { + access_levels = ( + spec.value.access_levels == null ? null : [ + for k in spec.value.access_levels : + try(google_access_context_manager_access_level.basic[k].id, k) + ] + ) + resources = spec.value.resources + restricted_services = spec.value.restricted_services + + dynamic "egress_policies" { + for_each = spec.value.egress_policies == null ? {} : { + for k in spec.value.egress_policies : + k => lookup(var.egress_policies, k, null) + if contains(keys(var.egress_policies), k) + } + iterator = policy + content { + dynamic "egress_from" { + for_each = policy.value.from == null ? [] : [""] + content { + identity_type = policy.value.from.identity_type + identities = policy.value.from.identities + } + } + dynamic "egress_to" { + for_each = policy.value.to == null ? [] : [""] + content { + resources = policy.value.to.resources + dynamic "operations" { + for_each = toset(policy.value.to.operations) + iterator = o + content { + service_name = o.value.service_name + dynamic "method_selectors" { + for_each = toset(coalesce(o.value.method_selectors, [])) + content { + method = method_selectors.key + } + } + dynamic "method_selectors" { + for_each = toset(coalesce(o.value.permission_selectors, [])) + content { + permission = method_selectors.key + } + } + } + } + } + } + } + } + + dynamic "ingress_policies" { + for_each = spec.value.ingress_policies == null ? {} : { + for k in spec.value.ingress_policies : + k => lookup(var.ingress_policies, k, null) + if contains(keys(var.ingress_policies), k) + } + iterator = policy + content { + dynamic "ingress_from" { + for_each = policy.value.from == null ? [] : [""] + content { + identity_type = policy.value.from.identity_type + identities = policy.value.from.identities + dynamic "sources" { + for_each = toset(policy.value.from.access_levels) + iterator = s + content { + access_level = try( + google_access_context_manager_access_level.basic[s.value].id, s.value + ) + } + } + dynamic "sources" { + for_each = toset(policy.value.from.resources) + content { + resource = sources.key + } + } + } + } + dynamic "ingress_to" { + for_each = policy.value.to == null ? [] : [""] + content { + resources = policy.value.to.resources + dynamic "operations" { + for_each = toset(policy.value.to.operations) + iterator = o + content { + service_name = o.value.service_name + dynamic "method_selectors" { + for_each = toset(coalesce(o.value.method_selectors, [])) + content { + method = method_selectors.value + } + } + dynamic "method_selectors" { + for_each = toset(coalesce(o.value.permission_selectors, [])) + content { + permission = method_selectors.value + } + } + } + } + } + } + } + } + + dynamic "vpc_accessible_services" { + for_each = spec.value.vpc_accessible_services == null ? {} : { 1 = 1 } + content { + allowed_services = spec.value.vpc_accessible_services.allowed_services + enable_restriction = spec.value.vpc_accessible_services.enable_restriction + } + } + + } + } + dynamic "status" { + for_each = each.value.status == null ? [] : [each.value.status] + iterator = status + content { + access_levels = ( + status.value.access_levels == null ? null : [ + for k in status.value.access_levels : + try(google_access_context_manager_access_level.basic[k].id, k) + ] + ) + resources = status.value.resources + restricted_services = status.value.restricted_services + + dynamic "egress_policies" { + for_each = status.value.egress_policies == null ? {} : { + for k in status.value.egress_policies : + k => lookup(var.egress_policies, k, null) + if contains(keys(var.egress_policies), k) + } + iterator = policy + content { + dynamic "egress_from" { + for_each = policy.value.from == null ? [] : [""] + content { + identity_type = policy.value.from.identity_type + identities = policy.value.from.identities + } + } + dynamic "egress_to" { + for_each = policy.value.to == null ? [] : [""] + content { + resources = policy.value.to.resources + dynamic "operations" { + for_each = toset(policy.value.to.operations) + iterator = o + content { + service_name = o.value.service_name + dynamic "method_selectors" { + for_each = toset(coalesce(o.value.method_selectors, [])) + content { + method = method_selectors.key + } + } + dynamic "method_selectors" { + for_each = toset(coalesce(o.value.permission_selectors, [])) + content { + permission = method_selectors.key + } + } + } + } + } + } + } + } + + dynamic "ingress_policies" { + for_each = status.value.ingress_policies == null ? {} : { + for k in status.value.ingress_policies : + k => lookup(var.ingress_policies, k, null) + if contains(keys(var.ingress_policies), k) + } + iterator = policy + content { + dynamic "ingress_from" { + for_each = policy.value.from == null ? [] : [""] + content { + identity_type = policy.value.from.identity_type + identities = policy.value.from.identities + dynamic "sources" { + for_each = toset(policy.value.from.access_levels) + iterator = s + content { + access_level = try( + google_access_context_manager_access_level.basic[s.value].id, + s.value + ) + } + } + dynamic "sources" { + for_each = toset(policy.value.from.resources) + content { + resource = sources.key + } + } + } + } + dynamic "ingress_to" { + for_each = policy.value.to == null ? [] : [""] + content { + resources = policy.value.to.resources + dynamic "operations" { + for_each = toset(policy.value.to.operations) + iterator = o + content { + service_name = o.value.service_name + dynamic "method_selectors" { + for_each = toset(coalesce(o.value.method_selectors, [])) + content { + method = method_selectors.value + } + } + dynamic "method_selectors" { + for_each = toset(coalesce(o.value.permission_selectors, [])) + content { + permission = method_selectors.value + } + } + } + } + } + } + } + } + + dynamic "vpc_accessible_services" { + for_each = status.value.vpc_accessible_services == null ? {} : { 1 = 1 } + content { + allowed_services = status.value.vpc_accessible_services.allowed_services + enable_restriction = status.value.vpc_accessible_services.enable_restriction + } + } + + } + } + # lifecycle { + # ignore_changes = [spec[0].resources, status[0].resources] + # } + depends_on = [ + google_access_context_manager_access_policy.default, + google_access_context_manager_access_level.basic + ] +} diff --git a/assets/modules-fabric/v26/vpc-sc/variables.tf b/assets/modules-fabric/v26/vpc-sc/variables.tf new file mode 100644 index 0000000..8ce4b41 --- /dev/null +++ b/assets/modules-fabric/v26/vpc-sc/variables.tf @@ -0,0 +1,173 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "access_levels" { + description = "Access level definitions." + type = map(object({ + combining_function = optional(string) + conditions = optional(list(object({ + device_policy = optional(object({ + allowed_device_management_levels = optional(list(string)) + allowed_encryption_statuses = optional(list(string)) + require_admin_approval = bool + require_corp_owned = bool + require_screen_lock = optional(bool) + os_constraints = optional(list(object({ + os_type = string + minimum_version = optional(string) + require_verified_chrome_os = optional(bool) + }))) + })) + ip_subnetworks = optional(list(string), []) + members = optional(list(string), []) + negate = optional(bool) + regions = optional(list(string), []) + required_access_levels = optional(list(string), []) + })), []) + description = optional(string) + })) + default = {} + nullable = false + validation { + condition = alltrue([ + for k, v in var.access_levels : ( + v.combining_function == null || + v.combining_function == "AND" || + v.combining_function == "OR" + ) + ]) + error_message = "Invalid `combining_function` value (null, \"AND\", \"OR\" accepted)." + } +} + +variable "access_policy" { + description = "Access Policy name, set to null if creating one." + type = string +} + +variable "access_policy_create" { + description = "Access Policy configuration, fill in to create. Parent is in 'organizations/123456' format, scopes are in 'folders/456789' or 'projects/project_id' format." + type = object({ + parent = string + title = string + scopes = optional(list(string), null) + }) + default = null +} + +variable "egress_policies" { + description = "Egress policy definitions that can be referenced in perimeters." + type = map(object({ + from = object({ + identity_type = optional(string) + identities = optional(list(string)) + }) + to = object({ + operations = optional(list(object({ + method_selectors = optional(list(string)) + permission_selectors = optional(list(string)) + service_name = string + })), []) + resources = optional(list(string)) + resource_type_external = optional(bool, false) + }) + })) + default = {} + nullable = false + validation { + condition = alltrue([ + for k, v in var.egress_policies : + v.from.identity_type == null || contains([ + "IDENTITY_TYPE_UNSPECIFIED", "ANY_IDENTITY", + "ANY_USER", "ANY_SERVICE_ACCOUNT" + ], coalesce(v.from.identity_type, "-")) + ]) + error_message = "Invalid `from.identity_type` value in egress policy." + } +} + +variable "ingress_policies" { + description = "Ingress policy definitions that can be referenced in perimeters." + type = map(object({ + from = object({ + access_levels = optional(list(string), []) + identity_type = optional(string) + identities = optional(list(string)) + resources = optional(list(string), []) + }) + to = object({ + operations = optional(list(object({ + method_selectors = optional(list(string)) + permission_selectors = optional(list(string)) + service_name = string + })), []) + resources = optional(list(string)) + }) + })) + default = {} + nullable = false + validation { + condition = alltrue([ + for k, v in var.ingress_policies : + v.from.identity_type == null || contains([ + "IDENTITY_TYPE_UNSPECIFIED", "ANY_IDENTITY", + "ANY_USER", "ANY_SERVICE_ACCOUNT" + ], coalesce(v.from.identity_type, "-")) + ]) + error_message = "Invalid `from.identity_type` value in ingress policy." + } +} + +variable "service_perimeters_bridge" { + description = "Bridge service perimeters." + type = map(object({ + spec_resources = optional(list(string)) + status_resources = optional(list(string)) + use_explicit_dry_run_spec = optional(bool, false) + })) + default = {} +} + +variable "service_perimeters_regular" { + description = "Regular service perimeters." + type = map(object({ + spec = optional(object({ + access_levels = optional(list(string)) + resources = optional(list(string)) + restricted_services = optional(list(string)) + egress_policies = optional(list(string)) + ingress_policies = optional(list(string)) + vpc_accessible_services = optional(object({ + allowed_services = list(string) + enable_restriction = bool + })) + })) + status = optional(object({ + access_levels = optional(list(string)) + resources = optional(list(string)) + restricted_services = optional(list(string)) + egress_policies = optional(list(string)) + ingress_policies = optional(list(string)) + vpc_accessible_services = optional(object({ + allowed_services = list(string) + enable_restriction = bool + })) + })) + use_explicit_dry_run_spec = optional(bool, false) + })) + default = {} + nullable = false +} diff --git a/assets/modules-fabric/v26/vpc-sc/versions.tf b/assets/modules-fabric/v26/vpc-sc/versions.tf new file mode 100644 index 0000000..91a91a3 --- /dev/null +++ b/assets/modules-fabric/v26/vpc-sc/versions.tf @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.4.4" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.82.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0" # tftest + } + } +} + + diff --git a/assets/providers.tpl b/assets/providers.tpl new file mode 100644 index 0000000..aa71948 --- /dev/null +++ b/assets/providers.tpl @@ -0,0 +1,44 @@ +terraform { + backend "gcs" { + bucket = "${bucket}" + %{~ if sa != null ~} + impersonate_service_account = "${sa}" + %{~ endif ~} + %{~ if prefix != null ~} + prefix = "${prefix}" + %{~ endif ~} + } + + required_version = "~> 1.6.5" + + required_providers { + google = { + source = "hashicorp/google" + version = "4.84.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = "4.84.0" + } + local = { + source = "hashicorp/local" + version = "~> 2.4.0" + } + } +} + +%{ if sa != null ~} +provider "google" { + impersonate_service_account = "${sa}" +} + +provider "google-beta" { + impersonate_service_account = "${sa}" +} + +provider "google" { + alias = "no-impersonate" +} +%{~ endif } + +provider "local" {} \ No newline at end of file diff --git a/assets/providers/00-organization.providers.tf b/assets/providers/00-organization.providers.tf new file mode 100644 index 0000000..1773dc0 --- /dev/null +++ b/assets/providers/00-organization.providers.tf @@ -0,0 +1,37 @@ +terraform { + backend "gcs" { + bucket = "d4science-com-ew8-foundation-tforg-bkt" + impersonate_service_account = "d4science-com-tforg-sa@d4science-com-automation-prj.iam.gserviceaccount.com" + } + + required_version = "~> 1.6.5" + + required_providers { + google = { + source = "hashicorp/google" + version = "4.84.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = "4.84.0" + } + local = { + source = "hashicorp/local" + version = "~> 2.4.0" + } + } +} + +provider "google" { + impersonate_service_account = "d4science-com-tforg-sa@d4science-com-automation-prj.iam.gserviceaccount.com" +} + +provider "google-beta" { + impersonate_service_account = "d4science-com-tforg-sa@d4science-com-automation-prj.iam.gserviceaccount.com" +} + +provider "google" { + alias = "no-impersonate" +} + +provider "local" {} \ No newline at end of file diff --git a/assets/providers/01-networking.providers.tf b/assets/providers/01-networking.providers.tf new file mode 100644 index 0000000..8fdee56 --- /dev/null +++ b/assets/providers/01-networking.providers.tf @@ -0,0 +1,37 @@ +terraform { + backend "gcs" { + bucket = "d4science-com-ew8-foundation-tfnet-bkt" + impersonate_service_account = "d4science-com-tfnet-sa@d4science-com-automation-prj.iam.gserviceaccount.com" + } + + required_version = "~> 1.6.5" + + required_providers { + google = { + source = "hashicorp/google" + version = "4.84.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = "4.84.0" + } + local = { + source = "hashicorp/local" + version = "~> 2.4.0" + } + } +} + +provider "google" { + impersonate_service_account = "d4science-com-tfnet-sa@d4science-com-automation-prj.iam.gserviceaccount.com" +} + +provider "google-beta" { + impersonate_service_account = "d4science-com-tfnet-sa@d4science-com-automation-prj.iam.gserviceaccount.com" +} + +provider "google" { + alias = "no-impersonate" +} + +provider "local" {} \ No newline at end of file diff --git a/assets/providers/02-security.providers.tf b/assets/providers/02-security.providers.tf new file mode 100644 index 0000000..af21b30 --- /dev/null +++ b/assets/providers/02-security.providers.tf @@ -0,0 +1,37 @@ +terraform { + backend "gcs" { + bucket = "d4science-com-ew8-foundation-tfsec-bkt" + impersonate_service_account = "d4science-com-tfsec-sa@d4science-com-automation-prj.iam.gserviceaccount.com" + } + + required_version = "~> 1.6.5" + + required_providers { + google = { + source = "hashicorp/google" + version = "4.84.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = "4.84.0" + } + local = { + source = "hashicorp/local" + version = "~> 2.4.0" + } + } +} + +provider "google" { + impersonate_service_account = "d4science-com-tfsec-sa@d4science-com-automation-prj.iam.gserviceaccount.com" +} + +provider "google-beta" { + impersonate_service_account = "d4science-com-tfsec-sa@d4science-com-automation-prj.iam.gserviceaccount.com" +} + +provider "google" { + alias = "no-impersonate" +} + +provider "local" {} \ No newline at end of file diff --git a/assets/providers/03-project-factory-prod.providers.tf b/assets/providers/03-project-factory-prod.providers.tf new file mode 100644 index 0000000..404e264 --- /dev/null +++ b/assets/providers/03-project-factory-prod.providers.tf @@ -0,0 +1,37 @@ +terraform { + backend "gcs" { + bucket = "d4science-prod-ew8-foundation-tfprj-bkt" + impersonate_service_account = "d4science-prod-tfprj-sa@d4science-com-automation-prj.iam.gserviceaccount.com" + } + + required_version = "~> 1.6.5" + + required_providers { + google = { + source = "hashicorp/google" + version = "4.84.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = "4.84.0" + } + local = { + source = "hashicorp/local" + version = "~> 2.4.0" + } + } +} + +provider "google" { + impersonate_service_account = "d4science-prod-tfprj-sa@d4science-com-automation-prj.iam.gserviceaccount.com" +} + +provider "google-beta" { + impersonate_service_account = "d4science-prod-tfprj-sa@d4science-com-automation-prj.iam.gserviceaccount.com" +} + +provider "google" { + alias = "no-impersonate" +} + +provider "local" {} \ No newline at end of file diff --git a/assets/providers/03-project-factory-test.providers.tf b/assets/providers/03-project-factory-test.providers.tf new file mode 100644 index 0000000..145605f --- /dev/null +++ b/assets/providers/03-project-factory-test.providers.tf @@ -0,0 +1,37 @@ +terraform { + backend "gcs" { + bucket = "d4science-test-ew8-foundation-tfprj-bkt" + impersonate_service_account = "d4science-test-tfprj-sa@d4science-com-automation-prj.iam.gserviceaccount.com" + } + + required_version = "~> 1.6.5" + + required_providers { + google = { + source = "hashicorp/google" + version = "4.84.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = "4.84.0" + } + local = { + source = "hashicorp/local" + version = "~> 2.4.0" + } + } +} + +provider "google" { + impersonate_service_account = "d4science-test-tfprj-sa@d4science-com-automation-prj.iam.gserviceaccount.com" +} + +provider "google-beta" { + impersonate_service_account = "d4science-test-tfprj-sa@d4science-com-automation-prj.iam.gserviceaccount.com" +} + +provider "google" { + alias = "no-impersonate" +} + +provider "local" {} \ No newline at end of file diff --git a/assets/tfvars/00-organization.auto.tfvars.json b/assets/tfvars/00-organization.auto.tfvars.json new file mode 100644 index 0000000..0ef89ff --- /dev/null +++ b/assets/tfvars/00-organization.auto.tfvars.json @@ -0,0 +1 @@ +{"billing_account_id":"018258-BCA804-E9D4C6","folders":{"networking":{"id":"folders/362572285278","name":"Networking"},"prod":{"id":"folders/325860131894","name":"Prod"},"test":{"id":"folders/943728866491","name":"Test"}},"groups":{},"labels":{},"monitoring":{"bucket-sink-prod":{"id":"projects/d4science-com-monitoring-prj/locations/europe-west4/buckets/d4science-prod-monitoring-sink-bucket"},"bucket-sink-test":{"id":"projects/d4science-com-monitoring-prj/locations/europe-west8/buckets/d4science-test-monitoring-sink-bucket"},"channels":{"budget-alerting":"projects/d4science-com-monitoring-prj/notificationChannels/11833687484587417800"},"project_id":"d4science-com-monitoring-prj"},"organization":{"domain":"d4sscience.org","id":392184451762},"prefix":null,"seed-project":{"project_id":"d4science-com-automation-prj","project_number":"867377562537"},"service_accounts":{"networking":"serviceAccount:d4science-com-tfnet-sa@d4science-com-automation-prj.iam.gserviceaccount.com","project_factory_prod":"serviceAccount:d4science-prod-tfprj-sa@d4science-com-automation-prj.iam.gserviceaccount.com","project_factory_test":"serviceAccount:d4science-test-tfprj-sa@d4science-com-automation-prj.iam.gserviceaccount.com","security":"serviceAccount:d4science-com-tfsec-sa@d4science-com-automation-prj.iam.gserviceaccount.com"}} \ No newline at end of file diff --git a/assets/tfvars/01-networking.auto.tfvars.json b/assets/tfvars/01-networking.auto.tfvars.json new file mode 100644 index 0000000..ceb5dfe --- /dev/null +++ b/assets/tfvars/01-networking.auto.tfvars.json @@ -0,0 +1 @@ +{"spoke-prod":{"nat-address":"34.90.97.108","network":"https://www.googleapis.com/compute/v1/projects/d4science-prod-spoke-prj/global/networks/d4science-prod-vpc","number":"434997396061","project_id":"d4science-prod-spoke-prj","subnets":{"europe-west4/d4science-prod-ew4-vpc-con-sub":{"ip":"10.254.2.0/28","secondary_ranges":{},"self_link":"https://www.googleapis.com/compute/v1/projects/d4science-prod-spoke-prj/regions/europe-west4/subnetworks/d4science-prod-ew4-vpc-con-sub"},"europe-west4/d4science-prod-ew4-vre-sub":{"ip":"10.254.0.0/25","secondary_ranges":{"pods":"10.250.0.0/17","services":"10.250.128.0/17"},"self_link":"https://www.googleapis.com/compute/v1/projects/d4science-prod-spoke-prj/regions/europe-west4/subnetworks/d4science-prod-ew4-vre-sub"}}},"spoke-test":{"nat-address":"34.154.15.153","network":"https://www.googleapis.com/compute/v1/projects/d4science-test-spoke-prj/global/networks/d4science-test-vpc","number":"54253924625","project_id":"d4science-test-spoke-prj","subnets":{"europe-west8/d4science-test-ew8-vpc-con-sub":{"ip":"10.254.66.0/28","secondary_ranges":{},"self_link":"https://www.googleapis.com/compute/v1/projects/d4science-test-spoke-prj/regions/europe-west8/subnetworks/d4science-test-ew8-vpc-con-sub"},"europe-west8/d4science-test-ew8-vre-sub":{"ip":"10.254.64.0/25","secondary_ranges":{"pods":"10.251.0.0/17","services":"10.251.128.0/17"},"self_link":"https://www.googleapis.com/compute/v1/projects/d4science-test-spoke-prj/regions/europe-west8/subnetworks/d4science-test-ew8-vre-sub"}}}} \ No newline at end of file