From 7d4395118cfc6029d30375da3923760a414a100d Mon Sep 17 00:00:00 2001 From: Gregg Ellis Schofield Date: Sun, 14 May 2023 19:11:35 +0000 Subject: [PATCH 1/2] Allow Dataflow job to deploy worker instances into Subnet of shared VPC - Add subnet_complete_url Terraform Variable - Update README to include new variable --- README.md | 1 + pipeline.tf | 2 +- variables.tf | 6 ++++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index eaeb898..95a782b 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ These deployment templates are provided as is, without warranty. See [Copyright | [splunk_hec_token_secret_id](#input_splunk_hec_token_secret_id) | Id of the Secret for Splunk HEC token. Required if `splunk_hec_token_source` is type of SECRET_MANAGER | `string` | `""` | no | | [splunk_hec_token_source](#input_splunk_hec_token_source) | Define in which type HEC token is provided. Possible options: [PLAINTEXT, KMS, SECRET_MANAGER]. | `string` | `"PLAINTEXT"` | no | | [subnet](#input_subnet) | Subnet to deploy into. This is required when deploying into existing network (`create_network=false`) (e.g. Shared VPC) | `string` | `""` | no | +| [subnet_complete_url](#input_subnet) | Complete URL of the Subnet to deploy into. This is required when deploying into a Subnet of a shared VPC network residing in a separate project. | `string` | `""` | no | | [use_externally_managed_dataflow_sa](#input_use_externally_managed_dataflow_sa) | Determines if the worker service account provided by `dataflow_worker_service_account` variable should be created by this module (default) or is managed outside of the module. In the latter case, user is expected to apply and manage the service account IAM permissions over external resources (e.g. Cloud KMS key or Secret version) before running this module. | `bool` | `false` | no | #### Outputs diff --git a/pipeline.tf b/pipeline.tf index 9574df0..1e7d709 100644 --- a/pipeline.tf +++ b/pipeline.tf @@ -100,7 +100,7 @@ resource "google_dataflow_job" "dataflow_job" { ) region = var.region network = var.network - subnetwork = "regions/${var.region}/subnetworks/${local.subnet_name}" + subnetwork = coalesce(var.subnet_complete_url, "regions/${var.region}/subnetworks/${local.subnet_name}") ip_configuration = "WORKER_IP_PRIVATE" lifecycle { diff --git a/variables.tf b/variables.tf index a2a2985..d1e2c8f 100644 --- a/variables.tf +++ b/variables.tf @@ -39,6 +39,12 @@ variable "subnet" { default = "" } +variable "subnet_complete_url" { + type = string + description = "Complete URL of the Subnet to deploy into. This is required when deploying into a Subnet of a shared VPC network residing in a separate project." + default = "" +} + variable "primary_subnet_cidr" { type = string description = "The CIDR Range of the primary subnet" From 3aac5760a5789fa27ad22ed0d9d3048094cf526c Mon Sep 17 00:00:00 2001 From: Gregg Ellis Schofield Date: Tue, 16 May 2023 17:53:06 +0100 Subject: [PATCH 2/2] Add ternary to only use complete subnet URL when we are using an existing network --- README.md | 1 - pipeline.tf | 2 +- variables.tf | 6 ------ 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/README.md b/README.md index 95a782b..eaeb898 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,6 @@ These deployment templates are provided as is, without warranty. See [Copyright | [splunk_hec_token_secret_id](#input_splunk_hec_token_secret_id) | Id of the Secret for Splunk HEC token. Required if `splunk_hec_token_source` is type of SECRET_MANAGER | `string` | `""` | no | | [splunk_hec_token_source](#input_splunk_hec_token_source) | Define in which type HEC token is provided. Possible options: [PLAINTEXT, KMS, SECRET_MANAGER]. | `string` | `"PLAINTEXT"` | no | | [subnet](#input_subnet) | Subnet to deploy into. This is required when deploying into existing network (`create_network=false`) (e.g. Shared VPC) | `string` | `""` | no | -| [subnet_complete_url](#input_subnet) | Complete URL of the Subnet to deploy into. This is required when deploying into a Subnet of a shared VPC network residing in a separate project. | `string` | `""` | no | | [use_externally_managed_dataflow_sa](#input_use_externally_managed_dataflow_sa) | Determines if the worker service account provided by `dataflow_worker_service_account` variable should be created by this module (default) or is managed outside of the module. In the latter case, user is expected to apply and manage the service account IAM permissions over external resources (e.g. Cloud KMS key or Secret version) before running this module. | `bool` | `false` | no | #### Outputs diff --git a/pipeline.tf b/pipeline.tf index 1e7d709..6efc494 100644 --- a/pipeline.tf +++ b/pipeline.tf @@ -100,7 +100,7 @@ resource "google_dataflow_job" "dataflow_job" { ) region = var.region network = var.network - subnetwork = coalesce(var.subnet_complete_url, "regions/${var.region}/subnetworks/${local.subnet_name}") + subnetwork = var.create_network ? "regions/${var.region}/subnetworks/${local.subnet_name}" : "https://www.googleapis.com/compute/v1/projects/${var.project}/regions/${var.region}/subnetworks/${var.subnet}" ip_configuration = "WORKER_IP_PRIVATE" lifecycle { diff --git a/variables.tf b/variables.tf index d1e2c8f..a2a2985 100644 --- a/variables.tf +++ b/variables.tf @@ -39,12 +39,6 @@ variable "subnet" { default = "" } -variable "subnet_complete_url" { - type = string - description = "Complete URL of the Subnet to deploy into. This is required when deploying into a Subnet of a shared VPC network residing in a separate project." - default = "" -} - variable "primary_subnet_cidr" { type = string description = "The CIDR Range of the primary subnet"