Small bug fixes and improvements

Author: bobbyiliev

README.md

@@ -44,8 +44,8 @@ No resources.
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| <a name="input_database_config"></a> [database\_config](#input\_database\_config) | Cloud SQL configuration | <pre>object({<br/>    tier     = string<br/>    version  = string<br/>    password = string<br/>  })</pre> | <pre>{<br/>  "password": null,<br/>  "tier": "db-custom-2-4096",<br/>  "version": "POSTGRES_15"<br/>}</pre> | no |
-| <a name="input_gke_config"></a> [gke\_config](#input\_gke\_config) | GKE cluster configuration | <pre>object({<br/>    node_count     = number<br/>    machine_type   = string<br/>    disk_size_gb   = number<br/>    min_nodes      = number<br/>    max_nodes      = number<br/>    node_locations = list(string)<br/>  })</pre> | <pre>{<br/>  "disk_size_gb": 100,<br/>  "machine_type": "e2-standard-4",<br/>  "max_nodes": 5,<br/>  "min_nodes": 1,<br/>  "node_count": 3,<br/>  "node_locations": []<br/>}</pre> | no |
+| <a name="input_database_config"></a> [database\_config](#input\_database\_config) | Cloud SQL configuration | <pre>object({<br/>    tier     = optional(string, "db-custom-2-4096")<br/>    version  = optional(string, "POSTGRES_15")<br/>    password = string<br/>    username = optional(string, "materialize")<br/>    db_name  = optional(string, "materialize")<br/>  })</pre> | n/a | yes |
+| <a name="input_gke_config"></a> [gke\_config](#input\_gke\_config) | GKE cluster configuration | <pre>object({<br/>    node_count     = number<br/>    machine_type   = string<br/>    disk_size_gb   = number<br/>    min_nodes      = number<br/>    max_nodes      = number<br/>    node_locations = list(string)<br/>  })</pre> | <pre>{<br/>  "disk_size_gb": 100,<br/>  "machine_type": "e2-standard-2",<br/>  "max_nodes": 5,<br/>  "min_nodes": 1,<br/>  "node_count": 3,<br/>  "node_locations": []<br/>}</pre> | no |
 | <a name="input_labels"></a> [labels](#input\_labels) | Labels to apply to all resources | `map(string)` | `{}` | no |
 | <a name="input_namespace"></a> [namespace](#input\_namespace) | Kubernetes namespace for Materialize | `string` | `"materialize"` | no |
 | <a name="input_network_config"></a> [network\_config](#input\_network\_config) | Network configuration for the GKE cluster | <pre>object({<br/>    subnet_cidr   = string<br/>    pods_cidr     = string<br/>    services_cidr = string<br/>  })</pre> | <pre>{<br/>  "pods_cidr": "10.48.0.0/14",<br/>  "services_cidr": "10.52.0.0/20",<br/>  "subnet_cidr": "10.0.0.0/20"<br/>}</pre> | no |
@@ -57,6 +57,7 @@ No resources.
 
 | Name | Description |
 |------|-------------|
+| <a name="output_connection_strings"></a> [connection\_strings](#output\_connection\_strings) | Formatted connection strings for Materialize |
 | <a name="output_database"></a> [database](#output\_database) | Cloud SQL instance details |
 | <a name="output_gke_cluster"></a> [gke\_cluster](#output\_gke\_cluster) | GKE cluster details |
 | <a name="output_service_accounts"></a> [service\_accounts](#output\_service\_accounts) | Service account details |

main.tf

@@ -28,6 +28,11 @@ module "gke" {
 module "database" {
   source = "./modules/database"
 
+  depends_on = [ module.gke ]
+
+  database_name = var.database_config.db_name
+  database_user = var.database_config.username
+
   project_id = var.project_id
   region     = var.region
   prefix     = var.prefix

modules/database/main.tf

@@ -29,17 +29,17 @@ resource "google_sql_database_instance" "materialize" {
     user_labels = var.labels
   }
 
-  deletion_protection = true
+  deletion_protection = false
 }
 
 resource "google_sql_database" "materialize" {
-  name     = "materialize_db"
+  name     = var.database_name
   instance = google_sql_database_instance.materialize.name
   project  = var.project_id
 }
 
 resource "google_sql_user" "materialize" {
-  name     = "materialize_user"
+  name     = var.database_user
   instance = google_sql_database_instance.materialize.name
   password = var.password
   project  = var.project_id

modules/database/variables.tf

@@ -38,6 +38,18 @@ variable "password" {
   sensitive   = true
 }
 
+variable "database_user" {
+  description = "The name of the database user"
+  default     = "materialize"
+  type        = string
+}
+
+variable "database_name" {
+  description = "The name of the database"
+  default     = "materialize"
+  type        = string
+}
+
 variable "labels" {
   description = "Labels to apply to resources"
   type        = map(string)

modules/gke/main.tf

@@ -2,6 +2,26 @@ resource "google_compute_network" "vpc" {
   name                    = "${var.prefix}-network"
   auto_create_subnetworks = false
   project                 = var.project_id
+
+  lifecycle {
+    create_before_destroy = true
+  }
+}
+
+resource "google_compute_route" "default_route" {
+  name             = "${var.prefix}-default-route"
+  project          = var.project_id
+  network         = google_compute_network.vpc.name
+  dest_range      = "0.0.0.0/0"
+  priority        = 1000
+  next_hop_gateway = "default-internet-gateway"
+
+  # Ensure this is destroyed before the network
+  depends_on = [google_compute_network.vpc]
+
+  lifecycle {
+    create_before_destroy = true
+  }
 }
 
 resource "google_compute_subnetwork" "subnet" {
@@ -30,6 +50,23 @@ resource "google_service_account" "gke_sa" {
   display_name = "GKE Service Account for Materialize"
 }
 
+resource "google_compute_global_address" "private_ip_address" {
+  provider = google
+  project  = var.project_id
+  name     = "${var.prefix}-private-ip"
+  purpose  = "VPC_PEERING"
+  address_type = "INTERNAL"
+  prefix_length = 16
+  network = google_compute_network.vpc.id
+}
+
+resource "google_service_networking_connection" "private_vpc_connection" {
+  provider = google
+  network                 = google_compute_network.vpc.id
+  service                 = "servicenetworking.googleapis.com"
+  reserved_peering_ranges = [google_compute_global_address.private_ip_address.name]
+}
+
 resource "google_service_account" "workload_identity_sa" {
   project      = var.project_id
   account_id   = "${var.prefix}-materialize-sa"
@@ -39,6 +76,16 @@ resource "google_service_account" "workload_identity_sa" {
 resource "google_container_cluster" "primary" {
   provider = google
 
+  deletion_protection = false
+
+  depends_on = [
+    google_service_account.gke_sa,
+    google_service_account.workload_identity_sa,
+    google_service_networking_connection.private_vpc_connection,
+    google_compute_subnetwork.subnet,
+    google_compute_route.default_route
+  ]
+
   name     = "${var.prefix}-gke"
   location = var.region
   project  = var.project_id
@@ -110,9 +157,17 @@ resource "google_container_node_pool" "primary_nodes" {
       mode = "GKE_METADATA"
     }
   }
+
+  lifecycle {
+    create_before_destroy = true
+
+    prevent_destroy = false
+  }
+
 }
 
 resource "google_service_account_iam_binding" "workload_identity" {
+  depends_on         = [google_service_account.workload_identity_sa]
   service_account_id = google_service_account.workload_identity_sa.name
   role               = "roles/iam.workloadIdentityUser"
   members = [

modules/storage/main.tf

@@ -34,3 +34,8 @@ resource "google_storage_bucket_iam_member" "materialize_storage" {
   role   = "roles/storage.admin"
   member = "serviceAccount:${var.service_account}"
 }
+
+resource "google_storage_hmac_key" "materialize" {
+  project               = var.project_id
+  service_account_email = var.service_account
+}

modules/storage/outputs.tf

@@ -12,3 +12,13 @@ output "bucket_self_link" {
   description = "The self_link of the GCS bucket"
   value       = google_storage_bucket.materialize.self_link
 }
+
+output "hmac_access_id" {
+  value     = google_storage_hmac_key.materialize.access_id
+  sensitive = true
+}
+
+output "hmac_secret" {
+  value     = google_storage_hmac_key.materialize.secret
+  sensitive = true
+}

outputs.tf

@@ -34,3 +34,34 @@ output "service_accounts" {
     materialize_sa = module.gke.workload_identity_sa_email
   }
 }
+
+locals {
+  metadata_backend_url = format(
+    "postgres://%s:%s@%s:5432/%s?sslmode=disable",
+    var.database_config.username,
+    var.database_config.password,
+    module.database.private_ip,
+    var.database_config.db_name
+  )
+
+  encoded_endpoint = urlencode("https://storage.googleapis.com")
+  encoded_secret   = urlencode(module.storage.hmac_secret)
+
+  persist_backend_url = format(
+    "s3://%s:%s@%s/materialize?endpoint=%s&region=%s",
+    module.storage.hmac_access_id,
+    local.encoded_secret,
+    module.storage.bucket_name,
+    local.encoded_endpoint,
+    var.region
+  )
+}
+
+output "connection_strings" {
+  description = "Formatted connection strings for Materialize"
+  value = {
+    metadata_backend_url = local.metadata_backend_url
+    persist_backend_url  = local.persist_backend_url
+  }
+  sensitive = true
+}

terraform.tfvars.example

@@ -13,7 +13,7 @@ network_config = {
 # GKE Configuration
 gke_config = {
   node_count   = 3
-  machine_type = "e2-standard-4"
+  machine_type = "e2-standard-2"
   disk_size_gb = 100
   min_nodes    = 1
   max_nodes    = 5

variables.tf

@@ -41,7 +41,7 @@ variable "gke_config" {
   })
   default = {
     node_count     = 3
-    machine_type   = "e2-standard-4"
+    machine_type   = "e2-standard-2"
     disk_size_gb   = 100
     min_nodes      = 1
     max_nodes      = 5
@@ -52,15 +52,13 @@ variable "gke_config" {
 variable "database_config" {
   description = "Cloud SQL configuration"
   type = object({
-    tier     = string
-    version  = string
+    tier     = optional(string, "db-custom-2-4096")
+    version  = optional(string, "POSTGRES_15")
     password = string
+    username = optional(string, "materialize")
+    db_name  = optional(string, "materialize")
   })
-  default = {
-    tier     = "db-custom-2-4096"
-    version  = "POSTGRES_15"
-    password = null # Must be provided
-  }
+
   validation {
     condition     = var.database_config.password != null
     error_message = "database_config.password must be provided"