configurable bucket versioning

README.md

@@ -74,6 +74,8 @@ No resources.
 | <a name="input_prefix"></a> [prefix](#input\_prefix) | Prefix to be used for resource names | `string` | `"materialize"` | no |
 | <a name="input_project_id"></a> [project\_id](#input\_project\_id) | The ID of the project where resources will be created | `string` | n/a | yes |
 | <a name="input_region"></a> [region](#input\_region) | The region where resources will be created | `string` | `"us-central1"` | no |
+| <a name="input_storage_bucket_version_ttl"></a> [storage\_bucket\_version\_ttl](#input\_storage\_bucket\_version\_ttl) | Sets the TTL (in days) on non current storage bucket objects. This must be set if storage\_bucket\_versioning is turned on. | `number` | `7` | no |
+| <a name="input_storage_bucket_versioning"></a> [storage\_bucket\_versioning](#input\_storage\_bucket\_versioning) | Enable bucket versioning. | `bool` | `true` | no |
 | <a name="input_use_local_chart"></a> [use\_local\_chart](#input\_use\_local\_chart) | Whether to use a local chart instead of one from a repository | `bool` | `false` | no |
 | <a name="input_use_self_signed_cluster_issuer"></a> [use\_self\_signed\_cluster\_issuer](#input\_use\_self\_signed\_cluster\_issuer) | Whether to install and use a self-signed ClusterIssuer for TLS. Due to limitations in Terraform, this may not be enabled before the cert-manager CRDs are installed. | `bool` | `false` | no |
 
@@ -111,3 +113,11 @@ More advanced TLS support using user-provided CAs or per-Materialize `Issuer`s a
 
 Due to limitations in Terraform, it cannot plan Kubernetes resources using CRDs that do not exist yet. We need to first install `cert-manager` in the first `terraform apply`, before defining any `ClusterIssuer` or `Certificate` resources which get created in the second `terraform apply`.
 <!-- END_TF_DOCS -->
+
+
+
+#### Storage Bucket Versioning
+By default storage bucket versioning is turned off. This both reduces
+costs and allows for easier cleanup of resources for testing. When running in
+production, versioning should be turned on with a sufficient TTL to meet any
+data-recovery requirements. See `storage_bucket_versioning` and `storage_bucket_version_ttl`.

main.tf

@@ -67,6 +67,8 @@ module "storage" {
   region          = var.region
   prefix          = var.prefix
   service_account = module.gke.workload_identity_sa_email
+  versioning      = var.storage_bucket_versioning
+  version_ttl     = var.storage_bucket_version_ttl
 
   labels = local.common_labels
 }

modules/storage/main.tf

@@ -1,3 +1,17 @@
+locals {
+  version_ttl = (var.versioning && var.version_ttl != null) ? [{
+    action = {
+      type = "delete"
+    }
+    condition = {
+      daysSinceNoncurrentTime = var.version_ttl
+    }
+  }] : []
+
+  lifecycle_rules = concat(var.lifecycle_rules, local.version_ttl)
+
+}
+
 resource "google_storage_bucket" "materialize" {
   name          = "${var.prefix}-storage-${var.project_id}"
   location      = var.region
@@ -7,11 +21,11 @@ resource "google_storage_bucket" "materialize" {
   uniform_bucket_level_access = true
 
   versioning {
-    enabled = true
+    enabled = var.versioning
   }
 
   dynamic "lifecycle_rule" {
-    for_each = var.lifecycle_rules
+    for_each = local.lifecycle_rules
     content {
       action {
         type          = lifecycle_rule.value.action.type
@@ -26,6 +40,7 @@ resource "google_storage_bucket" "materialize" {
     }
   }
 
+
   labels = var.labels
 }
 

modules/storage/variables.tf

@@ -24,6 +24,13 @@ variable "labels" {
   default     = {}
 }
 
+
+variable "versioning" {
+  description = "Enable bucket versioning. This should be enabled for production deployments."
+  type        = bool
+  default     = true
+}
+
 variable "lifecycle_rules" {
   description = "List of lifecycle rules to configure"
   type = list(object({
@@ -50,3 +57,10 @@ variable "lifecycle_rules" {
     }
   ]
 }
+
+variable "version_ttl" {
+  description = "Sets the TTL (in days) on non current storage bucket objects. This must be set if versioning is turned on."
+  type        = number
+  default     = 7
+
+}

variables.tf

@@ -158,6 +158,18 @@ variable "install_metrics_server" {
   default     = false
 }
 
+variable "storage_bucket_versioning" {
+  description = "Enable bucket versioning. This should be enabled for production deployments."
+  type        = bool
+  default     = false
+}
+
+variable "storage_bucket_version_ttl" {
+  description = "Sets the TTL (in days) on non current storage bucket objects. This must be set if storage_bucket_versioning is turned on."
+  type        = number
+  default     = 7
+}
+
 variable "install_cert_manager" {
   description = "Whether to install cert-manager."
   type        = bool