Skip to content

Commit de38c44

Browse files
igoochigooch
committed
Creates a separate terraform module for the upgrade test clusters
1 parent 29a2f78 commit de38c44

File tree

7 files changed

+290
-47
lines changed

7 files changed

+290
-47
lines changed

‎build/terraform/e2e/gke-autopilot/module.tf

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,20 +32,18 @@ terraform {
3232

3333
variable "project" {}
3434
variable "kubernetesVersion" {}
35-
variable "testName" {}
3635
variable "location" {}
3736
variable "releaseChannel" {}
3837

3938
module "gke_cluster" {
4039
source = "../../../../install/terraform/modules/gke-autopilot"
4140

4241
cluster = {
43-
"name" = format("gke-autopilot-%s-test-cluster-%s", var.testName, replace(var.kubernetesVersion, ".", "-"))
42+
"name" = format("gke-autopilot-e2e-test-cluster-%s", replace(var.kubernetesVersion, ".", "-"))
4443
"project" = var.project
4544
"location" = var.location
4645
"releaseChannel" = var.releaseChannel
4746
"kubernetesVersion" = var.kubernetesVersion
48-
"testName" = var.testName
4947
"deletionProtection" = false
5048
"maintenanceExclusionStartTime" = timestamp()
5149
"maintenanceExclusionEndTime" = timeadd(timestamp(), "2640h") # 110 days

‎build/terraform/e2e/gke-standard/module.tf

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,15 +32,17 @@ terraform {
3232

3333
variable "project" {}
3434
variable "kubernetesVersion" {}
35-
variable "testName" {}
3635
variable "location" {}
3736
variable "releaseChannel" {}
38-
variable "initialNodeCount" {}
3937

4038
variable "machineType" {
4139
default = "e2-standard-4"
4240
}
4341

42+
variable "initialNodeCount" {
43+
default = 10
44+
}
45+
4446
variable "overrideName" {
4547
default = ""
4648
}
@@ -49,15 +51,14 @@ module "gke_cluster" {
4951
source = "../../../../install/terraform/modules/gke"
5052

5153
cluster = {
52-
"name" = var.overrideName != "" ? var.overrideName : format("standard-%s-test-cluster-%s", var.testName, replace(var.kubernetesVersion, ".", "-"))
54+
"name" = var.overrideName != "" ? var.overrideName : format("standard-e2e-test-cluster-%s", replace(var.kubernetesVersion, ".", "-"))
5355
"location" = var.location
5456
"releaseChannel" = var.releaseChannel
5557
"machineType" = var.machineType
5658
"initialNodeCount" = var.initialNodeCount
5759
"enableImageStreaming" = true
5860
"project" = var.project
5961
"kubernetesVersion" = var.kubernetesVersion
60-
"testName" = var.testName
6162
"maintenanceExclusionStartTime" = timestamp()
6263
"maintenanceExclusionEndTime" = timeadd(timestamp(), "2640h") # 110 days
6364
}

‎build/terraform/e2e/module.tf

Lines changed: 12 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -62,50 +62,22 @@ variable "kubernetes_versions" {
6262
}
6363
}
6464

65-
variable "test_names" {
66-
description = "Use the same terraform templates for both e2e and upgrade tests. Includes test name and initial node counts for standard clusters."
67-
type = map(number)
68-
default = {
69-
"e2e" = 10
70-
"upgrade" = 4
71-
}
72-
}
73-
74-
// Handle nested loop in terraform. Flatten combines the two maps into a list.
75-
locals {
76-
test_versions = distinct(flatten([
77-
for name, nodes in var.test_names : [
78-
for version, val in var.kubernetes_versions : {
79-
test = name
80-
version = version
81-
location = val[0]
82-
releaseChannel = val[1]
83-
numNodes = nodes
84-
}
85-
]
86-
]))
87-
}
88-
8965
module "gke_standard_cluster" {
90-
// local.test_versions is a list, but to use `for_each` it need to be a changed to a map.
91-
for_each = { for config in local.test_versions: "${config.test}.${config.version}" => config }
92-
source = "./gke-standard"
93-
project = var.project
94-
testName = each.value.test
95-
kubernetesVersion = each.value.version
96-
location = each.value.location
97-
releaseChannel = each.value.releaseChannel
98-
initialNodeCount = each.value.numNodes
66+
for_each = var.kubernetes_versions
67+
source = "./gke-standard"
68+
project = var.project
69+
kubernetesVersion = each.key
70+
location = each.value[0]
71+
releaseChannel = each.value[1]
9972
}
10073

10174
module "gke_autopilot_cluster" {
102-
for_each = { for entry in local.test_versions: "${entry.test}.${entry.version}" => entry }
103-
source = "./gke-autopilot"
104-
project = var.project
105-
testName = each.value.test
106-
kubernetesVersion = each.value.version
107-
location = each.value.location
108-
releaseChannel = each.value.releaseChannel
75+
for_each = var.kubernetes_versions
76+
source = "./gke-autopilot"
77+
project = var.project
78+
kubernetesVersion = each.key
79+
location = each.value[0]
80+
releaseChannel = each.value[1]
10981
}
11082

11183
resource "google_compute_firewall" "udp" {

‎build/terraform/upgrade/gke-autopilot/module.tf

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
// Copyright 2024 Google LLC All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
16+
// Run:
17+
// terraform apply -var project="<YOUR_GCP_ProjectID>"
18+
19+
terraform {
20+
required_version = ">= 1.0.0"
21+
required_providers {
22+
google = {
23+
source = "hashicorp/google"
24+
version = "~> 4.25.0"
25+
}
26+
helm = {
27+
source = "hashicorp/helm"
28+
version = "~> 2.3"
29+
}
30+
}
31+
}
32+
33+
variable "project" {}
34+
variable "kubernetesVersion" {}
35+
variable "location" {}
36+
variable "releaseChannel" {}
37+
38+
module "gke_cluster" {
39+
source = "../../../../install/terraform/modules/gke-autopilot"
40+
41+
cluster = {
42+
"name" = format("gke-autopilot-upgrade-test-cluster-%s", replace(var.kubernetesVersion, ".", "-"))
43+
"project" = var.project
44+
"location" = var.location
45+
"releaseChannel" = var.releaseChannel
46+
"kubernetesVersion" = var.kubernetesVersion
47+
"deletionProtection" = false
48+
"maintenanceExclusionStartTime" = timestamp()
49+
"maintenanceExclusionEndTime" = timeadd(timestamp(), "2640h") # 110 days
50+
}
51+
52+
udpFirewall = false // firewall is created at the project module level
53+
}

‎build/terraform/upgrade/gke-standard/module.tf

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
// Copyright 2024 Google LLC All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
16+
// Run:
17+
// terraform apply -var project="<YOUR_GCP_ProjectID>"
18+
19+
terraform {
20+
required_version = ">= 1.0.0"
21+
required_providers {
22+
google = {
23+
source = "hashicorp/google"
24+
version = "~> 4.25.0"
25+
}
26+
helm = {
27+
source = "hashicorp/helm"
28+
version = "~> 2.3"
29+
}
30+
}
31+
}
32+
33+
variable "project" {}
34+
variable "kubernetesVersion" {}
35+
variable "location" {}
36+
variable "releaseChannel" {}
37+
38+
variable "machineType" {
39+
default = "e2-standard-4"
40+
}
41+
42+
variable "initialNodeCount" {
43+
default = 4
44+
}
45+
46+
variable "overrideName" {
47+
default = ""
48+
}
49+
50+
module "gke_cluster" {
51+
source = "../../../../install/terraform/modules/gke"
52+
53+
cluster = {
54+
"name" = var.overrideName != "" ? var.overrideName : format("standard-upgrade-test-cluster-%s", replace(var.kubernetesVersion, ".", "-"))
55+
"location" = var.location
56+
"releaseChannel" = var.releaseChannel
57+
"machineType" = var.machineType
58+
"initialNodeCount" = var.initialNodeCount
59+
"enableImageStreaming" = true
60+
"project" = var.project
61+
"kubernetesVersion" = var.kubernetesVersion
62+
"maintenanceExclusionStartTime" = timestamp()
63+
"maintenanceExclusionEndTime" = timeadd(timestamp(), "2640h") # 110 days
64+
}
65+
66+
udpFirewall = false // firewall is created at the project module level
67+
}

‎build/terraform/upgrade/module.tf

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
// Copyright 2024 Google LLC All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
16+
// Run:
17+
// terraform init -backend-config="bucket=<YOUR_GCP_ProjectID>-upgrade-infra-bucket-tfstate" -backend-config="prefix=terraform/state"
18+
// terraform apply -var project="<YOUR_GCP_ProjectID>"
19+
20+
terraform {
21+
required_version = ">= 1.0.0"
22+
required_providers {
23+
google = {
24+
source = "hashicorp/google"
25+
version = "~> 4.25.0"
26+
}
27+
helm = {
28+
source = "hashicorp/helm"
29+
version = "~> 2.3"
30+
}
31+
}
32+
backend "gcs" {
33+
}
34+
}
35+
36+
variable "project" {}
37+
variable "kubernetes_versions" {
38+
description = "Create upgrade test clusters with these k8s versions in these regions"
39+
type = map(list(string))
40+
default = {
41+
"1.28" = ["us-west1", "RAPID"]
42+
"1.29" = ["europe-west1", "RAPID"]
43+
"1.30" = ["asia-east1", "RAPID"]
44+
// "1.31" = ["us-east1", "RAPID"]
45+
//
46+
// Before merge: When adding Kubernetes version 1.{N}, first uncomment the line above, extending
47+
// the infrastructure to 4 versions temporarily. Come back to these instructions after the
48+
// update PR merges.
49+
//
50+
// After merge: After the Kubernetes update PR merges, and all active PRs are updated:
51+
//
52+
// * Move the 1.{N-3} line to the bottom and comment it out
53+
// * Change the (commented out) 1.{N-3} to 1.{N+1}
54+
// * You should now have 3 versions uncommented (versions 1.{N-2} .. 1.{N}),
55+
// and 1.{N+1} commented out for the next update. The new, commented out 1.{N+1}
56+
// should be using the region of the previous 1.{N-3} - this region will become
57+
// unused.
58+
//
59+
// Rationale: We cycle the regions us-east1 -> us-west1 -> europe-west1 -> asia-east1 -> us-east1
60+
// as versions are added, using 4 regions so that the PR adding 1.{N} is in a unique region to
61+
// 1.{N-3} .. 1.{N-1}, meaning versions never need to share a region in CI.
62+
}
63+
}
64+
65+
module "gke_standard_cluster" {
66+
for_each = var.kubernetes_versions
67+
source = "./gke-standard"
68+
project = var.project
69+
kubernetesVersion = each.key
70+
location = each.value[0]
71+
releaseChannel = each.value[1]
72+
}
73+
74+
module "gke_autopilot_cluster" {
75+
for_each = var.kubernetes_versions
76+
source = "./gke-autopilot"
77+
project = var.project
78+
kubernetesVersion = each.key
79+
location = each.value[0]
80+
releaseChannel = each.value[1]
81+
}
82+
83+
resource "google_compute_firewall" "udp" {
84+
name = "gke-game-server-firewall"
85+
project = var.project
86+
network = "default"
87+
88+
allow {
89+
protocol = "udp"
90+
ports = ["7000-8000"]
91+
}
92+
93+
target_tags = ["game-server"]
94+
source_ranges = ["0.0.0.0/0"]
95+
}
96+
97+
resource "google_compute_firewall" "tcp" {
98+
name = "gke-game-server-firewall-tcp"
99+
project = var.project
100+
network = "default"
101+
102+
allow {
103+
protocol = "tcp"
104+
ports = ["7000-8000"]
105+
}
106+
107+
target_tags = ["game-server"]
108+
source_ranges = ["0.0.0.0/0"]
109+
}

‎build/terraform/upgrade/state-bucket/main.tf

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
// Copyright 2024 Google LLC All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
16+
// Run:
17+
// terraform apply -var project="<YOUR_GCP_ProjectID>"
18+
19+
// GCS bucket for holding the Terraform state of the upgrade test Terraform config.
20+
21+
terraform {
22+
required_version = ">= 1.0.0"
23+
required_providers {
24+
google = {
25+
source = "hashicorp/google"
26+
version = "~> 4.25.0"
27+
}
28+
}
29+
}
30+
31+
variable "project" {}
32+
33+
resource "google_storage_bucket" "default" {
34+
project = var.project
35+
name = "${var.project}-upgrade-infra-bucket-tfstate"
36+
force_destroy = false
37+
uniform_bucket_level_access = true
38+
location = "US"
39+
storage_class = "STANDARD"
40+
versioning {
41+
enabled = true
42+
}
43+
}

0 commit comments

Comments
 (0)