Skip to content

Commit b98679a

Browse files
in-1911in-1911
authored
feat: public ingress option (#181)
1 parent 57f23ab commit b98679a

File tree

9 files changed

+403
-0
lines changed

9 files changed

+403
-0
lines changed

‎modules/roks-ingress/main.tf

Lines changed: 225 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,225 @@
1+
locals {
2+
3+
prefix_used = var.prefix != "" && var.prefix != null ? "${var.prefix}-" : ""
4+
# First entry in NLB DNS is the cluster's private ingress
5+
ingress_alb_hostname = data.ibm_container_nlb_dns.cluster_nlb_dns.nlb_config[0].lb_hostname
6+
7+
workload_nlb_dns_data = jsondecode(restapi_object.workload_nlb_dns.create_response)
8+
9+
# NLB DNS API returns different structures in create and read operations
10+
ingress_subdomain = can(local.workload_nlb_dns_data.Nlb) ? local.workload_nlb_dns_data.Nlb.nlbSubdomain : local.workload_nlb_dns_data.nlbSubdomain
11+
cluster_id = can(local.workload_nlb_dns_data.Nlb) ? local.workload_nlb_dns_data.Nlb.cluster : local.workload_nlb_dns_data.cluster
12+
ingress_secret_name = split(".", local.ingress_subdomain)[0]
13+
14+
cluster_workload_ingress_alb_hostname = data.kubernetes_service.ingress_router_service.status[0].load_balancer[0].ingress[0].hostname
15+
16+
nlb_dns_data = jsonencode({
17+
cluster = var.cluster_name
18+
dnsType = "public"
19+
nlbSubdomain = local.ingress_subdomain
20+
lbHostname = local.cluster_workload_ingress_alb_hostname # new ALB provisioned by the ingress controller
21+
#checkov:skip=CKV_SECRET_6:This is a designated namespace for igress TLS certificate
22+
secretNamespace = "openshift-ingress"
23+
type = "public"
24+
})
25+
26+
# Pick the first "Deny all" rule in the ACL to place new rules before that
27+
cluster_acl_deny_rule = [for rule in data.ibm_is_network_acl_rules.alb_acl_rules.rules : rule.rule_id if rule.action == "deny"][0]
28+
}
29+
30+
data "ibm_container_nlb_dns" "cluster_nlb_dns" {
31+
cluster = var.cluster_name
32+
}
33+
34+
# First create an NLB DNS entry - it will pick the next NLB DNS index for the name like
35+
# <cluster name>-<guid>-<NLB DNS index>.<region>.containers.appdomain.cloud
36+
# Initially the NLB DNS will point to the cluster's default load balancer
37+
# Once the ingress controller provisions new public ALB, this NLB DNS entry will be updated
38+
resource "restapi_object" "workload_nlb_dns" {
39+
path = "/v2/nlb-dns/getNlbDetails"
40+
read_path = "/v2/nlb-dns/getNlbDetails?cluster=${var.cluster_name}&nlbSubdomain={id}"
41+
read_method = "GET"
42+
create_path = "/v2/nlb-dns/vpc/createNlbDNS"
43+
create_method = "POST"
44+
data = jsonencode({
45+
cluster = var.cluster_name
46+
dnsType = "public"
47+
lbHostname = local.ingress_alb_hostname
48+
secretNamespace = "openshift-ingress"
49+
type = "public"
50+
})
51+
id_attribute = "nlbSubdomain"
52+
# The destruction cannot be handled in this resources because the {id} needs to be in the request body
53+
# and the create and read API data structures do not match so copy_keys are not working as intended
54+
# This is just a stub to avoid API errors, the actual remove operation will be done in the "patch/cleanup" resource
55+
destroy_method = "GET"
56+
destroy_path = "/v2/nlb-dns/getNlbDetails?cluster=${var.cluster_name}&nlbSubdomain={id}"
57+
# Update is also a stub - it needs the {id} (nlbSubdomain in the request body)
58+
update_method = "GET"
59+
update_path = "/v2/nlb-dns/getNlbDetails?cluster=${var.cluster_name}&nlbSubdomain={id}"
60+
}
61+
62+
# ALB private IPs take some time to get available - needed for ACL update
63+
resource "time_sleep" "wait_for_alb_provisioning" {
64+
depends_on = [restapi_object.workload_nlb_dns]
65+
66+
destroy_duration = "5s"
67+
create_duration = "10m"
68+
}
69+
70+
# Public ingress controller will result in provisioning a public load balancer in the cluster's VPC
71+
# assigned to worker nodes subnets
72+
resource "kubernetes_manifest" "workload_ingress" {
73+
manifest = {
74+
apiVersion = "operator.openshift.io/v1"
75+
kind = "IngressController"
76+
metadata = {
77+
name = "${local.prefix_used}${var.public_ingress_controller_name}"
78+
namespace = "openshift-ingress-operator"
79+
}
80+
spec = {
81+
replicas = 2
82+
domain = local.ingress_subdomain
83+
defaultCertificate = {
84+
name = local.ingress_secret_name
85+
}
86+
routeSelector = {
87+
matchLabels = {
88+
ingress = var.public_ingress_selector_label
89+
}
90+
}
91+
endpointPublishingStrategy = {
92+
type = "LoadBalancerService"
93+
loadBalancer = {
94+
dnsManagementPolicy = "Managed"
95+
scope = "External"
96+
}
97+
}
98+
}
99+
}
100+
wait {
101+
# Wait until the ingress controller is fully available
102+
# This requires a TLS secret to be created
103+
# and will give time for the ALB to finish provisioning (private IPs are available)
104+
condition {
105+
type = "Available"
106+
status = "True"
107+
}
108+
}
109+
}
110+
111+
data "kubernetes_service" "ingress_router_service" {
112+
metadata {
113+
name = "router-${kubernetes_manifest.workload_ingress.object.metadata.name}"
114+
namespace = "openshift-ingress"
115+
}
116+
}
117+
118+
# The "patch" resource is needed to update the ALB hostname in the NLB DNS entry
119+
# to point to the new ALB created by the ingress controller
120+
# It will update the resource created by restapi_object" "workload_nlb_dns
121+
# and on destry will delete the TLS secret
122+
resource "restapi_object" "workload_nlb_dns_patch" {
123+
path = "/v2/nlb-dns/getNlbDetails"
124+
query_string = "cluster=${var.cluster_name}&nlbSubdomain=${local.ingress_subdomain}"
125+
read_path = "/v2/nlb-dns/getNlbDetails"
126+
read_method = "GET"
127+
# Update existing entry with new ALB hostname instead of creating a new one
128+
create_path = "/v2/nlb-dns/vpc/replaceLBHostname"
129+
create_method = "POST"
130+
data = local.nlb_dns_data
131+
object_id = local.ingress_subdomain
132+
destroy_method = "POST"
133+
destroy_path = "/v2/nlb-dns/deleteSecret"
134+
# Delete ingress secret, the NLB DNS is removed by the resource creating it
135+
destroy_data = jsonencode({
136+
cluster = var.cluster_name
137+
subdomain = local.ingress_subdomain
138+
})
139+
update_method = "POST"
140+
update_path = "/v2/nlb-dns/vpc/replaceLBHostname"
141+
update_data = local.nlb_dns_data
142+
}
143+
144+
# This resource is just to implement the destroy operation because it cannot be done
145+
# in the resource that creates the NLB DNS entry (see comment on restapi_object.workload_nlb_dns)
146+
resource "restapi_object" "workload_nlb_dns_cleanup" {
147+
path = "/v2/nlb-dns/getNlbDetails"
148+
query_string = "cluster=${var.cluster_name}&nlbSubdomain=${local.ingress_subdomain}"
149+
read_path = "/v2/nlb-dns/getNlbDetails"
150+
read_method = "GET"
151+
# Update existing entry with new ALB hostname instead of creating a new one
152+
create_path = "/v2/nlb-dns/vpc/replaceLBHostname"
153+
create_method = "POST"
154+
data = local.nlb_dns_data
155+
object_id = local.ingress_subdomain
156+
# Delete the ALB assignment from the NLB DNS, needs to use ingress/v2/dns API, otherwise the NLB DNS entry is not deleted
157+
destroy_method = "POST"
158+
destroy_path = "/ingress/v2/dns/deleteDomain"
159+
destroy_data = jsonencode({
160+
cluster = var.cluster_name
161+
subdomain = local.ingress_subdomain
162+
})
163+
update_method = "POST"
164+
update_path = "/v2/nlb-dns/vpc/replaceLBHostname"
165+
update_data = local.nlb_dns_data
166+
depends_on = [restapi_object.workload_nlb_dns_patch]
167+
}
168+
169+
# Need to get private IPs (private_ips) of the ALB to include in ACL
170+
data "ibm_is_lb" "ingress_vpc_alb" {
171+
name = "kube-${local.cluster_id}-${replace(data.kubernetes_service.ingress_router_service.metadata[0].uid, "-", "")}"
172+
depends_on = [time_sleep.wait_for_alb_provisioning]
173+
}
174+
175+
# Assuming all SLZ zones for the ALB subnet will have the same ACL
176+
data "ibm_is_subnet" "cluster_subnet" {
177+
identifier = tolist(data.ibm_is_lb.ingress_vpc_alb.subnets)[0]
178+
}
179+
180+
data "ibm_is_network_acl" "alb_acl" {
181+
network_acl = data.ibm_is_subnet.cluster_subnet.network_acl
182+
}
183+
data "ibm_is_network_acl_rules" "alb_acl_rules" {
184+
network_acl = data.ibm_is_network_acl.alb_acl.id
185+
}
186+
187+
# Add ACL rules to allow HTTPS requests from the outside to the new public load balancer
188+
resource "ibm_is_network_acl_rule" "alb_https_req" {
189+
count = var.cluster_zone_count
190+
network_acl = data.ibm_is_network_acl.alb_acl.id
191+
before = local.cluster_acl_deny_rule
192+
name = "${local.prefix_used}public-ingress-lba-zone${count.index + 1}-https-req"
193+
action = "allow"
194+
source = "0.0.0.0/0"
195+
destination = "${data.ibm_is_lb.ingress_vpc_alb.private_ips[count.index]}/32"
196+
direction = "inbound"
197+
tcp {
198+
port_max = 443
199+
port_min = 443
200+
source_port_max = 65535
201+
source_port_min = 1024
202+
}
203+
lifecycle {
204+
ignore_changes = [before]
205+
}
206+
}
207+
resource "ibm_is_network_acl_rule" "alb_https_resp" {
208+
count = var.cluster_zone_count
209+
network_acl = data.ibm_is_network_acl.alb_acl.id
210+
before = local.cluster_acl_deny_rule
211+
name = "${local.prefix_used}public-ingress-lba-zone${count.index + 1}-https-resp"
212+
action = "allow"
213+
source = "${data.ibm_is_lb.ingress_vpc_alb.private_ips[count.index]}/32"
214+
destination = "0.0.0.0/0"
215+
direction = "outbound"
216+
tcp {
217+
port_max = 65535
218+
port_min = 1024
219+
source_port_max = 443
220+
source_port_min = 443
221+
}
222+
lifecycle {
223+
ignore_changes = [before]
224+
}
225+
}

‎modules/roks-ingress/outputs.tf

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
output "cluster_workload_ingress_subdomain" {
2+
description = "Public ingress subdomain"
3+
value = local.ingress_subdomain
4+
}
5+
6+
output "cluster_workload_ingress_controller" {
7+
description = "Ingress controller attributes"
8+
value = kubernetes_manifest.workload_ingress
9+
}
10+
11+
output "cluster_workload_ingress_service" {
12+
description = "Ingress controller service attributes"
13+
value = data.kubernetes_service.ingress_router_service
14+
}
15+
16+
output "vpc_public_load_balancer" {
17+
description = "Public ALB attributes "
18+
value = data.ibm_is_lb.ingress_vpc_alb
19+
}
20+
21+
output "vpc_alb_acl" {
22+
description = "ACL attributes for public ALB"
23+
value = data.ibm_is_network_acl.alb_acl
24+
}

‎modules/roks-ingress/variables.tf

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
variable "prefix" {
2+
description = "Prefix for resources to be created"
3+
type = string
4+
}
5+
6+
variable "cluster_name" {
7+
description = "Cluster name"
8+
type = string
9+
}
10+
11+
# Need to have the count of zones to determine how many rules to add to the ACL for public ingress
12+
variable "cluster_zone_count" {
13+
description = "Number of zones the cluster nodes are deployed in"
14+
type = number
15+
default = 2
16+
}
17+
18+
variable "public_ingress_controller_name" {
19+
description = "Name for the ingress controller to be created"
20+
type = string
21+
default = "ingress-public"
22+
}
23+
24+
variable "public_ingress_selector_label" {
25+
description = "Value for ingress label to select routes admitted to public ingress"
26+
type = string
27+
default = "ingress-public"
28+
}

‎modules/roks-ingress/version.tf

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
terraform {
2+
required_providers {
3+
ibm = {
4+
source = "IBM-Cloud/ibm"
5+
version = ">= 1.67.1"
6+
}
7+
restapi = {
8+
source = "Mastercard/restapi"
9+
version = ">= 1.19.1"
10+
}
11+
time = {
12+
source = "hashicorp/time"
13+
version = ">= 0.9.1"
14+
}
15+
kubernetes = {
16+
source = "hashicorp/kubernetes"
17+
version = ">= 2.32.0"
18+
}
19+
}
20+
required_version = ">= 1.3.0"
21+
}

‎solutions/banking/main.tf

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,21 @@ resource "ibm_resource_instance" "cd_instance" {
106106
resource_group_id = data.ibm_resource_group.toolchain_resource_group_id.id
107107
}
108108

109+
# ----------------------- Cluster deployment options
110+
module "cluster_ingress" {
111+
providers = {
112+
restapi = restapi.oc_api
113+
ibm = ibm.ibm_resources
114+
}
115+
count = var.cluster_name != null && var.provision_public_ingress ? 1 : 0
116+
source = "../../modules/roks-ingress"
117+
prefix = var.prefix
118+
cluster_name = var.cluster_name
119+
cluster_zone_count = var.cluster_zone_count
120+
}
121+
122+
123+
# ----------------------- Watson services configuration
109124
module "configure_wml_project" {
110125
providers = {
111126
ibm = ibm.ibm_resources
@@ -270,6 +285,28 @@ resource "ibm_cd_tekton_pipeline_property" "watsonx_assistant_integration_id_pip
270285
value = module.configure_watson_assistant.watsonx_assistant_integration_id
271286
}
272287

288+
# Update CI pipeline with public ingress subdomain
289+
resource "ibm_cd_tekton_pipeline_property" "cluster_public_ingress_subdomain_pipeline_property_ci" {
290+
count = var.cluster_name != null && var.provision_public_ingress ? 1 : 0
291+
provider = ibm.ibm_resources
292+
depends_on = [local.cd_instance]
293+
name = "cluster_public_ingress_subdomain"
294+
pipeline_id = var.ci_pipeline_id
295+
type = "text"
296+
value = module.cluster_ingress[0].cluster_workload_ingress_subdomain
297+
}
298+
299+
# Update CD pipeline with public ingress subdomain
300+
resource "ibm_cd_tekton_pipeline_property" "cluster_public_ingress_subdomain_pipeline_property_cd" {
301+
count = var.cluster_name != null && var.provision_public_ingress ? 1 : 0
302+
provider = ibm.ibm_resources
303+
depends_on = [local.cd_instance]
304+
name = "cluster_public_ingress_subdomain"
305+
pipeline_id = var.cd_pipeline_id
306+
type = "text"
307+
value = module.cluster_ingress[0].cluster_workload_ingress_subdomain
308+
}
309+
273310
# Random string for webhook token
274311
resource "random_string" "webhook_secret" {
275312
length = 48

‎solutions/banking/outputs.tf

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,3 +47,13 @@ output "elastic_collection_count" {
4747
description = "Count of sample data items uplaoded to elastic index"
4848
value = local.use_elastic_index ? module.configure_elastic_index[0].elastic_upload_count : 0
4949
}
50+
51+
output "cluster_workload_ingress_subdomain" {
52+
description = "Subdomain of the cluster's public ingress"
53+
value = var.cluster_name != null && var.provision_public_ingress ? module.cluster_ingress[0].cluster_workload_ingress_subdomain : null
54+
}
55+
56+
output "sample_app_public_url" {
57+
description = "URL of the public route of the sample app deployed on ROKS cluster"
58+
value = var.cluster_name != null && var.provision_public_ingress ? "https://gen-ai-rag-sample-app-tls-dev.${module.cluster_ingress[0].cluster_workload_ingress_subdomain}" : null
59+
}

0 commit comments

Comments
 (0)