Skip to content

Commit 7c36469

Browse files
matthchrmatthchr
committed
Change how we read configuration
Moving from a model where we have to explicitly set each secret in the pod env to a model where we just mount the whole secret into the pod. This has the added benefit of allowing us to autodetect secret changes and restart the pod AND means that users can totally omit the secret now as opposed to having to create the secret with empty values.
1 parent 75e617d commit 7c36469

File tree

7 files changed

+205
-254
lines changed

7 files changed

+205
-254
lines changed

‎v2/charts/azure-service-operator/templates/apps_v1_deployment_azureserviceoperator-controller-manager.yaml

Lines changed: 6 additions & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -59,111 +59,6 @@ spec:
5959
- --webhook-cert-dir={{ .Values.webhook.certDir }}
6060
{{- end }}
6161
env:
62-
- name: AZURE_CLIENT_ID
63-
valueFrom:
64-
secretKeyRef:
65-
key: AZURE_CLIENT_ID
66-
name: aso-controller-settings
67-
- name: AZURE_CLIENT_SECRET
68-
valueFrom:
69-
secretKeyRef:
70-
key: AZURE_CLIENT_SECRET
71-
name: aso-controller-settings
72-
optional: true
73-
- name: AZURE_TENANT_ID
74-
valueFrom:
75-
secretKeyRef:
76-
key: AZURE_TENANT_ID
77-
name: aso-controller-settings
78-
- name: AZURE_SUBSCRIPTION_ID
79-
valueFrom:
80-
secretKeyRef:
81-
key: AZURE_SUBSCRIPTION_ID
82-
name: aso-controller-settings
83-
- name: AZURE_CLIENT_CERTIFICATE
84-
valueFrom:
85-
secretKeyRef:
86-
key: AZURE_CLIENT_CERTIFICATE
87-
name: aso-controller-settings
88-
optional: true
89-
- name: AZURE_CLIENT_CERTIFICATE_PASSWORD
90-
valueFrom:
91-
secretKeyRef:
92-
key: AZURE_CLIENT_CERTIFICATE_PASSWORD
93-
name: aso-controller-settings
94-
optional: true
95-
- name: AZURE_AUTHORITY_HOST
96-
valueFrom:
97-
secretKeyRef:
98-
key: AZURE_AUTHORITY_HOST
99-
name: aso-controller-settings
100-
optional: true
101-
- name: AZURE_RESOURCE_MANAGER_ENDPOINT
102-
valueFrom:
103-
secretKeyRef:
104-
key: AZURE_RESOURCE_MANAGER_ENDPOINT
105-
name: aso-controller-settings
106-
optional: true
107-
- name: AZURE_RESOURCE_MANAGER_AUDIENCE
108-
valueFrom:
109-
secretKeyRef:
110-
key: AZURE_RESOURCE_MANAGER_AUDIENCE
111-
name: aso-controller-settings
112-
optional: true
113-
- name: AZURE_TARGET_NAMESPACES
114-
valueFrom:
115-
secretKeyRef:
116-
key: AZURE_TARGET_NAMESPACES
117-
name: aso-controller-settings
118-
optional: true
119-
- name: AZURE_OPERATOR_MODE
120-
valueFrom:
121-
secretKeyRef:
122-
key: AZURE_OPERATOR_MODE
123-
name: aso-controller-settings
124-
optional: true
125-
- name: AZURE_SYNC_PERIOD
126-
valueFrom:
127-
secretKeyRef:
128-
key: AZURE_SYNC_PERIOD
129-
name: aso-controller-settings
130-
optional: true
131-
- name: USE_WORKLOAD_IDENTITY_AUTH
132-
valueFrom:
133-
secretKeyRef:
134-
key: USE_WORKLOAD_IDENTITY_AUTH
135-
name: aso-controller-settings
136-
optional: true
137-
- name: AZURE_USER_AGENT_SUFFIX
138-
valueFrom:
139-
secretKeyRef:
140-
key: AZURE_USER_AGENT_SUFFIX
141-
name: aso-controller-settings
142-
optional: true
143-
- name: MAX_CONCURRENT_RECONCILES
144-
valueFrom:
145-
secretKeyRef:
146-
key: MAX_CONCURRENT_RECONCILES
147-
name: aso-controller-settings
148-
optional: true
149-
- name: RATE_LIMIT_MODE
150-
valueFrom:
151-
secretKeyRef:
152-
key: RATE_LIMIT_MODE
153-
name: aso-controller-settings
154-
optional: true
155-
- name: RATE_LIMIT_QPS
156-
valueFrom:
157-
secretKeyRef:
158-
key: RATE_LIMIT_QPS
159-
name: aso-controller-settings
160-
optional: true
161-
- name: RATE_LIMIT_BUCKET_SIZE
162-
valueFrom:
163-
secretKeyRef:
164-
key: RATE_LIMIT_BUCKET_SIZE
165-
name: aso-controller-settings
166-
optional: true
16762
- name: POD_NAMESPACE
16863
valueFrom:
16964
fieldRef:
@@ -210,6 +105,9 @@ spec:
210105
name: cert
211106
readOnly: true
212107
{{- end }}
108+
- name: settings-volume
109+
readOnly: true
110+
mountPath: "/etc/aso-controller-settings"
213111
nodeSelector:
214112
kubernetes.io/os: linux
215113
serviceAccountName: azureserviceoperator-default
@@ -229,3 +127,6 @@ spec:
229127
audience: api://AzureADTokenExchange
230128
expirationSeconds: 3600
231129
path: azure-identity
130+
- name: settings-volume
131+
secret:
132+
secretName: aso-controller-settings

‎v2/charts/azure-service-operator/templates/secret.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,15 @@ metadata:
66
namespace: {{.Release.Namespace}}
77
type: Opaque
88
data:
9+
{{- if .Values.azureSubscriptionID }}
910
AZURE_SUBSCRIPTION_ID: {{ .Values.azureSubscriptionID | b64enc | quote }}
11+
{{- end }}
12+
{{- if .Values.azureTenantID }}
1013
AZURE_TENANT_ID: {{ .Values.azureTenantID | b64enc | quote }}
14+
{{- end }}
15+
{{- if .Values.azureClientID }}
1116
AZURE_CLIENT_ID: {{ .Values.azureClientID | b64enc | quote }}
17+
{{- end }}
1218
{{- if .Values.azureClientSecret }}
1319
AZURE_CLIENT_SECRET: {{ .Values.azureClientSecret | b64enc | quote }}
1420
{{- end }}

‎v2/cmd/controller/app/setup.go

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,21 @@ func SetupPreUpgradeCheck(ctx context.Context) error {
107107
return kerrors.NewAggregate(errs)
108108
}
109109

110-
func SetupControllerManager(ctx context.Context, setupLog logr.Logger, flgs Flags) manager.Manager {
110+
type ManagerWrapper struct {
111+
mgr manager.Manager
112+
watcher *config.Watcher
113+
}
114+
115+
func (w *ManagerWrapper) Start(ctx context.Context) error {
116+
err := w.watcher.Start(ctx)
117+
if err != nil {
118+
return errors.Wrap(err, "failed to start config watcher")
119+
}
120+
121+
return w.mgr.Start(ctx) // This blocks
122+
}
123+
124+
func SetupControllerManager(ctx context.Context, setupLog logr.Logger, flgs Flags) ManagerWrapper {
111125
scheme := controllers.CreateScheme()
112126
_ = apiextensions.AddToScheme(scheme) // Used for managing CRDs
113127

@@ -263,7 +277,14 @@ func SetupControllerManager(ctx context.Context, setupLog logr.Logger, flgs Flag
263277
setupLog.Error(err, "Failed setting up readyz check")
264278
os.Exit(1)
265279
}
266-
return mgr
280+
281+
// This watches the mounted secret and restart the pod if it changes
282+
configWatcher := config.NewWatcher(setupLog)
283+
284+
return ManagerWrapper{
285+
mgr: mgr,
286+
watcher: configWatcher,
287+
}
267288
}
268289

269290
func getMetricsOpts(flags Flags) server.Options {

‎v2/config/manager/manager_image_patch.yaml

Lines changed: 8 additions & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -8,119 +8,22 @@ spec:
88
spec:
99
nodeSelector:
1010
"kubernetes.io/os": linux
11+
volumes:
12+
- name: settings-volume
13+
secret:
14+
secretName: aso-controller-settings
1115
containers:
1216
# Change the value of image field below to your controller image URL
1317
- image: localhost:5000/azureserviceoperator:latest
1418
name: manager
1519
env:
16-
- name: AZURE_CLIENT_ID
17-
valueFrom:
18-
secretKeyRef:
19-
name: aso-controller-settings
20-
key: AZURE_CLIENT_ID
21-
- name: AZURE_CLIENT_SECRET
22-
valueFrom:
23-
secretKeyRef:
24-
name: aso-controller-settings
25-
key: AZURE_CLIENT_SECRET
26-
optional: true
27-
- name: AZURE_TENANT_ID
28-
valueFrom:
29-
secretKeyRef:
30-
name: aso-controller-settings
31-
key: AZURE_TENANT_ID
32-
- name: AZURE_SUBSCRIPTION_ID
33-
valueFrom:
34-
secretKeyRef:
35-
name: aso-controller-settings
36-
key: AZURE_SUBSCRIPTION_ID
37-
- name: AZURE_CLIENT_CERTIFICATE
38-
valueFrom:
39-
secretKeyRef:
40-
name: aso-controller-settings
41-
key: AZURE_CLIENT_CERTIFICATE
42-
optional: true
43-
- name: AZURE_CLIENT_CERTIFICATE_PASSWORD
44-
valueFrom:
45-
secretKeyRef:
46-
name: aso-controller-settings
47-
key: AZURE_CLIENT_CERTIFICATE_PASSWORD
48-
optional: true
49-
- name: AZURE_AUTHORITY_HOST
50-
valueFrom:
51-
secretKeyRef:
52-
name: aso-controller-settings
53-
key: AZURE_AUTHORITY_HOST
54-
optional: true
55-
- name: AZURE_RESOURCE_MANAGER_ENDPOINT
56-
valueFrom:
57-
secretKeyRef:
58-
name: aso-controller-settings
59-
key: AZURE_RESOURCE_MANAGER_ENDPOINT
60-
optional: true
61-
- name: AZURE_RESOURCE_MANAGER_AUDIENCE
62-
valueFrom:
63-
secretKeyRef:
64-
name: aso-controller-settings
65-
key: AZURE_RESOURCE_MANAGER_AUDIENCE
66-
optional: true
67-
- name: AZURE_TARGET_NAMESPACES
68-
valueFrom:
69-
secretKeyRef:
70-
name: aso-controller-settings
71-
key: AZURE_TARGET_NAMESPACES
72-
optional: true
73-
- name: AZURE_OPERATOR_MODE
74-
valueFrom:
75-
secretKeyRef:
76-
name: aso-controller-settings
77-
key: AZURE_OPERATOR_MODE
78-
optional: true
79-
- name: AZURE_SYNC_PERIOD
80-
valueFrom:
81-
secretKeyRef:
82-
name: aso-controller-settings
83-
key: AZURE_SYNC_PERIOD
84-
optional: true
85-
- name: USE_WORKLOAD_IDENTITY_AUTH
86-
valueFrom:
87-
secretKeyRef:
88-
key: USE_WORKLOAD_IDENTITY_AUTH
89-
name: aso-controller-settings
90-
optional: true
91-
- name: AZURE_USER_AGENT_SUFFIX
92-
valueFrom:
93-
secretKeyRef:
94-
key: AZURE_USER_AGENT_SUFFIX
95-
name: aso-controller-settings
96-
optional: true
97-
- name: MAX_CONCURRENT_RECONCILES
98-
valueFrom:
99-
secretKeyRef:
100-
key: MAX_CONCURRENT_RECONCILES
101-
name: aso-controller-settings
102-
optional: true
103-
- name: RATE_LIMIT_MODE
104-
valueFrom:
105-
secretKeyRef:
106-
key: RATE_LIMIT_MODE
107-
name: aso-controller-settings
108-
optional: true
109-
- name: RATE_LIMIT_QPS
110-
valueFrom:
111-
secretKeyRef:
112-
key: RATE_LIMIT_QPS
113-
name: aso-controller-settings
114-
optional: true
115-
- name: RATE_LIMIT_BUCKET_SIZE
116-
valueFrom:
117-
secretKeyRef:
118-
key: RATE_LIMIT_BUCKET_SIZE
119-
name: aso-controller-settings
120-
optional: true
12120
# Used for setting the operator-namespace annotation (and
12221
# for aad-pod-identity once we support it).
12322
- name: POD_NAMESPACE
12423
valueFrom:
12524
fieldRef:
12625
fieldPath: metadata.namespace
26+
volumeMounts:
27+
- name: settings-volume
28+
readOnly: true
29+
mountPath: "/etc/aso-controller-settings"

0 commit comments

Comments
 (0)