helm: fix kubeadm bugs caused by CoreDNS installation (#3353)

* helm: rename CoreDNS configmap

* upgrade-agent: ignore CoreDNS preflight errors

* fixup! helm: rename CoreDNS configmap

Author: burgerdev

cli/internal/cmd/apply.go

@@ -428,6 +428,9 @@ func (a *applyCmd) apply(
 		if err := a.runHelmApply(cmd, conf, stateFile, upgradeDir); err != nil {
 			return err
 		}
+		if err := a.applier.CleanupCoreDNSResources(cmd.Context()); err != nil {
+			return fmt.Errorf("cleaning up CoreDNS: %w", err)
+		}
 	}
 
 	// Upgrade node image
@@ -847,6 +850,7 @@ type applier interface {
 	// methods required to install/upgrade Helm charts
 
 	AnnotateCoreDNSResources(context.Context) error
+	CleanupCoreDNSResources(context.Context) error
 	PrepareHelmCharts(
 		flags helm.Options, state *state.State, serviceAccURI string, masterSecret uri.MasterSecret,
 	) (helm.Applier, bool, error)

cli/internal/cmd/apply_test.go

@@ -554,6 +554,7 @@ func (s *stubConstellApplier) Init(context.Context, atls.Validator, *state.State
 
 type helmApplier interface {
 	AnnotateCoreDNSResources(context.Context) error
+	CleanupCoreDNSResources(ctx context.Context) error
 	PrepareHelmCharts(
 		flags helm.Options, stateFile *state.State, serviceAccURI string, masterSecret uri.MasterSecret,
 	) (

cli/internal/cmd/init_test.go

@@ -282,6 +282,10 @@ func (s stubHelmApplier) AnnotateCoreDNSResources(_ context.Context) error {
 	return nil
 }
 
+func (s stubHelmApplier) CleanupCoreDNSResources(_ context.Context) error {
+	return nil
+}
+
 func (s stubHelmApplier) PrepareHelmCharts(
 	_ helm.Options, _ *state.State, _ string, _ uri.MasterSecret,
 ) (helm.Applier, bool, error) {

cli/internal/cmd/upgradeapply_test.go

@@ -379,6 +379,10 @@ func (m *mockApplier) AnnotateCoreDNSResources(_ context.Context) error {
 	return nil
 }
 
+func (m *mockApplier) CleanupCoreDNSResources(_ context.Context) error {
+	return nil
+}
+
 func (m *mockApplier) PrepareHelmCharts(
 	helmOpts helm.Options, stateFile *state.State, str string, masterSecret uri.MasterSecret,
 ) (helm.Applier, bool, error) {

internal/constellation/helm.go

@@ -60,6 +60,21 @@ func (a *Applier) AnnotateCoreDNSResources(ctx context.Context) error {
 	return nil
 }
 
+// CleanupCoreDNSResources removes CoreDNS resources that are not managed by Helm.
+//
+// This is only required when CoreDNS was installed by kubeadm directly.
+// TODO(burgerdev): remove after v2.19 is released.
+func (a *Applier) CleanupCoreDNSResources(ctx context.Context) error {
+	err := a.dynamicClient.
+		Resource(schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"}).
+		Namespace("kube-system").
+		Delete(ctx, "coredns", v1.DeleteOptions{})
+	if !k8serrors.IsNotFound(err) {
+		return err
+	}
+	return nil
+}
+
 // PrepareHelmCharts loads Helm charts for Constellation and returns an executor to apply them.
 func (a *Applier) PrepareHelmCharts(
 	flags helm.Options, state *state.State, serviceAccURI string, masterSecret uri.MasterSecret,

internal/constellation/helm/charts/coredns/templates/configmap.yaml

@@ -1,9 +1,4 @@
-
 apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: coredns
-  namespace: kube-system
 data:
   Corefile: |
     .:53 {
@@ -26,3 +21,8 @@ data:
         reload
         loadbalance
     }
+kind: ConfigMap
+metadata:
+  creationTimestamp: null
+  name: edg-coredns
+  namespace: kube-system

internal/constellation/helm/charts/coredns/templates/deployment.yaml

@@ -104,6 +104,6 @@ spec:
           items:
           - key: Corefile
             path: Corefile
-          name: coredns
+          name: edg-coredns
         name: config-volume
 status: {}

internal/constellation/helm/corednsgen/corednsgen.go

@@ -29,6 +29,8 @@ import (
 	"sigs.k8s.io/yaml"
 )
 
+const configMapName = "edg-coredns"
+
 var chartDir = flag.String("charts", "./charts", "target directory to create charts in")
 
 func main() {
@@ -44,9 +46,9 @@ func main() {
 	writeTemplate(kubedns.CoreDNSServiceAccount, "serviceaccount.yaml")
 	writeTemplate(kubedns.CoreDNSClusterRole, "clusterrole.yaml")
 	writeTemplate(kubedns.CoreDNSClusterRoleBinding, "clusterrolebinding.yaml")
-	writeTemplate(kubedns.CoreDNSConfigMap, "configmap.yaml")
 	writeTemplate(kubedns.CoreDNSService, "service.yaml")
 
+	writeFileRelativeToChartDir(patchedConfigMap(), "templates", "configmap.yaml")
 	writeFileRelativeToChartDir(patchedDeployment(), "templates", "deployment.yaml")
 }
 
@@ -92,7 +94,25 @@ func valuesYAML() []byte {
 	return data
 }
 
-// patchedDeployment extracts the CoreDNS deployment from kubeadm and adds necessary tolerations.
+// patchedConfigMap renames the CoreDNS ConfigMap such that kubeadm does not find it.
+//
+// See https://github.com/kubernetes/kubeadm/issues/2846#issuecomment-1899942683.
+func patchedConfigMap() []byte {
+	var cm corev1.ConfigMap
+	if err := yaml.Unmarshal(parseTemplate(kubedns.CoreDNSConfigMap), &cm); err != nil {
+		log.Fatalf("Could not parse configmap: %v", err)
+	}
+
+	cm.Name = configMapName
+
+	out, err := yaml.Marshal(cm)
+	if err != nil {
+		log.Fatalf("Could not marshal patched deployment: %v", err)
+	}
+	return out
+}
+
+// patchedDeployment extracts the CoreDNS Deployment from kubeadm, adds necessary tolerations and updates the ConfigMap reference.
 func patchedDeployment() []byte {
 	var d appsv1.Deployment
 	if err := yaml.Unmarshal(parseTemplate(kubedns.CoreDNSDeployment), &d); err != nil {
@@ -104,6 +124,14 @@ func patchedDeployment() []byte {
 		{Key: "node.kubernetes.io/unreachable", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoExecute, TolerationSeconds: toPtr(int64(10))},
 	}
 	d.Spec.Template.Spec.Tolerations = append(d.Spec.Template.Spec.Tolerations, tolerations...)
+
+	for i, vol := range d.Spec.Template.Spec.Volumes {
+		if vol.ConfigMap != nil {
+			vol.ConfigMap.Name = configMapName
+		}
+		d.Spec.Template.Spec.Volumes[i] = vol
+	}
+
 	out, err := yaml.Marshal(d)
 	if err != nil {
 		log.Fatalf("Could not marshal patched deployment: %v", err)

terraform-provider-constellation/internal/provider/cluster_resource.go

@@ -1311,6 +1311,11 @@ func (r *ClusterResource) applyHelmCharts(ctx context.Context, applier *constell
 		diags.AddError("Applying Helm charts", err.Error())
 		return diags
 	}
+
+	if err := applier.CleanupCoreDNSResources(ctx); err != nil {
+		diags.AddError("Cleaning up CoreDNS resources", err.Error())
+		return diags
+	}
 	return diags
 }
 

upgrade-agent/internal/server/server.go

@@ -111,12 +111,18 @@ func (s *Server) ExecuteUpdate(ctx context.Context, updateRequest *upgradeproto.
 		return nil, status.Errorf(codes.Internal, "unable to install the kubeadm binary: %s", err)
 	}
 
-	upgradeCmd := exec.CommandContext(ctx, "kubeadm", "upgrade", "plan", updateRequest.WantedKubernetesVersion)
+	// CoreDNS addon status is checked even though we did not install it.
+	// TODO(burgerdev): Use kubeadm phases once supported: https://github.com/kubernetes/kubeadm/issues/1318.
+	commonArgs := []string{"--ignore-preflight-errors", "CoreDNSMigration,CoreDNSUnsupportedPlugins", updateRequest.WantedKubernetesVersion}
+	planArgs := append([]string{"upgrade", "plan"}, commonArgs...)
+	applyArgs := append([]string{"upgrade", "apply", "--yes", "--patches", constants.KubeadmPatchDir}, commonArgs...)
+
+	upgradeCmd := exec.CommandContext(ctx, "kubeadm", planArgs...)
 	if out, err := upgradeCmd.CombinedOutput(); err != nil {
 		return nil, status.Errorf(codes.Internal, "unable to execute kubeadm upgrade plan %s: %s: %s", updateRequest.WantedKubernetesVersion, err, string(out))
 	}
 
-	applyCmd := exec.CommandContext(ctx, "kubeadm", "upgrade", "apply", "--yes", "--patches", constants.KubeadmPatchDir, updateRequest.WantedKubernetesVersion)
+	applyCmd := exec.CommandContext(ctx, "kubeadm", applyArgs...)
 	if out, err := applyCmd.CombinedOutput(); err != nil {
 		return nil, status.Errorf(codes.Internal, "unable to execute kubeadm upgrade apply: %s: %s", err, string(out))
 	}