Testing: Support k3d/k3s/rancher in deployer/e2e tests

.buildkite/e2e/pipeline-gen/main.go

@@ -44,19 +44,19 @@ const (
 	EnvVarOperatorImage        = "OPERATOR_IMAGE"
 	EnvVarE2EImage             = "E2E_IMG"
 
-	KindAgentsMachineType = "n1-standard-16"
+	KindAgentsMachineType = "n2d-standard-16"
 )
 
 var (
 	//go:embed pipeline.tpl.yaml
 	pipelineTemplate string
 
 	// providersInDocker are k8s providers that require the deployer to run in Docker
-	providersInDocker = []string{"kind", "aks", "ocp"}
+	providersInDocker = []string{"kind", "aks", "ocp", "k3d"}
 	// providersNoCleanup are k8s providers that do not require the cluster to be deleted after use
-	providersNoCleanup = []string{"kind"}
+	providersNoCleanup = []string{"kind", "k3d"}
 	// providers are k8s providers for which it is not possible to retrieve the kube config after cluster creation
-	providersNoRemoteConfig = []string{"kind"}
+	providersNoRemoteConfig = []string{"kind", "k3d"}
 
 	semverRE = regexp.MustCompile(`\d*\.\d*\.\d*(-\w*)?`)
 	chars    = []rune("abcdefghijklmnopqrstuvwxyz")

.buildkite/e2e/pipeline-gen/pipeline.tpl.yaml

@@ -19,7 +19,6 @@ steps:
 
     commands:
       - .buildkite/scripts/test/set-deployer-config.sh
-
       {{- if $test.Dind }}
       - make -C .buildkite TARGET="run-deployer" ci
       {{- else }}
@@ -29,9 +28,9 @@ steps:
     agents:
       {{- if $test.Dind }}
       provider: "gcp"
-      image: "family/core-ubuntu-2004"
-      {{- if eq $test.Provider "kind" }}
-      machineType: "{{ .KindAgentsMachineType }}"
+      image: "family/core-ubuntu-2204"
+      {{- if or (eq $test.Provider "kind") (eq $test.Provider "k3d") }}
+      machineType: "{{ $.KindAgentsMachineType }}"
       {{- end }}
       {{- else }}
       image: docker.elastic.co/ci-agent-images/cloud-k8s-operator/buildkite-agent:c39fad65
@@ -71,6 +70,10 @@ steps:
         {{- $deployerCommand = "set-kubeconfig" }}
       {{- end }}
 
+      # Check k3s configuration
+      - wget https://github.com/k3s-io/k3s/releases/download/v1.33.3%2Bk3s1/k3s && chmod +x k3s
+      - ./k3s check-config
+
       {{- if $test.Dind }}
       - make -C .buildkite TARGET="{{ $deployerCommand }} e2e-run" ci
       {{- else }}
@@ -81,7 +84,7 @@ steps:
       {{- if $test.Dind }}
       provider: "gcp"
       image: "family/core-ubuntu-2004"
-      {{- if eq $test.Provider "kind" }}
+      {{- if or (eq $test.Provider "kind") (eq $test.Provider "k3d") }}
       machineType: "{{ $.KindAgentsMachineType }}"
       diskSizeGb: 150
       {{- end }}
@@ -119,7 +122,11 @@ steps:
         soft_fail: true
         commands:
           - .buildkite/scripts/test/set-deployer-config.sh
+        {{- if eq $test.Provider "k3d" }}
+          - "wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=v5.8.3 bash"
+        {{- end }}
         {{- if not $test.Dind }}
+          - "wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=v5.8.3 bash"
           - make run-deployer
         agents:
           image: docker.elastic.co/ci-agent-images/cloud-k8s-operator/buildkite-agent:c39fad65

.buildkite/e2e/release-branch-matrix.yaml

@@ -1,4 +1,3 @@
-
 - label: stack
   fixed:
     E2E_PROVIDER: gke
@@ -41,6 +40,12 @@
     - DEPLOYER_KIND_NODE_IMAGE: kindest/node:v1.33.1@sha256:050072256b9a903bd914c0b2866828150cb229cea0efe5892e2b644d5dd3b34f
       DEPLOYER_KIND_IP_FAMILY: ipv6
 
+- label: k3d
+  fixed:
+    E2E_PROVIDER: k3d
+  mixed:
+    - DEPLOYER_K3D_NODE_IMAGE: "rancher/k3s:v1.33.3+k3s1"
+
 - label: gke
   fixed:
     E2E_PROVIDER: gke

.buildkite/scripts/test/set-deployer-config.sh

@@ -25,7 +25,7 @@ ROOT="$WD/../../.."
 
 w()  { echo "$@" >> "$ROOT/deployer-config.yml"; }
 
-write_deployer_config() { 
+write_deployer_config() {
     :> "$ROOT/deployer-config.yml"
 
     w "id: ${E2E_PROVIDER}-ci"
@@ -36,7 +36,7 @@ write_deployer_config() {
     w "  operation: ${DEPLOYER_OPERATION:-create}"
     w "  clusterName: ${CLUSTER_NAME}"
 
-    # k8s version for ocp, kind    
+    # k8s version for ocp, kind
     if [[ "${DEPLOYER_CLIENT_VERSION:-}" != "" ]]; then
     w '  clientVersion: "'"${DEPLOYER_CLIENT_VERSION}"'"'
     fi
@@ -58,6 +58,11 @@ write_deployer_config() {
     w "    nodeImage: ${DEPLOYER_KIND_NODE_IMAGE}"
     w "    ipFamily: ${DEPLOYER_KIND_IP_FAMILY:-ipv4}"
     fi
+
+    if [[ "${DEPLOYER_K3D_NODE_IMAGE:-}" ]]; then
+    w "  k3d:"
+    w "    nodeImage: ${DEPLOYER_K3D_NODE_IMAGE}"
+    fi
 }
 
 write_deployer_config

hack/deployer/cmd/create.go

@@ -61,6 +61,8 @@ func CreateCommand() *cobra.Command {
 				cfgData = fmt.Sprintf(runner.DefaultEKSRunConfigTemplate, user, vaultAddr, token)
 			case runner.KindDriverID:
 				cfgData = fmt.Sprintf(runner.DefaultKindRunConfigTemplate, user)
+			case runner.K3dDriverID:
+				cfgData = fmt.Sprintf(runner.DefaultK3dRunConfigTemplate, user)
 			default:
 				return fmt.Errorf("unknown provider %s", provider)
 			}