[test] E2E test for kubeadm-full ETCD (#464)

prajwalvathreya · web-flow · commit ca9571e76114 · 2024-08-28T15:36:06.000-04:00
* - Cleaned up the flatcar chainsaw-test.yaml
- Previously the passed keys were kubeadm and flatcar

* - added etcd StatefulSet into the assert list

* - added test for checking if etcd-disk is added to spec

* - added filter to the curl command

* - fixed json parse error

* - printing disk response to debug the issue

* - updated logging mechanism and removed other tests to debug the particular test

* - added filter to the linode api call and updated logging

* - uncommented tests for other resources

* - testing filter for API call

* - updated cron job time for backup
- added tests to check if objects were being backed up into the bucket

* - added retry loops to tests which have scripts
- delay of 30 seconds per retry

* - fixed makefile error caused by stray test

* - updated make file to generate a 32 char ephemeral key
- updated chainsaw-test.yaml to use said 32 char ephemeral key
- updated bucket retry time to 60 seconds per try

* - removed additional logging

* - fixed SSE_KEY generation in Makefile

* - removed logging using echo
- updated retry time to 40 seconds

* - made SSE_KEY static
- updated API calls to use environment variable LINODE_REGION

* - gowrap_version set to v1.3.7 in Makefile

* - fixed bucket object deletion using s5cmd
- added steps to install s5cmd in Makefile

* - updated s3cmd ref to s5cmd

* - changed LOCALBIN to CACHE_BIN in Makefile.
- this change was made to test and see if s5cmd path was accessible when cache_bin was used instead of localbin

* - added s5cmd to make e2e test

* - added API call to delete the bucket
- added comments to bash scripts
diff --git a/Makefile b/Makefile
@@ -152,8 +152,8 @@ test: generate fmt vet envtest ## Run tests.
 	rm cover.out.tmp
 
 .PHONY: e2etest
-e2etest: generate local-release local-deploy chainsaw
-	GIT_REF=$(GIT_REF) SSE_KEY=$$(openssl rand -base64 32) $(CHAINSAW) test ./e2e --selector $(E2E_SELECTOR) $(E2E_FLAGS)
+e2etest: generate local-release local-deploy chainsaw s5cmd
+	GIT_REF=$(GIT_REF) SSE_KEY=$$(openssl rand -base64 32) LOCALBIN=$(CACHE_BIN) $(CHAINSAW) test ./e2e --selector $(E2E_SELECTOR) $(E2E_FLAGS)
 
 local-deploy: kind ctlptl tilt kustomize clusterctl
 	$(CTLPTL) apply -f .tilt/ctlptl-config.yaml
@@ -316,6 +316,7 @@ NILAWAY        ?= $(LOCALBIN)/nilaway
 GOVULNC        ?= $(LOCALBIN)/govulncheck
 MOCKGEN        ?= $(LOCALBIN)/mockgen
 GOWRAP         ?= $(CACHE_BIN)/gowrap
+S5CMD 		   ?= $(CACHE_BIN)/s5cmd
 
 ## Tool Versions
 KUSTOMIZE_VERSION        ?= v5.4.1
@@ -332,6 +333,7 @@ NILAWAY_VERSION          ?= latest
 GOVULNC_VERSION          ?= v1.1.1
 MOCKGEN_VERSION          ?= v0.4.0
 GOWRAP_VERSION           ?= v1.3.7
+S5CMD_VERSION            ?= v2.2.2
 
 .PHONY: tools
 tools: $(KUSTOMIZE) $(CTLPTL) $(CLUSTERCTL) $(KUBECTL) $(CONTROLLER_GEN) $(CONVERSION_GEN) $(TILT) $(KIND) $(CHAINSAW) $(ENVTEST) $(HUSKY) $(NILAWAY) $(GOVULNC) $(MOCKGEN) $(GOWRAP)
@@ -427,3 +429,7 @@ gowrap: $(GOWRAP) ## Download gowrap locally if necessary.
 $(GOWRAP): $(CACHE_BIN)
 	GOBIN=$(CACHE_BIN) go install github.com/hexdigest/gowrap/cmd/gowrap@$(GOWRAP_VERSION)
 
+.PHONY: s5cmd
+s5cmd: $(S5CMD)
+$(S5CMD): $(CACHE_BIN)
+	GOBIN=$(CACHE_BIN) go install github.com/peak/s5cmd/v2@$(S5CMD_VERSION)
diff --git a/e2e/capl-cluster-flavors/kubeadm-flatcar-vpcless-capl-cluster/chainsaw-test.yaml b/e2e/capl-cluster-flavors/kubeadm-flatcar-vpcless-capl-cluster/chainsaw-test.yaml
@@ -7,9 +7,8 @@ metadata:
   # Labels to allow the test to be triggered based on selector flag
   labels:
     all:
-    kubeadm:
-    flavors:
     flatcar:
+    flavors:
 spec:
   bindings:
     # A short identifier for the E2E test run
diff --git a/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-child-cluster-statefulsets.yaml b/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-child-cluster-statefulsets.yaml
@@ -5,3 +5,11 @@ metadata:
   namespace: kube-system
 status:
   replicas: 1
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: etcd-backup
+  namespace: kube-system
+status:
+  replicas: 1
diff --git a/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/chainsaw-test.yaml b/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/chainsaw-test.yaml
@@ -13,7 +13,7 @@ spec:
   bindings:
     # Identifier for the E2E test run
     - name: run
-      value: (join('-', ['e2e', 'kdm-ft', env('GIT_REF')]))
+      value: (join('-', ['e2e', 'kdmf-tst', env('GIT_REF')]))
     - name: cluster
       # Format the cluster name
       value: (trim((truncate(($run), `32`)), '-'))
@@ -37,15 +37,17 @@ spec:
               - name: CLUSTERCTL_CONFIG
                 value: (env('CLUSTERCTL_CONFIG'))
               - name: SSE_KEY
-                value: (env('SSE_KEY'))
+                value: "jU$n@AXXf3xseNd&U9ko4J#3B84D6YR6"
               - name: KONNECTIVITY_AGENT_REPLICAS
                 value: '1' # Here, 1 is set for testing purposes. Default is 3.
               - name: CLUSTER_AUTOSCALER_VERSION
                 value: 'v1.29.4'
+              - name: ETCD_BACKUP_SCHEDULE
+                value: '* * * * *'
             content: |
               set -e
               if [ -z "$SSE_KEY" ]; then
-                echo "SSE_KEY not set" >&2 
+                echo "SSE_KEY not set" >&2
                 exit 1
               else
                 clusterctl generate cluster $CLUSTER -n $NAMESPACE \
@@ -203,17 +205,198 @@ spec:
             namespace: kube-system
 
     # Test to check if konnectivity is working
-    - clusters:
-        kubeadm-full-cluster:
-          kubeconfig: ./kubeadm-full-cluster-kubeconfig.yaml
-      name: Testing to check if logs are retrievable
+    - name: Testing to check if logs are retrievable
       try:
         - script:
             content: |
-              KUBECONFIG=./kubeadm-full-cluster-kubeconfig.yaml kubectl logs csi-linode-controller-0 -n kube-system
+
+              MAX_RETRIES=5
+              RETRY_DELAY=30 # 30 seconds delay before retrying
+              ATTEMPT=0
+              
+              # retry loop to check if logs are retrievable from a pod
+              while [ $ATTEMPT -lt $MAX_RETRIES ]; do
+                KUBECONFIG=./kubeadm-full-cluster-kubeconfig.yaml kubectl logs csi-linode-controller-0 -n kube-system
+
+                if [ -z "$error" ]; then
+                  break
+                else
+                  ATTEMPT=$((ATTEMPT + 1))
+                  sleep $RETRY_DELAY
+                fi
+              done
+
             check:
               ($error == null): true
 
+    # Test to check if disks are added to the etcd spec
+    - name: Testing to check if disks are added to the spec
+      try:
+        - script:
+            env:
+              - name: TARGET_API
+                value: api.linode.com
+              - name:  TARGET_API_VERSION
+                value: v4beta
+              - name: URI
+                value: linode/instances
+              - name: FILTER
+                value: (to_string({"tags":($cluster)}))
+            content: |
+              set -e
+
+              MAX_RETRIES=5
+              RETRY_DELAY=30 # 30 second delay before retrying
+              ATTEMPT=0
+              ETCD_LABEL=""
+              ETCD_STATUS=""
+              
+              # retry loop to check if etcd disk is attached 
+              while [ $ATTEMPT -lt $MAX_RETRIES ]; do
+                
+                # API call to get the linode ID
+                RESPONSE=$(curl -s \
+                  -H "Authorization: Bearer $LINODE_TOKEN" \
+                  -H "X-Filter: $FILTER" \
+                  -H "Content-Type: application/json" \
+                  "https://$TARGET_API/$TARGET_API_VERSION/$URI")
+
+                LINODE_ID=$(echo "$RESPONSE" | jq -r '.data[0].id')
+
+                if [ -z "$LINODE_ID" ]; then
+                  echo "Error: LINODE_ID is empty. Response was: $RESPONSE"
+                  exit 1
+                fi
+                
+                # API call to check the list of disks attached 
+                DISK_RESPONSE=$(curl -s \
+                  -H "Authorization: Bearer $LINODE_TOKEN" \
+                  -H "Content-Type: application/json" \
+                  "https://$TARGET_API/$TARGET_API_VERSION/$URI/$LINODE_ID/disks?page=1&page_size=100")
+
+                ETCD_LABEL=$(echo "$DISK_RESPONSE" | jq -r '.data[2].label')
+                ETCD_STATUS=$(echo "$DISK_RESPONSE" | jq -r '.data[2].status')
+          
+                if [ "$ETCD_LABEL" == "etcd_disk" ] && [ "$ETCD_STATUS" == "ready" ]; then
+                  break
+                else
+                  ATTEMPT=$((ATTEMPT + 1))
+                  sleep $RETRY_DELAY
+                fi
+              done
+
+              echo "{
+                \"label\": \"$ETCD_LABEL\",
+                \"status\": \"$ETCD_STATUS\"
+              }"
+
+            check:
+              ($error): ~
+              (json_parse($stdout)):
+                label: etcd_disk
+                status: ready
+
+    # Test to query the object storage bucket and check for data
+    - name: Testing the object storage buckets
+      try:
+        - script:
+            env:
+              - name: TARGET_API
+                value: api.linode.com
+              - name: TARGET_API_VERSION
+                value: v4beta
+              - name: BUCKET_NAME
+                value: (join('-', ['e2e', 'kdmf-tst', env('GIT_REF'), 'etcd-backup']))
+              - name: URI
+                value: (join('/', ['object-storage', 'buckets', env('LINODE_REGION')]))
+            content: |
+              set -e
+
+              MAX_RETRIES=5
+              RETRY_DELAY=40 # 40 second delay before retrying
+              ATTEMPT=0
+              VALID_BUCKET=false
+
+              # retry loop to check if bucket is created and objects are present
+              while [ $ATTEMPT -lt $MAX_RETRIES ]; do
+                
+                # API call to get the bucket details
+                RESPONSE=$(curl -s \
+                  -H "Authorization: Bearer $LINODE_TOKEN" \
+                  -H "Content-Type: application/json" \
+                  "https://$TARGET_API/$TARGET_API_VERSION/$URI/$BUCKET_NAME")
+
+                BUCKET_SIZE=$(echo "$RESPONSE" | jq -r '.size')
+                BUCKET_OBJECTS=$(echo "$RESPONSE" | jq -r '.objects')
+
+                if [ "$BUCKET_SIZE" -gt 0 ] && [ "$BUCKET_OBJECTS" -gt 0 ]; then
+                  VALID_BUCKET=true
+                  break
+                else
+                  ATTEMPT=$((ATTEMPT + 1))
+                  sleep $RETRY_DELAY
+                fi
+              done
+
+              echo "{
+                \"status\": \"$VALID_BUCKET\"
+              }"
+
+            check:
+              ($error): ~
+              (json_parse($stdout)):
+                status: "true"
+
+    # Test to check object storage buckets and delete it
+    - name: Deleting object storage bucket
+      try:
+        - script:
+            env:
+              - name: CAPL_KUBECONFIG
+                value: ./kubeadm-full-cluster-kubeconfig.yaml
+              - name: BUCKET_NAME
+                value: (join('-', ['e2e', 'kdmf-tst', env('GIT_REF'), 'etcd-backup']))
+              - name: SECRET_NAME
+                value: (join('-', ['e2e', 'kdmf-tst', env('GIT_REF'), 'etcd-backup-obj-key']))
+              - name: BUCKET_ENDPOINT
+                value: (join('.', [(join('-', [env('LINODE_REGION'), '1'])), 'linodeobjects', 'com' ]))
+              - name: LOCAL_BIN
+                value: (env('LOCALBIN'))
+              - name: BUCKET_REGION
+                value: env('LINODE_REGION')
+              - name: TARGET_API
+                value: api.linode.com
+              - name: TARGET_API_VERSION
+                value: v4beta
+              - name: URI
+                value: (join('/', ['object-storage', 'buckets', env('LINODE_REGION')]))
+            content: |
+              set -e
+            
+              # Getting the keys from the CAPL cluster
+              access_key=$(KUBECONFIG=$CAPL_KUBECONFIG kubectl get secret $SECRET_NAME -n kube-system -o=jsonpath='{.data.access_key}' | base64 -d)
+              secret_key=$(KUBECONFIG=$CAPL_KUBECONFIG kubectl get secret $SECRET_NAME -n kube-system -o=jsonpath='{.data.secret_key}' | base64 -d)
+
+              #Storing the keys into a config file
+              cat <<EOL > .s5cfg
+              [default]
+              aws_access_key_id=$access_key
+              aws_secret_access_key=$secret_key
+              aws_default_region=$BUCKET_REGION-1
+              EOL
+              
+              # delete the objects
+              $LOCAL_BIN/s5cmd --credentials-file .s5cfg --endpoint-url https://$BUCKET_ENDPOINT rm s3://$BUCKET_NAME/etcd-backup/v2/*
+              
+              # delete the bucket 
+              curl --request DELETE \
+                -H "Authorization: Bearer $LINODE_TOKEN" \
+                -H "Content-Type: application/json" \
+                "https://$TARGET_API/$TARGET_API_VERSION/$URI/$BUCKET_NAME"
+
+            check:
+              ($error): ~
+
     # Test to check if child cluster is deleted
     - name: Testing to see if child cluster is deleted
       try:
@@ -262,5 +445,6 @@ spec:
             content: |
               rm -f kubeadm-full-cluster.yaml
               rm -f kubeadm-full-cluster-kubeconfig.yaml
+              rm -f .s5cfg
             check:
               ($error == null): true
