test/workflow: Refactor make test commands and update related workflows (#716)

Author: ykim-akamai

.github/workflows/e2e-suite-windows.yml

@@ -73,7 +73,7 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-      - run: make MODULE="${{ inputs.module }}" RUN_LONG_TESTS="${{ inputs.run_long_tests }}" testint
+      - run: make MODULE="${{ inputs.module }}" RUN_LONG_TESTS="${{ inputs.run_long_tests }}" test-int
         env:
           LINODE_CLI_TOKEN: ${{ secrets.LINODE_TOKEN_2 }}
 

.github/workflows/e2e-suite.yml

@@ -4,36 +4,36 @@ on:
   workflow_dispatch:
     inputs:
       use_minimal_test_account:
-        description: 'Use minimal test account'
+        description: 'Indicate whether to use a minimal test account with limited resources for testing. Defaults to "false"'
         required: false
         default: 'false'
-      module:
-        description: "The module from 'test/integration' to the target to be tested, e.g. 'cli, domains, events, etc'"
+      test_suite:
+        description: "Specify  test suite to run from the 'tests/integration' directory. Examples: 'cli', 'domains', 'events', etc. If not provided, all suites are executed"
         required: false
       run_long_tests:
-        description: "Select True to run long tests, e.g. database, rebuild, etc"
+        description: "Select 'True' to include long-running tests (e.g., database provisioning, server rebuilds). Defaults to 'False'"
         required: false
         type: choice
         options:
           - "True"
           - "False"
         default: "False"
       sha:
-        description: 'The hash value of the commit.'
+        description: 'Specify commit hash to test. This value is mandatory to ensure the tests run against a specific commit'
         required: true
         default: ''
       pull_request_number:
-        description: 'The number of the PR. Ensure sha value is provided'
+        description: 'Specify pull request number associated with the commit. Optional, but recommended when providing a commit hash (sha)'
         required: false
       openapi_spec_url:
-        description: 'URL of the OpenAPI spec to use for the tests'
+        description: 'Specify URL of the OpenAPI specification file to use for testing. Useful for validating tests against a specific API version or custom specification'
         required: false
         default: ''
       python-version:
-        description: 'Specify Python version to use'
+        description: 'Specify the Python version to use for running tests. Leave empty to use the default Python version configured in the environment'
         required: false
       run-eol-python-version:
-        description: 'Run EOL python version?'
+        description: 'Indicates whether to run tests using an End-of-Life (EOL) Python version. Defaults to "false". Choose "true" to include tests for deprecated Python versions'
         required: false
         default: 'false'
         type: choice
@@ -124,24 +124,18 @@ jobs:
         run: |
           timestamp=$(date +'%Y%m%d%H%M')
           report_filename="${timestamp}_cli_test_report.xml"
-          make testint TEST_ARGS="--junitxml=${report_filename}" MODULE="${{ inputs.module }}" RUN_LONG_TESTS="${{ inputs.run_long_tests }}"
+          make test-int TEST_ARGS="--junitxml=${report_filename}" TEST_SUITE="${{ inputs.test_suite }}" RUN_LONG_TESTS="${{ inputs.run_long_tests }}"
         env:
           LINODE_CLI_TOKEN: ${{ env.LINODE_CLI_TOKEN }}
 
-      - name: Upload test results
+      - name: Upload Test Report as Artifact
         if: always()
-        run: |
-          filename=$(ls | grep -E '^[0-9]{12}_cli_test_report\.xml$')
-          python3 e2e_scripts/tod_scripts/xml_to_obj_storage/scripts/add_gha_info_to_xml.py \
-          --branch_name "${GITHUB_REF#refs/*/}" \
-          --gha_run_id "$GITHUB_RUN_ID" \
-          --gha_run_number "$GITHUB_RUN_NUMBER" \
-          --xmlfile "${filename}"
-          sync
-          python3 e2e_scripts/tod_scripts/xml_to_obj_storage/scripts/xml_to_obj.py "${filename}"
-        env:
-          LINODE_CLI_OBJ_ACCESS_KEY: ${{ secrets.LINODE_CLI_OBJ_ACCESS_KEY }}
-          LINODE_CLI_OBJ_SECRET_KEY: ${{ secrets.LINODE_CLI_OBJ_SECRET_KEY }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-report-file
+          if-no-files-found: ignore
+          path: '*.xml'
+          retention-days: 1
 
       - name: Update PR Check Run
         uses: actions/github-script@v7
@@ -237,6 +231,51 @@ jobs:
         env:
           LINODE_CLI_TOKEN: ${{ env.LINODE_CLI_TOKEN }}
 
+  process-upload-report:
+    runs-on: ubuntu-latest
+    needs: [integration_tests]
+    if: always() && github.repository == 'linode/linode-cli' # Run even if integration tests fail and only on main repository
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: 'recursive'
+
+      - name: Download test report
+        uses: actions/download-artifact@v4
+        with:
+          name: test-report-file
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+
+      - name: Install Python dependencies
+        run: pip3 install requests wheel boto3==1.35.99
+
+      - name: Set release version env
+        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+
+
+      - name: Add variables and upload test results
+        if: always()
+        run: |
+          filename=$(ls | grep -E '^[0-9]{12}_cli_test_report\.xml$')
+          python3 e2e_scripts/tod_scripts/xml_to_obj_storage/scripts/add_gha_info_to_xml.py \
+          --branch_name "${GITHUB_REF#refs/*/}" \
+          --gha_run_id "$GITHUB_RUN_ID" \
+          --gha_run_number "$GITHUB_RUN_NUMBER" \
+          --xmlfile "${filename}"
+          sync
+          python3 e2e_scripts/tod_scripts/xml_to_obj_storage/scripts/xml_to_obj.py "${filename}"
+        env:
+          LINODE_CLI_OBJ_ACCESS_KEY: ${{ secrets.LINODE_CLI_OBJ_ACCESS_KEY }}
+          LINODE_CLI_OBJ_SECRET_KEY: ${{ secrets.LINODE_CLI_OBJ_SECRET_KEY }}
+
+
   notify-slack:
     runs-on: ubuntu-latest
     needs: [integration_tests]

.github/workflows/nightly-smoke-tests.yml

@@ -40,7 +40,7 @@ jobs:
       - name: Run smoke tests
         id: smoke_tests
         run: |
-          make smoketest
+          make test-smoke
         env:
           LINODE_CLI_TOKEN: ${{ secrets.LINODE_TOKEN }}
 

Makefile

@@ -2,15 +2,6 @@
 # Makefile for more convenient building of the Linode CLI and its baked content
 #
 
-# Test-related arguments
-MODULE :=
-TEST_CASE_COMMAND :=
-TEST_ARGS :=
-
-ifdef TEST_CASE
-TEST_CASE_COMMAND = -k $(TEST_CASE)
-endif
-
 SPEC_VERSION ?= latest
 ifndef SPEC
 override SPEC = $(shell ./resolve_spec_url ${SPEC_VERSION})
@@ -66,8 +57,8 @@ clean:
 	rm -f data-*
 	rm -rf dist linode_cli.egg-info build
 
-.PHONY: testunit
-testunit:
+.PHONY: test-unit
+test-unit:
 	@mkdir -p /tmp/linode/.config
 	@orig_xdg_config_home=$${XDG_CONFIG_HOME:-}; \
 	export LINODE_CLI_TEST_MODE=1 XDG_CONFIG_HOME=/tmp/linode/.config; \
@@ -76,17 +67,22 @@ testunit:
 	export XDG_CONFIG_HOME=$$orig_xdg_config_home; \
 	exit $$exit_code
 
-.PHONY: testint
-testint:
-	pytest tests/integration/${MODULE} ${TEST_CASE_COMMAND} ${TEST_ARGS}
+# Integration Test Arguments
+# TEST_SUITE: Optional, specify a test suite (e.g. domains), Default to run everything if not set
+# TEST_CASE: Optional, specify a test case (e.g. 'test_create_a_domain')
+# TEST_ARGS: Optional, additional arguments for pytest (e.g. '-v' for verbose mode)
+
+.PHONY: test-int
+test-int:
+	pytest tests/integration/$(TEST_SUITE) $(if $(TEST_CASE),-k $(TEST_CASE)) $(TEST_ARGS)
 
 .PHONY: testall
 testall:
 	pytest tests
 
 # Alias for unit; integration tests should be explicit
 .PHONY: test
-test: testunit
+test: test-unit
 
 .PHONY: black
 black:
@@ -103,6 +99,6 @@ autoflake:
 .PHONY: format
 format: black isort autoflake
 
-@PHONEY: smoketest
-smoketest:
+@PHONEY: test-smoke
+test-smoke:
 	pytest -m smoke tests/integration