test: Update smoke test coverage and improve nightly test workflow (#637)

ykim-akamai · web-flow · commit 9d99566c96c8 · 2024-09-26T10:50:40.000-07:00
diff --git a/.github/workflows/e2e-suite.yml b/.github/workflows/e2e-suite.yml
@@ -23,7 +23,7 @@ on:
       - dev
 
 jobs:
-  integration-tests:
+  integration_tests:
     name: Run integration tests on Ubuntu
     runs-on: ubuntu-latest
     if: github.event_name == 'workflow_dispatch' && inputs.sha != '' || github.event_name == 'push' || github.event_name == 'pull_request'
@@ -167,3 +167,69 @@ jobs:
               conclusion: process.env.conclusion
             });
             return result;
+
+  notify-slack:
+    runs-on: ubuntu-latest
+    needs: [integration_tests]
+    if: always() && github.repository == 'linode/linode-cli' # Run even if integration tests fail and only on main repository
+
+    steps:
+      - name: Notify Slack
+        uses: slackapi/slack-github-action@v1.27.0
+        with:
+          channel-id: ${{ secrets.SLACK_CHANNEL_ID }}
+          payload: |
+            {
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": ":rocket: *${{ github.workflow }} Completed in: ${{ github.repository }}* :white_check_mark:"
+                  }
+                },
+                {
+                  "type": "divider"
+                },
+                {
+                  "type": "section",
+                  "fields": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Build Result:*\n${{ steps.integration_tests.outcome == 'success' && ':large_green_circle: Build Passed' || ':red_circle: Build Failed' }}"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Branch:*\n`${{ github.ref_name }}`"
+                    }
+                  ]
+                },
+                {
+                  "type": "section",
+                  "fields": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Commit Hash:*\n<${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}>"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Run URL:*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run Details>"
+                    }
+                  ]
+                },
+                {
+                  "type": "divider"
+                },
+                {
+                  "type": "context",
+                  "elements": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "Triggered by: :bust_in_silhouette: `${{ github.actor }}`"
+                    }
+                  ]
+                }
+              ]
+            }
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
diff --git a/.github/workflows/nightly-smoke-tests.yml b/.github/workflows/nightly-smoke-tests.yml
@@ -4,16 +4,25 @@ on:
   schedule:
     - cron: "0 0 * * *"
   workflow_dispatch:
+    inputs:
+      sha:
+        description: 'Commit SHA to test'
+        required: false
+        default: ''
+        type: string
 
 jobs:
   smoke_tests:
+    if: github.repository == 'linode/linode-cli' || github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
-          ref: dev
+          fetch-depth: 0
+          submodules: 'recursive'
+          ref: ${{ github.event.inputs.sha || github.ref }}
 
       - name: Set up Python
         uses: actions/setup-python@v4
@@ -29,7 +38,69 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Run smoke tests
+        id: smoke_tests
         run: |
           make smoketest
         env:
           LINODE_CLI_TOKEN: ${{ secrets.LINODE_TOKEN }}
+
+      - name: Notify Slack
+        if: always() && github.repository == 'linode/linode-cli' # Run even if integration tests fail and only on main repository
+        uses: slackapi/slack-github-action@v1.26.0
+        with:
+          channel-id: ${{ secrets.SLACK_CHANNEL_ID }}
+          payload: |
+            {
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": ":rocket: *${{ github.workflow }} Completed in: ${{ github.repository }}* :white_check_mark:"
+                  }
+                },
+                {
+                  "type": "divider"
+                },
+                {
+                  "type": "section",
+                  "fields": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Build Result:*\n${{ steps.smoke_tests.outcome == 'success' && ':large_green_circle: Build Passed' || ':red_circle: Build Failed' }}"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Branch:*\n`${{ github.ref_name }}`"
+                    }
+                  ]
+                },
+                {
+                  "type": "section",
+                  "fields": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Commit Hash:*\n<${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}>"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Run URL:*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run Details>"
+                    }
+                  ]
+                },
+                {
+                  "type": "divider"
+                },
+                {
+                  "type": "context",
+                  "elements": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "Triggered by: :bust_in_silhouette: `${{ github.actor }}`"
+                    }
+                  ]
+                }
+              ]
+            }
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
diff --git a/.pylintrc b/.pylintrc
@@ -424,7 +424,8 @@ disable=raw-checker-failed,
         duplicate-code,
         too-few-public-methods,
         too-many-instance-attributes,
-        use-symbolic-message-instead
+        use-symbolic-message-instead,
+        too-many-positional-arguments
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
diff --git a/e2e_scripts b/e2e_scripts
@@ -1 +1 @@
-Subproject commit b56178520fae446a0a4f38df6259deb845efa667
+Subproject commit 6b71cb72eb20a18ace82f9e73a0f99fe1141d625
diff --git a/tests/integration/domains/test_slave_domains.py b/tests/integration/domains/test_slave_domains.py
@@ -102,7 +102,4 @@ def test_update_slave_domain(slave_domain_setup):
         ]
     )
 
-    assert (
-        result.returncode == SUCCESS_STATUS_CODE,
-        "Failed to update slave domain",
-    )
+    assert result.returncode == SUCCESS_STATUS_CODE
diff --git a/tests/integration/linodes/test_types.py b/tests/integration/linodes/test_types.py
@@ -2,6 +2,8 @@
 import subprocess
 from typing import List
 
+import pytest
+
 env = os.environ.copy()
 env["COLUMNS"] = "200"
 
@@ -16,6 +18,7 @@ def exec_test_command(args: List[str]):
 
 
 # verifying the DC pricing changes along with types
+@pytest.mark.smoke
 def test_linode_type():
     process = exec_test_command(["linode-cli", "linodes", "types"])
     output = process.stdout.decode()
diff --git a/tests/integration/longview/test_longview.py b/tests/integration/longview/test_longview.py
@@ -11,18 +11,23 @@
 BASE_CMD = ["linode-cli", "longview"]
 
 
+@pytest.mark.smoke
 def test_create_longview_client():
     new_label = str(time.time_ns()) + "label"
-    exec_test_command(
+    result = exec_test_command(
         BASE_CMD
         + [
             "create",
             "--label",
             new_label,
             "--text",
             "--no-headers",
+            "--delimiter",
+            ",",
         ]
-    )
+    ).stdout.decode()
+
+    assert new_label in result
 
 
 def test_longview_client_list():
diff --git a/tests/integration/regions/test_plugin_region_table.py b/tests/integration/regions/test_plugin_region_table.py
@@ -44,6 +44,7 @@ def test_regions_list():
     assert_headers_in_lines(headers, lines)
 
 
+@pytest.mark.smoke
 def test_regions_list_avail():
     res = (
         exec_test_command(BASE_CMD + ["list-avail", "--text", "--delimiter=,"])
@@ -78,6 +79,7 @@ def get_region_id():
     yield first_id
 
 
+@pytest.mark.smoke
 def test_regions_view(get_region_id):
     region_id = get_region_id
     res = (
diff --git a/tests/integration/vpc/test_vpc.py b/tests/integration/vpc/test_vpc.py
@@ -1,6 +1,8 @@
 import re
 import time
 
+import pytest
+
 from linodecli.exit_codes import ExitCodes
 from tests.integration.conftest import get_regions_with_capabilities
 from tests.integration.helpers import (
@@ -35,6 +37,7 @@ def test_view_vpc(test_vpc_wo_subnet):
     assert vpc_id in res
 
 
+@pytest.mark.smoke
 def test_update_vpc(test_vpc_wo_subnet):
     vpc_id = test_vpc_wo_subnet
 
@@ -120,6 +123,7 @@ def test_view_subnet(test_vpc_wo_subnet, test_subnet):
     assert vpc_subnet_id in output
 
 
+@pytest.mark.smoke
 def test_update_subnet(test_vpc_w_subnet):
     vpc_id = test_vpc_w_subnet
 

Original file line number	Diff line number	Diff line change
`@@ -102,7 +102,4 @@ def test_update_slave_domain(slave_domain_setup):`
`102`	`102`	`]`
`103`	`103`	`)`
`104`	`104`
`105`		`- assert (`
`106`		`- result.returncode == SUCCESS_STATUS_CODE,`
`107`		`- "Failed to update slave domain",`
`108`		`- )`
	`105`	`+ assert result.returncode == SUCCESS_STATUS_CODE`