Merge pull request #557 from linode/dev

Release v5.46.0

Author: ykim-akamai

.github/workflows/e2e-suite.yml

@@ -47,16 +47,11 @@ jobs:
         env:
           LINODE_CLI_TOKEN: ${{ secrets.LINODE_TOKEN }}
 
-      - name: Set release version env
-        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-
       - name: Add additional information to XML report
         run: |
-          echo $RELEASE_VERSION
-          echo ${{ env.RELEASE_VERSION }}
           filename=$(ls | grep -E '^[0-9]{12}_cli_test_report\.xml$') 
           python scripts/add_to_xml_test_report.py \
-          --branch_name "${{ env.RELEASE_VERSION }}" \
+          --branch_name "${GITHUB_REF#refs/*/}" \
           --gha_run_id "$GITHUB_RUN_ID" \
           --gha_run_number "$GITHUB_RUN_NUMBER" \
           --xmlfile "${filename}"

.github/workflows/oci-build.yml

@@ -16,6 +16,9 @@ jobs:
         with:
           python-version: '3.x'
 
+      - name: Install deps
+        run: make requirements
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # pin@v2.2.0
 

linodecli/plugins/obj/__init__.py

@@ -78,6 +78,18 @@
 INVALID_PAGE_MSG = "No result to show in this page."
 
 
+def get_available_cluster(cli: CLI):
+    """Get list of possible clusters for the account"""
+    return [
+        c["id"]
+        for c in _do_get_request(  # pylint: disable=protected-access
+            cli.config.base_url,
+            "/object-storage/clusters",
+            token=cli.config.get_token(),
+        )["data"]
+    ]
+
+
 def flip_to_page(iterable: Iterable, page: int = 1):
     """Given a iterable object and return a specific iteration (page)"""
     iterable = iter(iterable)
@@ -451,7 +463,7 @@ def print_help(parser: ArgumentParser):
     print("See --help for individual commands for more information")
 
 
-def get_obj_args_parser():
+def get_obj_args_parser(clusters: List[str]):
     """
     Initialize and return the argument parser for the obj plug-in.
     """
@@ -468,6 +480,7 @@ def get_obj_args_parser():
         "--cluster",
         metavar="CLUSTER",
         type=str,
+        choices=clusters,
         help="The cluster to use for the operation",
     )
 
@@ -527,7 +540,8 @@ def call(
 
         sys.exit(2)  # requirements not met - we can't go on
 
-    parser = get_obj_args_parser()
+    clusters = get_available_cluster(context.client)
+    parser = get_obj_args_parser(clusters)
     parsed, args = parser.parse_known_args(args)
 
     # don't mind --no-defaults if it's there; the top-level parser already took care of it
@@ -710,14 +724,7 @@ def _configure_plugin(client: CLI):
     """
     Configures a default cluster value.
     """
-    clusters = [
-        c["id"]
-        for c in _do_get_request(  # pylint: disable=protected-access
-            client.config.base_url,
-            "/object-storage/clusters",
-            token=client.config.get_value("token"),
-        )["data"]
-    ]
+    clusters = get_available_cluster(client)
 
     cluster = _default_thing_input(  # pylint: disable=protected-access
         "Configure a default Cluster for operations.",

linodecli/plugins/obj/config.py

@@ -1,6 +1,7 @@
 """
 The config of the object storage plugin.
 """
+import shutil
 
 ENV_ACCESS_KEY_NAME = "LINODE_CLI_OBJ_ACCESS_KEY"
 ENV_SECRET_KEY_NAME = "LINODE_CLI_OBJ_SECRET_KEY"
@@ -15,7 +16,8 @@
 # for help commands
 PLUGIN_BASE = "linode-cli obj"
 
-PROGRESS_BAR_WIDTH = 100
+columns = shutil.get_terminal_size(fallback=(80, 24)).columns
+PROGRESS_BAR_WIDTH = columns - 20 if columns > 30 else columns
 
 # constant error messages
 NO_SCOPES_ERROR = """Your OAuth token isn't authorized to create Object Storage keys.

linodecli/plugins/obj/helpers.py

@@ -26,8 +26,9 @@ def __call__(self, bytes_amount: int):
         if not self.size:
             return
         self.uploaded += bytes_amount
-        percentage = self.bar_width * (self.uploaded / self.size)
-        progress = int(percentage)
+        percentage = 100 * (self.uploaded / self.size)
+        uploaded = self.bar_width * (self.uploaded / self.size)
+        progress = int(uploaded)
         progress_bar = ("#" * progress) + ("-" * (self.bar_width - progress))
         print(f"\r |{progress_bar}| {percentage:.1f}%", end="\r")
         if self.uploaded == self.size:

linodecli/plugins/obj/objects.py

@@ -85,8 +85,14 @@ def upload_object(
 
     chunk_size = 1024 * 1024 * parsed.chunk_size
 
+    prefix = None
+    bucket = parsed.bucket
+    if "/" in parsed.bucket:
+        bucket = parsed.bucket.split("/")[0]
+        prefix = parsed.bucket.lstrip(f"{bucket}/")
+
     upload_options = {
-        "Bucket": parsed.bucket,
+        "Bucket": bucket,
         "Config": TransferConfig(multipart_chunksize=chunk_size * MB),
     }
 
@@ -95,8 +101,10 @@ def upload_object(
 
     for file_path in to_upload:
         print(f"Uploading {file_path.name}:")
+        upload_options["Key"] = (
+            file_path.name if not prefix else f"{prefix}/{file_path.name}"
+        )
         upload_options["Filename"] = str(file_path.resolve())
-        upload_options["Key"] = file_path.name
         upload_options["Callback"] = ProgressPercentage(
             file_path.stat().st_size, PROGRESS_BAR_WIDTH
         )

scripts/add_to_xml_test_report.py

@@ -1,11 +1,38 @@
 import argparse
 import xml.etree.ElementTree as ET
+import requests
+
+latest_release_url = "https://api.github.com/repos/linode/linode-cli/releases/latest"
+
+
+def get_release_version():
+    url = latest_release_url
+
+    try:
+        response = requests.get(url)
+        response.raise_for_status()  # Check for HTTP errors
+
+        release_info = response.json()
+        version = release_info["tag_name"]
+
+        # Remove 'v' prefix if it exists
+        if version.startswith("v"):
+            version = version[1:]
+
+        return str(version)
+
+    except requests.exceptions.RequestException as e:
+        print("Error:", e)
+    except KeyError:
+        print("Error: Unable to fetch release information from GitHub API.")
+
 
 # Parse command-line arguments
 parser = argparse.ArgumentParser(description='Modify XML with workflow information')
 parser.add_argument('--branch_name', required=True)
 parser.add_argument('--gha_run_id', required=True)
 parser.add_argument('--gha_run_number', required=True)
+parser.add_argument('--release_tag', required=False)
 parser.add_argument('--xmlfile', required=True)  # Added argument for XML file path
 
 args = parser.parse_args()
@@ -25,10 +52,14 @@
 gha_run_number_element = ET.Element('gha_run_number')
 gha_run_number_element.text = args.gha_run_number
 
+gha_release_tag_element = ET.Element('release_tag')
+gha_release_tag_element.text = get_release_version()
+
 # Add the new elements to the root of the XML
 root.append(branch_name_element)
 root.append(gha_run_id_element)
 root.append(gha_run_number_element)
+root.append(gha_release_tag_element)
 
 # Save the modified XML
 modified_xml_file_path = xml_file_path  # Overwrite it

tests/integration/conftest.py

@@ -374,3 +374,31 @@ def nodebalancer_with_default_conf():
     res_arr = result.split(",")
     nodebalancer_id = res_arr[0]
     delete_target_id(target="nodebalancers", id=nodebalancer_id)
+
+
+def get_regions_with_capabilities(capabilities):
+    regions = (
+        exec_test_command(
+            [
+                "linode-cli",
+                "regions",
+                "ls",
+                "--text",
+                "--no-headers",
+                "--format=id,capabilities",
+            ]
+        )
+        .stdout.decode()
+        .rstrip()
+    )
+
+    regions = regions.split("\n")
+
+    regions_with_all_caps = []
+
+    for region in regions:
+        region_name = region.split()[0]
+        if all(capability in region for capability in capabilities):
+            regions_with_all_caps.append(region_name)
+
+    return regions_with_all_caps

tests/integration/obj/test_obj_plugin.py

@@ -1,3 +1,4 @@
+import json
 import logging
 from dataclasses import dataclass
 from typing import Callable, Optional
@@ -6,16 +7,16 @@
 import requests
 from pytest import MonkeyPatch
 
-from linodecli.configuration.auth import _do_request
 from linodecli.plugins.obj import (
     ENV_ACCESS_KEY_NAME,
     ENV_SECRET_KEY_NAME,
     TRUNCATED_MSG,
 )
 from tests.integration.fixture_types import GetTestFilesType, GetTestFileType
-from tests.integration.helpers import BASE_URL, count_lines, exec_test_command
+from tests.integration.helpers import count_lines, exec_test_command
 
 REGION = "us-southeast-1"
+CLI_CMD = ["linode-cli", "object-storage"]
 BASE_CMD = ["linode-cli", "obj", "--cluster", REGION]
 
 
@@ -50,27 +51,24 @@ def static_site_error():
 
 
 @pytest.fixture(scope="session")
-def keys(token: str):
-    response = _do_request(
-        BASE_URL,
-        requests.post,
-        "object-storage/keys",
-        token,
-        False,
-        {"label": "cli-integration-test-obj-key"},
-    )
-
+def keys():
+    response = json.loads(
+        exec_test_command(
+            CLI_CMD
+            + [
+                "keys-create",
+                "--label",
+                "cli-integration-test-obj-key",
+                "--json",
+            ],
+        ).stdout.decode()
+    )[0]
     _keys = Keys(
         access_key=response.get("access_key"),
         secret_key=response.get("secret_key"),
     )
     yield _keys
-    _do_request(
-        BASE_URL,
-        requests.delete,
-        f"object-storage/keys/{response['id']}",
-        token,
-    )
+    exec_test_command(CLI_CMD + ["keys-delete", str(response.get("id"))])
 
 
 def patch_keys(keys: Keys, monkeypatch: MonkeyPatch):
@@ -150,6 +148,51 @@ def test_obj_single_file_single_bucket(
         assert f1.read() == f2.read()
 
 
+def test_obj_single_file_single_bucket_with_prefix(
+    create_bucket: Callable[[Optional[str]], str],
+    generate_test_files: GetTestFilesType,
+    keys: Keys,
+    monkeypatch: MonkeyPatch,
+):
+    patch_keys(keys, monkeypatch)
+    file_path = generate_test_files()[0]
+    bucket_name = create_bucket()
+    exec_test_command(
+        BASE_CMD + ["put", str(file_path), f"{bucket_name}/prefix"]
+    )
+    process = exec_test_command(BASE_CMD + ["la"])
+    output = process.stdout.decode()
+
+    assert f"{bucket_name}/prefix/{file_path.name}" in output
+
+    file_size = file_path.stat().st_size
+    assert str(file_size) in output
+
+    process = exec_test_command(BASE_CMD + ["ls"])
+    output = process.stdout.decode()
+    assert bucket_name in output
+    assert file_path.name not in output
+
+    process = exec_test_command(BASE_CMD + ["ls", bucket_name])
+    output = process.stdout.decode()
+    assert bucket_name not in output
+    assert "prefix" in output
+
+    downloaded_file_path = file_path.parent / f"downloaded_{file_path.name}"
+    process = exec_test_command(
+        BASE_CMD
+        + [
+            "get",
+            bucket_name,
+            "prefix/" + file_path.name,
+            str(downloaded_file_path),
+        ]
+    )
+    output = process.stdout.decode()
+    with open(downloaded_file_path) as f2, open(file_path) as f1:
+        assert f1.read() == f2.read()
+
+
 def test_multi_files_multi_bucket(
     create_bucket: Callable[[Optional[str]], str],
     generate_test_files: GetTestFilesType,