add support for delete tag and retry decorator for requests (#73)

* add support for delete tag and retry decorator for requests

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

Author: vsoch

.github/workflows/auth-tests.yaml

@@ -23,6 +23,7 @@ jobs:
           registry_port: 5000
           with_auth: true
           REGISTRY_AUTH: "{htpasswd: {realm: localhost, path: /etc/docker/registry/auth.htpasswd}}"
+          REGISTRY_STORAGE_DELETE_ENABLED: "true"
         run: |
            htpasswd -cB -b auth.htpasswd myuser mypass
            cp auth.htpasswd /etc/docker/registry/auth.htpasswd

.github/workflows/main.yaml

@@ -36,6 +36,7 @@ jobs:
       env:
         registry_host: localhost
         registry_port: ${{ job.services.registry.ports[5000] }}
+        REGISTRY_STORAGE_DELETE_ENABLED: "true"
       run: |
         export PATH="/usr/share/miniconda/bin:$PATH"
         source activate oras

CHANGELOG.md

@@ -14,6 +14,7 @@ and **Merged pull requests**. Critical items to know are:
 The versions coincide with releases on pip. Only major versions will be released as tags on Github.
 
 ## [0.0.x](https://github.com/oras-project/oras-py/tree/main) (0.0.x)
+ - add support for tag deletion and retry decorators (0.1.16)
  - bugfix that pagination sets upper limit of 10K (0.1.15)
  - pagination for tags (and general function for pagination) (0.1.14)
  - expose upload_blob function to be consistent (0.1.13)

oras/client.py

@@ -105,6 +105,25 @@ def get_tags(self, name: str, N=None) -> List[str]:
         """
         return self.remote.get_tags(name, N=N)
 
+    def delete_tags(self, name: str, tags=Union[str, list]) -> List[str]:
+        """
+        Delete one or more tags for a unique resource identifier.
+
+        Returns those successfully deleted.
+
+        :param name: container URI to parse
+        :type name: str
+        :param tags: single or multiple tags name to delete
+        :type N: string or list
+        """
+        if isinstance(tags, str):
+            tags = [tags]
+        deleted = []
+        for tag in tags:
+            if self.remote.delete_tag(name, tag):
+                deleted.append(tag)
+        return deleted
+
     def push(self, *args, **kwargs):
         """
         Push a container to the remote.

oras/container.py

@@ -59,11 +59,17 @@ def tags_url(self, N=None) -> str:
             return f"{self.registry}/v2/{self.api_prefix}/tags/list"
         return f"{self.registry}/v2/{self.api_prefix}/tags/list?n={N}"
 
-    def put_manifest_url(self) -> str:
-        return f"{self.registry}/v2/{self.api_prefix}/manifests/{self.tag}"
+    def manifest_url(self, tag: Optional[str] = None) -> str:
+        """
+        Get the manifest url for a specific tag, or the one for this container.
+
+        The tag provided can also correspond to a digest.
 
-    def get_manifest_url(self) -> str:
-        return f"{self.registry}/v2/{self.api_prefix}/manifests/{self.tag}"
+        :param tag: an optional tag to provide (if not provided defaults to container)
+        :type tag: None or str
+        """
+        tag = tag or self.tag
+        return f"{self.registry}/v2/{self.api_prefix}/manifests/{tag}"
 
     def __str__(self) -> str:
         return self.uri

oras/decorator.py

@@ -2,12 +2,15 @@
 __copyright__ = "Copyright The ORAS Authors."
 __license__ = "Apache-2.0"
 
+import time
 from functools import partial, update_wrapper
 
+from oras.logger import logger
 
-class ensure_container:
+
+class Decorator:
     """
-    Ensure the first argument is a container, and not a string.
+    Shared parent decorator class
     """
 
     def __init__(self, func):
@@ -17,10 +20,64 @@ def __init__(self, func):
     def __get__(self, obj, objtype):
         return partial(self.__call__, obj)
 
+
+class ensure_container(Decorator):
+    """
+    Ensure the first argument is a container, and not a string.
+    """
+
     def __call__(self, cls, *args, **kwargs):
         if "container" in kwargs:
             kwargs["container"] = cls.get_container(kwargs["container"])
         elif args:
             container = cls.get_container(args[0])
             args = (container, *args[1:])
         return self.func(cls, *args, **kwargs)
+
+
+class classretry(Decorator):
+    """
+    Retry a function that is part of a class
+    """
+
+    def __init__(self, func, attempts=5, timeout=2):
+        super().__init__(func)
+        self.attempts = attempts
+        self.timeout = timeout
+
+    def __call__(self, cls, *args, **kwargs):
+        attempt = 0
+        attempts = self.attempts
+        timeout = self.timeout
+        while attempt < attempts:
+            try:
+                return self.func(cls, *args, **kwargs)
+            except Exception as e:
+                sleep = timeout + 3**attempt
+                logger.info(f"Retrying in {sleep} seconds - error: {e}")
+                time.sleep(sleep)
+                attempt += 1
+        return self.func(cls, *args, **kwargs)
+
+
+def retry(attempts, timeout=2):
+    """
+    A simple retry decorator
+    """
+
+    def decorator(func):
+        def inner(*args, **kwargs):
+            attempt = 0
+            while attempt < attempts:
+                try:
+                    return func(*args, **kwargs)
+                except Exception as e:
+                    sleep = timeout + 3**attempt
+                    logger.info(f"Retrying in {sleep} seconds - error: {e}")
+                    time.sleep(sleep)
+                    attempt += 1
+            return func(*args, **kwargs)
+
+        return inner
+
+    return decorator

oras/provider.py

@@ -12,12 +12,15 @@
 
 import oras.auth
 import oras.container
+import oras.decorator as decorator
 import oras.oci
 import oras.schemas
 import oras.utils
-from oras.decorator import ensure_container
 from oras.logger import logger
 
+# container type can be string or container
+container_type = Union[str, oras.container.Container]
+
 
 class Registry:
     """
@@ -66,12 +69,8 @@ def logout(self, hostname: str):
                 return
         logger.info(f"You are not logged in to {hostname}")
 
-    @ensure_container
-    def load_configs(
-        self,
-        container: Union[str, oras.container.Container],
-        configs: Optional[list] = None,
-    ):
+    @decorator.ensure_container
+    def load_configs(self, container: container_type, configs: Optional[list] = None):
         """
         Load configs to discover credentials for a specific container.
 
@@ -185,7 +184,7 @@ def _parse_manifest_ref(self, ref: str) -> Union[Tuple[str, str], List[str]]:
     def upload_blob(
         self,
         blob: str,
-        container: Union[str, oras.container.Container],
+        container: container_type,
         layer: dict,
         do_chunked: bool = False,
     ) -> requests.Response:
@@ -225,10 +224,42 @@ def upload_blob(
             response.status_code = 200
         return response
 
-    @ensure_container
-    def get_tags(
-        self, container: Union[str, oras.container.Container], N=None
-    ) -> List[str]:
+    @decorator.ensure_container
+    def delete_tag(self, container: container_type, tag: str) -> bool:
+        """
+        Delete a tag for a container.
+
+        :param container:  parsed container URI
+        :type container: oras.container.Container or str
+        :param tag: name of tag to delete
+        :type tag: str
+        """
+        logger.debug(f"Deleting tag {tag} for {container}")
+
+        head_url = f"{self.prefix}://{container.manifest_url(tag)}"  # type: ignore
+
+        # get digest of manifest to delete
+        response = self.do_request(
+            head_url,
+            "HEAD",
+            headers={"Accept": "application/vnd.oci.image.manifest.v1+json"},
+        )
+        if response.status_code == 404:
+            logger.error(f"Cannot find tag {container}:{tag}")
+            return False
+
+        digest = response.headers.get("Docker-Content-Digest")
+        if not digest:
+            raise RuntimeError("Expected to find Docker-Content-Digest header.")
+
+        delete_url = f"{self.prefix}://{container.manifest_url(digest)}"  # type: ignore
+        response = self.do_request(delete_url, "DELETE")
+        if response.status_code != 202:
+            raise RuntimeError("Delete was not successful: {response.json()}")
+        return True
+
+    @decorator.ensure_container
+    def get_tags(self, container: container_type, N=None) -> List[str]:
         """
         Retrieve tags for a package.
 
@@ -291,10 +322,10 @@ def _do_paginated_request(
             # use link + base url to continue with next page
             url = f"{base_url}{link}"
 
-    @ensure_container
+    @decorator.ensure_container
     def get_blob(
         self,
-        container: Union[str, oras.container.Container],
+        container: container_type,
         digest: str,
         stream: bool = False,
         head: bool = False,
@@ -315,9 +346,7 @@ def get_blob(
         blob_url = f"{self.prefix}://{container.get_blob_url(digest)}"  # type: ignore
         return self.do_request(blob_url, method, headers=self.headers, stream=stream)
 
-    def get_container(
-        self, name: Union[str, oras.container.Container]
-    ) -> oras.container.Container:
+    def get_container(self, name: container_type) -> oras.container.Container:
         """
         Courtesy function to get a container from a URI.
 
@@ -329,9 +358,9 @@ def get_container(
         return oras.container.Container(name, registry=self.hostname)
 
     # Functions to be deprecated in favor of exposed ones
-    @ensure_container
+    @decorator.ensure_container
     def _download_blob(
-        self, container: Union[str, oras.container.Container], digest: str, outfile: str
+        self, container: container_type, digest: str, outfile: str
     ) -> str:
         logger.warning(
             "This function is deprecated in favor of download_blob and will be removed by 0.1.2"
@@ -365,7 +394,7 @@ def _upload_manifest(
     def _upload_blob(
         self,
         blob: str,
-        container: Union[str, oras.container.Container],
+        container: container_type,
         layer: dict,
         do_chunked: bool = False,
     ) -> requests.Response:
@@ -374,9 +403,9 @@ def _upload_blob(
         )
         return self.upload_blob(blob, container, layer, do_chunked)
 
-    @ensure_container
+    @decorator.ensure_container
     def download_blob(
-        self, container: Union[str, oras.container.Container], digest: str, outfile: str
+        self, container: container_type, digest: str, outfile: str
     ) -> str:
         """
         Stream download a blob into an output file.
@@ -563,8 +592,12 @@ def upload_manifest(
             "Content-Type": oras.defaults.default_manifest_media_type,
             "Content-Length": str(len(manifest)),
         }
-        put_url = f"{self.prefix}://{container.put_manifest_url()}"
-        return self.do_request(put_url, "PUT", headers=headers, json=manifest)
+        return self.do_request(
+            f"{self.prefix}://{container.manifest_url()}",  # noqa
+            "PUT",
+            headers=headers,
+            json=manifest,
+        )
 
     def push(self, *args, **kwargs) -> requests.Response:
         """
@@ -740,11 +773,9 @@ def pull(self, *args, **kwargs) -> List[str]:
             files.append(outfile)
         return files
 
-    @ensure_container
+    @decorator.ensure_container
     def get_manifest(
-        self,
-        container: Union[str, oras.container.Container],
-        allowed_media_type: list = None,
+        self, container: container_type, allowed_media_type: list = None
     ) -> dict:
         """
         Retrieve a manifest for a package.
@@ -757,13 +788,15 @@ def get_manifest(
         if not allowed_media_type:
             allowed_media_type = [oras.defaults.default_manifest_media_type]
         headers = {"Accept": ";".join(allowed_media_type)}
-        url = f"{self.prefix}://{container.get_manifest_url()}"  # type: ignore
-        response = self.do_request(url, "GET", headers=headers)
+
+        get_manifest = f"{self.prefix}://{container.manifest_url()}"  # type: ignore
+        response = self.do_request(get_manifest, "GET", headers=headers)
         self._check_200_response(response)
         manifest = response.json()
         jsonschema.validate(manifest, schema=oras.schemas.manifest)
         return manifest
 
+    @decorator.classretry
     def do_request(
         self,
         url: str,

oras/tests/run_registry.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+# A helper script to easily run a development, local registry
+docker run -it -e REGISTRY_STORAGE_DELETE_ENABLED=true --rm -p 5000:5000 ghcr.io/oras-project/registry:latest

oras/tests/test_oras.py

@@ -70,10 +70,6 @@ def test_basic_push_pull(tmp_path):
     res = client.push(files=[artifact], target=target)
     assert res.status_code in [200, 201]
 
-    # Test getting tags
-    tags = client.get_tags(target)
-    assert "v1" in tags
-
     # Test pulling elsewhere
     files = client.pull(target=target, outdir=tmp_path)
     assert len(files) == 1
@@ -92,6 +88,29 @@ def test_basic_push_pull(tmp_path):
     assert res.status_code == 201
 
 
+@pytest.mark.skipif(with_auth, reason="token auth is needed for push and pull")
+def test_get_delete_tags(tmp_path):
+    """
+    Test creationg, getting, and deleting tags.
+    """
+    client = oras.client.OrasClient(hostname=registry, insecure=True)
+    artifact = os.path.join(here, "artifact.txt")
+    assert os.path.exists(artifact)
+
+    res = client.push(files=[artifact], target=target)
+    assert res.status_code in [200, 201]
+
+    # Test getting tags
+    tags = client.get_tags(target)
+    assert "v1" in tags
+
+    # Test deleting not-existence tag
+    assert not client.delete_tags(target, "v1-boop-boop")
+    assert "v1" in client.delete_tags(target, "v1")
+    tags = client.get_tags(target)
+    assert not tags
+
+
 def test_get_many_tags():
     """
     Test getting many tags