--zarrman-cache-bytes

Author: jwodder

CHANGELOG.md

@@ -4,6 +4,12 @@ In Development
 - Added doc comments to much of the code
 - Return 502 status when a backend returns an invalid response
 - Require `--api-url` (and other URLs retrieved from APIs) to be HTTP(S)
+- Format all log lines as JSON
+- Add logging of Zarr manifest cache events
+- Limit Zarr manifest cache by total size of entries
+    - Add a `-Z`/`--zarrman-cache-bytes` option for setting the cache size
+- Expire idle Zarr manifest cache entries
+- Log Zarr manifest cache entries every hour
 
 v0.4.0 (2024-07-09)
 -------------------

README.md

@@ -119,3 +119,7 @@ Options
 - `-T <TITLE>`, `--title <TITLE>` — Specify the site name to use in HTML/web
   views of collections (used inside `<title>`'s and as the root breadcrumb
   text) [default: dandidav]
+
+- `-Z <INT>`, `--zarrman-cache-bytes <INT>` — Specify the maximum number of
+  bytes of parsed Zarr manifest files to store in the Zarr manifest cache at
+  once [default: 100 MiB]

src/consts.rs

@@ -61,6 +61,10 @@ pub(crate) static FAST_NOT_EXIST: &[&str] = &[".bzr", ".git", ".nols", ".svn"];
 /// Interval between periodic logging of the Zarr manifest cache's contents
 pub(crate) const ZARR_MANIFEST_CACHE_DUMP_PERIOD: Duration = Duration::from_secs(3600);
 
+/// Default size of the Zarr manifest cache; the cache is limited to storing no
+/// more than this many bytes of parsed manifests at once
+pub(crate) const ZARR_MANIFEST_CACHE_TOTAL_BYTES: u64 = 100 * 1024 * 1024; // 100 MiB
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/main.rs

@@ -9,9 +9,7 @@ mod paths;
 mod s3;
 mod streamutil;
 mod zarrman;
-use crate::consts::{
-    CSS_CONTENT_TYPE, DEFAULT_API_URL, SERVER_VALUE, ZARR_MANIFEST_CACHE_DUMP_PERIOD,
-};
+use crate::consts::*;
 use crate::dandi::DandiClient;
 use crate::dav::{DandiDav, Templater};
 use crate::httputil::HttpUrl;
@@ -67,6 +65,11 @@ struct Arguments {
     /// Site name to use in HTML collection pages
     #[arg(short = 'T', long, default_value = env!("CARGO_PKG_NAME"))]
     title: String,
+
+    /// Limit the Zarr manifest cache to storing no more than this many bytes
+    /// of parsed manifests at once
+    #[arg(short = 'Z', long, default_value_t = ZARR_MANIFEST_CACHE_TOTAL_BYTES, value_name = "INT")]
+    zarrman_cache_bytes: u64,
 }
 
 // See
@@ -99,7 +102,7 @@ fn main() -> anyhow::Result<()> {
 async fn run() -> anyhow::Result<()> {
     let args = Arguments::parse();
     let dandi = DandiClient::new(args.api_url)?;
-    let zarrfetcher = ManifestFetcher::new()?;
+    let zarrfetcher = ManifestFetcher::new(args.zarrman_cache_bytes)?;
     zarrfetcher.install_periodic_dump(ZARR_MANIFEST_CACHE_DUMP_PERIOD);
     let zarrman = ZarrManClient::new(zarrfetcher);
     let templater = Templater::new(args.title)?;

src/zarrman/consts.rs

@@ -19,9 +19,5 @@ pub(super) static MANIFEST_ROOT_URL: &str =
 /// `{ENTRY_DOWNLOAD_PREFIX}/{zarr_id}/{entry_path}`.
 pub(super) static ENTRY_DOWNLOAD_PREFIX: &str = "https://dandiarchive.s3.amazonaws.com/zarr/";
 
-/// Limit the manifest cache to storing no more than this many bytes of parsed
-/// manifests at once
-pub(super) const MANIFEST_CACHE_TOTAL_BYTES: u64 = 100 * 1024 * 1024; // 100 MiB
-
 /// Expire any manifest cache entries that haven't been accessed for this long
 pub(super) const MANIFEST_CACHE_IDLE_EXPIRY: Duration = Duration::from_secs(300);

src/zarrman/fetcher.rs

@@ -1,4 +1,4 @@
-use super::consts::{MANIFEST_CACHE_IDLE_EXPIRY, MANIFEST_CACHE_TOTAL_BYTES, MANIFEST_ROOT_URL};
+use super::consts::{MANIFEST_CACHE_IDLE_EXPIRY, MANIFEST_ROOT_URL};
 use super::manifest::Manifest;
 use super::resources::ManifestPath;
 use super::util::{Index, ZarrManError};
@@ -33,26 +33,25 @@ impl ManifestFetcher {
     /// # Errors
     ///
     /// Returns an error if construction of the inner `reqwest::Client` fails
-    pub(crate) fn new() -> Result<Self, BuildClientError> {
+    pub(crate) fn new(cache_size: u64) -> Result<Self, BuildClientError> {
         let inner = Client::new()?;
-        let cache: Cache<ManifestPath, Arc<Manifest>> =
-            CacheBuilder::new(MANIFEST_CACHE_TOTAL_BYTES)
-                .name("zarr-manifests")
-                .weigher(|_, manifest: &Arc<Manifest>| {
-                    u32::try_from(manifest.get_size()).unwrap_or(u32::MAX)
-                })
-                .time_to_idle(MANIFEST_CACHE_IDLE_EXPIRY)
-                .eviction_listener(|path, manifest, cause| {
-                    tracing::debug!(
-                        cache_event = "evict",
-                        cache = "zarr-manifests",
-                        manifest = %path,
-                        manifest_size = manifest.get_size(),
-                        ?cause,
-                        "Zarr manifest evicted from cache",
-                    );
-                })
-                .build();
+        let cache: Cache<ManifestPath, Arc<Manifest>> = CacheBuilder::new(cache_size)
+            .name("zarr-manifests")
+            .weigher(|_, manifest: &Arc<Manifest>| {
+                u32::try_from(manifest.get_size()).unwrap_or(u32::MAX)
+            })
+            .time_to_idle(MANIFEST_CACHE_IDLE_EXPIRY)
+            .eviction_listener(|path, manifest, cause| {
+                tracing::debug!(
+                    cache_event = "evict",
+                    cache = "zarr-manifests",
+                    manifest = %path,
+                    manifest_size = manifest.get_size(),
+                    ?cause,
+                    "Zarr manifest evicted from cache",
+                );
+            })
+            .build();
         let manifest_root_url = MANIFEST_ROOT_URL
             .parse::<HttpUrl>()
             .expect("MANIFEST_ROOT_URL should be a valid HTTP URL");