fix: This commit fixes the error where users request a crawl that doesn't exist and cc-downloader downloaded the body of the response

Now this action will produce an error

Author: pjox

Cargo.toml

@@ -1,33 +1,33 @@
 [package]
 name = "cc-downloader"
 version = "0.5.2"
-edition = "2021"
+edition = "2024"
 authors = ["Pedro Ortiz Suarez <pedro@commoncrawl.org>"]
 description = "A polite and user-friendly downloader for Common Crawl data."
 license = "MIT OR Apache-2.0"
-rust-version = "1.83"
+rust-version = "1.85"
 readme = "README.md"
 homepage = "https://commoncrawl.org"
 repository = "https://github.com/commoncrawl/cc-downloader"
 documentation = "https://docs.rs/cc-downloader"
 
 [dependencies]
-clap = { version = "4.5.29", features = ["derive"] }
-flate2 = "1.0.35"
+clap = { version = "4.5.31", features = ["derive"] }
+flate2 = "1.1.0"
 futures = "0.3.31"
 indicatif = "0.17.11"
 reqwest = { version = "0.12.12", default-features = false, features = [
     "stream",
     "rustls-tls",
 ] }
-reqwest-middleware = "0.4.0"
+reqwest-middleware = "0.4.1"
 reqwest-retry = "0.7.0"
 tokio = { version = "1.43.0", features = ["full"] }
 tokio-util = { version = "0.7.13", features = ["compat"] }
 url = "2.5.4"
 
 [dev-dependencies]
-serde = { version = "1.0.217", features = ["derive"] }
+serde = { version = "1.0.218", features = ["derive"] }
 reqwest = { version = "0.12.12", default-features = false, features = [
     "stream",
     "rustls-tls",

src/download.rs

@@ -89,6 +89,23 @@ pub async fn download_paths(options: DownloadOptions<'_>) -> Result<(), Download
         .and_then(|segments| segments.last()) // Retrieves the last segment
         .unwrap_or("file.download"); // Fallback to generic filename
 
+    let resp = client.head(url.as_str()).send().await?;
+    match resp.status() {
+        status if status.is_success() => (),
+        status if status.is_client_error() => {
+            return Err(format!(
+                "\n\nThe reference combination you requested:\n\tCRAWL: {}\n\tSUBSET:{}\n\tULR: {}\n\nDoesn't seem to exist or it is not accessible.\n\tError Code: {:?}",
+                options.snapshot, options.data_type, url, status
+            )
+            .into());
+        }
+        _ => {
+            return Err(
+                format!("Couldn't download URL: {}. Error: {:?}", url, resp.status()).into(),
+            )
+        }
+    }
+
     let request = client.get(url.as_str());
 
     let mut dst = options.dst.to_path_buf();
@@ -134,7 +151,7 @@ async fn download_task(
         } else {
             // We return an Error if something goes wrong here
             return Err(
-                format!("Couldn't download URL: {}. Error: {:?}", url, resp.status(),).into(),
+                format!("Couldn't download URL: {}. Error: {:?}", url, resp.status()).into(),
             );
         }
     };