fix(search): bound response bodies and detect DuckDuckGo HTTP 202 challenges

h2m-search/Cargo.toml

@@ -11,18 +11,19 @@ categories = ["web-programming", "command-line-utilities"]
 
 [features]
 default = ["duckduckgo", "wikipedia"]
-duckduckgo = ["dep:scraper", "dep:percent-encoding"]
-wikipedia = ["dep:percent-encoding"]
-searxng = []
-brave = []
-tavily = []
+duckduckgo = ["dep:scraper"]
+wikipedia = ["dep:percent-encoding", "dep:serde_json"]
+searxng = ["dep:serde_json"]
+brave = ["dep:serde_json"]
+tavily = ["dep:serde_json"]
 all = ["duckduckgo", "wikipedia", "searxng", "brave", "tavily"]
 
 [dependencies]
 reqwest.workspace = true
 scraper = { workspace = true, optional = true }
 percent-encoding = { workspace = true, optional = true }
 serde.workspace = true
+serde_json = { workspace = true, optional = true }
 thiserror.workspace = true
 tokio = { workspace = true, features = ["time"] }
 tracing.workspace = true

h2m-search/src/error.rs

@@ -71,6 +71,15 @@ pub enum SearchError {
         message: String,
     },
 
+    /// Response exceeded the configured decoded body size limit. Not retryable.
+    #[error("{provider} response exceeds {limit_bytes} bytes")]
+    ResponseTooLarge {
+        /// Provider identifier.
+        provider: &'static str,
+        /// Maximum permitted response size in bytes.
+        limit_bytes: usize,
+    },
+
     /// HTML scraping detected an anti-bot challenge (e.g. `DuckDuckGo` CAPTCHA).
     ///
     /// Retryable: callers can back off and try again with rotated UA or a
@@ -117,6 +126,7 @@ impl SearchError {
             | Self::AuthFailed { provider, .. }
             | Self::MissingApiKey { provider, .. }
             | Self::InvalidResponse { provider, .. }
+            | Self::ResponseTooLarge { provider, .. }
             | Self::CaptchaDetected { provider }
             | Self::ParseFailed { provider, .. } => Some(provider),
             Self::MissingSearxngUrl | Self::ProviderUnavailable { .. } | Self::Config { .. } => {
@@ -146,6 +156,7 @@ impl SearchError {
             Self::MissingApiKey { .. } => "missingApiKey",
             Self::MissingSearxngUrl => "missingSearxngUrl",
             Self::InvalidResponse { .. } => "invalidResponse",
+            Self::ResponseTooLarge { .. } => "responseTooLarge",
             Self::CaptchaDetected { .. } => "captchaDetected",
             Self::ParseFailed { .. } => "parseFailed",
             Self::ProviderUnavailable { .. } => "providerUnavailable",
@@ -181,6 +192,9 @@ impl Serialize for SearchError {
         if let Some(s) = self.status() {
             map.serialize_entry("status", &s)?;
         }
+        if let Self::ResponseTooLarge { limit_bytes, .. } = self {
+            map.serialize_entry("limitBytes", limit_bytes)?;
+        }
         map.end()
     }
 }

h2m-search/src/http.rs

@@ -5,6 +5,8 @@
 //! their own client via [`HttpConfig::with_client`] when callers need
 //! custom middleware, proxies, or shared pools with the host application.
 
+mod body;
+
 use std::sync::Arc;
 use std::time::Duration;
 
@@ -13,6 +15,9 @@ use crate::error::SearchError;
 /// Default per-request timeout applied to every provider.
 pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30);
 
+/// Maximum decoded response size per request by default (2 MiB).
+pub const DEFAULT_MAX_RESPONSE_BYTES: usize = 2 * 1024 * 1024;
+
 /// User agent sent with every request.
 pub const USER_AGENT: &str = concat!("h2m-search/", env!("CARGO_PKG_VERSION"));
 
@@ -23,6 +28,7 @@ pub const USER_AGENT: &str = concat!("h2m-search/", env!("CARGO_PKG_VERSION"));
 #[derive(Debug, Clone)]
 pub struct HttpConfig {
     client: Arc<reqwest::Client>,
+    max_response_bytes: usize,
 }
 
 impl HttpConfig {
@@ -42,6 +48,7 @@ impl HttpConfig {
             })?;
         Ok(Self {
             client: Arc::new(client),
+            max_response_bytes: DEFAULT_MAX_RESPONSE_BYTES,
         })
     }
 
@@ -50,9 +57,20 @@ impl HttpConfig {
     pub fn with_client(client: reqwest::Client) -> Self {
         Self {
             client: Arc::new(client),
+            max_response_bytes: DEFAULT_MAX_RESPONSE_BYTES,
         }
     }
 
+    /// Sets the maximum decoded response size for each request.
+    ///
+    /// Enforced while reading, including chunked responses without a length header.
+    /// A zero limit permits only empty bodies. Oversized responses fail without retry.
+    #[must_use]
+    pub const fn with_max_response_bytes(mut self, limit: usize) -> Self {
+        self.max_response_bytes = limit;
+        self
+    }
+
     /// Shared client handle.
     #[must_use]
     pub fn client(&self) -> &reqwest::Client {
@@ -65,6 +83,7 @@ impl Default for HttpConfig {
     fn default() -> Self {
         Self::new().unwrap_or_else(|_| Self {
             client: Arc::new(reqwest::Client::new()),
+            max_response_bytes: DEFAULT_MAX_RESPONSE_BYTES,
         })
     }
 }

h2m-search/src/http/body.rs

@@ -0,0 +1,68 @@
+//! Bounded response collection shared by HTML and JSON search providers.
+
+use super::HttpConfig;
+use crate::error::SearchError;
+use crate::providers::common::classify_transport;
+
+impl HttpConfig {
+    async fn read_body(
+        &self,
+        provider: &'static str,
+        mut response: reqwest::Response,
+    ) -> Result<Vec<u8>, SearchError> {
+        let too_large = || SearchError::ResponseTooLarge {
+            provider,
+            limit_bytes: self.max_response_bytes,
+        };
+        if response
+            .content_length()
+            .is_some_and(|length| length > self.max_response_bytes as u64)
+        {
+            return Err(too_large());
+        }
+        let mut body = Vec::new();
+        while let Some(chunk) = response
+            .chunk()
+            .await
+            .map_err(|error| classify_transport(provider, &error))?
+        {
+            // Check decoded chunks before accumulation; length headers may be absent.
+            if chunk.len() > self.max_response_bytes.saturating_sub(body.len()) {
+                return Err(too_large());
+            }
+            body.extend_from_slice(&chunk);
+        }
+        Ok(body)
+    }
+
+    #[cfg(feature = "duckduckgo")]
+    pub(crate) async fn read_text(
+        &self,
+        provider: &'static str,
+        response: reqwest::Response,
+    ) -> Result<String, SearchError> {
+        let body = self.read_body(provider, response).await?;
+        Ok(String::from_utf8_lossy(&body).into_owned())
+    }
+
+    #[cfg(any(
+        feature = "wikipedia",
+        feature = "searxng",
+        feature = "brave",
+        feature = "tavily"
+    ))]
+    pub(crate) async fn read_json<T: serde::de::DeserializeOwned>(
+        &self,
+        provider: &'static str,
+        response: reqwest::Response,
+    ) -> Result<T, SearchError> {
+        let body = self.read_body(provider, response).await?;
+        serde_json::from_slice(&body).map_err(|error| SearchError::InvalidResponse {
+            provider,
+            message: error.to_string(),
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests;

h2m-search/src/http/body/tests.rs

@@ -0,0 +1,112 @@
+use super::*;
+use std::io::{Read, Write};
+use std::net::TcpListener;
+use std::thread;
+use std::time::Duration;
+use wiremock::{Mock, MockServer, ResponseTemplate, matchers::any};
+
+#[tokio::test]
+async fn known_length_accepts_boundary_and_rejects_overflow() {
+    let server = MockServer::start().await;
+    Mock::given(any())
+        .respond_with(ResponseTemplate::new(200).set_body_bytes(b"12345678"))
+        .mount(&server)
+        .await;
+    let http = HttpConfig::new().unwrap().with_max_response_bytes(8);
+    let response = http.client().get(server.uri()).send().await.unwrap();
+    assert_eq!(http.read_body("test", response).await.unwrap(), b"12345678");
+
+    let http = http.with_max_response_bytes(7);
+    let oversized_response = http.client().get(server.uri()).send().await.unwrap();
+    let error = http
+        .read_body("test", oversized_response)
+        .await
+        .unwrap_err();
+    assert!(matches!(
+        error,
+        SearchError::ResponseTooLarge { limit_bytes: 7, .. }
+    ));
+    assert!(!error.is_retryable());
+    assert_eq!(error.provider(), Some("test"));
+    let json = serde_json::to_value(error).unwrap();
+    assert_eq!(
+        json.get("kind").and_then(serde_json::Value::as_str),
+        Some("responseTooLarge")
+    );
+    assert_eq!(
+        json.get("limitBytes").and_then(serde_json::Value::as_u64),
+        Some(7)
+    );
+}
+
+fn serve_chunked(listener: &TcpListener) {
+    let (mut socket, _) = listener.accept().unwrap();
+    socket
+        .set_read_timeout(Some(Duration::from_secs(5)))
+        .unwrap();
+    socket
+        .set_write_timeout(Some(Duration::from_secs(5)))
+        .unwrap();
+    let mut request = Vec::new();
+    while !request.ends_with(b"\r\n\r\n") {
+        let mut byte = [0];
+        socket.read_exact(&mut byte).unwrap();
+        request.extend_from_slice(&byte);
+    }
+    socket.write_all(b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n4\r\n1234\r\n4\r\n5678\r\n0\r\n\r\n").unwrap();
+}
+
+#[tokio::test]
+async fn chunked_bodies_are_bounded_without_content_length() {
+    for limit in [7, 8] {
+        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+        let address = listener.local_addr().unwrap();
+        let server = thread::spawn(move || serve_chunked(&listener));
+        let http = HttpConfig::new().unwrap().with_max_response_bytes(limit);
+        let response = http
+            .client()
+            .get(format!("http://{address}"))
+            .send()
+            .await
+            .unwrap();
+        assert_eq!(response.content_length(), None);
+        let result = http.read_body("test", response).await;
+        server.join().unwrap();
+        if limit == 8 {
+            assert_eq!(result.unwrap(), b"12345678");
+        } else {
+            assert!(matches!(
+                result,
+                Err(SearchError::ResponseTooLarge { limit_bytes: 7, .. })
+            ));
+        }
+    }
+}
+
+#[cfg(any(
+    feature = "wikipedia",
+    feature = "searxng",
+    feature = "brave",
+    feature = "tavily"
+))]
+#[tokio::test]
+async fn json_checks_size_before_parsing_and_preserves_parse_errors() {
+    let server = MockServer::start().await;
+    Mock::given(any())
+        .respond_with(ResponseTemplate::new(200).set_body_bytes(b"not json"))
+        .mount(&server)
+        .await;
+    for limit in [7, 8] {
+        let http = HttpConfig::new().unwrap().with_max_response_bytes(limit);
+        let response = http.client().get(server.uri()).send().await.unwrap();
+        let error = http
+            .read_json::<serde_json::Value>("test", response)
+            .await
+            .unwrap_err();
+        if limit == 7 {
+            assert!(matches!(error, SearchError::ResponseTooLarge { .. }));
+        } else {
+            assert!(matches!(error, SearchError::InvalidResponse { .. }));
+        }
+    }
+}

h2m-search/src/lib.rs

@@ -73,7 +73,7 @@ pub use client::ENV_SEARXNG_URL;
 pub use client::ENV_TAVILY_API_KEY;
 pub use client::{ENV_PROVIDER, ProviderId, SearchClient, SearchClientBuilder};
 pub use error::SearchError;
-pub use http::{DEFAULT_TIMEOUT, HttpConfig, USER_AGENT};
+pub use http::{DEFAULT_MAX_RESPONSE_BYTES, DEFAULT_TIMEOUT, HttpConfig, USER_AGENT};
 #[cfg(test)]
 use insta as _;
 #[cfg(test)]

h2m-search/src/providers/brave.rs

@@ -15,7 +15,7 @@ use std::time::Instant;
 use serde::Deserialize;
 use tracing::instrument;
 
-use super::common::{classify_parse, classify_status, classify_transport};
+use super::common::{classify_status, classify_transport};
 use crate::error::SearchError;
 use crate::http::HttpConfig;
 use crate::query::{SafeSearch, SearchQuery, SearchSource, TimeRange};
@@ -163,10 +163,7 @@ impl Brave {
         if !response.status().is_success() {
             return Err(classify_status(PROVIDER_ID, &response));
         }
-        response
-            .json::<BraveResponse>()
-            .await
-            .map_err(|e| classify_parse(PROVIDER_ID, &e))
+        self.http.read_json(PROVIDER_ID, response).await
     }
 
     fn build_endpoint(&self, query: &SearchQuery, offset: u32) -> Result<url::Url, SearchError> {

h2m-search/src/providers/common.rs

@@ -69,14 +69,6 @@ pub(crate) fn classify_transport(provider: &'static str, err: &reqwest::Error) -
     }
 }
 
-/// Maps a deserialisation failure to [`SearchError::InvalidResponse`].
-pub(crate) fn classify_parse(provider: &'static str, err: &reqwest::Error) -> SearchError {
-    SearchError::InvalidResponse {
-        provider,
-        message: err.to_string(),
-    }
-}
-
 /// Parses the `Retry-After` header into a [`Duration`].
 ///
 /// The header can be either a number of seconds or an HTTP-date. We only