diff --git a/h2m-search/Cargo.toml b/h2m-search/Cargo.toml index 6081da0..e37c8c5 100644 --- a/h2m-search/Cargo.toml +++ b/h2m-search/Cargo.toml @@ -11,11 +11,11 @@ categories = ["web-programming", "command-line-utilities"] [features] default = ["duckduckgo", "wikipedia"] -duckduckgo = ["dep:scraper", "dep:percent-encoding"] -wikipedia = ["dep:percent-encoding"] -searxng = [] -brave = [] -tavily = [] +duckduckgo = ["dep:scraper"] +wikipedia = ["dep:percent-encoding", "dep:serde_json"] +searxng = ["dep:serde_json"] +brave = ["dep:serde_json"] +tavily = ["dep:serde_json"] all = ["duckduckgo", "wikipedia", "searxng", "brave", "tavily"] [dependencies] @@ -23,6 +23,7 @@ reqwest.workspace = true scraper = { workspace = true, optional = true } percent-encoding = { workspace = true, optional = true } serde.workspace = true +serde_json = { workspace = true, optional = true } thiserror.workspace = true tokio = { workspace = true, features = ["time"] } tracing.workspace = true diff --git a/h2m-search/src/error.rs b/h2m-search/src/error.rs index 8cebd17..37ac75c 100644 --- a/h2m-search/src/error.rs +++ b/h2m-search/src/error.rs @@ -71,6 +71,15 @@ pub enum SearchError { message: String, }, + /// Response exceeded the configured decoded body size limit. Not retryable. + #[error("{provider} response exceeds {limit_bytes} bytes")] + ResponseTooLarge { + /// Provider identifier. + provider: &'static str, + /// Maximum permitted response size in bytes. + limit_bytes: usize, + }, + /// HTML scraping detected an anti-bot challenge (e.g. `DuckDuckGo` CAPTCHA). /// /// Retryable: callers can back off and try again with rotated UA or a @@ -117,6 +126,7 @@ impl SearchError { | Self::AuthFailed { provider, .. } | Self::MissingApiKey { provider, .. } | Self::InvalidResponse { provider, .. } + | Self::ResponseTooLarge { provider, .. } | Self::CaptchaDetected { provider } | Self::ParseFailed { provider, .. } => Some(provider), Self::MissingSearxngUrl | Self::ProviderUnavailable { .. } | Self::Config { .. } => { @@ -146,6 +156,7 @@ impl SearchError { Self::MissingApiKey { .. } => "missingApiKey", Self::MissingSearxngUrl => "missingSearxngUrl", Self::InvalidResponse { .. } => "invalidResponse", + Self::ResponseTooLarge { .. } => "responseTooLarge", Self::CaptchaDetected { .. } => "captchaDetected", Self::ParseFailed { .. } => "parseFailed", Self::ProviderUnavailable { .. } => "providerUnavailable", @@ -181,6 +192,9 @@ impl Serialize for SearchError { if let Some(s) = self.status() { map.serialize_entry("status", &s)?; } + if let Self::ResponseTooLarge { limit_bytes, .. } = self { + map.serialize_entry("limitBytes", limit_bytes)?; + } map.end() } } diff --git a/h2m-search/src/http.rs b/h2m-search/src/http.rs index ebe705a..08ffd1b 100644 --- a/h2m-search/src/http.rs +++ b/h2m-search/src/http.rs @@ -5,6 +5,8 @@ //! their own client via [`HttpConfig::with_client`] when callers need //! custom middleware, proxies, or shared pools with the host application. +mod body; + use std::sync::Arc; use std::time::Duration; @@ -13,6 +15,9 @@ use crate::error::SearchError; /// Default per-request timeout applied to every provider. pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30); +/// Maximum decoded response size per request by default (2 MiB). +pub const DEFAULT_MAX_RESPONSE_BYTES: usize = 2 * 1024 * 1024; + /// User agent sent with every request. pub const USER_AGENT: &str = concat!("h2m-search/", env!("CARGO_PKG_VERSION")); @@ -23,6 +28,7 @@ pub const USER_AGENT: &str = concat!("h2m-search/", env!("CARGO_PKG_VERSION")); #[derive(Debug, Clone)] pub struct HttpConfig { client: Arc, + max_response_bytes: usize, } impl HttpConfig { @@ -42,6 +48,7 @@ impl HttpConfig { })?; Ok(Self { client: Arc::new(client), + max_response_bytes: DEFAULT_MAX_RESPONSE_BYTES, }) } @@ -50,9 +57,20 @@ impl HttpConfig { pub fn with_client(client: reqwest::Client) -> Self { Self { client: Arc::new(client), + max_response_bytes: DEFAULT_MAX_RESPONSE_BYTES, } } + /// Sets the maximum decoded response size for each request. + /// + /// Enforced while reading, including chunked responses without a length header. + /// A zero limit permits only empty bodies. Oversized responses fail without retry. + #[must_use] + pub const fn with_max_response_bytes(mut self, limit: usize) -> Self { + self.max_response_bytes = limit; + self + } + /// Shared client handle. #[must_use] pub fn client(&self) -> &reqwest::Client { @@ -65,6 +83,7 @@ impl Default for HttpConfig { fn default() -> Self { Self::new().unwrap_or_else(|_| Self { client: Arc::new(reqwest::Client::new()), + max_response_bytes: DEFAULT_MAX_RESPONSE_BYTES, }) } } diff --git a/h2m-search/src/http/body.rs b/h2m-search/src/http/body.rs new file mode 100644 index 0000000..0718860 --- /dev/null +++ b/h2m-search/src/http/body.rs @@ -0,0 +1,68 @@ +//! Bounded response collection shared by HTML and JSON search providers. + +use super::HttpConfig; +use crate::error::SearchError; +use crate::providers::common::classify_transport; + +impl HttpConfig { + async fn read_body( + &self, + provider: &'static str, + mut response: reqwest::Response, + ) -> Result, SearchError> { + let too_large = || SearchError::ResponseTooLarge { + provider, + limit_bytes: self.max_response_bytes, + }; + if response + .content_length() + .is_some_and(|length| length > self.max_response_bytes as u64) + { + return Err(too_large()); + } + let mut body = Vec::new(); + while let Some(chunk) = response + .chunk() + .await + .map_err(|error| classify_transport(provider, &error))? + { + // Check decoded chunks before accumulation; length headers may be absent. + if chunk.len() > self.max_response_bytes.saturating_sub(body.len()) { + return Err(too_large()); + } + body.extend_from_slice(&chunk); + } + Ok(body) + } + + #[cfg(feature = "duckduckgo")] + pub(crate) async fn read_text( + &self, + provider: &'static str, + response: reqwest::Response, + ) -> Result { + let body = self.read_body(provider, response).await?; + Ok(String::from_utf8_lossy(&body).into_owned()) + } + + #[cfg(any( + feature = "wikipedia", + feature = "searxng", + feature = "brave", + feature = "tavily" + ))] + pub(crate) async fn read_json( + &self, + provider: &'static str, + response: reqwest::Response, + ) -> Result { + let body = self.read_body(provider, response).await?; + serde_json::from_slice(&body).map_err(|error| SearchError::InvalidResponse { + provider, + message: error.to_string(), + }) + } +} + +#[cfg(test)] +mod tests; diff --git a/h2m-search/src/http/body/tests.rs b/h2m-search/src/http/body/tests.rs new file mode 100644 index 0000000..dd81b23 --- /dev/null +++ b/h2m-search/src/http/body/tests.rs @@ -0,0 +1,112 @@ +use super::*; +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::thread; +use std::time::Duration; +use wiremock::{Mock, MockServer, ResponseTemplate, matchers::any}; + +#[tokio::test] +async fn known_length_accepts_boundary_and_rejects_overflow() { + let server = MockServer::start().await; + Mock::given(any()) + .respond_with(ResponseTemplate::new(200).set_body_bytes(b"12345678")) + .mount(&server) + .await; + let http = HttpConfig::new().unwrap().with_max_response_bytes(8); + let response = http.client().get(server.uri()).send().await.unwrap(); + assert_eq!(http.read_body("test", response).await.unwrap(), b"12345678"); + + let http = http.with_max_response_bytes(7); + let oversized_response = http.client().get(server.uri()).send().await.unwrap(); + let error = http + .read_body("test", oversized_response) + .await + .unwrap_err(); + assert!(matches!( + error, + SearchError::ResponseTooLarge { limit_bytes: 7, .. } + )); + assert!(!error.is_retryable()); + assert_eq!(error.provider(), Some("test")); + let json = serde_json::to_value(error).unwrap(); + assert_eq!( + json.get("kind").and_then(serde_json::Value::as_str), + Some("responseTooLarge") + ); + assert_eq!( + json.get("limitBytes").and_then(serde_json::Value::as_u64), + Some(7) + ); +} + +fn serve_chunked(listener: &TcpListener) { + let (mut socket, _) = listener.accept().unwrap(); + socket + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + socket + .set_write_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let mut request = Vec::new(); + while !request.ends_with(b"\r\n\r\n") { + let mut byte = [0]; + socket.read_exact(&mut byte).unwrap(); + request.extend_from_slice(&byte); + } + socket.write_all(b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n4\r\n1234\r\n4\r\n5678\r\n0\r\n\r\n").unwrap(); +} + +#[tokio::test] +async fn chunked_bodies_are_bounded_without_content_length() { + for limit in [7, 8] { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = thread::spawn(move || serve_chunked(&listener)); + let http = HttpConfig::new().unwrap().with_max_response_bytes(limit); + let response = http + .client() + .get(format!("http://{address}")) + .send() + .await + .unwrap(); + assert_eq!(response.content_length(), None); + let result = http.read_body("test", response).await; + server.join().unwrap(); + if limit == 8 { + assert_eq!(result.unwrap(), b"12345678"); + } else { + assert!(matches!( + result, + Err(SearchError::ResponseTooLarge { limit_bytes: 7, .. }) + )); + } + } +} + +#[cfg(any( + feature = "wikipedia", + feature = "searxng", + feature = "brave", + feature = "tavily" +))] +#[tokio::test] +async fn json_checks_size_before_parsing_and_preserves_parse_errors() { + let server = MockServer::start().await; + Mock::given(any()) + .respond_with(ResponseTemplate::new(200).set_body_bytes(b"not json")) + .mount(&server) + .await; + for limit in [7, 8] { + let http = HttpConfig::new().unwrap().with_max_response_bytes(limit); + let response = http.client().get(server.uri()).send().await.unwrap(); + let error = http + .read_json::("test", response) + .await + .unwrap_err(); + if limit == 7 { + assert!(matches!(error, SearchError::ResponseTooLarge { .. })); + } else { + assert!(matches!(error, SearchError::InvalidResponse { .. })); + } + } +} diff --git a/h2m-search/src/lib.rs b/h2m-search/src/lib.rs index f2bbbb6..522fe23 100644 --- a/h2m-search/src/lib.rs +++ b/h2m-search/src/lib.rs @@ -73,7 +73,7 @@ pub use client::ENV_SEARXNG_URL; pub use client::ENV_TAVILY_API_KEY; pub use client::{ENV_PROVIDER, ProviderId, SearchClient, SearchClientBuilder}; pub use error::SearchError; -pub use http::{DEFAULT_TIMEOUT, HttpConfig, USER_AGENT}; +pub use http::{DEFAULT_MAX_RESPONSE_BYTES, DEFAULT_TIMEOUT, HttpConfig, USER_AGENT}; #[cfg(test)] use insta as _; #[cfg(test)] diff --git a/h2m-search/src/providers/brave.rs b/h2m-search/src/providers/brave.rs index bc2e42c..ff332e6 100644 --- a/h2m-search/src/providers/brave.rs +++ b/h2m-search/src/providers/brave.rs @@ -15,7 +15,7 @@ use std::time::Instant; use serde::Deserialize; use tracing::instrument; -use super::common::{classify_parse, classify_status, classify_transport}; +use super::common::{classify_status, classify_transport}; use crate::error::SearchError; use crate::http::HttpConfig; use crate::query::{SafeSearch, SearchQuery, SearchSource, TimeRange}; @@ -163,10 +163,7 @@ impl Brave { if !response.status().is_success() { return Err(classify_status(PROVIDER_ID, &response)); } - response - .json::() - .await - .map_err(|e| classify_parse(PROVIDER_ID, &e)) + self.http.read_json(PROVIDER_ID, response).await } fn build_endpoint(&self, query: &SearchQuery, offset: u32) -> Result { diff --git a/h2m-search/src/providers/common.rs b/h2m-search/src/providers/common.rs index 561019e..7e845b6 100644 --- a/h2m-search/src/providers/common.rs +++ b/h2m-search/src/providers/common.rs @@ -69,14 +69,6 @@ pub(crate) fn classify_transport(provider: &'static str, err: &reqwest::Error) - } } -/// Maps a deserialisation failure to [`SearchError::InvalidResponse`]. -pub(crate) fn classify_parse(provider: &'static str, err: &reqwest::Error) -> SearchError { - SearchError::InvalidResponse { - provider, - message: err.to_string(), - } -} - /// Parses the `Retry-After` header into a [`Duration`]. /// /// The header can be either a number of seconds or an HTTP-date. We only diff --git a/h2m-search/src/providers/duckduckgo/http.rs b/h2m-search/src/providers/duckduckgo/http.rs index 2f9a415..0c146ef 100644 --- a/h2m-search/src/providers/duckduckgo/http.rs +++ b/h2m-search/src/providers/duckduckgo/http.rs @@ -99,7 +99,7 @@ pub(super) async fn post_html( .await .map_err(|e| classify_transport(PROVIDER_ID, &e))?; - finalize_body(response).await + finalize_body(http, response).await } /// POSTs a search request to the lite.duckduckgo.com endpoint. @@ -130,25 +130,22 @@ pub(super) async fn post_lite( .await .map_err(|e| classify_transport(PROVIDER_ID, &e))?; - finalize_body(response).await + finalize_body(http, response).await } /// Reads the response body, classifying non-success statuses, 202-soft-blocks, /// and CAPTCHA challenge pages into structured errors. -async fn finalize_body(response: Response) -> Result { +async fn finalize_body(http: &HttpConfig, response: Response) -> Result { + // DuckDuckGo uses a successful HTTP status for challenges, regardless of body shape. + if response.status() == reqwest::StatusCode::ACCEPTED { + return Err(SearchError::CaptchaDetected { + provider: PROVIDER_ID, + }); + } if !response.status().is_success() { - // 202 with empty body is DDG's classic soft-block signal. - if response.status().as_u16() == 202 { - return Err(SearchError::CaptchaDetected { - provider: PROVIDER_ID, - }); - } return Err(classify_status(PROVIDER_ID, &response)); } - let body = response.text().await.map_err(|e| SearchError::Transport { - provider: PROVIDER_ID, - message: format!("failed to read body: {e}"), - })?; + let body = http.read_text(PROVIDER_ID, response).await?; if looks_like_captcha(&body) { return Err(SearchError::CaptchaDetected { provider: PROVIDER_ID, diff --git a/h2m-search/src/providers/duckduckgo/mod.rs b/h2m-search/src/providers/duckduckgo/mod.rs index cec7a51..e48f773 100644 --- a/h2m-search/src/providers/duckduckgo/mod.rs +++ b/h2m-search/src/providers/duckduckgo/mod.rs @@ -308,7 +308,7 @@ mod tests { let server = MockServer::start().await; Mock::given(method("POST")) .and(path("/html")) - .respond_with(ResponseTemplate::new(202)) + .respond_with(ResponseTemplate::new(202).set_body_string(SAMPLE_HTML)) .mount(&server) .await; Mock::given(method("POST")) @@ -330,7 +330,7 @@ mod tests { let server = MockServer::start().await; Mock::given(method("POST")) .and(path("/")) - .respond_with(ResponseTemplate::new(202)) + .respond_with(ResponseTemplate::new(202).set_body_string(SAMPLE_HTML)) .mount(&server) .await; diff --git a/h2m-search/src/providers/searxng.rs b/h2m-search/src/providers/searxng.rs index f8637ad..37e0f84 100644 --- a/h2m-search/src/providers/searxng.rs +++ b/h2m-search/src/providers/searxng.rs @@ -15,7 +15,7 @@ use std::time::Instant; use serde::Deserialize; use tracing::instrument; -use super::common::{classify_parse, classify_status, classify_transport}; +use super::common::{classify_status, classify_transport}; use crate::error::SearchError; use crate::http::HttpConfig; use crate::query::{SearchQuery, SearchSource}; @@ -123,10 +123,7 @@ impl SearXNG { if !response.status().is_success() { return Err(classify_status(PROVIDER_ID, &response)); } - response - .json::() - .await - .map_err(|e| classify_parse(PROVIDER_ID, &e)) + self.http.read_json(PROVIDER_ID, response).await } fn build_endpoint(&self, query: &SearchQuery, pageno: u32) -> Result { diff --git a/h2m-search/src/providers/tavily.rs b/h2m-search/src/providers/tavily.rs index 63dbc4f..73596a5 100644 --- a/h2m-search/src/providers/tavily.rs +++ b/h2m-search/src/providers/tavily.rs @@ -10,7 +10,7 @@ use std::time::Instant; use serde::{Deserialize, Serialize}; use tracing::instrument; -use super::common::{classify_parse, classify_status, classify_transport}; +use super::common::{classify_status, classify_transport}; use crate::error::SearchError; use crate::http::HttpConfig; use crate::query::{SearchQuery, SearchSource, TimeRange}; @@ -185,10 +185,7 @@ impl Tavily { if !response.status().is_success() { return Err(classify_status(PROVIDER_ID, &response)); } - response - .json::() - .await - .map_err(|e| classify_parse(PROVIDER_ID, &e)) + self.http.read_json(PROVIDER_ID, response).await } } diff --git a/h2m-search/src/providers/wikipedia.rs b/h2m-search/src/providers/wikipedia.rs index f834278..391923d 100644 --- a/h2m-search/src/providers/wikipedia.rs +++ b/h2m-search/src/providers/wikipedia.rs @@ -23,7 +23,7 @@ use percent_encoding::{AsciiSet, CONTROLS, utf8_percent_encode}; use serde::Deserialize; use tracing::instrument; -use super::common::{classify_parse, classify_status, classify_transport}; +use super::common::{classify_status, classify_transport}; use crate::error::SearchError; use crate::http::HttpConfig; use crate::query::{SearchQuery, SearchSource}; @@ -149,10 +149,7 @@ impl Wikipedia { if !response.status().is_success() { return Err(classify_status(PROVIDER_ID, &response)); } - response - .json::() - .await - .map_err(|e| classify_parse(PROVIDER_ID, &e)) + self.http.read_json(PROVIDER_ID, response).await } } diff --git a/h2m-search/tests/integration.rs b/h2m-search/tests/integration.rs index 4f3ddd0..5d06419 100644 --- a/h2m-search/tests/integration.rs +++ b/h2m-search/tests/integration.rs @@ -16,15 +16,18 @@ // dev-deps of the host crate get linked into integration tests by default; // silence the `unused-crate-dependencies` lint by declaring them explicitly. +#[cfg(any(feature = "brave", feature = "searxng"))] use std::time::Duration; #[cfg(feature = "brave")] use h2m_search::Brave; +#[cfg(any(feature = "brave", feature = "searxng"))] +use h2m_search::RetryPolicy; #[cfg(feature = "searxng")] use h2m_search::SearXNG; -use h2m_search::{HttpConfig, RetryPolicy, SearchClient, SearchError, SearchQuery, SecretString}; +use h2m_search::{HttpConfig, SearchClient, SearchError, SearchQuery, SecretString}; use insta as _; -#[cfg(any(feature = "duckduckgo", feature = "wikipedia"))] +#[cfg(feature = "wikipedia")] use percent_encoding as _; use pretty_assertions as _; use reqwest as _; @@ -35,9 +38,75 @@ use thiserror as _; use tokio as _; use tracing as _; use url as _; -use wiremock::matchers::{any, header, method, path}; +#[cfg(all(feature = "brave", feature = "searxng"))] +use wiremock::matchers::header; +use wiremock::matchers::path; +#[cfg(any(feature = "brave", feature = "searxng"))] +use wiremock::matchers::{any, method}; use wiremock::{Mock, MockServer, ResponseTemplate}; +/// An oversized HTML result fails without falling back or retrying. +#[cfg(feature = "duckduckgo")] +#[tokio::test] +async fn duckduckgo_respects_host_response_limit() { + let server = MockServer::start().await; + Mock::given(path("/html")) + .respond_with(ResponseTemplate::new(200).set_body_string("x".repeat(1024))) + .expect(1) + .mount(&server) + .await; + Mock::given(path("/lite")) + .respond_with(ResponseTemplate::new(200)) + .expect(0) + .mount(&server) + .await; + let http = HttpConfig::new().unwrap().with_max_response_bytes(512); + let provider = h2m_search::DuckDuckGo::builder() + .http(http) + .html_endpoint(format!("{}/html", server.uri())) + .lite_endpoint(format!("{}/lite", server.uri())) + .build() + .unwrap(); + let error = provider + .search(&SearchQuery::new("rust")) + .await + .unwrap_err(); + assert!(matches!( + error, + SearchError::ResponseTooLarge { + provider: "duckduckgo", + limit_bytes: 512 + } + )); +} + +/// JSON providers use the same host limit through the public client builder. +#[cfg(feature = "searxng")] +#[tokio::test] +async fn json_search_respects_host_response_limit_without_retry() { + let server = MockServer::start().await; + Mock::given(any()) + .respond_with(ResponseTemplate::new(200).set_body_string(" ".repeat(1024))) + .expect(1) + .mount(&server) + .await; + let http = HttpConfig::new().unwrap().with_max_response_bytes(512); + let client = SearchClient::builder() + .provider("searxng") + .searxng_url(server.uri()) + .http(http) + .build() + .unwrap(); + let error = client.search(&SearchQuery::new("rust")).await.unwrap_err(); + assert!(matches!( + error, + SearchError::ResponseTooLarge { + provider: "searxng", + limit_bytes: 512 + } + )); +} + /// `SearchClient::builder()` resolves an unknown provider to a dedicated /// [`SearchError::ProviderUnavailable`] variant instead of a transport error. #[test] diff --git a/h2m/src/html/mod.rs b/h2m/src/html/mod.rs index 274a29f..4c1f515 100644 --- a/h2m/src/html/mod.rs +++ b/h2m/src/html/mod.rs @@ -52,7 +52,10 @@ mod tests { #[test] fn links_empty_href_excluded() { - assert!(extract_links(r#"empty"#).is_empty()); + assert_eq!( + extract_links(r#"empty"#), + Vec::::new() + ); } #[test]