diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 7d28c8ba..077e121e 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -26,7 +26,7 @@ jobs: - name: Clean docs folder run: cargo clean --doc - name: Build docs - run: RUSTFLAGS='--cfg reqwest_unstable' cargo doc --no-deps --manifest-path impit/Cargo.toml + run: RUSTFLAGS='--cfg reqwest_unstable' cargo doc --no-deps --features http3 --manifest-path impit/Cargo.toml - name: Remove lock file run: rm target/doc/.lock - name: Copy redirect diff --git a/.github/workflows/format.yaml b/.github/workflows/format.yaml index 0d6041d5..f030ea23 100644 --- a/.github/workflows/format.yaml +++ b/.github/workflows/format.yaml @@ -45,4 +45,7 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2 - name: Run tests - run: cargo test -p impit \ No newline at end of file + run: cargo test -p impit + + - name: Run tests with HTTP/3 + run: cargo test -p impit --features http3 diff --git a/README.md b/README.md index 15a7077e..757bd825 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,6 @@ use impit::{impit::Impit, fingerprint::database as fingerprints}; async fn main() { let impit = Impit::::builder() .with_fingerprint(fingerprints::firefox_144::fingerprint()) - .with_http3() .build() .unwrap(); @@ -106,5 +105,5 @@ h2 = { git="https://github.com/apify/h2.git" } Without the patched dependencies, the project won't build. -Note that you also have to build your project with `rustflags = "--cfg reqwest_unstable"`, otherwise, the build will also fail. -This is because `impit` uses unstable features of `reqwest` (namely `http3` support), which are not available in the stable version of the library. +HTTP/3 support is behind the `http3` feature (`impit = { ..., features = ["http3"] }`). With it, you also have to build your project with `rustflags = "--cfg reqwest_unstable"`, otherwise, the build will fail. +This is because HTTP/3 support in `reqwest` is unstable, so it's not available in the stable version of the library. diff --git a/impit-node/Cargo.toml b/impit-node/Cargo.toml index 8df73f1f..527f1b5c 100644 --- a/impit-node/Cargo.toml +++ b/impit-node/Cargo.toml @@ -9,7 +9,7 @@ crate-type = ["cdylib"] [dependencies] napi = { version = "=3.10.5", default-features = false, features = ["napi4", "napi5", "async", "web_stream", "tokio_rt"] } napi-derive = "=3.5.10" -impit = { path="../impit" } +impit = { path="../impit", features = ["http3"] } rustls = { version="0.23.36" } tokio = { version="1.41.1", features = ["full"] } h2 = "0.4.7" diff --git a/impit-python/Cargo.toml b/impit-python/Cargo.toml index bf15c003..ae6008cb 100644 --- a/impit-python/Cargo.toml +++ b/impit-python/Cargo.toml @@ -8,7 +8,7 @@ name = "impit" crate-type = ["cdylib"] [dependencies] -impit = { path="../impit" } +impit = { path="../impit", features = ["http3"] } rustls = { version="0.23.36" } tokio = { version="1.41.1", features = ["full"] } h2 = "0.4.7" diff --git a/impit/Cargo.toml b/impit/Cargo.toml index 176c6452..5f3b9d13 100644 --- a/impit/Cargo.toml +++ b/impit/Cargo.toml @@ -5,19 +5,25 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[features] +# HTTP/3 (QUIC) support, see `ImpitBuilder::with_http3`. Needs the `--cfg reqwest_unstable` rustflag. +# Without it, clients don't set up the unused QUIC endpoint (a UDP socket and a receive buffer of +# at least 2 MB per client). +http3 = ["reqwest/http3", "dep:hickory-proto", "dep:hickory-resolver", "dep:tokio"] + [dependencies] bytes = "1" encoding = "0.2.33" futures-core = "0.3" -hickory-proto = "0.26.1" -hickory-resolver = "0.26.1" +hickory-proto = { version = "0.26.1", optional = true } +hickory-resolver = { version = "0.26.1", optional = true } log = "0.4.22" mime = "0.3.17" -reqwest = { version="0.13.1", features = ["json", "gzip", "brotli", "zstd", "deflate", "http3", "cookies", "stream", "socks"] } +reqwest = { version="0.13.1", features = ["json", "gzip", "brotli", "zstd", "deflate", "cookies", "stream", "socks"] } rustls = { version="0.23.36", features=["impit"] } lol_html = "2.7.2" thiserror = "2.0.12" -tokio = { version="1.40.0", features = ["full"] } +tokio = { version="1.40.0", features = ["full"], optional = true } url = "2.5.2" rustls-platform-verifier = "0.6" webpki-root-certs = "1.0.10" @@ -27,4 +33,5 @@ rand = "0.9" uuid = { version = "1", features = ["v4"] } [dev-dependencies] +tokio = { version="1.40.0", features = ["full"] } futures-util = "0.3" diff --git a/impit/README.md b/impit/README.md index f14cce22..fd898828 100644 --- a/impit/README.md +++ b/impit/README.md @@ -13,7 +13,6 @@ use reqwest::cookie::Jar; async fn main() { let impit = Impit::::builder() .with_browser(Browser::Firefox) - .with_http3() .build() .unwrap(); @@ -54,5 +53,5 @@ h2 = { git="https://github.com/apify/h2.git" } Without the patched dependencies, the project won't build. -Note that you also have to build your project with `rustflags = "--cfg reqwest_unstable"`, otherwise, the build will also fail. -This is because `impit` uses unstable features of `reqwest` (namely `http3` support), which are not available in the stable version of the library. \ No newline at end of file +HTTP/3 support is behind the `http3` feature (`impit = { ..., features = ["http3"] }`). With it, you also have to build your project with `rustflags = "--cfg reqwest_unstable"`, otherwise, the build will fail. +This is because HTTP/3 support in `reqwest` is unstable, so it's not available in the stable version of the library. \ No newline at end of file diff --git a/impit/examples/basic.rs b/impit/examples/basic.rs index 6b010114..fdd8267e 100644 --- a/impit/examples/basic.rs +++ b/impit/examples/basic.rs @@ -5,7 +5,6 @@ use impit::{fingerprint::database as fingerprints, impit::Impit}; async fn main() { let impit = Impit::::builder() .with_fingerprint(fingerprints::firefox_144::fingerprint()) - .with_http3() .build() .unwrap(); diff --git a/impit/src/impit.rs b/impit/src/impit.rs index 4609ccef..559967b0 100644 --- a/impit/src/impit.rs +++ b/impit/src/impit.rs @@ -1,3 +1,4 @@ +#[cfg(feature = "http3")] use tokio::sync::RwLock; use log::debug; @@ -5,10 +6,11 @@ use reqwest::{cookie::CookieStore, header::HeaderMap, Method, Response, Version} use std::{fmt::Debug, net::IpAddr, sync::Arc, time::Duration}; use url::Url; +#[cfg(feature = "http3")] +use crate::http3::H3Engine; use crate::{ errors::{ErrorContext, ImpitError}, fingerprint::BrowserFingerprint, - http3::H3Engine, http_headers::HttpHeadersBuilder, request::{ImpitBody, ImpitRequest, RequestOptions}, tls, @@ -23,6 +25,7 @@ pub struct Impit { pub(self) base_client: reqwest::Client, pub(self) h3_client: Option, pub(self) vanilla_client: Option, + #[cfg(feature = "http3")] h3_engine: Arc>>, config: ImpitBuilder, } @@ -72,7 +75,6 @@ pub enum RedirectBehavior { /// .with_ignore_tls_errors(true) /// .with_proxy("http://localhost:8080".to_string()) /// .with_default_timeout(Duration::from_secs(10)) -/// .with_http3() /// .build() /// .unwrap(); /// @@ -175,7 +177,10 @@ impl ImpitBuilder { /// To enforce HTTP/3 usage, use the `http3_prior_knowledge` option in the `RequestOptions` struct when /// making the request. /// + /// Requires the `http3` feature of this crate (and the `--cfg reqwest_unstable` rustflag). + /// /// Note that this feature is experimental and may not work as expected with all servers. + #[cfg(feature = "http3")] pub fn with_http3(mut self) -> Self { self.max_http_version = Version::HTTP_3; self @@ -286,6 +291,7 @@ impl Impit { client = client.cookie_provider(cookie_provider.clone()); } + #[cfg(feature = "http3")] if config.max_http_version == Version::HTTP_3 { client = client.http3_prior_knowledge(); } @@ -359,6 +365,7 @@ impl Impit { h3_client, vanilla_client, config, + #[cfg(feature = "http3")] h3_engine: Arc::new(RwLock::new(None)), }) } @@ -379,6 +386,12 @@ impl Impit { } } + #[cfg(not(feature = "http3"))] + async fn should_use_h3(&self, _host: &str) -> bool { + false + } + + #[cfg(feature = "http3")] async fn should_use_h3(&self, host: &str) -> bool { if self.config.max_http_version < Version::HTTP_3 { debug!("HTTP/3 is disabled, falling back to TCP-based requests."); @@ -527,6 +540,7 @@ impl Impit { } }; + #[cfg(feature = "http3")] if !h3 { let engine_guard = self.h3_engine.read().await; if let Some(h3_engine) = engine_guard.as_ref() { @@ -700,3 +714,32 @@ impl Impit { } } } + +#[cfg(test)] +mod tests { + use reqwest::cookie::Jar; + + use super::Impit; + use crate::{errors::ImpitError, request::RequestOptions}; + + async fn get_with_http3_prior_knowledge(impit: &Impit) -> Result<(), ImpitError> { + let options = RequestOptions { + http3_prior_knowledge: true, + ..Default::default() + }; + impit + .get("https://example.com".to_string(), None, Some(options)) + .await + .map(|_| ()) + } + + #[tokio::test] + async fn http3_is_off_by_default() { + let impit = Impit::::builder().build().unwrap(); + assert!(impit.h3_client.is_none()); + assert!(matches!( + get_with_http3_prior_knowledge(&impit).await, + Err(ImpitError::Http3Disabled) + )); + } +} diff --git a/impit/src/lib.rs b/impit/src/lib.rs index 0af8cae7..4a1658fe 100644 --- a/impit/src/lib.rs +++ b/impit/src/lib.rs @@ -12,7 +12,6 @@ //! async fn main() { //! let impit = Impit::::builder() //! .with_fingerprint(fingerprints::firefox_144::fingerprint()) -//! .with_http3() //! .build() //! .unwrap(); //! @@ -53,14 +52,16 @@ //! //! Without the patched dependencies, the project won't build. //! -//! Note that you also have to build your project with `rustflags = "--cfg reqwest_unstable"`, otherwise, the build will also fail. -//! This is because `impit` uses unstable features of `reqwest` (namely `http3` support), which are not available in the stable version of the library. +//! HTTP/3 support is behind the `http3` feature (`impit = { ..., features = ["http3"] }`). With it, you also have to build your project +//! with `rustflags = "--cfg reqwest_unstable"`, otherwise, the build will fail. +//! This is because HTTP/3 support in `reqwest` is unstable, so it's not available in the stable version of the library. #![deny(unused_crate_dependencies)] mod http_headers; mod response_parsing; mod tls; +#[cfg(feature = "http3")] pub(crate) mod http3; /// Main module that contains the `Impit` struct and its methods.