diff --git a/README.md b/README.md index 60712fb..7ca0969 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,7 @@ rate_limit: | `CERTSTREAM_CT_LOG_REQUEST_TIMEOUT_SECS` | 30 | Request timeout | | `CERTSTREAM_CT_LOG_BATCH_SIZE` | 1024 | Entries requested per get-entries call (servers clamp to their own max) | | `CERTSTREAM_CT_LOG_FETCH_CONCURRENCY` | 4 | Concurrent range/tile fetches per watcher during catch-up (1-16) | +| `CERTSTREAM_USER_AGENT` | certstream-server-rust/{VERSION} | HTTP User-Agent for CT log fetches. Some operators (e.g. Geomys) apply a more generous rate limit tier to clients that include a contact email. | **Hot Reload** diff --git a/config.example.yaml b/config.example.yaml index f3eac41..e65eb90 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -54,6 +54,11 @@ ct_log: # any operator absent from operator_rate_limits. default_operator_rate_limit_ms: 500 operator_rate_limits: {} + # HTTP User-Agent for CT log fetches. Some operators (e.g. Geomys) apply a + # more generous rate limit tier to clients that include a contact email. + # When unset, defaults to certstream-server-rust/{VERSION}. + # (env: CERTSTREAM_USER_AGENT) + user_agent: null # Per-catalog-source runtime-authority overrides. Keys are # google_v3_usable, google_v3_all, and apple. An override can only grant # authority to a source that currently verifies; it cannot promote an diff --git a/docs/docs.html b/docs/docs.html index 1a88135..fbcd62e 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -495,6 +495,7 @@

CT log settings

CERTSTREAM_CT_LOG_REQUEST_TIMEOUT_SECS30Request timeout CERTSTREAM_CT_LOG_BATCH_SIZE1024Entries requested per get-entries call (servers clamp to their own max) CERTSTREAM_CT_LOG_FETCH_CONCURRENCY4Concurrent range/tile fetches per watcher during catch-up (1-16) + CERTSTREAM_USER_AGENTcertstream-server-rust/{VERSION}HTTP User-Agent for CT log fetches. Some operators (e.g. Geomys) apply a more generous rate limit tier to clients that include a contact email. CERTSTREAM_STATIC_CT_CHECKPOINT_SIGNATUREwarnCheckpoint signature policy: warn or enforce CERTSTREAM_DEDUP_CAPACITY200000Cross-log dedup capacity CERTSTREAM_DEDUP_TTL_SECS900Dedup window (seconds) @@ -549,6 +550,7 @@

YAML config file

poll_interval_ms: 500 retry_max_attempts: 3 request_timeout_secs: 30 + user_agent: "certstream-server-rust/1.5.3 (contact@example.com)" dedup: capacity: 200000 diff --git a/src/cli.rs b/src/cli.rs index b6d3604..898390d 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -45,6 +45,7 @@ impl CliArgs { println!(" CERTSTREAM_PORT Server port (default: 8080)"); println!(" CERTSTREAM_LOG_LEVEL Log level (default: info)"); println!(" CERTSTREAM_BUFFER_SIZE Broadcast buffer size (default: 1000)"); + println!(" CERTSTREAM_USER_AGENT Override HTTP User-Agent for CT log requests"); println!(); println!("For more information, see: https://github.com/reloading01/certstream-server-rust"); } diff --git a/src/config.rs b/src/config.rs index fda8e53..da71244 100644 --- a/src/config.rs +++ b/src/config.rs @@ -165,6 +165,12 @@ pub struct CtLogConfig { /// whitespace, or punctuation. Empty map means every operator uses the default. #[serde(default)] pub operator_rate_limits: std::collections::HashMap, + /// HTTP User-Agent for CT log fetches. Some CT log operators (e.g. + /// Geomys) apply a more generous rate limit tier to clients that include + /// a contact email. When unset, defaults to + /// `certstream-server-rust/{VERSION}`. + #[serde(default)] + pub user_agent: Option, /// Per-catalog-source runtime-authority overrides. Keys are the catalog /// registry source names (`google_v3_usable`, `google_v3_all`, `apple`). /// An override can only grant authority to a source that currently verifies; @@ -219,6 +225,7 @@ impl Default for CtLogConfig { static_ct_enabled: true, default_operator_rate_limit_ms: default_operator_rate_limit_ms(), operator_rate_limits: std::collections::HashMap::new(), + user_agent: None, catalog_authority_overrides: std::collections::HashMap::new(), } } @@ -586,6 +593,7 @@ impl Config { ct_log.checkpoint_signature_mode, "CERTSTREAM_STATIC_CT_CHECKPOINT_SIGNATURE" ); + env_override!(ct_log.user_agent, "CERTSTREAM_USER_AGENT", some_str); let mut connection_limit = yaml_config.connection_limit.unwrap_or_default(); env_override!(connection_limit.enabled, "CERTSTREAM_CONNECTION_LIMIT_ENABLED"); @@ -706,6 +714,14 @@ impl Config { message: "Fetch concurrency must be between 1 and 16".to_string(), }); } + if let Some(ua) = &self.ct_log.user_agent + && reqwest::header::HeaderValue::try_from(ua.as_str()).is_err() + { + errors.push(ConfigValidationError { + field: "ct_log.user_agent".to_string(), + message: "User-Agent must be a valid HTTP header value".to_string(), + }); + } if errors.is_empty() { Ok(()) @@ -801,6 +817,34 @@ mod tests { assert_eq!(config.start_overlap_leaves, 256); assert!(config.rfc6962_enabled); assert!(config.static_ct_enabled); + assert!(config.user_agent.is_none()); + } + + #[test] + fn test_ct_log_config_deserialize_user_agent() { + let yaml = r#" +user_agent: "certstream-server-rust/1.5.3 (contact@example.com)" +"#; + let config: CtLogConfig = serde_yaml::from_str(yaml).unwrap(); + assert_eq!( + config.user_agent.as_deref(), + Some("certstream-server-rust/1.5.3 (contact@example.com)") + ); + } + + #[test] + fn test_validate_user_agent_invalid_header() { + let config = Config { + ct_log: CtLogConfig { + user_agent: Some("bad\nuser-agent".to_string()), + ..CtLogConfig::default() + }, + ..test_config() + }; + let result = config.validate(); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors.iter().any(|e| e.field == "ct_log.user_agent")); } #[test] diff --git a/src/main.rs b/src/main.rs index 999d589..974f1ed 100644 --- a/src/main.rs +++ b/src/main.rs @@ -140,8 +140,14 @@ async fn main() { let tx: broadcast::Sender> = broadcast::channel(config.buffer_size).0; + let user_agent = config + .ct_log + .user_agent + .clone() + .unwrap_or_else(|| format!("certstream-server-rust/{}", VERSION)); + let client = Client::builder() - .user_agent(format!("certstream-server-rust/{}", VERSION)) + .user_agent(&user_agent) // Pre-1.5.0 kept 20 idle connections per host × 55 hosts = 1100 // hot TCP sockets, ~40-55 MiB of kernel + TLS state per process. // Watchers now pipeline up to `fetch_concurrency` range/tile fetches