From f1233c3c6af12a18e39a0b5e69479decd478f357 Mon Sep 17 00:00:00 2001 From: Deenkar Mahulkar <11814351+deenkar@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:03:15 +0530 Subject: [PATCH 1/7] fix: use sovereign Azure storage token scopes for managed identity The legacy azure_storage SDK hardcodes the public-cloud OAuth audience. Wrap token credentials with the correct scope for Azure Government and China, and fail fast when custom endpoints cannot be mapped. Follow-up to #6661. Fixes #6624. Co-authored-by: Cursor --- CHANGELOG.md | 1 + docs/configuration/storage-config.md | 12 ++ quickwit/Cargo.lock | 1 + quickwit/quickwit-config/src/lib.rs | 6 +- .../quickwit-config/src/storage_config.rs | 134 ++++++++++++++++++ quickwit/quickwit-storage/Cargo.toml | 1 + .../src/object_storage/azure_blob_storage.rs | 79 ++++++++++- .../object_storage/azure_token_credential.rs | 110 ++++++++++++++ .../src/object_storage/mod.rs | 2 + 9 files changed, 340 insertions(+), 6 deletions(-) create mode 100644 quickwit/quickwit-storage/src/object_storage/azure_token_credential.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 1981b1a9dfa..bb49026ffe9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Azure Blob Storage: support custom endpoints via `endpoint` and `endpoint_suffix` configuration options for sovereign clouds (#6624) +- Azure Blob Storage: use sovereign-cloud OAuth token scopes for managed identity on Azure Government and Azure China endpoints (#6624) ### Fixed - (Jaeger) Query resource attributes when Jaeger request carries tags diff --git a/docs/configuration/storage-config.md b/docs/configuration/storage-config.md index df26af4ac68..4dc95724832 100644 --- a/docs/configuration/storage-config.md +++ b/docs/configuration/storage-config.md @@ -149,6 +149,18 @@ storage: endpoint_suffix: core.chinacloudapi.cn ``` +#### Managed identity on sovereign clouds + +When using managed identity or other token-based authentication (without an `access_key`), Quickwit automatically selects the correct OAuth token scope for Azure Government and Azure China endpoints. + +For Azure US Government, you may also need to set the Entra authority host: + +```bash +export AZURE_AUTHORITY_HOST=https://login.microsoftonline.us/ +``` + +Custom endpoints that are not public Azure, Azure Government, or Azure China (for example Azure Stack) require an `access_key` when using token-based authentication. + ## Storage configuration examples for various object storage providers ### Garage diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index 2b3e7240524..1b28ef0158d 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -9438,6 +9438,7 @@ dependencies = [ "tantivy", "tempfile", "thiserror 2.0.18", + "time", "tokio", "tokio-rustls 0.26.4", "tokio-stream", diff --git a/quickwit/quickwit-config/src/lib.rs b/quickwit/quickwit-config/src/lib.rs index 4cbe506ddf4..7a426c5f810 100644 --- a/quickwit/quickwit-config/src/lib.rs +++ b/quickwit/quickwit-config/src/lib.rs @@ -85,9 +85,9 @@ pub use crate::node_config::{ pub use crate::serde_utils::HumanDuration; use crate::source_config::serialize::{SourceConfigV0_7, SourceConfigV0_8, VersionedSourceConfig}; pub use crate::storage_config::{ - AzureStorageConfig, ChecksumAlgorithm, FileStorageConfig, GoogleCloudStorageConfig, - RamStorageConfig, S3StorageConfig, StorageBackend, StorageBackendFlavor, StorageConfig, - StorageConfigs, + AzureNationalCloud, AzureStorageConfig, ChecksumAlgorithm, FileStorageConfig, + GoogleCloudStorageConfig, RamStorageConfig, S3StorageConfig, StorageBackend, + StorageBackendFlavor, StorageConfig, StorageConfigs, }; /// Returns true if the ingest API v2 is enabled. diff --git a/quickwit/quickwit-config/src/storage_config.rs b/quickwit/quickwit-config/src/storage_config.rs index 71684d350e7..99476362215 100644 --- a/quickwit/quickwit-config/src/storage_config.rs +++ b/quickwit/quickwit-config/src/storage_config.rs @@ -357,6 +357,86 @@ impl AzureStorageConfig { }; Some(uri) } + + /// Returns `true` when a custom blob endpoint is configured. + pub fn uses_custom_blob_endpoint(&self) -> bool { + self.endpoint().is_some() || self.endpoint_suffix().is_some() + } + + /// Classifies the Azure national cloud from the configured endpoint. + pub fn resolve_national_cloud(&self) -> AzureNationalCloud { + if let Some(endpoint) = self.endpoint() { + if let Some(host) = extract_azure_endpoint_host(&endpoint) { + return classify_azure_endpoint_host(host); + } + } + if let Some(endpoint_suffix) = self.endpoint_suffix() { + return classify_azure_endpoint_suffix(&endpoint_suffix); + } + AzureNationalCloud::Public + } + + /// Returns the OAuth token scope for Azure Storage in the configured national cloud. + /// + /// Returns `None` for custom endpoints that do not map to a known national cloud. + pub fn resolve_storage_token_scope(&self) -> Option<&'static str> { + match self.resolve_national_cloud() { + AzureNationalCloud::Public => Some(AZURE_PUBLIC_STORAGE_TOKEN_SCOPE), + AzureNationalCloud::UsGovernment => Some(AZURE_US_GOVERNMENT_STORAGE_TOKEN_SCOPE), + AzureNationalCloud::China => Some(AZURE_CHINA_STORAGE_TOKEN_SCOPE), + AzureNationalCloud::Custom => None, + } + } +} + +/// Azure national cloud classification for token authentication. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum AzureNationalCloud { + Public, + UsGovernment, + China, + Custom, +} + +pub const AZURE_PUBLIC_STORAGE_TOKEN_SCOPE: &str = "https://storage.azure.com/.default"; + +pub const AZURE_US_GOVERNMENT_STORAGE_TOKEN_SCOPE: &str = "https://storage.azure.us/.default"; + +pub const AZURE_CHINA_STORAGE_TOKEN_SCOPE: &str = "https://storage.azure.cn/.default"; + +fn extract_azure_endpoint_host(endpoint: &str) -> Option<&str> { + let endpoint = endpoint.trim(); + let host = endpoint + .strip_prefix("https://") + .or_else(|| endpoint.strip_prefix("http://")) + .unwrap_or(endpoint); + host.split('/').next().filter(|host| !host.is_empty()) +} + +fn classify_azure_endpoint_host(host: &str) -> AzureNationalCloud { + let host_lower = host.to_ascii_lowercase(); + if host_lower.contains("chinacloudapi.cn") { + AzureNationalCloud::China + } else if host_lower.contains("usgovcloudapi.net") { + AzureNationalCloud::UsGovernment + } else if host_lower.contains("core.windows.net") { + AzureNationalCloud::Public + } else { + AzureNationalCloud::Custom + } +} + +fn classify_azure_endpoint_suffix(endpoint_suffix: &str) -> AzureNationalCloud { + let endpoint_suffix_lower = endpoint_suffix.to_ascii_lowercase(); + if endpoint_suffix_lower.contains("chinacloudapi.cn") { + AzureNationalCloud::China + } else if endpoint_suffix_lower.contains("usgovcloudapi.net") { + AzureNationalCloud::UsGovernment + } else if endpoint_suffix_lower.contains("windows.net") { + AzureNationalCloud::Public + } else { + AzureNationalCloud::Custom + } } impl fmt::Debug for AzureStorageConfig { @@ -729,6 +809,60 @@ mod tests { assert!(config.resolve_blob_service_uri("my-account").is_none()); } + #[test] + fn test_storage_azure_config_resolve_national_cloud() { + let public_config = AzureStorageConfig::default(); + assert_eq!( + public_config.resolve_national_cloud(), + AzureNationalCloud::Public + ); + + let gov_config = AzureStorageConfig { + endpoint_suffix: Some("core.usgovcloudapi.net".to_string()), + ..Default::default() + }; + assert_eq!( + gov_config.resolve_national_cloud(), + AzureNationalCloud::UsGovernment + ); + + let china_config = AzureStorageConfig { + endpoint: Some("https://my-account.blob.core.chinacloudapi.cn".to_string()), + ..Default::default() + }; + assert_eq!( + china_config.resolve_national_cloud(), + AzureNationalCloud::China + ); + + let custom_config = AzureStorageConfig { + endpoint: Some("https://storage.example.com".to_string()), + ..Default::default() + }; + assert_eq!( + custom_config.resolve_national_cloud(), + AzureNationalCloud::Custom + ); + } + + #[test] + fn test_storage_azure_config_resolve_storage_token_scope() { + let gov_config = AzureStorageConfig { + endpoint_suffix: Some("core.usgovcloudapi.net".to_string()), + ..Default::default() + }; + assert_eq!( + gov_config.resolve_storage_token_scope(), + Some(AZURE_US_GOVERNMENT_STORAGE_TOKEN_SCOPE) + ); + + let custom_config = AzureStorageConfig { + endpoint: Some("https://storage.example.com".to_string()), + ..Default::default() + }; + assert!(custom_config.resolve_storage_token_scope().is_none()); + } + #[test] fn test_storage_google_config_serde() { { diff --git a/quickwit/quickwit-storage/Cargo.toml b/quickwit/quickwit-storage/Cargo.toml index 1c377eaacd5..61b61392397 100644 --- a/quickwit/quickwit-storage/Cargo.toml +++ b/quickwit/quickwit-storage/Cargo.toml @@ -75,6 +75,7 @@ aws-sdk-s3 = { workspace = true } aws-smithy-runtime = { workspace = true, features = ["test-util"] } quickwit-common = { workspace = true, features = ["testsuite"] } +time = { workspace = true } [features] azure = [ diff --git a/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs b/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs index cbbf6ea8ded..c05bce6d189 100644 --- a/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs +++ b/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs @@ -34,7 +34,7 @@ use md5::Digest; use quickwit_common::retry::{RetryParams, Retryable, retry}; use quickwit_common::uri::Uri; use quickwit_common::{chunk_range, ignore_error_kind, into_u64_range}; -use quickwit_config::{AzureStorageConfig, StorageBackend}; +use quickwit_config::{AzureNationalCloud, AzureStorageConfig, StorageBackend}; use quickwit_metrics::HistogramTimer; use regex::Regex; use tantivy::directory::OwnedBytes; @@ -46,6 +46,7 @@ use tracing::{info, instrument, warn}; use crate::debouncer::DebouncedStorage; use crate::metrics::object_storage_get_slice_in_flight_guards; +use crate::object_storage::azure_token_credential::ScopedTokenCredential; use crate::stable_deref_bytes::into_owned_bytes; use crate::storage::SendableAsync; use crate::{ @@ -185,7 +186,7 @@ impl AzureBlobStorage { { StorageCredentials::access_key(storage_account_name.clone(), access_key) } else if let Ok(credential) = azure_identity::create_credential() { - StorageCredentials::token_credential(credential) + build_azure_token_credentials(azure_storage_config, credential)? } else { return Err(StorageResolverError::InvalidConfig( "could not find Azure storage account credentials using the following credential \ @@ -563,6 +564,38 @@ async fn extract_range_data_and_hash( Ok((data, hash)) } +fn build_azure_token_credentials( + azure_storage_config: &AzureStorageConfig, + credential: Arc, +) -> Result { + match azure_storage_config.resolve_national_cloud() { + AzureNationalCloud::UsGovernment | AzureNationalCloud::China => { + let token_scope = azure_storage_config + .resolve_storage_token_scope() + .expect("sovereign cloud must resolve a storage token scope"); + info!( + token_scope = token_scope, + national_cloud = ?azure_storage_config.resolve_national_cloud(), + "using Azure storage token scope for sovereign cloud" + ); + Ok(StorageCredentials::token_credential(Arc::new( + ScopedTokenCredential::new(credential, token_scope), + ))) + } + AzureNationalCloud::Custom if azure_storage_config.uses_custom_blob_endpoint() => { + Err(StorageResolverError::InvalidConfig( + "custom Azure blob endpoints require an access key when using token-based \ + authentication; managed identity is only supported for public Azure, Azure \ + Government, and Azure China endpoints" + .to_string(), + )) + } + AzureNationalCloud::Public | AzureNationalCloud::Custom => { + Ok(StorageCredentials::token_credential(credential)) + } + } +} + fn build_container_client( storage_account_name: String, storage_credentials: StorageCredentials, @@ -690,9 +723,35 @@ impl From for StorageError { #[cfg(test)] mod tests { + use std::sync::Arc; + + use azure_core::auth::{AccessToken, Secret, TokenCredential}; use quickwit_common::uri::Uri; + use quickwit_config::AzureStorageConfig; + use time::OffsetDateTime; + + use crate::StorageResolverError; + use crate::object_storage::azure_blob_storage::{ + build_azure_token_credentials, parse_azure_uri, + }; + + #[derive(Debug)] + struct MockTokenCredential; + + #[cfg_attr(target_arch = "wasm32", async_trait::async_trait(?Send))] + #[cfg_attr(not(target_arch = "wasm32"), async_trait::async_trait)] + impl TokenCredential for MockTokenCredential { + async fn get_token(&self, _scopes: &[&str]) -> azure_core::Result { + Ok(AccessToken::new( + Secret::new("mock-token"), + OffsetDateTime::now_utc(), + )) + } - use crate::object_storage::azure_blob_storage::parse_azure_uri; + async fn clear_cache(&self) -> azure_core::Result<()> { + Ok(()) + } + } #[test] fn test_parse_azure_uri() { @@ -713,4 +772,18 @@ mod tests { assert_eq!(container, "test-container"); assert_eq!(prefix.to_str().unwrap(), "indexes"); } + + #[test] + fn test_build_azure_token_credentials_rejects_unknown_custom_endpoint() { + let azure_storage_config = AzureStorageConfig { + endpoint: Some("https://storage.example.com".to_string()), + ..Default::default() + }; + let credential = Arc::new(MockTokenCredential) as Arc; + + let error = build_azure_token_credentials(&azure_storage_config, credential) + .expect_err("custom endpoint with token auth should fail"); + + assert!(matches!(error, StorageResolverError::InvalidConfig(_))); + } } diff --git a/quickwit/quickwit-storage/src/object_storage/azure_token_credential.rs b/quickwit/quickwit-storage/src/object_storage/azure_token_credential.rs new file mode 100644 index 00000000000..d2537b2227b --- /dev/null +++ b/quickwit/quickwit-storage/src/object_storage/azure_token_credential.rs @@ -0,0 +1,110 @@ +// Copyright 2021-Present Datadog, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::fmt; +use std::sync::Arc; + +use azure_core::auth::{AccessToken, TokenCredential}; + +/// Wraps a [`TokenCredential`] and requests tokens for a fixed OAuth scope. +/// +/// The legacy `azure_storage` 0.21 SDK hardcodes the public-cloud storage scope when using token +/// credentials. Sovereign clouds require a different audience, so this wrapper ignores the scope +/// requested by the SDK and uses the configured national-cloud scope instead. +#[derive(Clone)] +pub struct ScopedTokenCredential { + inner: Arc, + scope: &'static str, +} + +impl ScopedTokenCredential { + pub fn new(inner: Arc, scope: &'static str) -> Self { + Self { inner, scope } + } +} + +impl fmt::Debug for ScopedTokenCredential { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ScopedTokenCredential") + .field("inner", &self.inner) + .field("scope", &self.scope) + .finish() + } +} + +#[cfg_attr(target_arch = "wasm32", async_trait::async_trait(?Send))] +#[cfg_attr(not(target_arch = "wasm32"), async_trait::async_trait)] +impl TokenCredential for ScopedTokenCredential { + async fn get_token(&self, _scopes: &[&str]) -> azure_core::Result { + self.inner.get_token(&[self.scope]).await + } + + async fn clear_cache(&self) -> azure_core::Result<()> { + self.inner.clear_cache().await + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use azure_core::auth::{AccessToken, Secret, TokenCredential}; + use time::OffsetDateTime; + + use super::ScopedTokenCredential; + + #[derive(Debug)] + struct MockTokenCredential { + requested_scopes: Arc>>, + } + + #[cfg_attr(target_arch = "wasm32", async_trait::async_trait(?Send))] + #[cfg_attr(not(target_arch = "wasm32"), async_trait::async_trait)] + impl TokenCredential for MockTokenCredential { + async fn get_token(&self, scopes: &[&str]) -> azure_core::Result { + self.requested_scopes + .lock() + .expect("lock poisoned") + .extend(scopes.iter().map(|scope| (*scope).to_string())); + Ok(AccessToken::new( + Secret::new("mock-token"), + OffsetDateTime::now_utc(), + )) + } + + async fn clear_cache(&self) -> azure_core::Result<()> { + Ok(()) + } + } + + #[tokio::test] + async fn test_scoped_token_credential_uses_configured_scope() { + let requested_scopes = Arc::new(std::sync::Mutex::new(Vec::new())); + let inner = Arc::new(MockTokenCredential { + requested_scopes: requested_scopes.clone(), + }) as Arc; + let credential = ScopedTokenCredential::new(inner, "https://storage.azure.us/.default"); + + let _token = credential + .get_token(&["https://storage.azure.com/.default"]) + .await + .expect("token request should succeed"); + + let requested_scopes = requested_scopes.lock().expect("lock poisoned"); + assert_eq!( + requested_scopes.as_slice(), + &["https://storage.azure.us/.default".to_string()] + ); + } +} diff --git a/quickwit/quickwit-storage/src/object_storage/mod.rs b/quickwit/quickwit-storage/src/object_storage/mod.rs index e914c107291..0f9e8a5ff08 100644 --- a/quickwit/quickwit-storage/src/object_storage/mod.rs +++ b/quickwit/quickwit-storage/src/object_storage/mod.rs @@ -26,4 +26,6 @@ mod s3_compatible_storage_resolver; #[cfg(feature = "azure")] mod azure_blob_storage; #[cfg(feature = "azure")] +mod azure_token_credential; +#[cfg(feature = "azure")] pub use self::azure_blob_storage::{AzureBlobStorage, AzureBlobStorageFactory}; From f8da328817950d47bf34aefe4398360c014ab9e5 Mon Sep 17 00:00:00 2001 From: Deenkar Mahulkar <11814351+deenkar@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:35:41 +0530 Subject: [PATCH 2/7] fix: tighten Azure sovereign cloud endpoint classification Use allowlisted suffix-boundary matching with port stripping so spoofed hostnames are rejected. Document Azure China AZURE_AUTHORITY_HOST, replace expect with InvalidConfig, and add sovereign token auth tests. Co-authored-by: Cursor --- CHANGELOG.md | 2 +- docs/configuration/storage-config.md | 6 +- .../quickwit-config/src/storage_config.rs | 112 ++++++++++++++---- .../src/object_storage/azure_blob_storage.rs | 36 +++++- 4 files changed, 128 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb49026ffe9..e754bb1e584 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Azure Blob Storage: support custom endpoints via `endpoint` and `endpoint_suffix` configuration options for sovereign clouds (#6624) -- Azure Blob Storage: use sovereign-cloud OAuth token scopes for managed identity on Azure Government and Azure China endpoints (#6624) ### Fixed +- Azure Blob Storage: use sovereign-cloud OAuth token scopes for managed identity on Azure Government and Azure China endpoints (#6624) - (Jaeger) Query resource attributes when Jaeger request carries tags ### Changed diff --git a/docs/configuration/storage-config.md b/docs/configuration/storage-config.md index 4dc95724832..17caeb8652c 100644 --- a/docs/configuration/storage-config.md +++ b/docs/configuration/storage-config.md @@ -153,10 +153,14 @@ storage: When using managed identity or other token-based authentication (without an `access_key`), Quickwit automatically selects the correct OAuth token scope for Azure Government and Azure China endpoints. -For Azure US Government, you may also need to set the Entra authority host: +Depending on the national cloud, you may also need to set the Entra authority host: ```bash +# Azure US Government export AZURE_AUTHORITY_HOST=https://login.microsoftonline.us/ + +# Azure China +export AZURE_AUTHORITY_HOST=https://login.chinacloudapi.cn/ ``` Custom endpoints that are not public Azure, Azure Government, or Azure China (for example Azure Stack) require an `access_key` when using token-based authentication. diff --git a/quickwit/quickwit-config/src/storage_config.rs b/quickwit/quickwit-config/src/storage_config.rs index 99476362215..4eac41d4803 100644 --- a/quickwit/quickwit-config/src/storage_config.rs +++ b/quickwit/quickwit-config/src/storage_config.rs @@ -365,10 +365,10 @@ impl AzureStorageConfig { /// Classifies the Azure national cloud from the configured endpoint. pub fn resolve_national_cloud(&self) -> AzureNationalCloud { - if let Some(endpoint) = self.endpoint() { - if let Some(host) = extract_azure_endpoint_host(&endpoint) { - return classify_azure_endpoint_host(host); - } + if let Some(endpoint) = self.endpoint() + && let Some(host) = extract_azure_endpoint_host(&endpoint) + { + return classify_azure_endpoint_host(host); } if let Some(endpoint_suffix) = self.endpoint_suffix() { return classify_azure_endpoint_suffix(&endpoint_suffix); @@ -404,6 +404,22 @@ pub const AZURE_US_GOVERNMENT_STORAGE_TOKEN_SCOPE: &str = "https://storage.azure pub const AZURE_CHINA_STORAGE_TOKEN_SCOPE: &str = "https://storage.azure.cn/.default"; +const AZURE_PUBLIC_BLOB_SUFFIXES: &[&str] = &["core.windows.net", "blob.core.windows.net"]; + +const AZURE_US_GOVERNMENT_BLOB_SUFFIXES: &[&str] = + &["core.usgovcloudapi.net", "blob.core.usgovcloudapi.net"]; + +const AZURE_CHINA_BLOB_SUFFIXES: &[&str] = &["core.chinacloudapi.cn", "blob.core.chinacloudapi.cn"]; + +const AZURE_BLOB_HOST_SUFFIXES: &[(&str, AzureNationalCloud)] = &[ + (".blob.core.chinacloudapi.cn", AzureNationalCloud::China), + ( + ".blob.core.usgovcloudapi.net", + AzureNationalCloud::UsGovernment, + ), + (".blob.core.windows.net", AzureNationalCloud::Public), +]; + fn extract_azure_endpoint_host(endpoint: &str) -> Option<&str> { let endpoint = endpoint.trim(); let host = endpoint @@ -413,30 +429,53 @@ fn extract_azure_endpoint_host(endpoint: &str) -> Option<&str> { host.split('/').next().filter(|host| !host.is_empty()) } +fn strip_host_port(host: &str) -> &str { + if let Some(stripped_host) = host.strip_prefix('[') { + if let Some(bracket_end) = stripped_host.find(']') { + return &host[..bracket_end + 1]; + } + return host; + } + match host.rsplit_once(':') { + Some((host_without_port, port)) + if port.chars().all(|character| character.is_ascii_digit()) => + { + host_without_port + } + _ => host, + } +} + +fn national_cloud_from_exact_blob_suffix(blob_suffix: &str) -> Option { + let blob_suffix_lower = blob_suffix.trim().to_ascii_lowercase(); + if AZURE_CHINA_BLOB_SUFFIXES.contains(&blob_suffix_lower.as_str()) { + return Some(AzureNationalCloud::China); + } + if AZURE_US_GOVERNMENT_BLOB_SUFFIXES.contains(&blob_suffix_lower.as_str()) { + return Some(AzureNationalCloud::UsGovernment); + } + if AZURE_PUBLIC_BLOB_SUFFIXES.contains(&blob_suffix_lower.as_str()) { + return Some(AzureNationalCloud::Public); + } + None +} + fn classify_azure_endpoint_host(host: &str) -> AzureNationalCloud { - let host_lower = host.to_ascii_lowercase(); - if host_lower.contains("chinacloudapi.cn") { - AzureNationalCloud::China - } else if host_lower.contains("usgovcloudapi.net") { - AzureNationalCloud::UsGovernment - } else if host_lower.contains("core.windows.net") { - AzureNationalCloud::Public - } else { - AzureNationalCloud::Custom + let host_without_port = strip_host_port(host); + let host_lower = host_without_port.to_ascii_lowercase(); + if let Some(national_cloud) = national_cloud_from_exact_blob_suffix(&host_lower) { + return national_cloud; + } + for (host_suffix, national_cloud) in AZURE_BLOB_HOST_SUFFIXES { + if host_lower.ends_with(host_suffix) { + return *national_cloud; + } } + AzureNationalCloud::Custom } fn classify_azure_endpoint_suffix(endpoint_suffix: &str) -> AzureNationalCloud { - let endpoint_suffix_lower = endpoint_suffix.to_ascii_lowercase(); - if endpoint_suffix_lower.contains("chinacloudapi.cn") { - AzureNationalCloud::China - } else if endpoint_suffix_lower.contains("usgovcloudapi.net") { - AzureNationalCloud::UsGovernment - } else if endpoint_suffix_lower.contains("windows.net") { - AzureNationalCloud::Public - } else { - AzureNationalCloud::Custom - } + national_cloud_from_exact_blob_suffix(endpoint_suffix).unwrap_or(AzureNationalCloud::Custom) } impl fmt::Debug for AzureStorageConfig { @@ -843,6 +882,33 @@ mod tests { custom_config.resolve_national_cloud(), AzureNationalCloud::Custom ); + + let spoofed_gov_config = AzureStorageConfig { + endpoint: Some("https://blob.core.usgovcloudapi.net.example.com".to_string()), + ..Default::default() + }; + assert_eq!( + spoofed_gov_config.resolve_national_cloud(), + AzureNationalCloud::Custom + ); + + let gov_with_port_config = AzureStorageConfig { + endpoint: Some("https://my-account.blob.core.usgovcloudapi.net:443".to_string()), + ..Default::default() + }; + assert_eq!( + gov_with_port_config.resolve_national_cloud(), + AzureNationalCloud::UsGovernment + ); + + let invalid_suffix_config = AzureStorageConfig { + endpoint_suffix: Some("evil.windows.net".to_string()), + ..Default::default() + }; + assert_eq!( + invalid_suffix_config.resolve_national_cloud(), + AzureNationalCloud::Custom + ); } #[test] diff --git a/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs b/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs index c05bce6d189..17a15e2fec2 100644 --- a/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs +++ b/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs @@ -568,14 +568,20 @@ fn build_azure_token_credentials( azure_storage_config: &AzureStorageConfig, credential: Arc, ) -> Result { - match azure_storage_config.resolve_national_cloud() { + let national_cloud = azure_storage_config.resolve_national_cloud(); + match national_cloud { AzureNationalCloud::UsGovernment | AzureNationalCloud::China => { let token_scope = azure_storage_config .resolve_storage_token_scope() - .expect("sovereign cloud must resolve a storage token scope"); + .ok_or_else(|| { + StorageResolverError::InvalidConfig(format!( + "could not resolve Azure storage token scope for national cloud \ + `{national_cloud:?}`" + )) + })?; info!( token_scope = token_scope, - national_cloud = ?azure_storage_config.resolve_national_cloud(), + national_cloud = ?national_cloud, "using Azure storage token scope for sovereign cloud" ); Ok(StorageCredentials::token_credential(Arc::new( @@ -786,4 +792,28 @@ mod tests { assert!(matches!(error, StorageResolverError::InvalidConfig(_))); } + + #[test] + fn test_build_azure_token_credentials_accepts_azure_government_endpoint() { + let azure_storage_config = AzureStorageConfig { + endpoint_suffix: Some("core.usgovcloudapi.net".to_string()), + ..Default::default() + }; + let credential = Arc::new(MockTokenCredential) as Arc; + + build_azure_token_credentials(&azure_storage_config, credential) + .expect("Azure Government endpoint with token auth should succeed"); + } + + #[test] + fn test_build_azure_token_credentials_accepts_azure_china_endpoint() { + let azure_storage_config = AzureStorageConfig { + endpoint_suffix: Some("core.chinacloudapi.cn".to_string()), + ..Default::default() + }; + let credential = Arc::new(MockTokenCredential) as Arc; + + build_azure_token_credentials(&azure_storage_config, credential) + .expect("Azure China endpoint with token auth should succeed"); + } } From fcb137a86061573b31faf9eb1a3a7054f9c253df Mon Sep 17 00:00:00 2001 From: Deenkar Mahulkar <11814351+deenkar@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:50:29 +0530 Subject: [PATCH 3/7] fix: keep public Azure Storage token audience for sovereign clouds Microsoft documents https://storage.azure.com/ as the all-account resource ID for Global, Government, and China. Remove ScopedTokenCredential overrides that requested invalid storage.azure.us/.cn audiences. Keep sovereign endpoint classification, fail-fast for unknown custom endpoints, and AZURE_AUTHORITY_HOST documentation. Co-authored-by: Cursor --- CHANGELOG.md | 2 +- docs/configuration/storage-config.md | 4 +- quickwit/Cargo.lock | 1 - .../quickwit-config/src/storage_config.rs | 38 +----- quickwit/quickwit-storage/Cargo.toml | 1 - .../src/object_storage/azure_blob_storage.rs | 61 ++++------ .../object_storage/azure_token_credential.rs | 110 ------------------ .../src/object_storage/mod.rs | 2 - 8 files changed, 27 insertions(+), 192 deletions(-) delete mode 100644 quickwit/quickwit-storage/src/object_storage/azure_token_credential.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index e754bb1e584..36575744c17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Azure Blob Storage: support custom endpoints via `endpoint` and `endpoint_suffix` configuration options for sovereign clouds (#6624) ### Fixed -- Azure Blob Storage: use sovereign-cloud OAuth token scopes for managed identity on Azure Government and Azure China endpoints (#6624) +- Azure Blob Storage: reject managed identity on unmapped custom endpoints and document sovereign-cloud `AZURE_AUTHORITY_HOST` requirements (#6624) - (Jaeger) Query resource attributes when Jaeger request carries tags ### Changed diff --git a/docs/configuration/storage-config.md b/docs/configuration/storage-config.md index 17caeb8652c..2a02753fd5e 100644 --- a/docs/configuration/storage-config.md +++ b/docs/configuration/storage-config.md @@ -151,9 +151,9 @@ storage: #### Managed identity on sovereign clouds -When using managed identity or other token-based authentication (without an `access_key`), Quickwit automatically selects the correct OAuth token scope for Azure Government and Azure China endpoints. +When using managed identity or other token-based authentication (without an `access_key`) against Azure Government or Azure China, Quickwit uses the standard Azure Storage resource ID (`https://storage.azure.com/`) for OAuth tokens, together with the sovereign blob endpoint configured via `endpoint` or `endpoint_suffix`. -Depending on the national cloud, you may also need to set the Entra authority host: +Depending on the national cloud, you must set the Entra authority host so tokens are acquired from the correct login endpoint: ```bash # Azure US Government diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index 1b28ef0158d..2b3e7240524 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -9438,7 +9438,6 @@ dependencies = [ "tantivy", "tempfile", "thiserror 2.0.18", - "time", "tokio", "tokio-rustls 0.26.4", "tokio-stream", diff --git a/quickwit/quickwit-config/src/storage_config.rs b/quickwit/quickwit-config/src/storage_config.rs index 4eac41d4803..3c3d87614d4 100644 --- a/quickwit/quickwit-config/src/storage_config.rs +++ b/quickwit/quickwit-config/src/storage_config.rs @@ -375,21 +375,9 @@ impl AzureStorageConfig { } AzureNationalCloud::Public } - - /// Returns the OAuth token scope for Azure Storage in the configured national cloud. - /// - /// Returns `None` for custom endpoints that do not map to a known national cloud. - pub fn resolve_storage_token_scope(&self) -> Option<&'static str> { - match self.resolve_national_cloud() { - AzureNationalCloud::Public => Some(AZURE_PUBLIC_STORAGE_TOKEN_SCOPE), - AzureNationalCloud::UsGovernment => Some(AZURE_US_GOVERNMENT_STORAGE_TOKEN_SCOPE), - AzureNationalCloud::China => Some(AZURE_CHINA_STORAGE_TOKEN_SCOPE), - AzureNationalCloud::Custom => None, - } - } } -/// Azure national cloud classification for token authentication. +/// Azure national cloud classification derived from the configured blob endpoint. #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub enum AzureNationalCloud { Public, @@ -398,12 +386,6 @@ pub enum AzureNationalCloud { Custom, } -pub const AZURE_PUBLIC_STORAGE_TOKEN_SCOPE: &str = "https://storage.azure.com/.default"; - -pub const AZURE_US_GOVERNMENT_STORAGE_TOKEN_SCOPE: &str = "https://storage.azure.us/.default"; - -pub const AZURE_CHINA_STORAGE_TOKEN_SCOPE: &str = "https://storage.azure.cn/.default"; - const AZURE_PUBLIC_BLOB_SUFFIXES: &[&str] = &["core.windows.net", "blob.core.windows.net"]; const AZURE_US_GOVERNMENT_BLOB_SUFFIXES: &[&str] = @@ -911,24 +893,6 @@ mod tests { ); } - #[test] - fn test_storage_azure_config_resolve_storage_token_scope() { - let gov_config = AzureStorageConfig { - endpoint_suffix: Some("core.usgovcloudapi.net".to_string()), - ..Default::default() - }; - assert_eq!( - gov_config.resolve_storage_token_scope(), - Some(AZURE_US_GOVERNMENT_STORAGE_TOKEN_SCOPE) - ); - - let custom_config = AzureStorageConfig { - endpoint: Some("https://storage.example.com".to_string()), - ..Default::default() - }; - assert!(custom_config.resolve_storage_token_scope().is_none()); - } - #[test] fn test_storage_google_config_serde() { { diff --git a/quickwit/quickwit-storage/Cargo.toml b/quickwit/quickwit-storage/Cargo.toml index 61b61392397..1c377eaacd5 100644 --- a/quickwit/quickwit-storage/Cargo.toml +++ b/quickwit/quickwit-storage/Cargo.toml @@ -75,7 +75,6 @@ aws-sdk-s3 = { workspace = true } aws-smithy-runtime = { workspace = true, features = ["test-util"] } quickwit-common = { workspace = true, features = ["testsuite"] } -time = { workspace = true } [features] azure = [ diff --git a/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs b/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs index 17a15e2fec2..27c4536372e 100644 --- a/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs +++ b/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs @@ -46,7 +46,6 @@ use tracing::{info, instrument, warn}; use crate::debouncer::DebouncedStorage; use crate::metrics::object_storage_get_slice_in_flight_guards; -use crate::object_storage::azure_token_credential::ScopedTokenCredential; use crate::stable_deref_bytes::into_owned_bytes; use crate::storage::SendableAsync; use crate::{ @@ -569,37 +568,24 @@ fn build_azure_token_credentials( credential: Arc, ) -> Result { let national_cloud = azure_storage_config.resolve_national_cloud(); - match national_cloud { - AzureNationalCloud::UsGovernment | AzureNationalCloud::China => { - let token_scope = azure_storage_config - .resolve_storage_token_scope() - .ok_or_else(|| { - StorageResolverError::InvalidConfig(format!( - "could not resolve Azure storage token scope for national cloud \ - `{national_cloud:?}`" - )) - })?; - info!( - token_scope = token_scope, - national_cloud = ?national_cloud, - "using Azure storage token scope for sovereign cloud" - ); - Ok(StorageCredentials::token_credential(Arc::new( - ScopedTokenCredential::new(credential, token_scope), - ))) - } - AzureNationalCloud::Custom if azure_storage_config.uses_custom_blob_endpoint() => { - Err(StorageResolverError::InvalidConfig( - "custom Azure blob endpoints require an access key when using token-based \ - authentication; managed identity is only supported for public Azure, Azure \ - Government, and Azure China endpoints" - .to_string(), - )) - } - AzureNationalCloud::Public | AzureNationalCloud::Custom => { - Ok(StorageCredentials::token_credential(credential)) - } + if national_cloud == AzureNationalCloud::Custom + && azure_storage_config.uses_custom_blob_endpoint() + { + return Err(StorageResolverError::InvalidConfig( + "custom Azure blob endpoints require an access key when using token-based \ + authentication; managed identity is only supported for public Azure, Azure \ + Government, and Azure China endpoints" + .to_string(), + )); + } + if national_cloud != AzureNationalCloud::Public { + info!( + national_cloud = ?national_cloud, + "using Azure blob storage endpoint for sovereign cloud; ensure AZURE_AUTHORITY_HOST \ + is configured for token acquisition when using managed identity" + ); } + Ok(StorageCredentials::token_credential(credential)) } fn build_container_client( @@ -731,10 +717,9 @@ impl From for StorageError { mod tests { use std::sync::Arc; - use azure_core::auth::{AccessToken, Secret, TokenCredential}; + use azure_core::auth::TokenCredential; use quickwit_common::uri::Uri; use quickwit_config::AzureStorageConfig; - use time::OffsetDateTime; use crate::StorageResolverError; use crate::object_storage::azure_blob_storage::{ @@ -747,11 +732,11 @@ mod tests { #[cfg_attr(target_arch = "wasm32", async_trait::async_trait(?Send))] #[cfg_attr(not(target_arch = "wasm32"), async_trait::async_trait)] impl TokenCredential for MockTokenCredential { - async fn get_token(&self, _scopes: &[&str]) -> azure_core::Result { - Ok(AccessToken::new( - Secret::new("mock-token"), - OffsetDateTime::now_utc(), - )) + async fn get_token( + &self, + _scopes: &[&str], + ) -> azure_core::Result { + unimplemented!("mock credential should not request tokens in these unit tests") } async fn clear_cache(&self) -> azure_core::Result<()> { diff --git a/quickwit/quickwit-storage/src/object_storage/azure_token_credential.rs b/quickwit/quickwit-storage/src/object_storage/azure_token_credential.rs deleted file mode 100644 index d2537b2227b..00000000000 --- a/quickwit/quickwit-storage/src/object_storage/azure_token_credential.rs +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2021-Present Datadog, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::fmt; -use std::sync::Arc; - -use azure_core::auth::{AccessToken, TokenCredential}; - -/// Wraps a [`TokenCredential`] and requests tokens for a fixed OAuth scope. -/// -/// The legacy `azure_storage` 0.21 SDK hardcodes the public-cloud storage scope when using token -/// credentials. Sovereign clouds require a different audience, so this wrapper ignores the scope -/// requested by the SDK and uses the configured national-cloud scope instead. -#[derive(Clone)] -pub struct ScopedTokenCredential { - inner: Arc, - scope: &'static str, -} - -impl ScopedTokenCredential { - pub fn new(inner: Arc, scope: &'static str) -> Self { - Self { inner, scope } - } -} - -impl fmt::Debug for ScopedTokenCredential { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("ScopedTokenCredential") - .field("inner", &self.inner) - .field("scope", &self.scope) - .finish() - } -} - -#[cfg_attr(target_arch = "wasm32", async_trait::async_trait(?Send))] -#[cfg_attr(not(target_arch = "wasm32"), async_trait::async_trait)] -impl TokenCredential for ScopedTokenCredential { - async fn get_token(&self, _scopes: &[&str]) -> azure_core::Result { - self.inner.get_token(&[self.scope]).await - } - - async fn clear_cache(&self) -> azure_core::Result<()> { - self.inner.clear_cache().await - } -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use azure_core::auth::{AccessToken, Secret, TokenCredential}; - use time::OffsetDateTime; - - use super::ScopedTokenCredential; - - #[derive(Debug)] - struct MockTokenCredential { - requested_scopes: Arc>>, - } - - #[cfg_attr(target_arch = "wasm32", async_trait::async_trait(?Send))] - #[cfg_attr(not(target_arch = "wasm32"), async_trait::async_trait)] - impl TokenCredential for MockTokenCredential { - async fn get_token(&self, scopes: &[&str]) -> azure_core::Result { - self.requested_scopes - .lock() - .expect("lock poisoned") - .extend(scopes.iter().map(|scope| (*scope).to_string())); - Ok(AccessToken::new( - Secret::new("mock-token"), - OffsetDateTime::now_utc(), - )) - } - - async fn clear_cache(&self) -> azure_core::Result<()> { - Ok(()) - } - } - - #[tokio::test] - async fn test_scoped_token_credential_uses_configured_scope() { - let requested_scopes = Arc::new(std::sync::Mutex::new(Vec::new())); - let inner = Arc::new(MockTokenCredential { - requested_scopes: requested_scopes.clone(), - }) as Arc; - let credential = ScopedTokenCredential::new(inner, "https://storage.azure.us/.default"); - - let _token = credential - .get_token(&["https://storage.azure.com/.default"]) - .await - .expect("token request should succeed"); - - let requested_scopes = requested_scopes.lock().expect("lock poisoned"); - assert_eq!( - requested_scopes.as_slice(), - &["https://storage.azure.us/.default".to_string()] - ); - } -} diff --git a/quickwit/quickwit-storage/src/object_storage/mod.rs b/quickwit/quickwit-storage/src/object_storage/mod.rs index 0f9e8a5ff08..e914c107291 100644 --- a/quickwit/quickwit-storage/src/object_storage/mod.rs +++ b/quickwit/quickwit-storage/src/object_storage/mod.rs @@ -26,6 +26,4 @@ mod s3_compatible_storage_resolver; #[cfg(feature = "azure")] mod azure_blob_storage; #[cfg(feature = "azure")] -mod azure_token_credential; -#[cfg(feature = "azure")] pub use self::azure_blob_storage::{AzureBlobStorage, AzureBlobStorageFactory}; From 0344ca50dba6047654cab15222e4ea0a4c775082 Mon Sep 17 00:00:00 2001 From: Deenkar Mahulkar <11814351+deenkar@users.noreply.github.com> Date: Thu, 6 Aug 2026 21:26:03 +0530 Subject: [PATCH 4/7] fix: harden Azure endpoint classification for URL parsing and DNS zones Parse blob endpoints with the url crate so query strings cannot spoof sovereign suffixes, and recognize public *.blob.storage.azure.net hosts. Co-authored-by: Cursor --- quickwit/Cargo.lock | 1 + quickwit/quickwit-config/Cargo.toml | 1 + .../quickwit-config/src/storage_config.rs | 41 +++++++++++++++---- 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index 2b3e7240524..0cdaeb3004c 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -8583,6 +8583,7 @@ dependencies = [ "tokio", "toml", "tracing", + "url", "utoipa", "vrl", ] diff --git a/quickwit/quickwit-config/Cargo.toml b/quickwit/quickwit-config/Cargo.toml index 44cae30260c..9a1dad75f3d 100644 --- a/quickwit/quickwit-config/Cargo.toml +++ b/quickwit/quickwit-config/Cargo.toml @@ -31,6 +31,7 @@ serde_yaml = { workspace = true } siphasher = { workspace = true } toml = { workspace = true } tracing = { workspace = true } +url = "2" utoipa = { workspace = true } vrl = { workspace = true, optional = true } diff --git a/quickwit/quickwit-config/src/storage_config.rs b/quickwit/quickwit-config/src/storage_config.rs index 3c3d87614d4..91548153f6b 100644 --- a/quickwit/quickwit-config/src/storage_config.rs +++ b/quickwit/quickwit-config/src/storage_config.rs @@ -368,7 +368,7 @@ impl AzureStorageConfig { if let Some(endpoint) = self.endpoint() && let Some(host) = extract_azure_endpoint_host(&endpoint) { - return classify_azure_endpoint_host(host); + return classify_azure_endpoint_host(&host); } if let Some(endpoint_suffix) = self.endpoint_suffix() { return classify_azure_endpoint_suffix(&endpoint_suffix); @@ -386,7 +386,11 @@ pub enum AzureNationalCloud { Custom, } -const AZURE_PUBLIC_BLOB_SUFFIXES: &[&str] = &["core.windows.net", "blob.core.windows.net"]; +const AZURE_PUBLIC_BLOB_SUFFIXES: &[&str] = &[ + "core.windows.net", + "blob.core.windows.net", + "blob.storage.azure.net", +]; const AZURE_US_GOVERNMENT_BLOB_SUFFIXES: &[&str] = &["core.usgovcloudapi.net", "blob.core.usgovcloudapi.net"]; @@ -399,16 +403,17 @@ const AZURE_BLOB_HOST_SUFFIXES: &[(&str, AzureNationalCloud)] = &[ ".blob.core.usgovcloudapi.net", AzureNationalCloud::UsGovernment, ), + (".blob.storage.azure.net", AzureNationalCloud::Public), (".blob.core.windows.net", AzureNationalCloud::Public), ]; -fn extract_azure_endpoint_host(endpoint: &str) -> Option<&str> { +fn extract_azure_endpoint_host(endpoint: &str) -> Option { let endpoint = endpoint.trim(); - let host = endpoint - .strip_prefix("https://") - .or_else(|| endpoint.strip_prefix("http://")) - .unwrap_or(endpoint); - host.split('/').next().filter(|host| !host.is_empty()) + let parsed_url = url::Url::parse(endpoint).ok()?; + if !parsed_url.username().is_empty() || parsed_url.password().is_some() { + return None; + } + parsed_url.host_str().map(str::to_string) } fn strip_host_port(host: &str) -> &str { @@ -891,6 +896,26 @@ mod tests { invalid_suffix_config.resolve_national_cloud(), AzureNationalCloud::Custom ); + + let query_spoof_config = AzureStorageConfig { + endpoint: Some( + "https://storage.example.com?x=.blob.core.usgovcloudapi.net".to_string(), + ), + ..Default::default() + }; + assert_eq!( + query_spoof_config.resolve_national_cloud(), + AzureNationalCloud::Custom + ); + + let dns_zone_config = AzureStorageConfig { + endpoint: Some("https://myaccount.z18.blob.storage.azure.net".to_string()), + ..Default::default() + }; + assert_eq!( + dns_zone_config.resolve_national_cloud(), + AzureNationalCloud::Public + ); } #[test] From ac1e2b23373167f766b41fabf9f5eb26bfa79932 Mon Sep 17 00:00:00 2001 From: Deenkar Mahulkar <11814351+deenkar@users.noreply.github.com> Date: Thu, 6 Aug 2026 22:35:07 +0530 Subject: [PATCH 5/7] fix: classify unparseable Azure endpoints as custom When endpoint is configured but host extraction fails, return Custom instead of falling through to endpoint_suffix or Public so managed identity cannot be used against unrecognized hosts. Co-authored-by: Cursor --- .../quickwit-config/src/storage_config.rs | 29 ++++++++++++++++--- .../src/object_storage/azure_blob_storage.rs | 15 ++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/quickwit/quickwit-config/src/storage_config.rs b/quickwit/quickwit-config/src/storage_config.rs index 91548153f6b..7181ee526f4 100644 --- a/quickwit/quickwit-config/src/storage_config.rs +++ b/quickwit/quickwit-config/src/storage_config.rs @@ -365,10 +365,11 @@ impl AzureStorageConfig { /// Classifies the Azure national cloud from the configured endpoint. pub fn resolve_national_cloud(&self) -> AzureNationalCloud { - if let Some(endpoint) = self.endpoint() - && let Some(host) = extract_azure_endpoint_host(&endpoint) - { - return classify_azure_endpoint_host(&host); + if let Some(endpoint) = self.endpoint() { + if let Some(host) = extract_azure_endpoint_host(&endpoint) { + return classify_azure_endpoint_host(&host); + } + return AzureNationalCloud::Custom; } if let Some(endpoint_suffix) = self.endpoint_suffix() { return classify_azure_endpoint_suffix(&endpoint_suffix); @@ -916,6 +917,26 @@ mod tests { dns_zone_config.resolve_national_cloud(), AzureNationalCloud::Public ); + + let userinfo_with_gov_suffix_config = AzureStorageConfig { + endpoint: Some("https://user@storage.example.com".to_string()), + endpoint_suffix: Some("core.usgovcloudapi.net".to_string()), + ..Default::default() + }; + assert_eq!( + userinfo_with_gov_suffix_config.resolve_national_cloud(), + AzureNationalCloud::Custom + ); + + let unparseable_endpoint_config = AzureStorageConfig { + endpoint: Some("not-a-valid-url".to_string()), + endpoint_suffix: Some("core.usgovcloudapi.net".to_string()), + ..Default::default() + }; + assert_eq!( + unparseable_endpoint_config.resolve_national_cloud(), + AzureNationalCloud::Custom + ); } #[test] diff --git a/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs b/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs index 27c4536372e..1a1055e19fa 100644 --- a/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs +++ b/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs @@ -778,6 +778,21 @@ mod tests { assert!(matches!(error, StorageResolverError::InvalidConfig(_))); } + #[test] + fn test_build_azure_token_credentials_rejects_unparseable_endpoint() { + let azure_storage_config = AzureStorageConfig { + endpoint: Some("https://user@storage.example.com".to_string()), + endpoint_suffix: Some("core.usgovcloudapi.net".to_string()), + ..Default::default() + }; + let credential = Arc::new(MockTokenCredential) as Arc; + + let error = build_azure_token_credentials(&azure_storage_config, credential) + .expect_err("unparseable endpoint with token auth should fail"); + + assert!(matches!(error, StorageResolverError::InvalidConfig(_))); + } + #[test] fn test_build_azure_token_credentials_accepts_azure_government_endpoint() { let azure_storage_config = AzureStorageConfig { From 4a9f3ab99b92f3a160dd5d28719bd1a8f9bb58f8 Mon Sep 17 00:00:00 2001 From: Deenkar Mahulkar <11814351+deenkar@users.noreply.github.com> Date: Tue, 11 Aug 2026 13:41:44 +0530 Subject: [PATCH 6/7] fix: reject HTTP Azure blob endpoints for token authentication Require HTTPS in extract_azure_endpoint_host so HTTP endpoints classify as Custom and managed identity cannot send bearer tokens over plaintext. Co-authored-by: Cursor --- .../quickwit-config/src/storage_config.rs | 21 +++++++++++++++++++ .../src/object_storage/azure_blob_storage.rs | 14 +++++++++++++ 2 files changed, 35 insertions(+) diff --git a/quickwit/quickwit-config/src/storage_config.rs b/quickwit/quickwit-config/src/storage_config.rs index 7181ee526f4..de830e31480 100644 --- a/quickwit/quickwit-config/src/storage_config.rs +++ b/quickwit/quickwit-config/src/storage_config.rs @@ -411,6 +411,9 @@ const AZURE_BLOB_HOST_SUFFIXES: &[(&str, AzureNationalCloud)] = &[ fn extract_azure_endpoint_host(endpoint: &str) -> Option { let endpoint = endpoint.trim(); let parsed_url = url::Url::parse(endpoint).ok()?; + if parsed_url.scheme() != "https" { + return None; + } if !parsed_url.username().is_empty() || parsed_url.password().is_some() { return None; } @@ -937,6 +940,24 @@ mod tests { unparseable_endpoint_config.resolve_national_cloud(), AzureNationalCloud::Custom ); + + let http_public_endpoint_config = AzureStorageConfig { + endpoint: Some("http://my-account.blob.core.windows.net".to_string()), + ..Default::default() + }; + assert_eq!( + http_public_endpoint_config.resolve_national_cloud(), + AzureNationalCloud::Custom + ); + + let http_gov_endpoint_config = AzureStorageConfig { + endpoint: Some("http://my-account.blob.core.usgovcloudapi.net".to_string()), + ..Default::default() + }; + assert_eq!( + http_gov_endpoint_config.resolve_national_cloud(), + AzureNationalCloud::Custom + ); } #[test] diff --git a/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs b/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs index 1a1055e19fa..5e1ffc3031d 100644 --- a/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs +++ b/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs @@ -793,6 +793,20 @@ mod tests { assert!(matches!(error, StorageResolverError::InvalidConfig(_))); } + #[test] + fn test_build_azure_token_credentials_rejects_http_azure_endpoint() { + let azure_storage_config = AzureStorageConfig { + endpoint: Some("http://my-account.blob.core.windows.net".to_string()), + ..Default::default() + }; + let credential = Arc::new(MockTokenCredential) as Arc; + + let error = build_azure_token_credentials(&azure_storage_config, credential) + .expect_err("HTTP Azure endpoint with token auth should fail"); + + assert!(matches!(error, StorageResolverError::InvalidConfig(_))); + } + #[test] fn test_build_azure_token_credentials_accepts_azure_government_endpoint() { let azure_storage_config = AzureStorageConfig { From 8fcab6873c484deeacc289efef7f41832f1a775b Mon Sep 17 00:00:00 2001 From: Deenkar Mahulkar <11814351+deenkar@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:08:01 +0530 Subject: [PATCH 7/7] refactor: enforce HTTPS for Azure token auth at credential boundary Keep host extraction scheme-agnostic so HTTP Azure endpoints classify correctly, and reject managed identity with an explicit HTTPS error in build_azure_token_credentials via endpoint_uses_non_https_transport(). Co-authored-by: Cursor --- .../quickwit-config/src/storage_config.rs | 30 +++++++++++++++---- .../src/object_storage/azure_blob_storage.rs | 7 ++++- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/quickwit/quickwit-config/src/storage_config.rs b/quickwit/quickwit-config/src/storage_config.rs index de830e31480..c7a6fdd7d2f 100644 --- a/quickwit/quickwit-config/src/storage_config.rs +++ b/quickwit/quickwit-config/src/storage_config.rs @@ -376,6 +376,15 @@ impl AzureStorageConfig { } AzureNationalCloud::Public } + + /// Returns `true` when a configured blob `endpoint` URL does not use HTTPS. + pub fn endpoint_uses_non_https_transport(&self) -> bool { + self.endpoint().is_some_and(|endpoint| { + url::Url::parse(endpoint.trim()) + .map(|parsed_url| !parsed_url.scheme().eq_ignore_ascii_case("https")) + .unwrap_or(false) + }) + } } /// Azure national cloud classification derived from the configured blob endpoint. @@ -411,9 +420,6 @@ const AZURE_BLOB_HOST_SUFFIXES: &[(&str, AzureNationalCloud)] = &[ fn extract_azure_endpoint_host(endpoint: &str) -> Option { let endpoint = endpoint.trim(); let parsed_url = url::Url::parse(endpoint).ok()?; - if parsed_url.scheme() != "https" { - return None; - } if !parsed_url.username().is_empty() || parsed_url.password().is_some() { return None; } @@ -947,8 +953,9 @@ mod tests { }; assert_eq!( http_public_endpoint_config.resolve_national_cloud(), - AzureNationalCloud::Custom + AzureNationalCloud::Public ); + assert!(http_public_endpoint_config.endpoint_uses_non_https_transport()); let http_gov_endpoint_config = AzureStorageConfig { endpoint: Some("http://my-account.blob.core.usgovcloudapi.net".to_string()), @@ -956,8 +963,21 @@ mod tests { }; assert_eq!( http_gov_endpoint_config.resolve_national_cloud(), - AzureNationalCloud::Custom + AzureNationalCloud::UsGovernment ); + assert!(http_gov_endpoint_config.endpoint_uses_non_https_transport()); + + let https_public_endpoint_config = AzureStorageConfig { + endpoint: Some("https://my-account.blob.core.windows.net".to_string()), + ..Default::default() + }; + assert!(!https_public_endpoint_config.endpoint_uses_non_https_transport()); + + let uppercase_https_endpoint_config = AzureStorageConfig { + endpoint: Some("HTTPS://my-account.blob.core.windows.net".to_string()), + ..Default::default() + }; + assert!(!uppercase_https_endpoint_config.endpoint_uses_non_https_transport()); } #[test] diff --git a/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs b/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs index 5e1ffc3031d..452b447da72 100644 --- a/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs +++ b/quickwit/quickwit-storage/src/object_storage/azure_blob_storage.rs @@ -567,6 +567,11 @@ fn build_azure_token_credentials( azure_storage_config: &AzureStorageConfig, credential: Arc, ) -> Result { + if azure_storage_config.endpoint_uses_non_https_transport() { + return Err(StorageResolverError::InvalidConfig( + "Azure token credentials require an HTTPS endpoint".to_string(), + )); + } let national_cloud = azure_storage_config.resolve_national_cloud(); if national_cloud == AzureNationalCloud::Custom && azure_storage_config.uses_custom_blob_endpoint() @@ -804,7 +809,7 @@ mod tests { let error = build_azure_token_credentials(&azure_storage_config, credential) .expect_err("HTTP Azure endpoint with token auth should fail"); - assert!(matches!(error, StorageResolverError::InvalidConfig(_))); + assert!(matches!(error, StorageResolverError::InvalidConfig(message) if message.contains("HTTPS"))); } #[test]