From 3dd157c3cd27997e99cf1841f8f32c7ee252ba2c Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Thu, 22 Oct 2026 10:38:01 +0200 Subject: [PATCH 1/2] Cherry-pick of https://github.com/Altinity/ClickHouse/pull/2329 with unresolved conflict markers (resolution in next commit) --- Original cherry-pick message follows: Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake Antalya 26.6: Forward auth tokens to data lake catalogs # Conflicts: # docs/reference/engines/database-engines/datalake.mdx # src/Access/AccessControl.h # src/Common/ErrorCodes.cpp # src/Core/ServerSettings.cpp # src/Databases/DataLake/DatabaseDataLake.cpp # src/Databases/DataLake/DatabaseDataLakeSettings.cpp # src/Databases/DataLake/GlueCatalog.cpp # src/Databases/DataLake/GlueCatalog.h # src/Databases/DataLake/HiveCatalog.cpp # src/Databases/DataLake/HiveCatalog.h # src/Databases/DataLake/ICatalog.cpp # src/Databases/DataLake/ICatalog.h # src/Databases/DataLake/PaimonRestCatalog.cpp # src/Databases/DataLake/PaimonRestCatalog.h # src/Databases/DataLake/RestCatalog.cpp # src/Databases/DataLake/RestCatalog.h # src/Databases/DataLake/S3TablesCatalog.cpp # src/Databases/DataLake/S3TablesCatalog.h # src/Databases/DataLake/UnityCatalog.cpp # src/Databases/DataLake/UnityCatalog.h # src/Databases/DataLake/tests/gtest_rest_catalog.cpp # src/IO/S3/Credentials.cpp # src/IO/S3/tests/TestPocoHTTPServer.h # src/Interpreters/Session.cpp # src/Server/TCPHandler.cpp # src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp # src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp --- .../external-authenticators/tokens.md | 33 + .../engines/database-engines/datalake.mdx | 130 ++++ src/Access/AccessControl.cpp | 12 + src/Access/AccessControl.h | 7 + src/Access/ForwardedAuthToken.cpp | 18 + src/Access/ForwardedAuthToken.h | 25 + src/Common/CurrentMetrics.cpp | 2 + src/Common/ErrorCodes.cpp | 5 + src/Common/FormUrlEncode.cpp | 15 + src/Common/FormUrlEncode.h | 11 + src/Common/ProfileEvents.cpp | 8 + src/Core/ServerSettings.cpp | 17 + src/Databases/DataLake/Common.cpp | 9 + src/Databases/DataLake/Common.h | 3 + src/Databases/DataLake/DatabaseDataLake.cpp | 202 ++++- src/Databases/DataLake/DatabaseDataLake.h | 9 +- .../DataLake/DatabaseDataLakeSettings.cpp | 13 + src/Databases/DataLake/GlueCatalog.cpp | 295 +++++-- src/Databases/DataLake/GlueCatalog.h | 77 +- src/Databases/DataLake/HiveCatalog.cpp | 14 +- src/Databases/DataLake/HiveCatalog.h | 14 +- src/Databases/DataLake/ICatalog.cpp | 65 +- src/Databases/DataLake/ICatalog.h | 61 +- src/Databases/DataLake/PaimonRestCatalog.cpp | 14 +- src/Databases/DataLake/PaimonRestCatalog.h | 14 +- src/Databases/DataLake/RestCatalog.cpp | 727 +++++++++++++++--- src/Databases/DataLake/RestCatalog.h | 217 +++++- src/Databases/DataLake/S3TablesCatalog.cpp | 53 +- src/Databases/DataLake/S3TablesCatalog.h | 21 + src/Databases/DataLake/UnityCatalog.cpp | 20 +- src/Databases/DataLake/UnityCatalog.h | 20 +- .../DataLake/tests/gtest_rest_catalog.cpp | 119 ++- .../gtest_rest_catalog_token_forwarding.cpp | 379 +++++++++ .../DataLake/tests/rest_catalog_test_server.h | 199 +++++ src/Databases/IDatabase.h | 4 +- src/IO/S3/Credentials.cpp | 111 ++- src/IO/S3/Credentials.h | 54 +- src/IO/S3/tests/TestPocoHTTPServer.h | 33 +- ...test_sts_assume_role_with_web_identity.cpp | 91 +++ .../Access/InterpreterExecuteAsQuery.cpp | 3 + src/Interpreters/AsynchronousInsertQueue.cpp | 12 + src/Interpreters/AsynchronousInsertQueue.h | 12 +- src/Interpreters/Context.cpp | 6 + src/Interpreters/Context.h | 5 + src/Interpreters/InterpreterCheckQuery.cpp | 2 +- src/Interpreters/InterpreterCreateQuery.cpp | 2 +- src/Interpreters/Session.cpp | 22 + src/Interpreters/Session.h | 3 + .../tests/gtest_async_insert_key.cpp | 40 +- src/Server/TCPHandler.cpp | 39 +- .../DataLakes/Iceberg/Compaction.cpp | 2 +- .../Iceberg/ExpireSnapshotsExecute.cpp | 2 +- .../DataLakes/Iceberg/IcebergMetadata.cpp | 12 +- .../DataLakes/Iceberg/IcebergWrites.cpp | 2 +- .../DataLakes/Iceberg/Mutations.cpp | 12 +- .../ObjectStorage/StorageObjectStorage.cpp | 6 +- .../ObjectStorage/StorageObjectStorage.h | 3 + .../__init__.py | 0 .../configs/token_forwarding.xml | 25 + .../configs/users.xml | 10 + .../s3_mocks/mock_sts.py | 73 ++ .../test.py | 155 ++++ .../__init__.py | 0 .../configs/token_forwarding.xml | 22 + .../configs/users.xml | 15 + .../test_datalake_token_forwarding/test.py | 226 ++++++ 66 files changed, 3504 insertions(+), 328 deletions(-) create mode 100644 src/Access/ForwardedAuthToken.cpp create mode 100644 src/Access/ForwardedAuthToken.h create mode 100644 src/Common/FormUrlEncode.cpp create mode 100644 src/Common/FormUrlEncode.h create mode 100644 src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp create mode 100644 src/Databases/DataLake/tests/rest_catalog_test_server.h create mode 100644 src/IO/S3/tests/gtest_sts_assume_role_with_web_identity.cpp create mode 100644 tests/integration/test_datalake_glue_token_forwarding/__init__.py create mode 100644 tests/integration/test_datalake_glue_token_forwarding/configs/token_forwarding.xml create mode 100644 tests/integration/test_datalake_glue_token_forwarding/configs/users.xml create mode 100644 tests/integration/test_datalake_glue_token_forwarding/s3_mocks/mock_sts.py create mode 100644 tests/integration/test_datalake_glue_token_forwarding/test.py create mode 100644 tests/integration/test_datalake_token_forwarding/__init__.py create mode 100644 tests/integration/test_datalake_token_forwarding/configs/token_forwarding.xml create mode 100644 tests/integration/test_datalake_token_forwarding/configs/users.xml create mode 100644 tests/integration/test_datalake_token_forwarding/test.py diff --git a/docs/concepts/features/security/external-authenticators/tokens.md b/docs/concepts/features/security/external-authenticators/tokens.md index 4bc2151c24986..d1c4eda9c6e9e 100644 --- a/docs/concepts/features/security/external-authenticators/tokens.md +++ b/docs/concepts/features/security/external-authenticators/tokens.md @@ -300,6 +300,39 @@ To reduce number of requests to IdP, tokens are cached internally for a maximum If token expires sooner than `token_cache_lifetime`, then cache entry for this token will only be valid while token is valid. If token lifetime is longer than `token_cache_lifetime`, cache entry for this token will be valid for `token_cache_lifetime`. +## Forwarding the token to external services {#token-forwarding} + +By default, the authenticated token is not retained in the session for forwarding. Set +`enable_token_forwarding` to `1` in `config.xml` to retain it for external-service authentication: + +```xml +1 +``` + +The setting is hot-reloadable and defaults to `false`. To use it with Iceberg REST or Glue, enable +`oauth_forward_user_token` on the [`DataLakeCatalog`](/engines/database-engines/datalakecatalog) +database. See [catalog token forwarding](/engines/database-engines/datalakecatalog#user-token-forwarding) +for configuration and examples. + +:::danger Restrict `CREATE DATABASE` +Database creators choose the endpoints that receive users' tokens. Grant `CREATE DATABASE` only +to trusted users and restrict `remote_url_allow_hosts` for catalog and token-exchange endpoints. +::: + +The forwarded token is sent to the catalog and token-exchange endpoints and, in passthrough mode, is +also valid for ClickHouse itself. Use HTTPS for these endpoints in production; an HTTP endpoint is +suitable only for local testing. + +Only the session's verified token is forwarded. It is not inherited by `EXECUTE AS` or +`DEFINER` views, forwarded to other ClickHouse nodes, or persisted to disk. + +HTTP requests authenticate separately, so a rotated token takes effect on the next request. +Native TCP sessions must reconnect with the new token. + +With `async_insert = 1`, each queued batch retains its token until the flush completes, even with +`wait_for_async_insert = 0`. Different tokens use separate batches; rotation does not change a +queued batch's token. + ## Enabling token authentication for a user in `users.xml` {#enabling-jwt-auth-in-users-xml} In order to enable token-based authentication for the user, specify `jwt` section instead of `password` or other similar sections in the user definition. diff --git a/docs/reference/engines/database-engines/datalake.mdx b/docs/reference/engines/database-engines/datalake.mdx index 07e732461ff12..6be5f84d242b6 100644 --- a/docs/reference/engines/database-engines/datalake.mdx +++ b/docs/reference/engines/database-engines/datalake.mdx @@ -55,6 +55,7 @@ The following settings are supported: | `storage_endpoint` | Endpoint URL for the underlying storage | | `oauth_server_uri` | URI of the OAuth2 authorization server for authentication | | `vended_credentials` | Boolean indicating whether to use vended credentials from the catalog (supports AWS S3 and Azure ADLS Gen2) | +<<<<<<< HEAD:docs/reference/engines/database-engines/datalake.mdx | `aws_access_key_id` | AWS access key ID for S3/Glue access (if not using vended credentials) | | `aws_secret_access_key` | AWS secret access key for S3/Glue access (if not using vended credentials) | | `aws_role_arn` | ARN of the IAM role to assume for AWS/Glue access. When set, ClickHouse uses AWS STS `AssumeRole` with base credentials from `aws_access_key_id` and `aws_secret_access_key` when both are provided, or from the default AWS credential chain otherwise (the role must trust the identity the server runs under). | @@ -64,7 +65,22 @@ The following settings are supported: | `dlf_access_key_id` | Access key ID for DLF access | | `dlf_access_key_secret` | Access key Secret for DLF access | | `force_add_bucket` | When constructing object-storage URLs from the catalog-provided table location and `storage_endpoint`, prepend the bucket/container name even if the endpoint already contains it. Default: `false`. Set to `true` for catalogs that hand back paths without the bucket and require it to be added at the URL-construction step (Polaris-style paths). | +======= +| `vended_credentials_cache_ttl` | Maximum cache entry lifetime (in seconds) for vended credentials (REST catalogs only). Default `300`; `0` disables caching. | +| `aws_access_key_id` | AWS access key ID for S3/Glue access (if not using vended credentials) | +| `aws_secret_access_key` | AWS secret access key for S3/Glue access (if not using vended credentials) | +| `region` | AWS region for the service (e.g., `us-east-1`) | +| `aws_sts_endpoint` | Custom AWS STS endpoint for Glue role assumption | +| `dlf_access_key_id` | Access key ID for DLF access | +| `dlf_access_key_secret` | Access key Secret for DLF access | +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake):docs/en/engines/database-engines/datalake.md | `namespaces` | Comma-separated list of namespaces, implemented for catalog types: `rest`, `glue` and `unity` | +| `oauth_forward_user_token` | Authenticate to the catalog as the user running the query instead of as the shared service identity. Iceberg REST and Glue only. See [Forwarding the user's identity to the catalog](#user-token-forwarding) | +| `oauth_token_exchange_uri` | Empty (the default) forwards the user's token unchanged; non-empty performs an RFC 8693 token exchange at this URL first | +| `oauth_subject_token_type` | RFC 8693 `subject_token_type` of the forwarded token. Default `urn:ietf:params:oauth:token-type:access_token` | +| `oauth_requested_token_type` | RFC 8693 `requested_token_type`; empty omits the field. Default `urn:ietf:params:oauth:token-type:access_token` | +| `oauth_forward_actor_token` | Send the service principal's own token as the RFC 8693 `actor_token`. Default `0`. See [Delegation with an actor token](#user-token-forwarding-actor-token) | +| `oauth_user_token_cache_ttl` | Maximum lifetime (in seconds) of a cached exchanged session token; `0` disables caching. Default `300` | ## Examples {#examples} @@ -93,6 +109,120 @@ SELECT count() from database_name.table_name; `onelake_client_id`/`onelake_client_secret`. ClickHouse does not refresh the token, so the database must be recreated after it expires. +## Forwarding the user's identity to the catalog {#user-token-forwarding} + +Set `oauth_forward_user_token = 1` to use the querying user's token for catalog authentication. +This requires: + +- the server-level [`enable_token_forwarding`](/operations/server-configuration-parameters/settings#enable_token_forwarding) + setting, which is `false` by default; +- `catalog_type = 'rest'` or `catalog_type = 'glue'`; +- token authentication through an `Authorization: Bearer` HTTP header or `--jwt` for the native + protocol. See [Token-based authentication](/en/operations/external-authenticators/oauth). + +:::danger Restrict `CREATE DATABASE` +Database creators choose the endpoints that receive users' tokens. Grant `CREATE DATABASE` only +to trusted users and restrict `remote_url_allow_hosts`, which applies to catalog and token-exchange +endpoints. +::: + +The forwarded token is sent to the catalog endpoint and, with token exchange, to `oauth_token_exchange_uri` +together with the `catalog_credential` client secret; in passthrough mode the token is also valid for +ClickHouse itself. Use HTTPS for these endpoints in production; an HTTP endpoint is suitable only for local testing. + +### Passthrough: the default {#user-token-forwarding-passthrough} + +For Iceberg REST catalogs, `oauth_forward_user_token = 1` forwards the user's bearer token +unchanged. No token endpoint or client credentials are required: + +```sql +CREATE DATABASE demo +ENGINE = DataLakeCatalog('http://lakekeeper:8181/catalog') +SETTINGS + catalog_type = 'rest', + warehouse = 'demo', + oauth_forward_user_token = 1; +``` + +The token's audience must cover both ClickHouse and the catalog. With Keycloak, add an audience +mapper to the ClickHouse client to include the catalog's audience. + +### Token exchange: opt-in {#user-token-forwarding-exchange} + +Set `oauth_token_exchange_uri` to exchange the user's token using +[RFC 8693](https://www.rfc-editor.org/rfc/rfc8693) before presenting it to the REST catalog. +Use an IdP token endpoint that issues tokens with an audience the catalog accepts: + +```sql +CREATE DATABASE demo +ENGINE = DataLakeCatalog('http://lakekeeper:8181/catalog') +SETTINGS + catalog_type = 'rest', + warehouse = 'demo', + catalog_credential = 'clickhouse:', + auth_scope = 'lakekeeper', + oauth_forward_user_token = 1, + oauth_token_exchange_uri = 'http://keycloak:8080/realms/demo/protocol/openid-connect/token'; +``` + +The exchange requires `catalog_credential`; its `client_id` and `client_secret` are sent in the +form body. `auth_scope` supplies the exchange `scope`. Override its Polaris-specific default, +`PRINCIPAL_ROLE:ALL`, for other providers. + +A catalog's `/v1/oauth/tokens` endpoint can also be used if supported. The Iceberg REST +specification deprecates this endpoint, and some catalogs, including Lakekeeper, do not implement it. + +### Delegation with an actor token {#user-token-forwarding-actor-token} + +Set `oauth_forward_actor_token = 1` to include the service principal's token as the RFC 8693 +`actor_token`. This requires `oauth_token_exchange_uri` and an endpoint that supports delegation +and can validate the actor token. + +ClickHouse obtains the actor token through a `client_credentials` grant using `catalog_credential` +at `oauth_server_uri`, or the catalog's `/v1/oauth/tokens` endpoint if that setting is empty. +The token is cached until expiry and used only for exchanges. If obtaining it fails, the query fails. + +### Glue {#user-token-forwarding-glue} + +For Glue, ClickHouse exchanges the user's token through AWS STS `AssumeRoleWithWebIdentity` +for temporary credentials of `aws_role_arn`. These credentials sign Glue and S3 requests using +SigV4. The ClickHouse user name supplies `RoleSessionName` for CloudTrail auditing. + +```sql +CREATE DATABASE glue_db +ENGINE = DataLakeCatalog +SETTINGS + catalog_type = 'glue', + region = 'us-east-1', + aws_role_arn = 'arn:aws:iam::123456789012:role/data-lake-reader', + aws_sts_endpoint = 'https://sts.us-east-1.amazonaws.com', + oauth_forward_user_token = 1; +``` + +Register the token issuer as an IAM OIDC identity provider and configure the role's trust policy +to accept the users' tokens, including their `aud` and `sub` claims. + +- `aws_role_arn` is required. +- `aws_access_key_id`, `aws_secret_access_key`, and RFC 8693 exchange settings are rejected. +- `aws_sts_endpoint` overrides the regional AWS STS endpoint. Use HTTPS in production; an HTTP endpoint is suitable only for local testing. +- Users receive the assumed role's permissions. Use separate roles or session-tag policies to + distinguish access. ClickHouse does not implement IAM Identity Center trusted identity propagation. + +### Scope and limitations {#user-token-forwarding-scope} + +- Forwarding covers catalog listings, table metadata, and write operations (`INSERT`, `ALTER`, + mutations, `DROP TABLE`, and snapshot expiry). +- Vended storage credentials and Glue sessions are cached separately for each user token. +- Requests without a user token fail with `CATALOG_USER_TOKEN_NOT_AVAILABLE`; ClickHouse does not + fall back to the service identity. Catalog listings may suppress these errors and appear empty, + depending on `database_datalake_require_metadata_access`. +- With `object_storage_cluster`, workers receive table-scoped storage credentials over the + interserver channel. Configure `interserver_https_port` or a cluster `` for cluster reads. +- For Iceberg REST, rotate `catalog_credential` with `ALTER DATABASE ... MODIFY SETTING`, + authenticated with a user token. ClickHouse validates the new credentials and reloads the catalog + configuration before applying the change, then invalidates cached session and storage credentials. + Glue settings cannot be altered. + ## Namespace filter {#namespace} By default, ClickHouse reads tables from all namespaces available in the catalog. You can limit this behavior using the `namespaces` database setting. The value should be a comma‑separated list of namespaces that are allowed to be read. diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp index b04b14ed9c8e7..5406fd534369b 100644 --- a/src/Access/AccessControl.cpp +++ b/src/Access/AccessControl.cpp @@ -295,6 +295,7 @@ void AccessControl::setupFromMainConfig(const Poco::Util::AbstractConfiguration setPasswordComplexityRulesFromConfig(config_); setTokenAuthEnabled(config_.getBool("enable_token_auth", true)); + setTokenForwardingEnabled(config_.getBool("enable_token_forwarding", false)); setBcryptWorkfactor(config_.getInt("bcrypt_workfactor", 12)); @@ -706,6 +707,7 @@ void AccessControl::setExternalAuthenticatorsConfig(const Poco::Util::AbstractCo /// value in place -- operators who toggle token auth off in response to an /// IdP outage or a credential leak would see no effect until restart. setTokenAuthEnabled(config.getBool("enable_token_auth", true)); + setTokenForwardingEnabled(config.getBool("enable_token_forwarding", false)); external_authenticators->setConfiguration(config, getLogger(), token_http_timeouts, isTokenAuthEnabled()); } @@ -1003,4 +1005,14 @@ bool AccessControl::isTokenAuthEnabled() const { return enable_token_auth; } + +void AccessControl::setTokenForwardingEnabled(bool enable) +{ + enable_token_forwarding = enable; +} + +bool AccessControl::isTokenForwardingEnabled() const +{ + return enable_token_forwarding; +} } diff --git a/src/Access/AccessControl.h b/src/Access/AccessControl.h index f912793fb4816..ab5a542ade5f4 100644 --- a/src/Access/AccessControl.h +++ b/src/Access/AccessControl.h @@ -287,6 +287,9 @@ class AccessControl : public MultipleAccessStorage void setTokenAuthEnabled(bool enable); bool isTokenAuthEnabled() const; + void setTokenForwardingEnabled(bool enable); + bool isTokenForwardingEnabled() const; + private: class ContextAccessCache; class CustomSettingsPrefixes; @@ -326,7 +329,11 @@ class AccessControl : public MultipleAccessStorage std::atomic_bool enable_read_write_grants = false; std::atomic_bool allow_impersonate_user = false; std::atomic_bool enable_token_auth = true; +<<<<<<< HEAD std::atomic_bool readonly_restricts_set_role = false; +======= + std::atomic_bool enable_token_forwarding = false; +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) }; } diff --git a/src/Access/ForwardedAuthToken.cpp b/src/Access/ForwardedAuthToken.cpp new file mode 100644 index 0000000000000..fdc3d6fa6dca1 --- /dev/null +++ b/src/Access/ForwardedAuthToken.cpp @@ -0,0 +1,18 @@ +#include + +#include +#include + +namespace DB +{ + +ForwardedAuthTokenPtr makeForwardedAuthToken(const TokenCredentials & credentials, const String & principal) +{ + auto result = std::make_shared(); + result->token = credentials.getToken(); + result->fingerprint = getSipHash128AsHexString(sipHash128(result->token.data(), result->token.size())); + result->principal = principal; + return result; +} + +} diff --git a/src/Access/ForwardedAuthToken.h b/src/Access/ForwardedAuthToken.h new file mode 100644 index 0000000000000..62657fc7a9170 --- /dev/null +++ b/src/Access/ForwardedAuthToken.h @@ -0,0 +1,25 @@ +#pragma once + +#include + +#include + +namespace DB +{ + +class TokenCredentials; + +struct ForwardedAuthToken +{ + String token; + /// Use the token fingerprint so rotation cannot reuse credentials cached for the previous token. + String fingerprint; + String principal; +}; + +using ForwardedAuthTokenPtr = std::shared_ptr; + +/// `principal` must be the canonical `AuthResult::user_name`, not the name the client sent. +ForwardedAuthTokenPtr makeForwardedAuthToken(const TokenCredentials & credentials, const String & principal); + +} diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index 0f8dc57b99726..22436859b93c9 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -401,6 +401,8 @@ M(DNSAddressesCacheSize, "Number of cached DNS addresses") \ M(MarkCacheBytes, "Total size of mark cache in bytes") \ M(MarkCacheFiles, "Total number of mark files cached in the mark cache") \ + M(DataLakeCatalogUserTokenCacheEntries, "Total number of per-user session tokens exchanged for data lake catalog access") \ + M(DataLakeCatalogUserClientCacheEntries, "Total number of per-user catalog clients built from forwarded user tokens") \ M(UniqueKeyIndexCacheBytes, "Total size of UNIQUE KEY index cache in bytes") \ M(UniqueKeyIndexCacheEntries, "Total number of UNIQUE KEY index blocks cached") \ M(DeleteBitmapCacheBytes, "Total size of the UNIQUE KEY delete-bitmap cache in bytes") \ diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index a99c662d331d3..503d148aa651f 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -658,11 +658,16 @@ M(776, RESOURCE_LIMIT_EXCEEDED) \ M(777, MEMORY_RESERVATION_KILLED) \ M(778, MEMORY_RESERVATION_FAILED) \ +<<<<<<< HEAD M(779, UNSUPPORTED_MEDIA_TYPE) \ M(780, SW_SERVER_NO_WORKERS_AVAILABLE) \ M(781, AI_PROVIDER_RESPONSE_TRUNCATED) \ M(782, AI_PROVIDER_RESPONSE_INCOMPLETE) \ M(783, CATALOG_NAMESPACE_DISABLED) \ +======= + M(779, CATALOG_NAMESPACE_DISABLED) \ + M(780, CATALOG_USER_TOKEN_NOT_AVAILABLE) \ +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) \ M(900, DISTRIBUTED_CACHE_ERROR) \ M(901, CANNOT_USE_DISTRIBUTED_CACHE) \ diff --git a/src/Common/FormUrlEncode.cpp b/src/Common/FormUrlEncode.cpp new file mode 100644 index 0000000000000..419337cc42bb2 --- /dev/null +++ b/src/Common/FormUrlEncode.cpp @@ -0,0 +1,15 @@ +#include + +#include + +namespace DB +{ + +std::string formUrlEncode(const std::string & value) +{ + std::string encoded; + Poco::URI::encode(value, "!$&'()*+,;=:@/?", encoded); + return encoded; +} + +} diff --git a/src/Common/FormUrlEncode.h b/src/Common/FormUrlEncode.h new file mode 100644 index 0000000000000..18602a9275d44 --- /dev/null +++ b/src/Common/FormUrlEncode.h @@ -0,0 +1,11 @@ +#pragma once + +#include + +namespace DB +{ + +/// `Poco::URI::encode` leaves form delimiters unescaped unless they are explicitly reserved. +std::string formUrlEncode(const std::string & value); + +} diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index bc6bb3c6be2a9..0199708f69395 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -1700,6 +1700,10 @@ The server successfully detected this situation and will download merged part fr \ M(DataLakeRestCatalogCredentialsVended, "Number of table metadata requests to REST catalog asking to vend storage credentials.", ValueType::Number) \ M(DataLakeRestCatalogCredentialsCacheHits, "Number of table metadata requests to REST catalog reusing cached storage credentials.", ValueType::Number) \ + M(DataLakeRestCatalogTokenExchange, "Number of RFC 8693 token exchanges performed to obtain a session token for the querying user.", ValueType::Number) \ + M(DataLakeRestCatalogTokenExchangeMicroseconds, "Total time of RFC 8693 token exchanges.", ValueType::Microseconds) \ + M(DataLakeRestCatalogTokenExchangeFailures, "Number of RFC 8693 token exchanges that failed.", ValueType::Number) \ + M(DataLakeRestCatalogUserTokenCacheHits, "Number of times a previously exchanged per-user session token was reused.", ValueType::Number) \ \ M(StatelessWorkerRequested, "Number of stateless workers requested by queries for distributed query execution.", ValueType::Number) \ M(StatelessWorkerProvided, "Number of stateless workers provided to queries for distributed query execution.", ValueType::Number) \ @@ -1806,6 +1810,10 @@ The server successfully detected this situation and will download merged part fr M(DataLakeGlueCatalogUpdateTableMicroseconds, "Total time of 'update table' requests to Iceberg Glue catalog.", ValueType::Microseconds) \ M(DataLakeGlueCatalogDropTable, "Number of 'drop table' requests to Iceberg Glue catalog.", ValueType::Number) \ M(DataLakeGlueCatalogDropTableMicroseconds, "Total time of 'drop table' requests to Iceberg Glue catalog.", ValueType::Microseconds) \ + M(DataLakeGlueCatalogAssumeRoleWithWebIdentity, "Number of AWS STS `AssumeRoleWithWebIdentity` calls made to turn a forwarded user token into credentials for the Glue catalog.", ValueType::Number) \ + M(DataLakeGlueCatalogAssumeRoleWithWebIdentityMicroseconds, "Total time of AWS STS `AssumeRoleWithWebIdentity` calls made for forwarded user tokens.", ValueType::Microseconds) \ + M(DataLakeGlueCatalogAssumeRoleWithWebIdentityFailures, "Number of AWS STS `AssumeRoleWithWebIdentity` calls that returned no credentials.", ValueType::Number) \ + M(DataLakeGlueCatalogUserClientCacheHits, "Number of times a Glue client built for a forwarded user token was reused.", ValueType::Number) \ \ M(DataLakeUnityCatalogGetTables, "Number of 'get tables' requests to Iceberg Unity catalog.", ValueType::Number) \ M(DataLakeUnityCatalogGetTablesMicroseconds, "Total time of 'get tables' requests to Iceberg Unity catalog.", ValueType::Microseconds) \ diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index 11d7e231de1bb..e1fc1b0b901b0 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -987,8 +987,23 @@ When disabled: - `TokenAccessStorage` (token user directory) is not added. - Authentication via tokens (`--jwt` option in clickhouse-client or `Authorization: Bearer` HTTP header) is rejected. +<<<<<<< HEAD Default value: `true` (token authentication is enabled). )", 0) \ +======= + Default value: `true` (token authentication is enabled). + )", 0) \ + DECLARE(Bool, enable_token_forwarding, false, R"( + Retain authenticated bearer tokens for forwarding through the `DataLakeCatalog` + setting `oauth_forward_user_token`. Supports Iceberg REST and AWS STS for Glue. + When disabled, tokens are not retained in sessions for forwarding. + + Database creators choose the endpoints that receive users' tokens. Grant `CREATE DATABASE` + only to trusted users and restrict `remote_url_allow_hosts`. + + Default value: `false`. + )", 0) \ +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) DECLARE(UInt64, concurrent_threads_soft_limit_num, 0, R"( The maximum number of query processing threads, excluding threads for retrieving data from remote servers, allowed to run all queries. This is not a hard limit. In case if the limit is reached the query will still get at least one thread to run. Query can upscale to desired number of threads during execution if more threads become available. @@ -3618,6 +3633,8 @@ ChangeableSettingsMap collectChangeableServerSettings(ContextPtr context) {"allow_feature_tier", {std::to_string(context->getAccessControl().getAllowTierSettings()), ChangeableWithoutRestart::Yes}}, + {"enable_token_forwarding", + {std::to_string(context->getAccessControl().isTokenForwardingEnabled()), ChangeableWithoutRestart::Yes}}, {"s3queue_disable_streaming", {std::to_string(context->getServerSettingsCopy().get("s3queue_disable_streaming").safeGet()), ChangeableWithoutRestart::Yes}}, {"message_queue_disable_insertion", {std::to_string(context->getMessageQueueDisableInsertion()), ChangeableWithoutRestart::No}}, diff --git a/src/Databases/DataLake/Common.cpp b/src/Databases/DataLake/Common.cpp index 7bc7ee7898b89..af798e438c340 100644 --- a/src/Databases/DataLake/Common.cpp +++ b/src/Databases/DataLake/Common.cpp @@ -1,5 +1,7 @@ #include +#include + #include #include #include @@ -113,6 +115,13 @@ DB::DataTypePtr getType(const String & type_name, bool nullable, DB::ContextPtr : DB::Iceberg::IcebergSchemaProcessor::getSimpleType(name, context); } +DB::ForwardedAuthTokenPtr getForwardedAuthToken(const DB::ContextPtr & context) +{ + if (!context) + return {}; + return context->getForwardedAuthToken(); +} + std::pair parseTableName(const std::string & name) { auto pos = name.rfind('.'); diff --git a/src/Databases/DataLake/Common.h b/src/Databases/DataLake/Common.h index 912758f7fae8b..463f3d96a7cfa 100644 --- a/src/Databases/DataLake/Common.h +++ b/src/Databases/DataLake/Common.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -20,6 +21,8 @@ DB::DataTypePtr getType(const String & type_name, bool nullable, DB::ContextPtr /// `E` is a table name. std::pair parseTableName(const std::string & name); +DB::ForwardedAuthTokenPtr getForwardedAuthToken(const DB::ContextPtr & context); + String constructTableLocation( const String & location_scheme, const String & storage_endpoint, diff --git a/src/Databases/DataLake/DatabaseDataLake.cpp b/src/Databases/DataLake/DatabaseDataLake.cpp index 952e49e465135..5860d0f876959 100644 --- a/src/Databases/DataLake/DatabaseDataLake.cpp +++ b/src/Databases/DataLake/DatabaseDataLake.cpp @@ -78,6 +78,12 @@ namespace DatabaseDataLakeSetting extern const DatabaseDataLakeSettingsS3UriStyle storage_uri_style; extern const DatabaseDataLakeSettingsString oauth_server_uri; extern const DatabaseDataLakeSettingsBool oauth_server_use_request_body; + extern const DatabaseDataLakeSettingsBool oauth_forward_user_token; + extern const DatabaseDataLakeSettingsString oauth_token_exchange_uri; + extern const DatabaseDataLakeSettingsString oauth_subject_token_type; + extern const DatabaseDataLakeSettingsString oauth_requested_token_type; + extern const DatabaseDataLakeSettingsBool oauth_forward_actor_token; + extern const DatabaseDataLakeSettingsUInt64 oauth_user_token_cache_ttl; extern const DatabaseDataLakeSettingsBool vended_credentials; extern const DatabaseDataLakeSettingsUInt64 vended_credentials_cache_ttl; extern const DatabaseDataLakeSettingsString object_storage_cluster; @@ -87,6 +93,7 @@ namespace DatabaseDataLakeSetting extern const DatabaseDataLakeSettingsString aws_role_arn; extern const DatabaseDataLakeSettingsString aws_role_session_name; extern const DatabaseDataLakeSettingsString aws_external_id; + extern const DatabaseDataLakeSettingsString aws_sts_endpoint; extern const DatabaseDataLakeSettingsString onelake_tenant_id; extern const DatabaseDataLakeSettingsString onelake_client_id; extern const DatabaseDataLakeSettingsString onelake_client_secret; @@ -258,6 +265,105 @@ void DatabaseDataLake::validateSettings() ErrorCodes::BAD_ARGUMENTS, "`warehouse` setting cannot be empty. " "Please specify 'SETTINGS warehouse=' in the CREATE DATABASE query"); } + + validateTokenForwardingSettings(); +} + +void DatabaseDataLake::validateTokenForwardingSettings() const +{ + const auto settings_version = database_settings.get(); + const DatabaseDataLakeSettings & settings = *settings_version; + + if (!settings[DatabaseDataLakeSetting::oauth_forward_user_token].value) + { + if (settings[DatabaseDataLakeSetting::oauth_forward_actor_token].value) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`oauth_forward_actor_token` requires `oauth_forward_user_token = 1`"); + return; + } + + const auto catalog_type = settings[DatabaseDataLakeSetting::catalog_type].value; + + if (catalog_type != DB::DatabaseDataLakeCatalogType::ICEBERG_REST && catalog_type != DB::DatabaseDataLakeCatalogType::GLUE) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`oauth_forward_user_token` is only supported for `catalog_type = 'rest'` and " + "`catalog_type = 'glue'`"); + + if (catalog_type == DB::DatabaseDataLakeCatalogType::GLUE) + { + validateGlueTokenForwardingSettings(settings); + return; + } + + if (!settings[DatabaseDataLakeSetting::auth_header].value.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`oauth_forward_user_token` cannot be combined with `auth_header`"); + + const auto & exchange_uri = settings[DatabaseDataLakeSetting::oauth_token_exchange_uri].value; + if (!exchange_uri.empty() && settings[DatabaseDataLakeSetting::catalog_credential].value.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`oauth_token_exchange_uri` requires a non-empty `catalog_credential`"); + + if (exchange_uri.empty() && settings[DatabaseDataLakeSetting::oauth_forward_actor_token].value) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`oauth_forward_actor_token` requires a non-empty `oauth_token_exchange_uri`"); + + static const std::array valid_token_types = { + "urn:ietf:params:oauth:token-type:access_token", + "urn:ietf:params:oauth:token-type:refresh_token", + "urn:ietf:params:oauth:token-type:id_token", + "urn:ietf:params:oauth:token-type:saml1", + "urn:ietf:params:oauth:token-type:saml2", + "urn:ietf:params:oauth:token-type:jwt", + }; + auto check_token_type = [&](std::string_view setting_name, const std::string & value, bool empty_allowed) + { + if (value.empty() && empty_allowed) + return; + if (std::find(valid_token_types.begin(), valid_token_types.end(), value) == valid_token_types.end()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`{}` must be one of the token type URNs defined by RFC 8693, got `{}`", + setting_name, value); + }; + check_token_type( + "oauth_subject_token_type", + settings[DatabaseDataLakeSetting::oauth_subject_token_type].value, + /* empty_allowed */ false); + check_token_type( + "oauth_requested_token_type", + settings[DatabaseDataLakeSetting::oauth_requested_token_type].value, + /* empty_allowed */ true); +} + +void DatabaseDataLake::validateGlueTokenForwardingSettings(const DatabaseDataLakeSettings & settings) +{ + if (settings[DatabaseDataLakeSetting::aws_role_arn].value.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`oauth_forward_user_token` requires a non-empty `aws_role_arn` for a Glue catalog"); + + if (!settings[DatabaseDataLakeSetting::aws_access_key_id].value.empty() + || !settings[DatabaseDataLakeSetting::aws_secret_access_key].value.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`oauth_forward_user_token` cannot be combined with `aws_access_key_id` / " + "`aws_secret_access_key` for a Glue catalog"); + + if (!settings[DatabaseDataLakeSetting::oauth_token_exchange_uri].value.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`oauth_token_exchange_uri` is only supported for `catalog_type = 'rest'`"); + + if (settings[DatabaseDataLakeSetting::oauth_forward_actor_token].value) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`oauth_forward_actor_token` is only supported for `catalog_type = 'rest'`"); } void DatabaseDataLake::initialize() const @@ -279,6 +385,8 @@ void DatabaseDataLake::initialize() const .aws_role_arn = settings[DatabaseDataLakeSetting::aws_role_arn].value, .aws_role_session_name = settings[DatabaseDataLakeSetting::aws_role_session_name].value, .aws_external_id = settings[DatabaseDataLakeSetting::aws_external_id].value, + .aws_sts_endpoint = settings[DatabaseDataLakeSetting::aws_sts_endpoint].value, + .forward_user_token = settings[DatabaseDataLakeSetting::oauth_forward_user_token].value, }; switch (settings[DatabaseDataLakeSetting::catalog_type].value) @@ -294,6 +402,7 @@ void DatabaseDataLake::initialize() const settings[DatabaseDataLakeSetting::oauth_server_uri].value, settings[DatabaseDataLakeSetting::oauth_server_use_request_body].value, settings[DatabaseDataLakeSetting::namespaces].value, +<<<<<<< HEAD settings[DatabaseDataLakeSetting::flat_namespaces].value, Context::getGlobalContextInstance()); break; @@ -331,6 +440,17 @@ void DatabaseDataLake::initialize() const settings[DatabaseDataLakeSetting::namespaces].value, settings[DatabaseDataLakeSetting::flat_namespaces].value, Context::getGlobalContextInstance()); +======= + Context::getGlobalContextInstance(), + DataLake::TokenForwardingConfig{ + .forward_user_token = settings[DatabaseDataLakeSetting::oauth_forward_user_token].value, + .token_exchange_uri = settings[DatabaseDataLakeSetting::oauth_token_exchange_uri].value, + .subject_token_type = settings[DatabaseDataLakeSetting::oauth_subject_token_type].value, + .requested_token_type = settings[DatabaseDataLakeSetting::oauth_requested_token_type].value, + .forward_actor_token = settings[DatabaseDataLakeSetting::oauth_forward_actor_token].value, + .user_token_cache_ttl = settings[DatabaseDataLakeSetting::oauth_user_token_cache_ttl].value, + }); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) break; } case DB::DatabaseDataLakeCatalogType::ICEBERG_ONELAKE: @@ -733,13 +853,13 @@ std::string DatabaseDataLake::getStorageEndpointForTable(const DataLake::TableMe bool DatabaseDataLake::empty() const { - return getCatalog()->empty(); + return getCatalog()->empty(/* auth_token */ {}); } -bool DatabaseDataLake::isTableExist(const String & name, ContextPtr /* context_ */) const +bool DatabaseDataLake::isTableExist(const String & name, ContextPtr context_) const { const auto [namespace_name, table_name] = DataLake::parseTableName(name); - return getCatalog()->existsTable(namespace_name, table_name); + return getCatalog()->existsTable(namespace_name, table_name, DataLake::getForwardedAuthToken(context_)); } StoragePtr DatabaseDataLake::tryGetTable(const String & name, ContextPtr context_) const @@ -991,7 +1111,15 @@ StoragePtr DatabaseDataLake::tryGetTableImpl(const String & name, ContextPtr con auto storage_cluster = std::make_shared( cluster_name, configuration, +<<<<<<< HEAD configuration->createObjectStorage(context_copy, /* is_readonly */ false, get_credentials_refresh_callback(StorageID(getDatabaseName(), name, table_uuid))), +======= + configuration->createObjectStorage( + context_copy, + /* is_readonly */ false, + catalog->getCredentialsConfigurationCallback( + StorageID(getDatabaseName(), name, table_uuid), DataLake::getForwardedAuthToken(context_))), +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) StorageID(getDatabaseName(), name, table_uuid), /* columns */columns, /* constraints */ConstraintsDescription{}, @@ -1016,7 +1144,7 @@ StoragePtr DatabaseDataLake::tryGetTableImpl(const String & name, ContextPtr con return storage_cluster; } -void DatabaseDataLake::validateCreateTableEngine(const ASTStorage & storage) const +void DatabaseDataLake::validateCreateTableEngine(const ASTStorage & storage, ContextPtr context_) const { const ASTFunction & engine = *storage.engine; const auto catalog = getCatalog(); @@ -1034,6 +1162,7 @@ void DatabaseDataLake::validateCreateTableEngine(const ASTStorage & storage) con "This DataLakeCatalog assigns table locations itself, so an explicit table `ENGINE` " "is not supported; omit the `ENGINE` clause"); + catalog->loadConfigIfNeeded(DataLake::getForwardedAuthToken(context_)); const auto catalog_storage_type = catalog->getStorageType(); std::optional engine_backend; @@ -1239,6 +1368,7 @@ void DatabaseDataLake::createTable( const auto compression_method_str = context_->getSettingsRef()[Setting::iceberg_metadata_compression_method].value; const auto compression_method = chooseCompressionMethod(compression_method_str, compression_method_str); +<<<<<<< HEAD if (compression_method != CompressionMethod::None) { /// A catalog that writes the first metadata file itself reads the codec from this property. @@ -1248,9 +1378,17 @@ void DatabaseDataLake::createTable( } catalog->createNamespaceIfNotExists(namespace_name, /* location */ ""); +======= + /// The namespace's default location is the namespace base, not this first table's directory. + String namespace_location = location; + if (const String table_suffix = "/" + table_name; namespace_location.ends_with(table_suffix)) + namespace_location.resize(namespace_location.size() - table_suffix.size()); + const auto auth_token = DataLake::getForwardedAuthToken(context_); + catalog->createNamespaceIfNotExists(namespace_name, namespace_location, auth_token); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) const bool created = catalog->createTable( - namespace_name, table_name, /* metadata_path */ "", metadata_content, compression_method, create.if_not_exists); + namespace_name, table_name, /* metadata_path */ "", metadata_content, compression_method, create.if_not_exists, auth_token); if (!created) throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Table {}.{} already exists in the catalog", namespace_name, table_name); @@ -1268,7 +1406,7 @@ void DatabaseDataLake::dropTable( /// NOLINT const auto [namespace_name, table_name] = DataLake::parseTableName(name); bool purge = context_->getSettingsRef()[Setting::data_lake_delete_data_on_drop]; - catalog->dropTable(namespace_name, table_name, purge, if_exists); + catalog->dropTable(namespace_name, table_name, purge, if_exists, DataLake::getForwardedAuthToken(context_)); LOG_INFO(log, "Dropped table {}.{} from DataLakeCatalog (purge={})", namespace_name, table_name, purge); } @@ -1319,7 +1457,11 @@ DatabaseTablesIteratorPtr DatabaseDataLake::getTablesIteratorImpl( throw Exception(ErrorCodes::DATALAKE_DATABASE_ERROR, "Injected catalog listing failure"); }); +<<<<<<< HEAD catalog_tables = getCatalog()->getTables(toCatalogTableNameFilter(tables_filter)); +======= + iceberg_tables = getCatalog()->getTables(toCatalogTableNameFilter(tables_filter), DataLake::getForwardedAuthToken(context_)); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } catch (...) { @@ -1468,7 +1610,11 @@ std::vector DatabaseDataLake::getLightweightTablesItera throw Exception(ErrorCodes::DATALAKE_DATABASE_ERROR, "Injected catalog listing failure"); }); +<<<<<<< HEAD catalog_tables = getCatalog()->getTables(toCatalogTableNameFilter(tables_filter)); +======= + iceberg_tables = getCatalog()->getTables(toCatalogTableNameFilter(tables_filter), DataLake::getForwardedAuthToken(context_)); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } catch (...) { @@ -1491,7 +1637,11 @@ std::vector DatabaseDataLake::getLightweightTablesItera return result; } +<<<<<<< HEAD VectorWithMemoryTracking DatabaseDataLake::getAllTableNames(ContextPtr /*context*/) const +======= +Strings DatabaseDataLake::getAllTableNames(ContextPtr context_) const +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { VectorWithMemoryTracking result; @@ -1500,6 +1650,7 @@ VectorWithMemoryTracking DatabaseDataLake::getAllTableNames(ContextPtr / /// must not fail even when the catalog is temporarily unreachable. try { +<<<<<<< HEAD DataLake::CatalogTables tables = getCatalog()->getTables(); result.reserve(tables.size()); for (auto & table : tables) @@ -1509,6 +1660,9 @@ VectorWithMemoryTracking DatabaseDataLake::getAllTableNames(ContextPtr / continue; result.push_back(std::move(table.name)); } +======= + result = getCatalog()->getTables(DataLake::getForwardedAuthToken(context_)); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } catch (...) { @@ -1527,18 +1681,18 @@ ASTPtr DatabaseDataLake::getCreateDatabaseQueryImpl() const return create_query; } -void DatabaseDataLake::checkDatabase() const +void DatabaseDataLake::checkDatabase(ContextPtr context_) const { auto catalog = getCatalog(); /// This function checks if we can access catalog and get tables list. /// We do not check if there are tables in catalog, because even if catalog is empty, it still can be valid and working. - std::ignore = catalog->empty(); + std::ignore = catalog->empty(DataLake::getForwardedAuthToken(context_)); LOG_TEST(log, "Database '{}' is OK", getDatabaseName()); } -void DatabaseDataLake::applySettingsChanges(const SettingsChanges & settings_changes, ContextPtr /*query_context*/) +void DatabaseDataLake::applySettingsChanges(const SettingsChanges & settings_changes, ContextPtr query_context) { const auto current_settings = database_settings.get(); @@ -1587,7 +1741,7 @@ void DatabaseDataLake::applySettingsChanges(const SettingsChanges & settings_cha /// fetch and the config reload may throw, and then nothing has changed yet. DataLake::ICatalog::PreparedSettingsChangesPtr prepared_catalog_changes; if (local_catalog_snapshot) - prepared_catalog_changes = local_catalog_snapshot->prepareSettingsChanges(settings_changes); + prepared_catalog_changes = local_catalog_snapshot->prepareSettingsChanges(settings_changes, DataLake::getForwardedAuthToken(query_context)); /// Persist the new metadata before publishing anything: if the write fails, the live /// state is untouched and matches the old metadata on disk. The create query is built @@ -1758,6 +1912,34 @@ void registerDatabaseDataLake(DatabaseFactory & factory) } } + /// Validate only on `CREATE` so older persisted databases can still attach at startup. + if (!args.create_query.attach) + { + const bool forwarding = database_settings[DatabaseDataLakeSetting::oauth_forward_user_token].value; + static constexpr std::array exchange_only_settings = { + "oauth_token_exchange_uri", + "oauth_subject_token_type", + "oauth_requested_token_type", + "oauth_forward_actor_token", + "oauth_user_token_cache_ttl", + }; + + for (const auto & change : database_settings.allChanged()) + { + const auto & name = change.name; + if (std::find(exchange_only_settings.begin(), exchange_only_settings.end(), name) == exchange_only_settings.end()) + continue; + if (!forwarding) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`{}` has no effect without `oauth_forward_user_token = 1`", name); + if (name != "oauth_token_exchange_uri" && database_settings[DatabaseDataLakeSetting::oauth_token_exchange_uri].value.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`{}` has no effect without `oauth_token_exchange_uri`", name); + } + } + auto catalog_type = database_settings[DB::DatabaseDataLakeSetting::catalog_type].value; /// Glue catalog is one per region, so it's fully identified by aws keys and region /// There is no URL you need to provide in constructor, even if we would want it diff --git a/src/Databases/DataLake/DatabaseDataLake.h b/src/Databases/DataLake/DatabaseDataLake.h index e1101c2232146..4d8c4e3dd93f2 100644 --- a/src/Databases/DataLake/DatabaseDataLake.h +++ b/src/Databases/DataLake/DatabaseDataLake.h @@ -3,6 +3,7 @@ #if USE_AVRO && USE_PARQUET +#include #include #include #include @@ -65,13 +66,13 @@ class DatabaseDataLake final : public IDatabase, WithContext VectorWithMemoryTracking getAllTableNames(ContextPtr context) const override; - void checkDatabase() const override; + void checkDatabase(ContextPtr context) const override; void shutdown() override {} std::vector> getTablesForBackup(const FilterByNameFunction &, const ContextPtr &) const override { return {}; } - void validateCreateTableEngine(const ASTStorage & storage) const override; + void validateCreateTableEngine(const ASTStorage & storage, ContextPtr context_) const override; void createTable( ContextPtr context, @@ -121,6 +122,10 @@ class DatabaseDataLake final : public IDatabase, WithContext void validateSettings(); + void validateTokenForwardingSettings() const; + + static void validateGlueTokenForwardingSettings(const DatabaseDataLakeSettings & settings); + void validateCreateTableEngineArguments(const ASTFunction & engine) const; /// Builds `catalog_impl` based on the configured catalog type. Constructing a catalog can diff --git a/src/Databases/DataLake/DatabaseDataLakeSettings.cpp b/src/Databases/DataLake/DatabaseDataLakeSettings.cpp index 18cadbb56bcdd..d74830fc84c31 100644 --- a/src/Databases/DataLake/DatabaseDataLakeSettings.cpp +++ b/src/Databases/DataLake/DatabaseDataLakeSettings.cpp @@ -25,14 +25,27 @@ namespace ErrorCodes DECLARE(String, auth_scope, "PRINCIPAL_ROLE:ALL", "Authorization scope for client credentials or token exchange", 0) \ DECLARE(String, oauth_server_uri, "", "OAuth server uri", 0) \ DECLARE(Bool, oauth_server_use_request_body, true, "Put parameters into request body or query params", 0) \ + DECLARE(Bool, oauth_forward_user_token, false, "Authenticate to Iceberg REST or Glue using the querying user's token. Requires `enable_token_forwarding`", 0) \ + DECLARE(String, oauth_token_exchange_uri, "", "RFC 8693 token-exchange endpoint for Iceberg REST. Empty forwards the token unchanged. Requires `oauth_forward_user_token` and a non-empty `catalog_credential`", 0) \ + DECLARE(String, oauth_subject_token_type, "urn:ietf:params:oauth:token-type:access_token", "RFC 8693 `subject_token_type` of the forwarded user token. Used only when `oauth_token_exchange_uri` is set", 0) \ + DECLARE(String, oauth_requested_token_type, "urn:ietf:params:oauth:token-type:access_token", "RFC 8693 `requested_token_type`; empty omits the field. Used only when `oauth_token_exchange_uri` is set", 0) \ + DECLARE(Bool, oauth_forward_actor_token, false, "Include the service principal token as the RFC 8693 `actor_token`. Requires `oauth_token_exchange_uri` and an endpoint that supports delegation", 0) \ + DECLARE(UInt64, oauth_user_token_cache_ttl, 300, "Maximum lifetime (in seconds) of a cached per-user session token obtained by token exchange; '0' disables caching. Used only when `oauth_token_exchange_uri` is set", 0) \ DECLARE(String, warehouse, "", "Warehouse name inside the catalog", 0) \ DECLARE(String, auth_header, "", "Authorization header of format 'Authorization: '", 0) \ DECLARE(String, aws_access_key_id, "", "AWS access key id used to connect to the Glue catalog, and forwarded as a static S3 storage credential for non-Glue DataLakeCatalog table reads when vended_credentials = false", 0) \ DECLARE(String, aws_secret_access_key, "", "AWS secret access key used to connect to the Glue catalog, and forwarded as a static S3 storage credential for non-Glue DataLakeCatalog table reads when vended_credentials = false", 0) \ DECLARE(String, region, "", "Region for Glue catalog", 0) \ +<<<<<<< HEAD DECLARE(String, aws_role_arn, "", "ARN of the IAM role to assume for AWS/Glue access. When set, ClickHouse uses AWS STS `AssumeRole` with base credentials from `aws_access_key_id` and `aws_secret_access_key` when both are provided, or from the default AWS credential chain otherwise (the role must trust the identity the server runs under)", 0) \ DECLARE(String, aws_role_session_name, "", "Session name used for the AWS STS `AssumeRole` call. Optional; defaults to `ClickHouseSession`", 0) \ DECLARE(String, aws_external_id, "", "External ID passed to AWS STS `AssumeRole`, matching the `sts:ExternalId` condition on the role's trust policy. Use this when the role is owned by a third party, such as ClickHouse Cloud", 0) \ +======= + DECLARE(String, aws_role_arn, "", "Role arn for AWS connection for Glue catalog", 0) \ + DECLARE(String, aws_role_session_name, "", "Role session name for AWS connection for Glue catalog", 0) \ + DECLARE(String, aws_external_id, "", "External id for the AWS STS AssumeRole trust policy for Glue catalog", 0) \ + DECLARE(String, aws_sts_endpoint, "", "Custom AWS STS endpoint for Glue catalog", 0) \ +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) DECLARE(String, storage_endpoint, "", "Object storage endpoint", 0) \ DECLARE(String, default_base_location, "", "Base URI under which CREATE TABLE places new tables. Used only when the catalog does not report `default-base-location`", 0) \ DECLARE(S3UriStyle, storage_uri_style, S3UriStyle::AUTO, "URL style used when constructing object storage URLs from catalog-provided table locations. Use 'virtual_hosted' when the object storage server requires the bucket in the hostname (e.g. https://bucket.endpoint.com/path/)", 0) \ diff --git a/src/Databases/DataLake/GlueCatalog.cpp b/src/Databases/DataLake/GlueCatalog.cpp index 764684e19c12d..73583d8b8c2d7 100644 --- a/src/Databases/DataLake/GlueCatalog.cpp +++ b/src/Databases/DataLake/GlueCatalog.cpp @@ -51,6 +51,8 @@ #include #include #include +#include +#include #include #include #include @@ -69,6 +71,7 @@ namespace DB::ErrorCodes extern const int DATALAKE_DATABASE_ERROR; extern const int FAULT_INJECTED; extern const int CATALOG_NAMESPACE_DISABLED; + extern const int CATALOG_USER_TOKEN_NOT_AVAILABLE; extern const int NOT_IMPLEMENTED; extern const int S3_ERROR; } @@ -112,12 +115,17 @@ namespace ProfileEvents extern const Event DataLakeGlueCatalogUpdateTableMicroseconds; extern const Event DataLakeGlueCatalogDropTable; extern const Event DataLakeGlueCatalogDropTableMicroseconds; + extern const Event DataLakeGlueCatalogUserClientCacheHits; + extern const Event DataLakeGlueCatalogAssumeRoleWithWebIdentity; + extern const Event DataLakeGlueCatalogAssumeRoleWithWebIdentityMicroseconds; + extern const Event DataLakeGlueCatalogAssumeRoleWithWebIdentityFailures; } namespace CurrentMetrics { extern const Metric MarkCacheBytes; extern const Metric MarkCacheFiles; + extern const Metric DataLakeCatalogUserClientCacheEntries; } namespace @@ -207,6 +215,7 @@ namespace DataLake namespace { +<<<<<<< HEAD /// A Glue table is readable only if it is an Iceberg table. `table_type` is available in /// both the bulk `GetTables` listing and `tryGetTableMetadata`, so the two stay consistent. bool isReadableGlueTable(const Aws::Glue::Model::Table & table) @@ -215,6 +224,28 @@ bool isReadableGlueTable(const Aws::Glue::Model::Table & table) auto it = parameters.find("table_type"); const std::string table_type = it != parameters.end() ? it->second : ""; return Poco::toUpper(table_type) == "ICEBERG"; +======= +/// STS `RoleSessionName`: 2-64 characters matching `[\w+=,.@-]*`. +/// If the principal does not fit as is, a hash of the full principal is appended to keep names of different principals distinct. +std::string makeRoleSessionName(const std::string & principal) +{ + static constexpr size_t max_length = 64; + static constexpr size_t hash_suffix_length = 1 + 16; + + std::string sanitized; + sanitized.reserve(principal.size()); + for (char c : principal) + { + if (isalnum(static_cast(c)) || c == '_' || c == '+' || c == '=' || c == ',' || c == '.' || c == '@' || c == '-') + sanitized += c; + } + + if (sanitized.size() == principal.size() && sanitized.size() >= 2 && sanitized.size() <= max_length) + return sanitized; + + std::string readable = sanitized.empty() ? "ClickHouseUser" : sanitized.substr(0, max_length - hash_suffix_length); + return readable + "-" + getHexUIntLowercase(sipHash64(principal)); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } } @@ -227,6 +258,10 @@ GlueCatalog::GlueCatalog( bool allow_server_credentials_in_user_queries_) : ICatalog("") , DB::WithContext(context_) + , user_clients( + CurrentMetrics::end(), + CurrentMetrics::DataLakeCatalogUserClientCacheEntries, + user_client_cache_max_entries) , log(getLogger("GlueCatalog(" + settings_.region + ")")) , region(settings_.region) , settings(settings_) @@ -238,6 +273,7 @@ GlueCatalog::GlueCatalog( creds_config.role_arn = settings.aws_role_arn; creds_config.role_session_name = settings.aws_role_session_name; creds_config.external_id = settings.aws_external_id; + creds_config.sts_endpoint_override = settings.aws_sts_endpoint; /// A Glue catalog is created by user SQL, so it must not reuse the server's credentials unless allowed at /// CREATE time. The cached catalog uses the global context, so pass the value captured then, not the live setting. @@ -279,14 +315,12 @@ GlueCatalog::GlueCatalog( client_configuration.connectTimeoutMs = static_cast(global_settings[DB::Setting::s3_connect_timeout_ms]); client_configuration.requestTimeoutMs = static_cast(global_settings[DB::Setting::s3_request_timeout_ms]); client_configuration.region = region; - auto endpoint_provider = std::make_shared(); Aws::Auth::AWSCredentials credentials(settings_.aws_access_key_id, settings_.aws_secret_access_key); /// Only for testing when we are mocking glue if (!endpoint.empty()) { client_configuration.endpointOverride = endpoint; - endpoint_provider->OverrideEndpoint(endpoint); if (credentials.IsEmpty() && !creds_config.forbid_implicit_credentials) { @@ -305,14 +339,95 @@ GlueCatalog::GlueCatalog( LOG_TRACE(log, "Creating AWS glue client with credentials empty {}, region '{}', endpoint '{}'", credentials.IsEmpty(), region, endpoint); } + if (!settings.aws_sts_endpoint.empty()) + { + Poco::URI uri(settings.aws_sts_endpoint); + getContext()->getRemoteHostFilter().checkHostAndPort(uri.getHost(), std::to_string(uri.getPort())); + if (uri.getScheme() == "http") + poco_config.scheme = Aws::Http::Scheme::HTTP; + } + boost::split(allowed_namespaces, settings.namespaces, boost::is_any_of(", "), boost::token_compress_on); - credentials_provider = DB::S3::getCredentialsProvider(poco_config, credentials, creds_config); - glue_client = std::make_unique(credentials_provider, endpoint_provider, client_configuration); + + /// Each `GlueClient` owns its endpoint resolver state. + auto build_glue_client = [client_configuration, endpoint](const std::shared_ptr & provider) + { + auto client_endpoint_provider = std::make_shared(); + if (!endpoint.empty()) + client_endpoint_provider->OverrideEndpoint(endpoint); + return std::make_shared(provider, client_endpoint_provider, client_configuration); + }; + + if (settings.forward_user_token) + { + make_user_client = [build_glue_client, + poco_config, + role_arn = settings.aws_role_arn, + sts_endpoint = settings.aws_sts_endpoint, + expiration_window_seconds = creds_config.expiration_window_seconds, + logger = log](const DB::ForwardedAuthToken & auth_token) + { + auto sts_client = std::make_shared( + std::make_shared(), poco_config, sts_endpoint); + + auto provider = std::make_shared( + role_arn, + makeRoleSessionName(auth_token.principal), + auth_token.token, + expiration_window_seconds, + std::move(sts_client)); + + bool assumed = false; + { + ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogAssumeRoleWithWebIdentity); + auto timer = DB::CurrentThread::getProfileEvents().timer( + ProfileEvents::DataLakeGlueCatalogAssumeRoleWithWebIdentityMicroseconds); + assumed = !provider->GetAWSCredentials().IsEmpty(); + } + + if (!assumed) + { + ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogAssumeRoleWithWebIdentityFailures); + throw DB::Exception( + DB::ErrorCodes::CATALOG_USER_TOKEN_NOT_AVAILABLE, + "Could not assume role `{}` with the token of user `{}`: {}. " + "Check the role's trust policy for the token's issuer, `sub`, and `aud`.", + role_arn, + auth_token.principal, + provider->getLastError()); + } + + LOG_DEBUG(logger, "Assumed role {} as user {}", role_arn, auth_token.principal); + return AuthenticatedClient{build_glue_client(provider), provider, auth_token.fingerprint}; + }; + } + else + { + auto service_credentials_provider = DB::S3::getCredentialsProvider(poco_config, credentials, creds_config); + service_client = AuthenticatedClient{build_glue_client(service_credentials_provider), service_credentials_provider, /* token_fingerprint */ {}}; + } +} + +GlueCatalog::AuthenticatedClient GlueCatalog::getClient(const DB::ForwardedAuthTokenPtr & auth_token) const +{ + if (!make_user_client) + return service_client; + + validateForwardedToken(getContext(), auth_token, fmt::format("Glue({})", region)); + + auto [client, outcome] = user_clients.getOrSetWithOutcome( + auth_token->fingerprint, + [&] { return std::make_shared(make_user_client(*auth_token)); }); + + if (outcome == DB::CacheGetOrSetOutcome::Hit) + ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogUserClientCacheHits); + + return *client; } GlueCatalog::~GlueCatalog() = default; -DataLake::ICatalog::Namespaces GlueCatalog::getDatabases(const std::string & prefix, size_t limit) const +DataLake::ICatalog::Namespaces GlueCatalog::getDatabases(const AuthenticatedClient & client, const std::string & prefix, size_t limit) const { DataLake::ICatalog::Namespaces result; Aws::Glue::Model::GetDatabasesRequest request; @@ -329,7 +444,7 @@ DataLake::ICatalog::Namespaces GlueCatalog::getDatabases(const std::string & pre { ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogGetDatabases); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogGetDatabasesMicroseconds); - outcome = glue_client->GetDatabases(request); + outcome = client.client->GetDatabases(request); } if (outcome.IsSuccess()) @@ -363,7 +478,11 @@ DataLake::ICatalog::Namespaces GlueCatalog::getDatabases(const std::string & pre return result; } +<<<<<<< HEAD CatalogTables GlueCatalog::getTablesForDatabase(const std::string & db_name, size_t limit) const +======= +DB::Names GlueCatalog::getTablesForDatabase(const AuthenticatedClient & client, const std::string & db_name, size_t limit) const +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { LOG_TEST(log, "Getting tables for database '{}' with limit {}", db_name, limit); CatalogTables result; @@ -385,7 +504,7 @@ CatalogTables GlueCatalog::getTablesForDatabase(const std::string & db_name, siz { ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogGetTables); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogGetTablesMicroseconds); - outcome = glue_client->GetTables(request); + outcome = client.client->GetTables(request); } if (outcome.IsSuccess()) { @@ -426,37 +545,78 @@ CatalogTables GlueCatalog::getTablesForDatabase(const std::string & db_name, siz return result; } +<<<<<<< HEAD CatalogTables GlueCatalog::getTables() const { auto databases = getDatabases(""); CatalogTables result; +======= +DB::Names GlueCatalog::getTables(const DB::ForwardedAuthTokenPtr & auth_token) const +{ + auto client = getClient(auth_token); + auto databases = getDatabases(client, ""); + DB::Names result; +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) for (const auto & database : databases) { - auto tables_in_database = getTablesForDatabase(database); + auto tables_in_database = getTablesForDatabase(client, database); result.insert(result.end(), tables_in_database.begin(), tables_in_database.end()); } return result; } -DataLake::ICatalog::Namespaces GlueCatalog::getNamespaces() const +DataLake::ICatalog::Namespaces GlueCatalog::getNamespaces(const DB::ForwardedAuthTokenPtr & auth_token) const { /// Glue databases are flat — they cannot contain nested namespaces. - return getDatabases(""); + return getDatabases(getClient(auth_token), ""); } +<<<<<<< HEAD CatalogTables GlueCatalog::listTablesInNamespaceDirect(const std::string & namespace_name) const +======= +DB::Names GlueCatalog::listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & auth_token) const +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { - return getTablesForDatabase(namespace_name); + return getTablesForDatabase(getClient(auth_token), namespace_name); } -bool GlueCatalog::existsTable(const std::string & database_name, const std::string & table_name) const +bool GlueCatalog::existsTable(const std::string & database_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const { +<<<<<<< HEAD if (!isNamespaceAllowed(database_name)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Namespace {} is filtered by `namespaces` database parameter", database_name); TableMetadata metadata; return tryGetTableMetadata(database_name, table_name, getContext(), metadata); } +======= + auto client = getClient(auth_token); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + + if (!isNamespaceAllowed(database_name)) + throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Namespace {} is filtered by `namespaces` database parameter", database_name); + + Aws::Glue::Model::GetTableRequest request; + request.SetDatabaseName(database_name); + request.SetName(table_name); + + Aws::Glue::Model::GetTableOutcome outcome; + { + ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogGetTable); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogGetTableMicroseconds); + outcome = client.client->GetTable(request); + } + if (outcome.IsSuccess()) + return true; + + if (outcome.GetError().GetErrorType() == Aws::Glue::GlueErrors::ENTITY_NOT_FOUND) + return false; + + throw DB::Exception( + DB::ErrorCodes::DATALAKE_DATABASE_ERROR, + "Exception calling GetTable for table {}: {}", + database_name + "." + table_name, outcome.GetError().GetMessage()); +} bool GlueCatalog::tryGetTableMetadata( const std::string & database_name, @@ -464,6 +624,8 @@ bool GlueCatalog::tryGetTableMetadata( DB::ContextPtr context_, TableMetadata & result) const { + auto client = getClient(getForwardedAuthToken(context_)); + if (!isNamespaceAllowed(database_name)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Namespace {} is filtered by `namespaces` database parameter", database_name); @@ -475,7 +637,7 @@ bool GlueCatalog::tryGetTableMetadata( { ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogGetTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogGetTableMicroseconds); - outcome = glue_client->GetTable(request); + outcome = client.client->GetTable(request); } if (outcome.IsSuccess()) { @@ -501,7 +663,7 @@ bool GlueCatalog::tryGetTableMetadata( } if (result.requiresCredentials()) - setCredentials(result); + setCredentials(client, result); auto setup_specific_properties = [&] { @@ -516,7 +678,7 @@ bool GlueCatalog::tryGetTableMetadata( if (!location_with_slash.ends_with('/')) location_with_slash += '/'; - String resolved_metadata_path = resolveMetadataPathFromTableLocation(location_with_slash, result); + String resolved_metadata_path = resolveMetadataPathFromTableLocation(client, location_with_slash, result); if (resolved_metadata_path.empty()) { result.setTableIsNotReadable(fmt::format("Could not determine metadata_location of table `{}`. ", @@ -561,7 +723,7 @@ bool GlueCatalog::tryGetTableMetadata( { if (!result.requiresDataLakeSpecificProperties()) setup_specific_properties(); - column_type = getActualTimestampType(column.GetName(), result, column_type); + column_type = getActualTimestampType(client, column.GetName(), result, column_type); } schema.push_back({column.GetName(), getType(column_type, can_be_nullable, getContext())}); @@ -578,7 +740,7 @@ bool GlueCatalog::tryGetTableMetadata( auto table_specific_properties = result.getDataLakeSpecificProperties(); if (table_specific_properties.has_value() && !table_specific_properties->iceberg_metadata_file_location.empty()) { - auto metadata_object = getOrFetchMetadataObject(table_specific_properties->iceberg_metadata_file_location, result); + auto metadata_object = getOrFetchMetadataObject(client, table_specific_properties->iceberg_metadata_file_location, result); const bool allow_geo_parser = getContext()->getSettingsRef()[DB::Setting::allow_experimental_geo_types_in_iceberg].value; auto schema_processor = DB::Iceberg::IcebergSchemaProcessor(context_, allow_geo_parser); @@ -597,7 +759,7 @@ bool GlueCatalog::tryGetTableMetadata( setup_specific_properties(); if (result.isDefaultReadableTable()) { - auto [partition_by, order_by, unsupported_properties] = DB::Iceberg::getPartitionAndSortingKeyASTsFromMetadata(getIcebergMetadataObject(result)); + auto [partition_by, order_by, unsupported_properties] = DB::Iceberg::getPartitionAndSortingKeyASTsFromMetadata(getIcebergMetadataObject(client, result)); result.setPartitionAndSortingKeys(std::move(partition_by), std::move(order_by), std::move(unsupported_properties)); } } @@ -631,13 +793,13 @@ void GlueCatalog::getTableMetadata( } } -void GlueCatalog::setCredentials(TableMetadata & metadata) const +void GlueCatalog::setCredentials(const AuthenticatedClient & client, TableMetadata & metadata) const { auto storage_type = parseStorageTypeFromLocation(metadata.getLocation()); if (storage_type == StorageType::S3) { - auto credentials = credentials_provider->GetAWSCredentials(); + auto credentials = client.credentials_provider->GetAWSCredentials(); auto s3_creds = std::make_shared(credentials.GetAWSAccessKeyId(), credentials.GetAWSSecretKey(), credentials.GetSessionToken()); metadata.setStorageCredentials(s3_creds); } @@ -648,7 +810,8 @@ void GlueCatalog::setCredentials(TableMetadata & metadata) const } } -ICatalog::CredentialsRefreshCallback GlueCatalog::getCredentialsConfigurationCallback(const DB::StorageID & storage_id) +ICatalog::CredentialsRefreshCallback GlueCatalog::getCredentialsConfigurationCallback( + const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) { /// The AWS SDK credentials provider chain (instance profile, STS assume-role, /// web-identity, etc.) refreshes its cached credentials internally before @@ -658,10 +821,11 @@ ICatalog::CredentialsRefreshCallback GlueCatalog::getCredentialsConfigurationCal /// S3 client is pinned to a snapshot that goes stale on long reads. This /// callback re-asks the same provider for current credentials each time /// `ReadBufferFromS3` reports an `ExpiredToken`, letting the read recover. - return [this, storage_id]() -> std::shared_ptr + /// `getClient` is called on each refresh to recheck the token forwarding switch before contacting STS. + return [this, storage_id, auth_token]() -> std::shared_ptr { LOG_DEBUG(log, "Refreshing AWS credentials for {} after expired token", storage_id.getNameForLogs()); - auto credentials = credentials_provider->GetAWSCredentials(); + auto credentials = getClient(auth_token).credentials_provider->GetAWSCredentials(); return std::make_shared( credentials.GetAWSAccessKeyId(), credentials.GetAWSSecretKey(), @@ -669,22 +833,25 @@ ICatalog::CredentialsRefreshCallback GlueCatalog::getCredentialsConfigurationCal }; } -bool GlueCatalog::empty() const +bool GlueCatalog::empty(const DB::ForwardedAuthTokenPtr & auth_token) const { - auto all_databases = getDatabases(""); + auto client = getClient(auth_token); + auto all_databases = getDatabases(client, ""); for (const auto & db : all_databases) { - if (!getTablesForDatabase(db, /* limit = */ 1).empty()) + if (!getTablesForDatabase(client, db, /* limit = */ 1).empty()) return false; } return true; } -Poco::JSON::Object::Ptr GlueCatalog::getOrFetchMetadataObject(const String & metadata_uri, const TableMetadata & table_metadata) const +Poco::JSON::Object::Ptr GlueCatalog::getOrFetchMetadataObject( + const AuthenticatedClient & client, const String & metadata_uri, const TableMetadata & table_metadata) const { - auto [value, _] = metadata_objects.getOrSet(metadata_uri, [&]() + /// The metadata is read with the user's credentials, so a user must not get an entry fetched by another user. + auto [value, _] = metadata_objects.getOrSet(client.token_fingerprint + metadata_uri, [&]() { - auto [object_storage, bucket_name, metadata_path] = createObjectStorageForEarlyTableAccess(metadata_uri, table_metadata); + auto [object_storage, bucket_name, metadata_path] = createObjectStorageForEarlyTableAccess(client, metadata_uri, table_metadata); auto compression_method = DB::Iceberg::getCompressionMethodFromMetadataFile(metadata_uri); auto metadata_object = DB::Iceberg::getMetadataJSONObject( metadata_path, object_storage, nullptr, getContext(), log, compression_method, std::nullopt); @@ -693,23 +860,31 @@ Poco::JSON::Object::Ptr GlueCatalog::getOrFetchMetadataObject(const String & met return *value; } -String GlueCatalog::getActualTimestampType(const String & column_name, const TableMetadata & table_metadata, const String & glue_column_type) const +String GlueCatalog::getActualTimestampType( + const AuthenticatedClient & client, + const String & column_name, + const TableMetadata & table_metadata, + const String & glue_column_type) const { +<<<<<<< HEAD auto table_specific_properties = table_metadata.getDataLakeSpecificProperties(); if (!table_specific_properties.has_value()) throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Failed to read table metadata, reason why table is unreadable: {}", table_metadata.getReasonWhyTableIsUnreadable()); auto metadata_object = getOrFetchMetadataObject(table_specific_properties->iceberg_metadata_file_location, table_metadata); return resolveTimestampTypeFromMetadata(metadata_object, column_name, glue_column_type); +======= + return resolveTimestampTypeFromMetadata(getIcebergMetadataObject(client, table_metadata), column_name, glue_column_type); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } -Poco::JSON::Object::Ptr GlueCatalog::getIcebergMetadataObject(const TableMetadata & table_metadata) const +Poco::JSON::Object::Ptr GlueCatalog::getIcebergMetadataObject(const AuthenticatedClient & client, const TableMetadata & table_metadata) const { auto table_specific_properties = table_metadata.getDataLakeSpecificProperties(); if (!table_specific_properties.has_value()) throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Failed to read table metadata, reason why table is unreadable: {}", table_metadata.getReasonWhyTableIsUnreadable()); - return getOrFetchMetadataObject(table_specific_properties->iceberg_metadata_file_location, table_metadata); + return getOrFetchMetadataObject(client, table_specific_properties->iceberg_metadata_file_location, table_metadata); } String GlueCatalog::resolveTimestampTypeFromMetadata( @@ -737,7 +912,8 @@ String GlueCatalog::resolveTimestampTypeFromMetadata( return glue_column_type == "timestamp_nano" ? "timestamp_ns" : "timestamp"; } -GlueCatalog::ObjectStorageWithPath GlueCatalog::createObjectStorageForEarlyTableAccess(const String & s3_location, const TableMetadata & table_metadata) const +GlueCatalog::ObjectStorageWithPath GlueCatalog::createObjectStorageForEarlyTableAccess( + const AuthenticatedClient & client, const String & s3_location, const TableMetadata & table_metadata) const { DB::ASTStorage * storage = table_engine_definition->as(); DB::ASTs args = storage->engine->arguments->children; @@ -756,7 +932,7 @@ GlueCatalog::ObjectStorageWithPath GlueCatalog::createObjectStorageForEarlyTable } else { - auto credentials = credentials_provider->GetAWSCredentials(); + auto credentials = client.credentials_provider->GetAWSCredentials(); DataLake::S3Credentials(credentials.GetAWSAccessKeyId(), credentials.GetAWSSecretKey(), credentials.GetSessionToken()).addCredentialsToEngineArgs(args); } } @@ -789,9 +965,10 @@ GlueCatalog::ObjectStorageWithPath GlueCatalog::createObjectStorageForEarlyTable return {object_storage, bucket_name, table_path}; } -String GlueCatalog::resolveMetadataPathFromTableLocation(const String & table_location, const TableMetadata & table_metadata) const +String GlueCatalog::resolveMetadataPathFromTableLocation( + const AuthenticatedClient & client, const String & table_location, const TableMetadata & table_metadata) const { - auto [object_storage, bucket_name, table_path] = createObjectStorageForEarlyTableAccess(table_location, table_metadata); + auto [object_storage, bucket_name, table_path] = createObjectStorageForEarlyTableAccess(client, table_location, table_metadata); auto storage_settings = std::make_shared(); storage_settings->loadFromSettingsChanges(settings.allChanged()); @@ -812,12 +989,14 @@ String GlueCatalog::resolveMetadataPathFromTableLocation(const String & table_lo } } -void GlueCatalog::createNamespaceIfNotExists(const String & namespace_name, const String & /*location*/) const +void GlueCatalog::createNamespaceIfNotExists(const String & namespace_name, const String & /*location*/, const DB::ForwardedAuthTokenPtr & auth_token) const { + auto client = getClient(auth_token); + Aws::Glue::Model::GetDatabaseRequest get_request; get_request.SetName(namespace_name); - auto get_outcome = glue_client->GetDatabase(get_request); + auto get_outcome = client.client->GetDatabase(get_request); if (get_outcome.IsSuccess()) return; @@ -836,7 +1015,7 @@ void GlueCatalog::createNamespaceIfNotExists(const String & namespace_name, cons ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogCreateDatabase); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogCreateDatabaseMicroseconds); - auto outcome = glue_client->CreateDatabase(create_request); + auto outcome = client.client->CreateDatabase(create_request); if (!outcome.IsSuccess() && outcome.GetError().GetErrorType() != Aws::Glue::GlueErrors::ALREADY_EXISTS) { throw DB::Exception( @@ -852,8 +1031,11 @@ bool GlueCatalog::createTable( const String & new_metadata_path, Poco::JSON::Object::Ptr metadata_content, DB::CompressionMethod metadata_compression_method, - bool if_not_exists) const + bool if_not_exists, + const DB::ForwardedAuthTokenPtr & auth_token) const { + auto client = getClient(auth_token); + if (!isNamespaceAllowed(namespace_name)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Failed to create table {}, namespace {} is filtered by `namespaces` database parameter", @@ -872,7 +1054,7 @@ bool GlueCatalog::createTable( Aws::Glue::Model::GetTableRequest get_request; get_request.SetDatabaseName(namespace_name); get_request.SetName(table_name); - auto get_outcome = glue_client->GetTable(get_request); + auto get_outcome = client.client->GetTable(get_request); if (get_outcome.IsSuccess()) { @@ -918,7 +1100,7 @@ bool GlueCatalog::createTable( table_location = table_location.substr(0, table_location.size() - 1); TableMetadata dummy_metadata; - auto [object_storage, bucket_name, table_path] = createObjectStorageForEarlyTableAccess(table_location, dummy_metadata); + auto [object_storage, bucket_name, table_path] = createObjectStorageForEarlyTableAccess(client, table_location, dummy_metadata); /// Name the file exactly like `IcebergMetadata::createInitial` does: the Iceberg spec extension /// (gzip -> "gz"), not the raw setting token. @@ -990,7 +1172,7 @@ bool GlueCatalog::createTable( { ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogCreateTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogCreateTableMicroseconds); - response = glue_client->CreateTable(request); + response = client.client->CreateTable(request); } if (!response.IsSuccess()) @@ -1009,8 +1191,11 @@ bool GlueCatalog::updateTableInGlue( const String & namespace_name, const String & table_name, const String & new_metadata_path, + const DB::ForwardedAuthTokenPtr & auth_token, const std::vector & columns) const { + auto client = getClient(auth_token); + Aws::Glue::Model::UpdateTableRequest request; request.SetDatabaseName(namespace_name); @@ -1046,7 +1231,7 @@ bool GlueCatalog::updateTableInGlue( { ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogUpdateTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogUpdateTableMicroseconds); - response = glue_client->UpdateTable(request); + response = client.client->UpdateTable(request); } if (!response.IsSuccess()) @@ -1055,9 +1240,9 @@ bool GlueCatalog::updateTableInGlue( return true; } -bool GlueCatalog::updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr /*new_snapshot*/) const +bool GlueCatalog::updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr /*new_snapshot*/, const DB::ForwardedAuthTokenPtr & auth_token) const { - return updateTableInGlue(namespace_name, table_name, new_metadata_path); + return updateTableInGlue(namespace_name, table_name, new_metadata_path, auth_token); } bool GlueCatalog::updateSchema( @@ -1067,16 +1252,24 @@ bool GlueCatalog::updateSchema( Poco::JSON::Object::Ptr new_schema, Int32 /*previous_schema_id*/, Int32 /*new_last_column_id*/, - Poco::JSON::Object::Ptr /*metadata*/) const + Poco::JSON::Object::Ptr /*metadata*/, + const DB::ForwardedAuthTokenPtr & auth_token) const { std::vector columns; if (new_schema) columns = icebergSchemaToGlueColumns(new_schema); - return updateTableInGlue(namespace_name, table_name, new_metadata_path, columns); + return updateTableInGlue(namespace_name, table_name, new_metadata_path, auth_token, columns); } -void GlueCatalog::dropTable(const String & namespace_name, const String & table_name, bool purge, bool if_exists) const +void GlueCatalog::dropTable( + const String & namespace_name, + const String & table_name, + bool purge, + bool if_exists, + const DB::ForwardedAuthTokenPtr & auth_token) const { + auto client = getClient(auth_token); + if (!isNamespaceAllowed(namespace_name)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Failed to drop table {}, namespace {} is filtered by `namespaces` database parameter", @@ -1087,7 +1280,7 @@ void GlueCatalog::dropTable(const String & namespace_name, const String & table_ /// TODO: implement the client-side purge so `data_lake_delete_data_on_drop` can be honored for Glue. if (purge) { - if (if_exists && !existsTable(namespace_name, table_name)) + if (if_exists && !existsTable(namespace_name, table_name, auth_token)) return; throw DB::Exception( @@ -1105,7 +1298,7 @@ void GlueCatalog::dropTable(const String & namespace_name, const String & table_ { ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogDropTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogDropTableMicroseconds); - response = glue_client->DeleteTable(request); + response = client.client->DeleteTable(request); } if (!response.IsSuccess() diff --git a/src/Databases/DataLake/GlueCatalog.h b/src/Databases/DataLake/GlueCatalog.h index 41c7deba8a003..6594e1ef4ec25 100644 --- a/src/Databases/DataLake/GlueCatalog.h +++ b/src/Databases/DataLake/GlueCatalog.h @@ -12,6 +12,8 @@ #include #include + +#include #include namespace Aws::Glue @@ -39,13 +41,17 @@ class GlueCatalog final : public ICatalog, private DB::WithContext ~GlueCatalog() override; - bool empty() const override; + bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const override; +<<<<<<< HEAD CatalogTables getTables() const override; +======= + DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) - Namespaces getNamespaces() const override; + Namespaces getNamespaces(const DB::ForwardedAuthTokenPtr & auth_token) const override; - bool existsTable(const std::string & database_name, const std::string & table_name) const override; + bool existsTable(const std::string & database_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; void getTableMetadata( const std::string & database_name, @@ -53,6 +59,8 @@ class GlueCatalog final : public ICatalog, private DB::WithContext DB::ContextPtr context_, TableMetadata & result) const override; + void onTokenForwardingDisabled() const override { user_clients.clear(); } + bool tryGetTableMetadata( const std::string & database_name, const std::string & table_name, @@ -76,11 +84,12 @@ class GlueCatalog final : public ICatalog, private DB::WithContext const String & new_metadata_path, Poco::JSON::Object::Ptr metadata_content, DB::CompressionMethod metadata_compression_method, - bool if_not_exists) const override; + bool if_not_exists, + const DB::ForwardedAuthTokenPtr & auth_token) const override; - void createNamespaceIfNotExists(const String & namespace_name, const String & location) const override; + void createNamespaceIfNotExists(const String & namespace_name, const String & location, const DB::ForwardedAuthTokenPtr & auth_token) const override; - bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot) const override; + bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const override; bool updateSchema( const String & namespace_name, @@ -89,15 +98,22 @@ class GlueCatalog final : public ICatalog, private DB::WithContext Poco::JSON::Object::Ptr new_schema, Int32 previous_schema_id, Int32 new_last_column_id, - Poco::JSON::Object::Ptr metadata = nullptr) const override; + Poco::JSON::Object::Ptr metadata, + const DB::ForwardedAuthTokenPtr & auth_token) const override; - void dropTable(const String & namespace_name, const String & table_name, bool purge, bool if_exists) const override; + void dropTable( + const String & namespace_name, + const String & table_name, + bool purge, + bool if_exists, + const DB::ForwardedAuthTokenPtr & auth_token) const override; /// Returns a callback that re-vends fresh AWS credentials from the configured /// credentials provider chain. Invoked by `ReadBufferFromS3` when an S3 call /// fails with `ExpiredToken`, so that a long-running read can recover without /// the user having to restart the query. - ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback(const DB::StorageID & storage_id) override; + ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback( + const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) override; /// Resolves the precise Iceberg timestamp type for `column_name` by searching the current schema /// in the Iceberg `metadata_object`. Falls back to `"timestamp_ns"` when `glue_column_type` is @@ -108,9 +124,23 @@ class GlueCatalog final : public ICatalog, private DB::WithContext const String & glue_column_type); private: - std::unique_ptr glue_client; + struct AuthenticatedClient + { + std::shared_ptr client; + std::shared_ptr credentials_provider; + /// Fingerprint of the forwarded token; empty for `service_client`. + String token_fingerprint; + }; + + AuthenticatedClient service_client; + std::function make_user_client; + + static constexpr size_t user_client_cache_max_entries = 1024; + mutable DB::CacheBase user_clients; + + AuthenticatedClient getClient(const DB::ForwardedAuthTokenPtr & auth_token) const; + const LoggerPtr log; - std::shared_ptr credentials_provider; std::string region; CatalogSettings settings; DB::ASTPtr table_engine_definition; @@ -118,19 +148,31 @@ class GlueCatalog final : public ICatalog, private DB::WithContext bool isNamespaceAllowed(const std::string & namespace_) const; +<<<<<<< HEAD DataLake::ICatalog::Namespaces getDatabases(const std::string & prefix, size_t limit = 0) const; CatalogTables getTablesForDatabase(const std::string & db_name, size_t limit = 0) const; CatalogTables listTablesInNamespaceDirect(const std::string & namespace_name) const override; void setCredentials(TableMetadata & metadata) const; +======= + DataLake::ICatalog::Namespaces getDatabases(const AuthenticatedClient & client, const std::string & prefix, size_t limit = 0) const; + DB::Names getTablesForDatabase(const AuthenticatedClient & client, const std::string & db_name, size_t limit = 0) const; + DB::Names listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; + void setCredentials(const AuthenticatedClient & client, TableMetadata & metadata) const; +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) /// The Glue catalog does not store detailed information about the types of timestamp columns, such as whether the column is timestamp or timestamptz. /// This method allows to clarify the actual type of the timestamp column. /// `glue_column_type` is the raw Glue type (`"timestamp"` or `"timestamp_nano"`) used as a fallback when the column is not found in Iceberg metadata. - String getActualTimestampType(const String & column_name, const TableMetadata & table_metadata, const String & glue_column_type) const; + String getActualTimestampType( + const AuthenticatedClient & client, + const String & column_name, + const TableMetadata & table_metadata, + const String & glue_column_type) const; - Poco::JSON::Object::Ptr getIcebergMetadataObject(const TableMetadata & table_metadata) const; + Poco::JSON::Object::Ptr getIcebergMetadataObject(const AuthenticatedClient & client, const TableMetadata & table_metadata) const; - String resolveMetadataPathFromTableLocation(const String & table_location, const TableMetadata & table_metadata) const; + String resolveMetadataPathFromTableLocation( + const AuthenticatedClient & client, const String & table_location, const TableMetadata & table_metadata) const; struct ObjectStorageWithPath { @@ -139,11 +181,13 @@ class GlueCatalog final : public ICatalog, private DB::WithContext String table_path; /// Path within bucket }; - ObjectStorageWithPath createObjectStorageForEarlyTableAccess(const String & s3_location, const TableMetadata & table_metadata) const; + ObjectStorageWithPath createObjectStorageForEarlyTableAccess( + const AuthenticatedClient & client, const String & s3_location, const TableMetadata & table_metadata) const; /// Fetches and caches the parsed Iceberg metadata JSON for `metadata_uri`. /// Returns the cached object on subsequent calls for the same URI. - Poco::JSON::Object::Ptr getOrFetchMetadataObject(const String & metadata_uri, const TableMetadata & table_metadata) const; + Poco::JSON::Object::Ptr getOrFetchMetadataObject( + const AuthenticatedClient & client, const String & metadata_uri, const TableMetadata & table_metadata) const; /// Shared implementation for updateMetadata / updateSchema that optionally /// sets StorageDescriptor columns in the Glue UpdateTable call. @@ -151,6 +195,7 @@ class GlueCatalog final : public ICatalog, private DB::WithContext const String & namespace_name, const String & table_name, const String & new_metadata_path, + const DB::ForwardedAuthTokenPtr & auth_token, const std::vector & columns = {}) const; mutable DB::CacheBase metadata_objects; diff --git a/src/Databases/DataLake/HiveCatalog.cpp b/src/Databases/DataLake/HiveCatalog.cpp index 42aeadc6fdc51..1c1e48c7184d3 100644 --- a/src/Databases/DataLake/HiveCatalog.cpp +++ b/src/Databases/DataLake/HiveCatalog.cpp @@ -141,7 +141,7 @@ void HiveCatalog::executeWithRetry(Func && func) const DB::ErrorCodes::NO_HIVEMETASTORE, "Hive Metastore connection failed after {} attempts. Last error: {}", max_retries, last_err_msg); } -bool HiveCatalog::empty() const +bool HiveCatalog::empty(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { fiu_do_on(DB::FailPoints::check_database_datalake_negative, { @@ -154,7 +154,11 @@ bool HiveCatalog::empty() const return result.empty(); } +<<<<<<< HEAD CatalogTables HiveCatalog::getTables() const +======= +DB::Names HiveCatalog::getTables(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { CatalogTables result; DB::Names databases; @@ -172,7 +176,7 @@ CatalogTables HiveCatalog::getTables() const return result; } -DataLake::ICatalog::Namespaces HiveCatalog::getNamespaces() const +DataLake::ICatalog::Namespaces HiveCatalog::getNamespaces(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { /// HMS databases are flat — they cannot contain nested namespaces. DB::Names databases; @@ -180,7 +184,11 @@ DataLake::ICatalog::Namespaces HiveCatalog::getNamespaces() const return databases; } +<<<<<<< HEAD CatalogTables HiveCatalog::listTablesInNamespaceDirect(const std::string & namespace_name) const +======= +DB::Names HiveCatalog::listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { DB::Names current_tables; /// Retry transient `TTransportException` (same wrapper as `getTables`) rather @@ -194,7 +202,7 @@ CatalogTables HiveCatalog::listTablesInNamespaceDirect(const std::string & names return result; } -bool HiveCatalog::existsTable(const std::string & namespace_name, const std::string & table_name) const +bool HiveCatalog::existsTable(const std::string & namespace_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { Apache::Hadoop::Hive::Table table; diff --git a/src/Databases/DataLake/HiveCatalog.h b/src/Databases/DataLake/HiveCatalog.h index aa8980fc7a14a..6c47b262b20e2 100644 --- a/src/Databases/DataLake/HiveCatalog.h +++ b/src/Databases/DataLake/HiveCatalog.h @@ -32,13 +32,17 @@ class HiveCatalog final : public ICatalog, private DB::WithContext ~HiveCatalog() override = default; - bool empty() const override; + bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const override; +<<<<<<< HEAD CatalogTables getTables() const override; +======= + DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) - Namespaces getNamespaces() const override; + Namespaces getNamespaces(const DB::ForwardedAuthTokenPtr & auth_token) const override; - bool existsTable(const std::string & namespace_name, const std::string & table_name) const override; + bool existsTable(const std::string & namespace_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; void getTableMetadata( const std::string & namespace_name, @@ -62,7 +66,11 @@ class HiveCatalog final : public ICatalog, private DB::WithContext template void executeWithRetry(Func && func) const; +<<<<<<< HEAD CatalogTables listTablesInNamespaceDirect(const std::string & namespace_name) const override; +======= + DB::Names listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) String base_url; diff --git a/src/Databases/DataLake/ICatalog.cpp b/src/Databases/DataLake/ICatalog.cpp index 75e200177199e..2f1b909ab27b4 100644 --- a/src/Databases/DataLake/ICatalog.cpp +++ b/src/Databases/DataLake/ICatalog.cpp @@ -14,11 +14,15 @@ #include #include +#include +#include + namespace DB::ErrorCodes { extern const int NOT_IMPLEMENTED; extern const int LOGICAL_ERROR; extern const int BAD_ARGUMENTS; + extern const int CATALOG_USER_TOKEN_NOT_AVAILABLE; } namespace DB::DatabaseDataLakeSetting @@ -405,20 +409,46 @@ DB::SettingsChanges CatalogSettings::allChanged() const return changes; } +<<<<<<< HEAD CatalogTables ICatalog::getTables(const TableNameFilter & filter) const +======= +void ICatalog::validateForwardedToken( + const DB::ContextPtr & context, const DB::ForwardedAuthTokenPtr & auth_token, const std::string & catalog_description) const +{ + /// Recheck the hot-reloadable switch so existing sessions stop forwarding when it is disabled. + if (!context->getGlobalContext()->getAccessControl().isTokenForwardingEnabled()) + { + onTokenForwardingDisabled(); + + throw DB::Exception( + DB::ErrorCodes::CATALOG_USER_TOKEN_NOT_AVAILABLE, + "Catalog `{}` requires token forwarding. Set `enable_token_forwarding = 1` and reconnect.", + catalog_description); + } + + if (!auth_token || auth_token->token.empty()) + throw DB::Exception( + DB::ErrorCodes::CATALOG_USER_TOKEN_NOT_AVAILABLE, + "Cannot authenticate to catalog `{}`: this session carries no bearer token. " + "Authenticate with an `Authorization: Bearer` HTTP header or `--jwt` for the native protocol.", + catalog_description); +} + +DB::Names ICatalog::getTables(const TableNameFilter & filter, const DB::ForwardedAuthTokenPtr & auth_token) const +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { switch (filter.kind) { case TableNameFilter::Kind::All: - return getTables(); + return getTables(auth_token); case TableNameFilter::Kind::Equals: { /// `name = 'ns.table'` -> list only namespace `ns`; the outer filter keeps the exact row. const auto pos = filter.value.rfind('.'); if (pos == std::string::npos) - return getTables(); - return listTablesInNamespaceDirect(filter.value.substr(0, pos)); + return getTables(auth_token); + return listTablesInNamespaceDirect(filter.value.substr(0, pos), auth_token); } case TableNameFilter::Kind::Like: @@ -437,15 +467,20 @@ CatalogTables ICatalog::getTables(const TableNameFilter & filter) const /// A leading wildcard (e.g. `%foo%`) yields an empty prefix, so we must list all namespaces and tables. /// Calling getTables() is better as its parallel. if (fixed_prefix.empty()) - return getTables(); + return getTables(auth_token); +<<<<<<< HEAD CatalogTables result; for (const auto & namespace_name : getNamespaces()) +======= + DB::Names result; + for (const auto & namespace_name : getNamespaces(auth_token)) +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { const std::string namespace_prefix = namespace_name + "."; if (!startsWith(namespace_prefix, fixed_prefix) && !startsWith(fixed_prefix, namespace_prefix)) continue; - auto tables = listTablesInNamespaceDirect(namespace_name); + auto tables = listTablesInNamespaceDirect(namespace_name, auth_token); std::move(tables.begin(), tables.end(), std::back_inserter(result)); } return result; @@ -460,19 +495,20 @@ bool ICatalog::createTable( const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr /*metadata_content*/, DB::CompressionMethod /*metadata_compression_method*/, - bool /*if_not_exists*/) const + bool /*if_not_exists*/, + const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "CREATE TABLE is not supported for this DataLakeCatalog catalog type; " "it is available only for Iceberg REST (including OneLake, BigLake, Delta Sharing) and Glue catalogs"); } -void ICatalog::createNamespaceIfNotExists(const String & /*namespace_name*/, const String & /*location*/) const +void ICatalog::createNamespaceIfNotExists(const String & /*namespace_name*/, const String & /*location*/, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "createNamespaceIfNotExists is not implemented"); } -bool ICatalog::updateMetadata(const String & /*namespace_name*/, const String & /*table_name*/, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr /*new_snapshot*/) const +bool ICatalog::updateMetadata(const String & /*namespace_name*/, const String & /*table_name*/, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr /*new_snapshot*/, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "updateMetadata is not implemented"); } @@ -484,19 +520,26 @@ bool ICatalog::updateSchema( Poco::JSON::Object::Ptr /*new_schema*/, Int32 /*previous_schema_id*/, Int32 /*new_last_column_id*/, - Poco::JSON::Object::Ptr /*metadata*/) const + Poco::JSON::Object::Ptr /*metadata*/, + const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "updateSchema is not implemented"); } -void ICatalog::dropTable(const String & /*namespace_name*/, const String & /*table_name*/, bool /*purge*/, bool /*if_exists*/) const +void ICatalog::dropTable( + const String & /*namespace_name*/, + const String & /*table_name*/, + bool /*purge*/, + bool /*if_exists*/, + const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "DROP TABLE is not supported for this DataLakeCatalog catalog type; " "it is available only for Iceberg REST (including OneLake, BigLake, Delta Sharing) and Glue catalogs"); } -ICatalog::PreparedSettingsChangesPtr ICatalog::prepareSettingsChanges(const DB::SettingsChanges & /*changes*/) +ICatalog::PreparedSettingsChangesPtr ICatalog::prepareSettingsChanges( + const DB::SettingsChanges & /*changes*/, const DB::ForwardedAuthTokenPtr & /*auth_token*/) { throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "Settings of a catalog of this type cannot be altered"); } diff --git a/src/Databases/DataLake/ICatalog.h b/src/Databases/DataLake/ICatalog.h index 131b200700bca..e7662e475aead 100644 --- a/src/Databases/DataLake/ICatalog.h +++ b/src/Databases/DataLake/ICatalog.h @@ -1,5 +1,6 @@ #pragma once #include +#include #include #include #include @@ -212,6 +213,8 @@ struct CatalogSettings String aws_role_arn; String aws_role_session_name; String aws_external_id; + String aws_sts_endpoint; + bool forward_user_token = false; DB::SettingsChanges allChanged() const; }; @@ -230,25 +233,38 @@ class ICatalog virtual ~ICatalog() = default; /// Does catalog have any tables? - virtual bool empty() const = 0; + virtual bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const = 0; +<<<<<<< HEAD /// Fetch the list of tables (names contain full namespaces). Each entry carries an /// `is_readable` flag so listings can drop unreadable tables without a metadata fetch. virtual CatalogTables getTables() const = 0; +======= + /// Fetch tables' names list. + /// Contains full namespaces in names. + virtual DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const = 0; +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) /// Enumerate every namespace as a full dot-separated path (hierarchical catalogs /// return every nested level; flat catalogs their single-level names). - virtual Namespaces getNamespaces() const = 0; + virtual Namespaces getNamespaces(const DB::ForwardedAuthTokenPtr & auth_token) const = 0; +<<<<<<< HEAD /// Fetch the list of tables restricted by the `name` predicate (see `TableNameFilter`); /// each entry carries an `is_readable` flag like getTables(). Default impl prunes /// namespaces via `getNamespaces()`. virtual CatalogTables getTables(const TableNameFilter & filter) const; +======= + /// Fetch fully-qualified table names, restricted by the `name` predicate (see + /// `TableNameFilter`). Default impl prunes namespaces via `getNamespaces()`. + virtual DB::Names getTables(const TableNameFilter & filter, const DB::ForwardedAuthTokenPtr & auth_token) const; +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) /// Check that a table exists in a given namespace. virtual bool existsTable( const std::string & namespace_naem, - const std::string & table_name) const = 0; + const std::string & table_name, + const DB::ForwardedAuthTokenPtr & auth_token) const = 0; /// Get table metadata in the given namespace. /// Throw exception if table does not exist. @@ -270,6 +286,9 @@ class ICatalog /// E.g. one of S3, Azure, Local, HDFS. virtual std::optional getStorageType() const = 0; + /// Loads catalog-wide configuration with the given token if it has not been loaded yet. + virtual void loadConfigIfNeeded(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const {} + /// Catalog-wide base location for new tables, e.g. `s3://warehouse/data`. Empty if unknown. virtual String getDefaultBaseLocation() const { return ""; } @@ -287,15 +306,16 @@ class ICatalog const String & new_metadata_path, Poco::JSON::Object::Ptr metadata_content, DB::CompressionMethod metadata_compression_method, - bool if_not_exists) const; + bool if_not_exists, + const DB::ForwardedAuthTokenPtr & auth_token) const; /// Creates the namespace unless it already exists. - virtual void createNamespaceIfNotExists(const String & namespace_name, const String & location) const; + virtual void createNamespaceIfNotExists(const String & namespace_name, const String & location, const DB::ForwardedAuthTokenPtr & auth_token) const; virtual bool managesTableLocation() const { return false; } /// Updates metadata in catalog. - virtual bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot) const; + virtual bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const; /// Commit a schema evolution (ADD/DROP/MODIFY/RENAME COLUMN) to the catalog. /// `new_metadata_path` is the path of the freshly written `vN.metadata.json`; it is used by @@ -311,11 +331,13 @@ class ICatalog Poco::JSON::Object::Ptr new_schema, Int32 previous_schema_id, Int32 new_last_column_id, - Poco::JSON::Object::Ptr metadata = nullptr) const; + Poco::JSON::Object::Ptr metadata, + const DB::ForwardedAuthTokenPtr & auth_token) const; /// Drop table from catalog. /// If `purge`, the catalog is also asked to delete the underlying data files. - virtual void dropTable(const String & namespace_name, const String & table_name, bool purge, bool if_exists) const; + virtual void dropTable( + const String & namespace_name, const String & table_name, bool purge, bool if_exists, const DB::ForwardedAuthTokenPtr & auth_token) const; /// Does the catalog support transactions or anything like that? /// For example, the Iceberg REST catalog supports atomic operations "compare if snapshot X is equal to" and "add new snapshot Y". @@ -323,11 +345,15 @@ class ICatalog /// The Glue catalog does not support such operation. virtual bool isTransactional() const { return false; } - virtual CredentialsRefreshCallback getCredentialsConfigurationCallback(const DB::StorageID & /*storage_id*/) + /// The callback outlives the query context, so retain the token directly. + virtual CredentialsRefreshCallback getCredentialsConfigurationCallback( + const DB::StorageID & /*storage_id*/, const DB::ForwardedAuthTokenPtr & /*auth_token*/) { return std::nullopt; } + virtual void onTokenForwardingDisabled() const {} + virtual void setVendedCredentialsCacheTTL(std::chrono::seconds /*ttl*/) {} virtual CredentialsRefreshCallback getWriteCredentialsConfigurationCallback(const DB::StorageID & /*storage_id*/) { @@ -346,20 +372,31 @@ class ICatalog /// state without publishing anything (may throw, may do network I/O). The state /// becomes visible only after `commitSettingsChanges`, so the caller can persist /// the changes in between and abandon the prepared state on failure. - virtual PreparedSettingsChangesPtr prepareSettingsChanges(const DB::SettingsChanges & changes); + virtual PreparedSettingsChangesPtr prepareSettingsChanges( + const DB::SettingsChanges & changes, const DB::ForwardedAuthTokenPtr & auth_token = {}); /// Publish the state built by `prepareSettingsChanges`. Must not fail. virtual void commitSettingsChanges(PreparedSettingsChangesPtr prepared); - void applySettingsChanges(const DB::SettingsChanges & changes) + void applySettingsChanges(const DB::SettingsChanges & changes, const DB::ForwardedAuthTokenPtr & auth_token = {}) { - commitSettingsChanges(prepareSettingsChanges(changes)); + commitSettingsChanges(prepareSettingsChanges(changes, auth_token)); } protected: + void validateForwardedToken( + const DB::ContextPtr & context, + const DB::ForwardedAuthTokenPtr & auth_token, + const std::string & catalog_description) const; + /// List tables directly in `namespace_name` (non-recursive), as fully-qualified +<<<<<<< HEAD /// `namespace.table` entries carrying an `is_readable` flag like getTables(). virtual CatalogTables listTablesInNamespaceDirect(const std::string & namespace_name) const = 0; +======= + /// `namespace.table` names. + virtual DB::Names listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & auth_token) const = 0; +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) /// Name of the warehouse, /// which is sometimes also called "catalog name". diff --git a/src/Databases/DataLake/PaimonRestCatalog.cpp b/src/Databases/DataLake/PaimonRestCatalog.cpp index 854532e8f0a7b..c2404c0c1f56e 100644 --- a/src/Databases/DataLake/PaimonRestCatalog.cpp +++ b/src/Databases/DataLake/PaimonRestCatalog.cpp @@ -417,7 +417,7 @@ void PaimonRestCatalog::forEachTables( } -bool PaimonRestCatalog::empty() const +bool PaimonRestCatalog::empty(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { DB::Strings databases; DB::Names tables; @@ -431,7 +431,11 @@ bool PaimonRestCatalog::empty() const return tables.empty(); } +<<<<<<< HEAD CatalogTables PaimonRestCatalog::getTables() const +======= +DB::Names PaimonRestCatalog::getTables(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { DB::Strings databases; DB::Names tables; @@ -446,7 +450,7 @@ CatalogTables PaimonRestCatalog::getTables() const return result; } -DataLake::ICatalog::Namespaces PaimonRestCatalog::getNamespaces() const +DataLake::ICatalog::Namespaces PaimonRestCatalog::getNamespaces(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { /// Paimon REST databases are flat — they cannot contain nested namespaces. DB::Strings databases; @@ -454,7 +458,11 @@ DataLake::ICatalog::Namespaces PaimonRestCatalog::getNamespaces() const return databases; } +<<<<<<< HEAD CatalogTables PaimonRestCatalog::listTablesInNamespaceDirect(const std::string & namespace_name) const +======= +DB::Names PaimonRestCatalog::listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { DB::Names tables; forEachTables(namespace_name, tables, {}); @@ -467,7 +475,7 @@ CatalogTables PaimonRestCatalog::listTablesInNamespaceDirect(const std::string & return result; } -bool PaimonRestCatalog::existsTable(const String & database_name, const String & table_name) const +bool PaimonRestCatalog::existsTable(const String & database_name, const String & table_name, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { try { diff --git a/src/Databases/DataLake/PaimonRestCatalog.h b/src/Databases/DataLake/PaimonRestCatalog.h index b77225a7636b9..e43969809ca7a 100644 --- a/src/Databases/DataLake/PaimonRestCatalog.h +++ b/src/Databases/DataLake/PaimonRestCatalog.h @@ -82,13 +82,17 @@ class PaimonRestCatalog final : public ICatalog, private DB::WithContext ~PaimonRestCatalog() override = default; - bool empty() const override; + bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const override; +<<<<<<< HEAD CatalogTables getTables() const override; +======= + DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) - Namespaces getNamespaces() const override; + Namespaces getNamespaces(const DB::ForwardedAuthTokenPtr & auth_token) const override; - bool existsTable(const String & database_name, const String & table_name) const override; + bool existsTable(const String & database_name, const String & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; void getTableMetadata(const String & database_name, const String & table_name, DB::ContextPtr context_, TableMetadata & result) const override; @@ -134,7 +138,11 @@ class PaimonRestCatalog final : public ICatalog, private DB::WithContext void forEachTables(const String & database, DB::Names & tables, StopCondition stop_condition = {}, ExecuteFunc execute_func = {}) const; +<<<<<<< HEAD CatalogTables listTablesInNamespaceDirect(const std::string & namespace_name) const override; +======= + DB::Names listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) Poco::JSON::Object::Ptr requestRest( const String & endpoint, diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index b7009376a81cb..37f8137bc1430 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -1,13 +1,22 @@ #include #include #include +#include +#include +#include #include +<<<<<<< HEAD +======= +#include +#include +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) #include #include #include #include #include #include +#include #include #include #include @@ -95,10 +104,17 @@ namespace ProfileEvents { extern const Event DataLakeRestCatalogCredentialsVended; extern const Event DataLakeRestCatalogCredentialsCacheHits; +<<<<<<< HEAD extern const Event OneLakeAccessTokenRequests; extern const Event OneLakeAccessTokenRequestFailures; extern const Event OneLakeAccessTokenRequestMicroseconds; extern const Event OneLakeAccessTokenExpirations; +======= + extern const Event DataLakeRestCatalogTokenExchange; + extern const Event DataLakeRestCatalogTokenExchangeMicroseconds; + extern const Event DataLakeRestCatalogTokenExchangeFailures; + extern const Event DataLakeRestCatalogUserTokenCacheHits; +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) extern const Event DataLakeRestCatalogLoadConfig; extern const Event DataLakeRestCatalogLoadConfigMicroseconds; extern const Event DataLakeRestCatalogGetNamespaces; @@ -123,6 +139,11 @@ namespace ProfileEvents extern const Event DataLakeRestCatalogDropTableMicroseconds; } +namespace CurrentMetrics +{ + extern const Metric DataLakeCatalogUserTokenCacheEntries; +} + namespace DB::DatabaseDataLakeSetting { extern const DatabaseDataLakeSettingsString catalog_credential; @@ -238,6 +259,14 @@ std::unordered_set getAllowedBigLakeMetadataServiceHosts( return allowed; } +/// Token is considered valid for 90% of `expires_in`, which is clamped to [0, 1 day]. +std::chrono::system_clock::time_point tokenExpiresAt(Int64 expires_in) +{ + static constexpr Int64 max_expires_in_seconds = 24 * 60 * 60; + expires_in = std::clamp(expires_in, 0, max_expires_in_seconds); + return std::chrono::system_clock::now() + std::chrono::seconds(expires_in * 9 / 10); +} + } namespace @@ -480,8 +509,13 @@ RestCatalog::RestCatalog( const std::string & oauth_server_uri_, bool oauth_server_use_request_body_, const std::string & namespaces_, +<<<<<<< HEAD bool flat_namespaces_, DB::ContextPtr context_) +======= + DB::ContextPtr context_, + const TokenForwardingConfig & token_forwarding_) +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) : ICatalog(warehouse_) , DB::WithContext(context_) , base_url(correctAPIURI(base_url_)) @@ -489,7 +523,15 @@ RestCatalog::RestCatalog( , auth_scope(auth_scope_) , oauth_server_uri(oauth_server_uri_) , oauth_server_use_request_body(oauth_server_use_request_body_) +<<<<<<< HEAD , flat_namespaces(flat_namespaces_) +======= + , token_forwarding(token_forwarding_) + , user_token_cache( + CurrentMetrics::end(), + CurrentMetrics::DataLakeCatalogUserTokenCacheEntries, + user_token_cache_max_entries) +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) , allowed_namespaces(namespaces_) { CatalogState initial_state; @@ -503,11 +545,21 @@ RestCatalog::RestCatalog( initial_state.auth_header = parseAuthHeader(auth_header_); validateAuthHeaders(initial_state.auth_header.value()); } +<<<<<<< HEAD /// `loadConfig` reaches the virtual `createReadBuffer`, but no catalog constructed through this /// ctor (`OneLakeCatalog`, `BigLakeCatalog`) overrides it, so the base implementation is the intended /// target. `S3TablesCatalog` overrides it and uses the separate ctor, calling `loadConfig` afterwards. /// NOLINTNEXTLINE(clang-analyzer-optin.cplusplus.VirtualCall) initial_state.config = loadConfig(initial_state); +======= + + /// Defer `/v1/config` until a query supplies the user token needed to authenticate it. + if (!token_forwarding.forward_user_token) + { + initial_state.config = loadConfig(initial_state, /* generation */ 0, /* auth_token */ {}); + initial_state.config_loaded = true; + } +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) state.set(std::make_unique(std::move(initial_state))); } @@ -527,13 +579,48 @@ RestCatalog::RestCatalog( , auth_scope(auth_scope_) , oauth_server_uri(oauth_server_uri_) , oauth_server_use_request_body(oauth_server_use_request_body_) +<<<<<<< HEAD , flat_namespaces(flat_namespaces_) +======= + , user_token_cache( + CurrentMetrics::end(), + CurrentMetrics::DataLakeCatalogUserTokenCacheEntries, + user_token_cache_max_entries) +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) , allowed_namespaces(namespaces_) { } -RestCatalog::Config RestCatalog::loadConfig(const CatalogState & catalog_state, const std::optional & auth_headers) +void RestCatalog::loadConfigIfNeeded(const DB::ForwardedAuthTokenPtr & auth_token) const +{ + const auto old_state = getStateSnapshot(); + if (old_state->config_loaded) + return; + + auto new_state = std::make_unique(*old_state); + new_state->config = loadConfig(*old_state, old_state.generation, auth_token); + new_state->config_loaded = true; + + /// `commitSettingsChanges` can publish during the config request. Discard a stale result + /// so it cannot restore old credentials or a warehouse resolved with them. + std::lock_guard publish_lock(auth_publish_mutex); + if (auth_generation.load(std::memory_order_acquire) != old_state.generation) + { + LOG_DEBUG(log, "Catalog credentials changed while `/v1/config` was loading; discarding it"); + return; + } + /// A concurrent first caller may have already published the config. + if (state.get()->config_loaded) + return; + state.set(std::move(new_state)); +} + +RestCatalog::Config RestCatalog::loadConfig( + const CatalogState & catalog_state, + UInt64 generation, + const DB::ForwardedAuthTokenPtr & auth_token, + const std::optional & auth_headers) const { Poco::URI::QueryParameters params = {{"warehouse", warehouse}}; @@ -542,7 +629,11 @@ RestCatalog::Config RestCatalog::loadConfig(const CatalogState & catalog_state, { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogLoadConfig); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogLoadConfigMicroseconds); +<<<<<<< HEAD auto buf = createReadBuffer(catalog_state, CONFIG_ENDPOINT, params, /* headers */ {}, auth_headers); +======= + auto buf = createReadBuffer(catalog_state, generation, CONFIG_ENDPOINT, auth_token, params, /* headers */{}, auth_headers); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) readJSONObjectPossiblyInvalid(json_str, *buf); } @@ -588,18 +679,24 @@ void RestCatalog::validateAuthHeaders(const DB::HTTPHeaderEntry & header) const getContext()->getGlobalContext()->getHTTPHeaderFilter().checkAndNormalizeHeaders(header_to_check); } +<<<<<<< HEAD DB::HTTPHeaderEntries RestCatalog::getAuthHeaders( const CatalogState & catalog_state, bool update_token, bool * used_cached_oauth_token) const +======= +DB::HTTPHeaderEntries RestCatalog::getAuthHeaders(const AuthContext & auth_context) const +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { fiu_do_on(DB::FailPoints::check_database_datalake_negative, { throw DB::Exception(DB::ErrorCodes::FAULT_INJECTED, "Injecting fault when checking database"); }); - if (used_cached_oauth_token) - *used_cached_oauth_token = false; + if (auth_context.used_cached_oauth_token) + *auth_context.used_cached_oauth_token = false; + + const auto & catalog_state = auth_context.catalog_state; /// Option 1: user specified auth header manually. /// Header has format: 'Authorization: '. @@ -608,7 +705,21 @@ DB::HTTPHeaderEntries RestCatalog::getAuthHeaders( return DB::HTTPHeaderEntries{catalog_state.auth_header.value()}; } +<<<<<<< HEAD /// Option 2: user provided grant_type and client credentials for OAuthClientCredentialsRequest. +======= + if (token_forwarding.forward_user_token) + { + DB::HTTPHeaderEntries headers; + headers.emplace_back( + "Authorization", + "Bearer " + + getForwardedToken(catalog_state, auth_context.generation, auth_context.auth_token, auth_context.update_token)); + return headers; + } + + /// We would make OAuthClientCredentialsRequest +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) /// https://github.com/apache/iceberg/blob/3badfe0c1fcf0c0adfc7aa4a10f0b50365c48cf9/open-api/rest-catalog-open-api.yaml#L3498C5-L3498C34 /// Horizon accepts a secret-only credential (PAT/JWT as `client_secret` with empty `client_id`). if (!catalog_state.client_id.empty() || !catalog_state.client_secret.empty()) @@ -618,14 +729,14 @@ DB::HTTPHeaderEntries RestCatalog::getAuthHeaders( /// request fails with 401/403 and is retried with `update_token = true`, fetching /// a token with the snapshot's credentials. auto current = access_token.get(); - if (!current || update_token || current->isExpired()) + if (!current || auth_context.update_token || current->isExpired()) { - access_token.set(std::make_unique(retrieveAccessToken(catalog_state.client_id, catalog_state.client_secret))); - current = access_token.get(); + current = publishServiceToken( + retrieveAccessToken(catalog_state.client_id, catalog_state.client_secret), auth_context.generation); } - else if (used_cached_oauth_token) + else if (auth_context.used_cached_oauth_token) { - *used_cached_oauth_token = true; + *auth_context.used_cached_oauth_token = true; } DB::HTTPHeaderEntries headers; @@ -635,6 +746,110 @@ DB::HTTPHeaderEntries RestCatalog::getAuthHeaders( return {}; } +MultiVersion::Version RestCatalog::publishServiceToken(AccessToken minted, UInt64 generation) const +{ + auto result = std::make_shared(std::move(minted)); + + /// Check and publish under one lock so a concurrent `ALTER` cannot restore a superseded token. + std::lock_guard lock(auth_publish_mutex); + if (auth_generation.load(std::memory_order_acquire) == generation) + access_token.set(std::make_unique(*result)); + + return result; +} + +void RestCatalog::validateForwardedToken(const DB::ForwardedAuthTokenPtr & auth_token) const +{ + ICatalog::validateForwardedToken(getContext(), auth_token, warehouse); +} + +String RestCatalog::getForwardedToken( + const CatalogState & catalog_state, UInt64 generation, const DB::ForwardedAuthTokenPtr & auth_token, bool update_token) const +{ + validateForwardedToken(auth_token); + + if (!token_forwarding.exchangeEnabled()) + return auth_token->token; + + if (token_forwarding.user_token_cache_ttl == 0) + return exchangeUserToken(catalog_state, generation, *auth_token).token; + + auto exchange = [&] + { + return std::make_shared(exchangeUserToken(catalog_state, generation, *auth_token)); + }; + + const String cache_key = fmt::format("{}:{}", generation, auth_token->fingerprint); + + if (!update_token) + { + if (auto cached = user_token_cache.get(cache_key); cached && !cached->isExpired()) + { + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUserTokenCacheHits); + return cached->token; + } + } + + /// Remove stale entries before `getOrSetWithOutcome` so concurrent refreshes share a fresh result. + user_token_cache.remove(cache_key); + auto [session_token, outcome] = user_token_cache.getOrSetWithOutcome(cache_key, exchange); + if (outcome == DB::CacheGetOrSetOutcome::Hit) + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUserTokenCacheHits); + return session_token->token; +} + +AccessToken RestCatalog::exchangeUserToken( + const CatalogState & catalog_state, UInt64 generation, const DB::ForwardedAuthToken & auth_token, + const AccessToken * prepared_actor_token) const +{ + const Poco::URI url(token_forwarding.token_exchange_uri); + Poco::URI::QueryParameters params = { + {"grant_type", "urn:ietf:params:oauth:grant-type:token-exchange"}, + {"subject_token", auth_token.token}, + {"subject_token_type", token_forwarding.subject_token_type}, + }; + if (!token_forwarding.requested_token_type.empty()) + params.emplace_back("requested_token_type", token_forwarding.requested_token_type); + if (!auth_scope.empty()) + params.emplace_back("scope", auth_scope); + if (token_forwarding.forward_actor_token) + { + const auto actor_token = prepared_actor_token ? prepared_actor_token->token : getServicePrincipalToken(catalog_state, generation); + if (!actor_token.empty()) + { + params.emplace_back("actor_token", actor_token); + params.emplace_back("actor_token_type", "urn:ietf:params:oauth:token-type:access_token"); + } + } + params.emplace_back("client_id", catalog_state.client_id); + params.emplace_back("client_secret", catalog_state.client_secret); + + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogTokenExchange); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogTokenExchangeMicroseconds); + + AccessToken exchanged; + try + { + exchanged = requestToken(url, params); + } + catch (...) + { + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogTokenExchangeFailures); + throw; + } + + /// Bound cached tokens even when the endpoint omits `expires_in`. + if (token_forwarding.user_token_cache_ttl > 0) + { + const auto ttl_bound = std::chrono::system_clock::now() + std::chrono::seconds(token_forwarding.user_token_cache_ttl); + if (!exchanged.expires_at.has_value() || exchanged.expires_at.value() > ttl_bound) + exchanged.expires_at = ttl_bound; + } + + LOG_DEBUG(log, "Exchanged the token of user `{}` for a catalog session token", auth_token.principal); + return exchanged; +} + OneLakeCatalog::OneLakeCatalog( const std::string & warehouse_, const std::string & base_url_, @@ -682,7 +897,8 @@ OneLakeCatalog::OneLakeCatalog( break; } } - initial_state.config = loadConfig(initial_state); + initial_state.config = loadConfig(initial_state, /* generation */ 0, /* auth_token */ {}); + initial_state.config_loaded = true; state.set(std::make_unique(std::move(initial_state))); } @@ -737,22 +953,39 @@ void RestCatalog::validateSettingsChanges(const DB::SettingsChanges & changes, b struct RestCatalog::PreparedAuthChanges : ICatalog::PreparedSettingsChanges { std::unique_ptr new_state; - /// Set only when the OAuth credentials changed. std::unique_ptr new_access_token; }; -ICatalog::PreparedSettingsChangesPtr RestCatalog::prepareSettingsChanges(const DB::SettingsChanges & changes) +ICatalog::PreparedSettingsChangesPtr RestCatalog::prepareSettingsChanges( + const DB::SettingsChanges & changes, const DB::ForwardedAuthTokenPtr & auth_token) { - const auto old_state = state.get(); + if (token_forwarding.forward_user_token) + validateForwardedToken(auth_token); + const auto old_state = getStateSnapshot(); CatalogState new_state = *old_state; auto prepared = std::make_unique(); std::optional new_auth_headers; applySettingsChangesToState(changes, *old_state, new_state, new_auth_headers, prepared->new_access_token); + if (token_forwarding.forward_user_token) + { + /// Preparation may fail or be abandoned; do not publish its tokens in the live cache. + if (token_forwarding.exchangeEnabled()) + { + if (token_forwarding.forward_actor_token && !prepared->new_access_token) + prepared->new_access_token = std::make_unique(retrieveAccessToken(new_state.client_id, new_state.client_secret)); + const auto session = exchangeUserToken(new_state, old_state.generation, *auth_token, prepared->new_access_token.get()); + new_auth_headers = DB::HTTPHeaderEntries{{"Authorization", "Bearer " + session.token}}; + } + else + new_auth_headers = DB::HTTPHeaderEntries{{"Authorization", "Bearer " + auth_token->token}}; + } + /// The config was loaded with the old credentials; the new ones may resolve the /// warehouse to a different prefix or base location, so reload it before publishing. - new_state.config = loadConfig(new_state, new_auth_headers); + new_state.config = loadConfig(new_state, old_state.generation, auth_token, new_auth_headers); + new_state.config_loaded = true; prepared->new_state = std::make_unique(std::move(new_state)); return prepared; } @@ -763,6 +996,7 @@ void RestCatalog::commitSettingsChanges(ICatalog::PreparedSettingsChangesPtr pre if (!prepared_auth || !prepared_auth->new_state) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Settings changes to commit were not prepared by this catalog"); +<<<<<<< HEAD state.set(std::move(prepared_auth->new_state)); if (prepared_auth->new_access_token) access_token.set(std::move(prepared_auth->new_access_token)); @@ -777,6 +1011,21 @@ void RestCatalog::setVendedCredentialsCacheTTL(std::chrono::seconds ttl) if (ttl != vended_credentials_cache_ttl) { vended_credentials_cache_ttl = ttl; +======= + { + std::lock_guard lock(auth_publish_mutex); + state.set(std::move(prepared_auth->new_state)); + if (prepared_auth->new_access_token) + access_token.set(std::move(prepared_auth->new_access_token)); + + /// Publish the state before its generation; readers load them in the opposite order. + auth_generation.fetch_add(1, std::memory_order_release); + } + + user_token_cache.clear(); + { + std::lock_guard lock(credentials_cache_mutex); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) credentials_cache.clear(); } } @@ -809,16 +1058,15 @@ void RestCatalog::applySettingsChangesToState( throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unexpected setting `{}` after validation", change.name); } - if (credential_mode && (new_state.client_id != old_state.client_id || new_state.client_secret != old_state.client_secret)) + if (credential_mode && (!token_forwarding.forward_user_token || token_forwarding.forward_actor_token) + && (new_state.client_id != old_state.client_id || new_state.client_secret != old_state.client_secret)) { - /// Eagerly fetch a token with the not-yet-published credentials: wrong credentials - /// fail the ALTER right here, and the config reload authenticates with that token - /// instead of the cached one. new_access_token = std::make_unique(retrieveAccessToken(new_state.client_id, new_state.client_secret)); new_auth_headers = DB::HTTPHeaderEntries{{"Authorization", "Bearer " + new_access_token->token}}; } } +<<<<<<< HEAD DB::HTTPHeaderEntries OneLakeCatalog::getAuthHeaders( const CatalogState & catalog_state, bool update_token, @@ -834,6 +1082,11 @@ DB::HTTPHeaderEntries OneLakeCatalog::getAuthHeaders( { headers = RestCatalog::getAuthHeaders(catalog_state, update_token, used_cached_oauth_token); } +======= +DB::HTTPHeaderEntries OneLakeCatalog::getAuthHeaders(const AuthContext & auth_context) const +{ + auto headers = RestCatalog::getAuthHeaders(auth_context); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) headers.emplace_back("User-Agent", fmt::format("ClickHouse/{}{} OneLake-Catalog", VERSION_STRING, VERSION_OFFICIAL)); return headers; } @@ -1064,22 +1317,11 @@ namespace } -AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, const std::string & client_secret) const +AccessToken RestCatalog::requestToken(Poco::URI url, const Poco::URI::QueryParameters & params, bool use_query_parameters) const { - ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogAuthTokenRetrieve); - auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogAuthTokenRefreshedMicroseconds); - - static constexpr auto oauth_tokens_endpoint = "oauth/tokens"; - - /// TODO: - /// 1. support oauth2-server-uri - /// https://github.com/apache/iceberg/blob/918f81f3c3f498f46afcea17c1ac9cdc6913cb5c/open-api/rest-catalog-open-api.yaml#L183C82-L183C99 - - Poco::URI url; - DB::ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback; - size_t body_size = 0; String body; +<<<<<<< HEAD if (oauth_server_uri.empty() && !oauth_server_use_request_body) { url = Poco::URI(base_url / oauth_tokens_endpoint); @@ -1092,10 +1334,13 @@ AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, cons /// Snowflake Horizon OAuth uses secret-only credentials (PAT/JWT); omit empty client_id. if (!client_id.empty()) params.emplace_back("client_id", client_id); +======= + if (use_query_parameters) +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) url.setQueryParameters(params); - } else { +<<<<<<< HEAD String encoded_auth_scope; String encoded_client_secret; Poco::URI::encode(auth_scope, auth_scope, encoded_auth_scope); @@ -1112,14 +1357,18 @@ AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, cons } body_size = body.size(); out_stream_callback = [&](std::ostream & os) +======= + DB::WriteBufferFromOwnString wb; + bool first = true; + for (const auto & [name, value] : params) +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { - os << body; - }; - - if (oauth_server_uri.empty()) - url = Poco::URI(base_url / oauth_tokens_endpoint); - else - url = Poco::URI(oauth_server_uri); + if (!first) + wb << "&"; + first = false; + wb << name << "=" << DB::formUrlEncode(value); + } + body = wb.str(); } const auto & context = getContext(); @@ -1130,13 +1379,11 @@ AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, cons Poco::Net::HTTPRequest request(Poco::Net::HTTPRequest::HTTP_POST, url.getPathAndQuery(), Poco::Net::HTTPMessage::HTTP_1_1); request.setContentType("application/x-www-form-urlencoded"); - request.setContentLength(body_size); + request.setContentLength(body.size()); request.set("Accept", "application/json"); - std::ostream & os = session->sendRequest(request); - /// The query-parameters flavor of the request has no body. - if (out_stream_callback) - out_stream_callback(os); + /// Strict OAuth servers reject bearer authentication alongside client credentials in the form. + session->sendRequest(request) << body; Poco::Net::HTTPResponse response; std::istream & rs = session->receiveResponse(response); @@ -1144,6 +1391,7 @@ AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, cons std::string json_str; Poco::StreamCopier::copyToString(rs, json_str); +<<<<<<< HEAD /// The body of a failed response is an OAuth error object, safe to show. /// The URL is omitted: its query string can carry `client_secret`. if (response.getStatus() != Poco::Net::HTTPResponse::HTTP_OK) @@ -1272,14 +1520,44 @@ AccessToken OneLakeCatalog::retrieveAccessTokenViaRefreshToken(const CatalogStat Poco::JSON::Parser parser; Poco::Dynamic::Var res_json = parser.parse(json_str); const Poco::JSON::Object::Ptr & object = res_json.extract(); +======= + /// OAuth error bodies may echo the subject token, so exclude them from exceptions. + const auto describe_endpoint = [&url, &response] + { + return fmt::format( + "OAuth token endpoint {}://{}:{}{} returned HTTP {}", + url.getScheme(), url.getHost(), url.getPort(), url.getPath(), + static_cast(response.getStatus())); + }; + + Poco::JSON::Object::Ptr object; + try + { + object = Poco::JSON::Parser().parse(json_str).extract(); + } + catch (const Poco::Exception &) + { + object = nullptr; + } + if (!object) + throw DB::Exception( + DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "{} with a body that is not a JSON object", describe_endpoint()); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) AccessToken token; + if (!object->has("access_token")) + throw DB::Exception( + DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "{} with no `access_token` field", describe_endpoint()); token.token = object->get("access_token").extract(); if (object->has("expires_in")) { +<<<<<<< HEAD Int64 expires_in = object->getValue("expires_in"); token.expires_at = std::chrono::system_clock::now() + std::chrono::seconds(expires_in * 9 / 10); +======= + token.expires_at = tokenExpiresAt(object->getValue("expires_in")); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } /// The response also carries a rotated `refresh_token`. It is deliberately ignored: @@ -1290,6 +1568,31 @@ AccessToken OneLakeCatalog::retrieveAccessTokenViaRefreshToken(const CatalogStat return token; } +AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, const std::string & client_secret) const +{ + static constexpr auto oauth_tokens_endpoint = "oauth/tokens"; + + const Poco::URI::QueryParameters params = { + {"grant_type", "client_credentials"}, + {"scope", auth_scope}, + {"client_id", client_id}, + {"client_secret", client_secret}, + }; + const auto url = oauth_server_uri.empty() ? Poco::URI(base_url / oauth_tokens_endpoint) : Poco::URI(oauth_server_uri); + + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogAuthTokenRetrieve); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogAuthTokenRefreshedMicroseconds); + return requestToken(url, params, oauth_server_uri.empty() && !oauth_server_use_request_body); +} + +String RestCatalog::getServicePrincipalToken(const CatalogState & catalog_state, UInt64 generation) const +{ + auto current = access_token.get(); + if (!current || current->isExpired()) + current = publishServiceToken(retrieveAccessToken(catalog_state.client_id, catalog_state.client_secret), generation); + return current->token; +} + BigLakeCatalog::BigLakeCatalog( const std::string & warehouse_, const std::string & base_url_, @@ -1320,14 +1623,19 @@ BigLakeCatalog::BigLakeCatalog( access_token.set(std::make_unique(retrieveGoogleCloudAccessToken())); } CatalogState initial_state; - initial_state.config = loadConfig(initial_state); + initial_state.config = loadConfig(initial_state, /* generation */ 0, /* auth_token */ {}); + initial_state.config_loaded = true; state.set(std::make_unique(std::move(initial_state))); } +<<<<<<< HEAD DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders( const CatalogState & catalog_state, bool update_token, bool * used_cached_oauth_token) const +======= +DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders(const AuthContext & auth_context) const +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { /// Google Cloud OAuth2 for BigLake. /// Uses GCP metadata service or Application Default Credentials to get access token. @@ -1335,18 +1643,18 @@ DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders( /// https://developers.google.com/identity/protocols/oauth2 if (!google_project_id.empty() || !google_adc_client_id.empty()) { - if (used_cached_oauth_token) - *used_cached_oauth_token = false; + if (auth_context.used_cached_oauth_token) + *auth_context.used_cached_oauth_token = false; auto current = access_token.get(); - if (!current || update_token || current->isExpired()) + if (!current || auth_context.update_token || current->isExpired()) { access_token.set(std::make_unique(retrieveGoogleCloudAccessToken())); current = access_token.get(); } - else if (used_cached_oauth_token) + else if (auth_context.used_cached_oauth_token) { - *used_cached_oauth_token = true; + *auth_context.used_cached_oauth_token = true; } DB::HTTPHeaderEntries headers; @@ -1366,7 +1674,11 @@ DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders( return headers; } +<<<<<<< HEAD return RestCatalog::getAuthHeaders(catalog_state, update_token, used_cached_oauth_token); +======= + return RestCatalog::getAuthHeaders(auth_context); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } AccessToken BigLakeCatalog::retrieveGoogleCloudAccessTokenFromRefreshToken() const @@ -1382,7 +1694,7 @@ AccessToken BigLakeCatalog::retrieveGoogleCloudAccessTokenFromRefreshToken() con AccessToken token; token.token = std::move(result.access_token); - token.expires_at = std::chrono::system_clock::now() + std::chrono::seconds(result.expires_in * 9 / 10); + token.expires_at = tokenExpiresAt(result.expires_in); return token; } @@ -1493,8 +1805,7 @@ AccessToken BigLakeCatalog::retrieveGoogleCloudAccessToken() const if (object->has("expires_in")) { - Int64 expires_in = object->getValue("expires_in"); - token.expires_at = std::chrono::system_clock::now() + std::chrono::seconds(expires_in * 9 / 10); + token.expires_at = tokenExpiresAt(object->getValue("expires_in")); } return token; @@ -1502,8 +1813,8 @@ AccessToken BigLakeCatalog::retrieveGoogleCloudAccessToken() const std::optional RestCatalog::getStorageType() const { - const auto state_snapshot = state.get(); - if (state_snapshot->config.default_base_location.empty()) + const auto state_snapshot = getStateSnapshot(); + if (!state_snapshot->config_loaded || state_snapshot->config.default_base_location.empty()) return std::nullopt; return parseStorageTypeFromLocation(state_snapshot->config.default_base_location); } @@ -1515,7 +1826,9 @@ String RestCatalog::getDefaultBaseLocation() const DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( const CatalogState & catalog_state, + UInt64 generation, const std::string & endpoint, + const DB::ForwardedAuthTokenPtr & auth_token, const Poco::URI::QueryParameters & params, const DB::HTTPHeaderEntries & headers, const std::optional & auth_headers) const @@ -1529,7 +1842,22 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( auto create_buffer = [&](bool update_token, bool & used_cached_oauth_token) { +<<<<<<< HEAD auto result_headers = auth_headers ? *auth_headers : getAuthHeaders(catalog_state, update_token, &used_cached_oauth_token); +======= + AuthContext auth_context{ + .catalog_state = catalog_state, + .generation = generation, + .update_token = update_token, + .method = Poco::Net::HTTPRequest::HTTP_GET, + .url = url, + .extra_headers = headers, + .body = {}, + .auth_token = auth_token, + .used_cached_oauth_token = &used_cached_oauth_token, + }; + auto result_headers = auth_headers ? *auth_headers : getAuthHeaders(auth_context); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) std::move(headers.begin(), headers.end(), std::back_inserter(result_headers)); return DB::BuilderRWBufferFromHTTP(url) @@ -1556,20 +1884,29 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( catch (const DB::HTTPException & e) { const auto status = e.getHTTPStatus(); - if (update_token_if_expired && - (status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_UNAUTHORIZED - || status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_FORBIDDEN)) - { - ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUnauthorized); - bool used_cached_oauth_token_on_retry = false; - return create_buffer(true, used_cached_oauth_token_on_retry); - } - throw; + if (!shouldRetryWithFreshToken(status)) + throw; + + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUnauthorized); + bool used_cached_oauth_token_on_retry = false; + return create_buffer(true, used_cached_oauth_token_on_retry); } } -bool RestCatalog::empty() const +bool RestCatalog::shouldRetryWithFreshToken(Poco::Net::HTTPResponse::HTTPStatus status) const +{ + if (token_forwarding.forward_user_token) + return token_forwarding.exchangeEnabled() && status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_UNAUTHORIZED; + + return update_token_if_expired + && (status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_UNAUTHORIZED + || status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_FORBIDDEN); +} + +bool RestCatalog::empty(const DB::ForwardedAuthTokenPtr & auth_token) const { + loadConfigIfNeeded(auth_token); + bool found_table = false; auto stop_condition = [&](const std::string & namespace_name) -> bool { @@ -1579,20 +1916,26 @@ bool RestCatalog::empty() const if (!allowed_namespaces.isNamespaceAllowed(namespace_name, /*nested*/ false)) return false; - const auto tables = listTablesInNamespace(namespace_name, /* limit */1); + const auto tables = listTablesInNamespace(namespace_name, auth_token, /* limit */1); if (!tables.empty()) found_table = true; return found_table; }; Namespaces namespaces; - getNamespacesRecursive("", namespaces, stop_condition, /* execute_func */{}); + getNamespacesRecursive("", namespaces, stop_condition, /* execute_func */{}, auth_token); return !found_table; } +<<<<<<< HEAD CatalogTables RestCatalog::getTables() const +======= +DB::Names RestCatalog::getTables(const DB::ForwardedAuthTokenPtr & auth_token) const +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { + loadConfigIfNeeded(auth_token); + auto & pool = getContext()->getIcebergCatalogThreadpool(); DB::Names tables; std::mutex mutex; @@ -1608,7 +1951,7 @@ CatalogTables RestCatalog::getTables() const runner.enqueueAndKeepTrack( [=, &tables, &mutex, this] { - auto tables_in_namespace = listTablesInNamespace(current_namespace); + auto tables_in_namespace = listTablesInNamespace(current_namespace, auth_token); std::lock_guard lock(mutex); std::move(tables_in_namespace.begin(), tables_in_namespace.end(), std::back_inserter(tables)); }); @@ -1619,7 +1962,8 @@ CatalogTables RestCatalog::getTables() const /* base_namespace */"", /// Empty base namespace means starting from root. namespaces, /* stop_condition */{}, - /* execute_func */execute_for_each_namespace); + /* execute_func */execute_for_each_namespace, + auth_token); runner.waitForAllToFinishAndRethrowFirstError(); } @@ -1632,19 +1976,23 @@ CatalogTables RestCatalog::getTables() const return result; } -RestCatalog::Namespaces RestCatalog::getNamespaces() const +RestCatalog::Namespaces RestCatalog::getNamespaces(const DB::ForwardedAuthTokenPtr & auth_token) const { /// Enumerate the whole namespace tree (every node at every level). Used by /// the `getTables(const TableNameFilter &)` namespace push-down. + loadConfigIfNeeded(auth_token); + Namespaces namespaces; getNamespacesRecursive( /* base_namespace */"", /// Empty base namespace means starting from root. namespaces, /* stop_condition */{}, - /* execute_func */{}); + /* execute_func */{}, + auth_token); return namespaces; } +<<<<<<< HEAD CatalogTables RestCatalog::listTablesInNamespaceDirect(const std::string & namespace_name) const { /// A REST catalog is Iceberg-only, so every listed table is readable. @@ -1654,17 +2002,24 @@ CatalogTables RestCatalog::listTablesInNamespaceDirect(const std::string & names for (auto & name : names) result.push_back(CatalogTable{.name = std::move(name)}); return result; +======= +DB::Names RestCatalog::listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & auth_token) const +{ + loadConfigIfNeeded(auth_token); + return listTablesInNamespace(namespace_name, auth_token); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } void RestCatalog::getNamespacesRecursive( const std::string & base_namespace, Namespaces & result, StopCondition stop_condition, - ExecuteFunc func) const + ExecuteFunc func, + const DB::ForwardedAuthTokenPtr & auth_token) const { checkStackSize(); - auto namespaces = listChildNamespaces(base_namespace); + auto namespaces = listChildNamespaces(base_namespace, auth_token); result.reserve(result.size() + namespaces.size()); result.insert(result.end(), namespaces.begin(), namespaces.end()); @@ -1692,7 +2047,13 @@ void RestCatalog::getNamespacesRecursive( } } +<<<<<<< HEAD if (!hasFlatNamespaces()) +======= + if (allowed_namespaces.isNamespaceAllowed(current_namespace, /*nested*/ true)) + getNamespacesRecursive(current_namespace, result, stop_condition, func, auth_token); + else +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { if (allowed_namespaces.isNamespaceAllowed(current_namespace, /*nested*/ true)) getNamespacesRecursive(current_namespace, result, stop_condition, func); @@ -1720,6 +2081,7 @@ Poco::URI::QueryParameters RestCatalog::createParentNamespaceParams(const std::s return {{"parent", parent_param}}; } +<<<<<<< HEAD bool RestCatalog::hasFlatNamespaces() const { /// Catalogs whose namespaces are single-level and which ignore or reject the `parent` filter when @@ -1735,8 +2097,11 @@ bool RestCatalog::hasFlatNamespaces() const } RestCatalog::Namespaces RestCatalog::listChildNamespaces(const std::string & base_namespace) const +======= +RestCatalog::Namespaces RestCatalog::listChildNamespaces(const std::string & base_namespace, const DB::ForwardedAuthTokenPtr & auth_token) const +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); Poco::URI::QueryParameters base_params; if (!base_namespace.empty()) @@ -1765,7 +2130,11 @@ RestCatalog::Namespaces RestCatalog::listChildNamespaces(const std::string & bas ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetNamespaces); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetNamespacesMicroseconds); auto buf = createReadBuffer( +<<<<<<< HEAD *state_snapshot, state_snapshot->config.prefix / NAMESPACES_ENDPOINT, params, /* headers */ {}, /* auth_headers */ std::nullopt); +======= + *state_snapshot, state_snapshot.generation, state_snapshot->config.prefix / NAMESPACES_ENDPOINT, auth_token, params); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) String next_page_token; auto page_namespaces = parseNamespaces(*buf, base_namespace, next_page_token); LOG_DEBUG( @@ -1899,13 +2268,13 @@ RestCatalog::Namespaces RestCatalog::parseNamespaces(DB::ReadBuffer & buf, const } } -DB::Names RestCatalog::listTablesInNamespace(const std::string & base_namespace, size_t limit) const +DB::Names RestCatalog::listTablesInNamespace(const std::string & base_namespace, const DB::ForwardedAuthTokenPtr & auth_token, size_t limit) const { if (!allowed_namespaces.isNamespaceAllowed(base_namespace, /*nested*/ false)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Namespace {} is filtered by `namespaces` database parameter", base_namespace); - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); auto encoded_namespace = encodeNamespaceForURI(base_namespace); const std::string endpoint = std::filesystem::path(NAMESPACES_ENDPOINT) / encoded_namespace / "tables"; @@ -1930,8 +2299,12 @@ DB::Names RestCatalog::listTablesInNamespace(const std::string & base_namespace, ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetTables); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetTablesMicroseconds); +<<<<<<< HEAD auto buf = createReadBuffer( *state_snapshot, state_snapshot->config.prefix / endpoint, params, /* headers */ {}, /* auth_headers */ std::nullopt); +======= + auto buf = createReadBuffer(*state_snapshot, state_snapshot.generation, state_snapshot->config.prefix / endpoint, auth_token, params); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) /// Pass through the remaining limit so that single-page short-circuiting still works /// when the caller is in `empty()` (limit=1) and the first page already contains a row. @@ -2016,10 +2389,10 @@ DB::Names RestCatalog::parseTables(DB::ReadBuffer & buf, const std::string & bas } } -bool RestCatalog::existsTable(const std::string & namespace_name, const std::string & table_name) const +bool RestCatalog::existsTable(const std::string & namespace_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const { TableMetadata table_metadata; - return tryGetTableMetadata(namespace_name, table_name, getContext(), table_metadata); + return tryGetTableMetadataImpl(namespace_name, table_name, getContext(), table_metadata, auth_token); } bool RestCatalog::tryGetTableMetadata( @@ -2027,10 +2400,20 @@ bool RestCatalog::tryGetTableMetadata( const std::string & table_name, DB::ContextPtr context_, TableMetadata & result) const +{ + return tryGetTableMetadataImpl(namespace_name, table_name, context_, result, getForwardedAuthToken(context_)); +} + +bool RestCatalog::tryGetTableMetadataImpl( + const std::string & namespace_name, + const std::string & table_name, + DB::ContextPtr context_, + TableMetadata & result, + const DB::ForwardedAuthTokenPtr & auth_token) const { try { - return getTableMetadataImpl(namespace_name, table_name, context_, result); + return getTableMetadataImpl(namespace_name, table_name, context_, result, auth_token); } catch (const DB::HTTPException & ex) { @@ -2053,7 +2436,7 @@ void RestCatalog::getTableMetadata( DB::ContextPtr context_, TableMetadata & result) const { - if (!getTableMetadataImpl(namespace_name, table_name, context_, result)) + if (!getTableMetadataImpl(namespace_name, table_name, context_, result, getForwardedAuthToken(context_))) throw DB::Exception(DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "No response from iceberg catalog"); } @@ -2142,23 +2525,30 @@ bool RestCatalog::getTableMetadataImpl( const std::string & table_name, DB::ContextPtr context_, TableMetadata & result, + const DB::ForwardedAuthTokenPtr & auth_token, bool allow_credentials_cache) const { LOG_DEBUG(log, "Checking table {} in namespace {}", table_name, namespace_name); + loadConfigIfNeeded(auth_token); + if (!allowed_namespaces.isNamespaceAllowed(namespace_name, /*nested*/ false)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Namespace {} is filtered by `namespaces` database parameter", namespace_name); DB::HTTPHeaderEntries headers; + const auto state_snapshot = getStateSnapshot(); + const bool want_credentials = result.requiresCredentials(); + const CredentialsCacheKey credentials_key{ + state_snapshot.generation, getCredentialsCachePrincipal(auth_token), namespace_name, table_name}; std::optional cached_credentials; if (want_credentials) { if (allow_credentials_cache) - cached_credentials = tryGetCachedCredentials(namespace_name, table_name); + cached_credentials = tryGetCachedCredentials(credentials_key); /// Header `X-Iceberg-Access-Delegation` tells catalog to include storage credentials in LoadTableResponse. /// Value can be one of the two: @@ -2173,7 +2563,6 @@ bool RestCatalog::getTableMetadataImpl( } } - const auto state_snapshot = state.get(); const std::string endpoint = std::filesystem::path(NAMESPACES_ENDPOINT) / encodeNamespaceForURI(namespace_name) / "tables" / table_name; String json_str; @@ -2181,7 +2570,11 @@ bool RestCatalog::getTableMetadataImpl( ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetTableMetadata); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetTableMetadataMicroseconds); auto buf = createReadBuffer( +<<<<<<< HEAD *state_snapshot, state_snapshot->config.prefix / endpoint, /* params */ {}, headers, /* auth_headers */ std::nullopt); +======= + *state_snapshot, state_snapshot.generation, state_snapshot->config.prefix / endpoint, auth_token, /* params */{}, headers); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) if (buf->eof()) { @@ -2243,9 +2636,9 @@ bool RestCatalog::getTableMetadataImpl( { { std::lock_guard lock(credentials_cache_mutex); - credentials_cache.erase({namespace_name, table_name}); + credentials_cache.erase(credentials_key); } - return getTableMetadataImpl(namespace_name, table_name, context_, result, /* allow_credentials_cache */ false); + return getTableMetadataImpl(namespace_name, table_name, context_, result, auth_token, /* allow_credentials_cache */ false); } ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogCredentialsCacheHits); result.setStorageCredentials(cached_credentials->credentials); @@ -2260,7 +2653,11 @@ bool RestCatalog::getTableMetadataImpl( if (parsed.credentials) { result.setStorageCredentials(parsed.credentials); +<<<<<<< HEAD cacheCredentials(namespace_name, table_name, parsed, state_snapshot); +======= + cacheCredentials(credentials_key, parsed); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } if (!parsed.endpoint.empty()) result.setEndpoint(parsed.endpoint); @@ -2282,7 +2679,14 @@ bool RestCatalog::getTableMetadataImpl( return true; } -void RestCatalog::sendRequest(const CatalogState & catalog_state, const String & endpoint, Poco::JSON::Object::Ptr request_body, const String & method, bool ignore_result) const +void RestCatalog::sendRequest( + const CatalogState & catalog_state, + UInt64 generation, + const String & endpoint, + Poco::JSON::Object::Ptr request_body, + const DB::ForwardedAuthTokenPtr & auth_token, + const String & method, + bool ignore_result) const { std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM if (request_body) @@ -2307,7 +2711,22 @@ void RestCatalog::sendRequest(const CatalogState & catalog_state, const String & auto create_buffer = [&](bool update_token, bool & used_cached_oauth_token) { +<<<<<<< HEAD DB::HTTPHeaderEntries headers = getAuthHeaders(catalog_state, update_token, &used_cached_oauth_token); +======= + AuthContext auth_context{ + .catalog_state = catalog_state, + .generation = generation, + .update_token = update_token, + .method = method, + .url = url, + .extra_headers = extra_headers, + .body = body_str, + .auth_token = auth_token, + .used_cached_oauth_token = &used_cached_oauth_token, + }; + DB::HTTPHeaderEntries headers = getAuthHeaders(auth_context); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) headers.emplace_back("Content-Type", "application/json"); headers.emplace_back("X-Iceberg-Access-Delegation", "vended-credentials"); return DB::BuilderRWBufferFromHTTP(url) @@ -2341,31 +2760,26 @@ void RestCatalog::sendRequest(const CatalogState & catalog_state, const String & } catch (const DB::HTTPException & e) { - const auto status = e.getHTTPStatus(); - if (update_token_if_expired && - (status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_UNAUTHORIZED - || status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_FORBIDDEN)) - { - ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUnauthorized); - bool used_cached_oauth_token_on_retry = false; - auto wb = create_buffer(true, used_cached_oauth_token_on_retry); + if (!shouldRetryWithFreshToken(e.getHTTPStatus())) + throw; - String response_str; - if (!ignore_result) - readJSONObjectPossiblyInvalid(response_str, *wb); - else - wb->ignoreAll(); - } + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUnauthorized); + bool used_cached_oauth_token_on_retry = false; + auto wb = create_buffer(true, used_cached_oauth_token_on_retry); + + String response_str; + if (!ignore_result) + readJSONObjectPossiblyInvalid(response_str, *wb); else - { - throw; - } + wb->ignoreAll(); } } -void RestCatalog::createNamespaceIfNotExists(const String & namespace_name, const String & location) const +void RestCatalog::createNamespaceIfNotExists(const String & namespace_name, const String & location, const DB::ForwardedAuthTokenPtr & auth_token) const { - const auto state_snapshot = state.get(); + loadConfigIfNeeded(auth_token); + + const auto state_snapshot = getStateSnapshot(); /// Check existence first: creation may be denied to a principal that is still /// allowed to use a pre-provisioned namespace. @@ -2373,7 +2787,9 @@ void RestCatalog::createNamespaceIfNotExists(const String & namespace_name, cons = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name)).generic_string(); try { - sendRequest(*state_snapshot, check_endpoint, /* request_body */ nullptr, Poco::Net::HTTPRequest::HTTP_GET, /* ignore_result */ true); + sendRequest( + *state_snapshot, state_snapshot.generation, check_endpoint, /* request_body */ nullptr, auth_token, + Poco::Net::HTTPRequest::HTTP_GET, /* ignore_result */ true); return; } catch (const DB::HTTPException & e) @@ -2398,7 +2814,11 @@ void RestCatalog::createNamespaceIfNotExists(const String & namespace_name, cons { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogCreateNamespace); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogCreateNamespaceMicroseconds); +<<<<<<< HEAD sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_POST, /* ignore_result */ false); +======= + sendRequest(*state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } catch (const DB::HTTPException & e) { @@ -2414,15 +2834,18 @@ bool RestCatalog::createTable( const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr metadata_content, DB::CompressionMethod /*metadata_compression_method*/, - bool if_not_exists) const + bool if_not_exists, + const DB::ForwardedAuthTokenPtr & auth_token) const { + loadConfigIfNeeded(auth_token); + if (!allowed_namespaces.isNamespaceAllowed(namespace_name, /*nested*/ false)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Failed to create table {}, namespace {} is filtered by `namespaces` database parameter", table_name, namespace_name); const String location = metadata_content->getValue("location"); - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); const std::string endpoint = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables").generic_string(); Poco::JSON::Object::Ptr request_body = new Poco::JSON::Object; @@ -2464,7 +2887,11 @@ bool RestCatalog::createTable( { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogCreateTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogCreateTableMicroseconds); +<<<<<<< HEAD sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_POST, /* ignore_result */ false); +======= + sendRequest(*state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } catch (const DB::HTTPException & ex) { @@ -2477,15 +2904,17 @@ bool RestCatalog::createTable( } -bool RestCatalog::updateMetadata(const String & namespace_name, const String & table_name, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr new_snapshot) const +bool RestCatalog::updateMetadata(const String & namespace_name, const String & table_name, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const { + loadConfigIfNeeded(auth_token); + if (!new_snapshot) throw DB::Exception( DB::ErrorCodes::NOT_IMPLEMENTED, "REST catalog does not support metadata-only updates without a snapshot " "(required for EXPIRE SNAPSHOTS)"); - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); const std::string endpoint = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables" / table_name).generic_string(); auto request_body = buildUpdateMetadataRequestBody(namespace_name, table_name, new_snapshot); @@ -2494,7 +2923,11 @@ bool RestCatalog::updateMetadata(const String & namespace_name, const String & t { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUpdateTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogUpdateTableMicroseconds); +<<<<<<< HEAD sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_POST, /* ignore_result */ false); +======= + sendRequest(*state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } catch (const DB::HTTPException & ex) { @@ -2523,11 +2956,14 @@ bool RestCatalog::updateSchema( Poco::JSON::Object::Ptr new_schema, Int32 previous_schema_id, Int32 new_last_column_id, - Poco::JSON::Object::Ptr metadata) const + Poco::JSON::Object::Ptr metadata, + const DB::ForwardedAuthTokenPtr & auth_token) const { fiu_do_on(DB::FailPoints::iceberg_alter_catalog_update_schema_fail, { return false; }); - const auto state_snapshot = state.get(); + loadConfigIfNeeded(auth_token); + + const auto state_snapshot = getStateSnapshot(); const std::string endpoint = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables" / table_name).generic_string(); auto request_body = buildUpdateSchemaRequestBody( @@ -2535,7 +2971,11 @@ bool RestCatalog::updateSchema( try { +<<<<<<< HEAD sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_POST, /* ignore_result */ false); +======= + sendRequest(*state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } catch (const DB::HTTPException & ex) { @@ -2556,14 +2996,21 @@ bool RestCatalog::updateSchema( return true; } -void RestCatalog::dropTable(const String & namespace_name, const String & table_name, bool purge, bool if_exists) const +void RestCatalog::dropTable( + const String & namespace_name, + const String & table_name, + bool purge, + bool if_exists, + const DB::ForwardedAuthTokenPtr & auth_token) const { + loadConfigIfNeeded(auth_token); + if (!allowed_namespaces.isNamespaceAllowed(namespace_name, /*nested*/ false)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Failed to drop table {}, namespace {} is filtered by `namespaces` database parameter", table_name, namespace_name); - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); const std::string base_endpoint = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables" / table_name).generic_string(); const std::string endpoint = fmt::format("{}?purgeRequested={}", base_endpoint, purge ? "true" : "false"); @@ -2573,7 +3020,8 @@ void RestCatalog::dropTable(const String & namespace_name, const String & table_ { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogDropTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogDropTableMicroseconds); - sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_DELETE, true); + sendRequest( + *state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token, Poco::Net::HTTPRequest::HTTP_DELETE, true); } catch (const DB::HTTPException & ex) { @@ -2708,14 +3156,25 @@ VendedStorageCredentials RestCatalog::getCredentialsAndEndpoint(Poco::JSON::Obje return {nullptr, "", std::nullopt}; } -std::optional RestCatalog::tryGetCachedCredentials( - const std::string & namespace_name, const std::string & table_name) const +String RestCatalog::getCredentialsCachePrincipal(const DB::ForwardedAuthTokenPtr & auth_token) const +{ + if (!token_forwarding.forward_user_token || !auth_token) + return {}; + return auth_token->fingerprint; +} + +std::optional RestCatalog::tryGetCachedCredentials(const CredentialsCacheKey & key) const { std::lock_guard lock(credentials_cache_mutex); if (vended_credentials_cache_ttl <= std::chrono::seconds::zero()) return std::nullopt; +<<<<<<< HEAD auto it = credentials_cache.find({namespace_name, table_name}); +======= + std::lock_guard lock(credentials_cache_mutex); + auto it = credentials_cache.find(key); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) if (it == credentials_cache.end()) return std::nullopt; if (std::chrono::system_clock::now() >= it->second.expires_at.value()) @@ -2727,11 +3186,15 @@ std::optional RestCatalog::tryGetCachedCredentials( return it->second; } +<<<<<<< HEAD void RestCatalog::cacheCredentials( const std::string & namespace_name, const std::string & table_name, const VendedStorageCredentials & parsed, const CatalogStateVersion & state_snapshot) const +======= +void RestCatalog::cacheCredentials(const CredentialsCacheKey & key, const VendedStorageCredentials & parsed) const +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { if (!parsed.credentials || parsed.credentials->isEmpty()) return; @@ -2754,20 +3217,36 @@ void RestCatalog::cacheCredentials( if (credentials_cache.size() >= credentials_cache_cleanup_threshold) std::erase_if(credentials_cache, [&now](const auto & entry) { return now >= entry.second.expires_at.value(); }); +<<<<<<< HEAD credentials_cache[{namespace_name, table_name}] = VendedStorageCredentials{parsed.credentials, parsed.endpoint, refresh_after, parsed.table_uuid, parsed.location}; +======= + + while (credentials_cache.size() >= credentials_cache_max_entries) + { + auto oldest = std::min_element( + credentials_cache.begin(), + credentials_cache.end(), + [](const auto & lhs, const auto & rhs) { return lhs.second.expires_at.value() < rhs.second.expires_at.value(); }); + credentials_cache.erase(oldest); + } + + credentials_cache[key] + = VendedStorageCredentials{parsed.credentials, parsed.endpoint, refresh_after, parsed.table_uuid}; +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } -ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCallback(const DB::StorageID & storage_id) +ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCallback( + const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) { - return [this, storage_id] () -> std::shared_ptr + return [this, storage_id, auth_token] () -> std::shared_ptr { LOG_DEBUG(log, "Update credentials in the catalog"); DB::HTTPHeaderEntries headers; headers.emplace_back("X-Iceberg-Access-Delegation", "vended-credentials"); - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); const auto & table = storage_id.getTableName(); auto [namespace_name, table_name] = DataLake::parseTableName(table); const std::string endpoint = std::filesystem::path(NAMESPACES_ENDPOINT) / encodeNamespaceForURI(namespace_name) / "tables" / table_name; @@ -2777,7 +3256,11 @@ ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCal ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetCredentials); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetCredentialsMicroseconds); auto buf = createReadBuffer( +<<<<<<< HEAD *state_snapshot, state_snapshot->config.prefix / endpoint, /* params */ {}, headers, /* auth_headers */ std::nullopt); +======= + *state_snapshot, state_snapshot.generation, state_snapshot->config.prefix / endpoint, auth_token, /* params */{}, headers); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) if (buf->eof()) { @@ -2822,9 +3305,17 @@ ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCal } auto parsed = getCredentialsAndEndpoint(config_object, location); +<<<<<<< HEAD parsed.table_uuid = table_uuid; parsed.location = location; cacheCredentials(namespace_name, table_name, parsed, state_snapshot); +======= + if (metadata_object) + parsed.table_uuid = parseTableUuid(metadata_object); + /// Refresh the per-table cache so subsequent queries reuse these freshly vended credentials. + cacheCredentials( + {state_snapshot.generation, getCredentialsCachePrincipal(auth_token), namespace_name, table_name}, parsed); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) return parsed.credentials; }; } diff --git a/src/Databases/DataLake/RestCatalog.h b/src/Databases/DataLake/RestCatalog.h index 1bd1466099e82..b6a1c44ecc3cb 100644 --- a/src/Databases/DataLake/RestCatalog.h +++ b/src/Databases/DataLake/RestCatalog.h @@ -2,8 +2,11 @@ #include "config.h" #if USE_AVRO +#include #include #include +#include +#include #include #include #include @@ -47,6 +50,28 @@ struct VendedStorageCredentials std::string location{}; }; +struct TokenForwardingConfig +{ + bool forward_user_token = false; + String token_exchange_uri; + String subject_token_type; + String requested_token_type; + bool forward_actor_token = false; + UInt64 user_token_cache_ttl = 0; + + bool exchangeEnabled() const { return forward_user_token && !token_exchange_uri.empty(); } +}; + +struct CredentialsCacheKey +{ + UInt64 generation = 0; + std::string principal; + std::string namespace_name; + std::string table_name; + + auto operator<=>(const CredentialsCacheKey &) const = default; +}; + class RestCatalog : public ICatalog, public DB::WithContext { public: @@ -59,18 +84,27 @@ class RestCatalog : public ICatalog, public DB::WithContext const std::string & oauth_server_uri_, bool oauth_server_use_request_body_, const std::string & namespaces_, +<<<<<<< HEAD bool flat_namespaces_, DB::ContextPtr context_); +======= + DB::ContextPtr context_, + const TokenForwardingConfig & token_forwarding_ = {}); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) ~RestCatalog() override = default; - bool empty() const override; + bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const override; +<<<<<<< HEAD CatalogTables getTables() const override; +======= + DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) - Namespaces getNamespaces() const override; + Namespaces getNamespaces(const DB::ForwardedAuthTokenPtr & auth_token) const override; - bool existsTable(const std::string & namespace_name, const std::string & table_name) const override; + bool existsTable(const std::string & namespace_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; void getTableMetadata( const std::string & namespace_name, @@ -99,9 +133,10 @@ class RestCatalog : public ICatalog, public DB::WithContext const String & new_metadata_path, Poco::JSON::Object::Ptr metadata_content, DB::CompressionMethod metadata_compression_method, - bool if_not_exists) const override; + bool if_not_exists, + const DB::ForwardedAuthTokenPtr & auth_token) const override; - bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot) const override; + bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const override; bool updateSchema( const String & namespace_name, @@ -110,13 +145,24 @@ class RestCatalog : public ICatalog, public DB::WithContext Poco::JSON::Object::Ptr new_schema, Int32 previous_schema_id, Int32 new_last_column_id, - Poco::JSON::Object::Ptr metadata = nullptr) const override; + Poco::JSON::Object::Ptr metadata, + const DB::ForwardedAuthTokenPtr & auth_token) const override; bool isTransactional() const override { return true; } - void dropTable(const String & namespace_name, const String & table_name, bool purge, bool if_exists) const override; + void dropTable( + const String & namespace_name, + const String & table_name, + bool purge, + bool if_exists, + const DB::ForwardedAuthTokenPtr & auth_token) const override; + + ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback( + const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) override; + + void onTokenForwardingDisabled() const override { user_token_cache.clear(); } - ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback(const DB::StorageID & storage_id) override; + void loadConfigIfNeeded(const DB::ForwardedAuthTokenPtr & auth_token) const override; void setVendedCredentialsCacheTTL(std::chrono::seconds ttl) override; @@ -144,12 +190,43 @@ class RestCatalog : public ICatalog, public DB::WithContext std::string bearer_token; std::string refresh_token; Config config; + bool config_loaded = false; }; using CatalogStateVersion = MultiVersion::Version; - CatalogStateVersion getStateSnapshot() const { return state.get(); } + struct AuthContext + { + /// Keep the endpoint and authentication from the same state snapshot. + const CatalogState & catalog_state; + UInt64 generation = 0; + bool update_token = false; + String method; + Poco::URI url; + DB::HTTPHeaderEntries extra_headers; + String body; + DB::ForwardedAuthTokenPtr auth_token; + /// The caller records cache hits only after the catalog request succeeds. + bool * used_cached_oauth_token = nullptr; + }; + + struct StateSnapshot + { + UInt64 generation = 0; + CatalogStateVersion state; - ICatalog::PreparedSettingsChangesPtr prepareSettingsChanges(const DB::SettingsChanges & changes) override; + const CatalogState & operator*() const { return *state; } + const CatalogState * operator->() const { return state.get(); } + }; + + /// Read the generation first so an old state cannot cache credentials under a new generation. + StateSnapshot getStateSnapshot() const + { + const UInt64 generation = auth_generation.load(std::memory_order_acquire); + return StateSnapshot{generation, state.get()}; + } + + ICatalog::PreparedSettingsChangesPtr prepareSettingsChanges( + const DB::SettingsChanges & changes, const DB::ForwardedAuthTokenPtr & auth_token = {}) override; void commitSettingsChanges(ICatalog::PreparedSettingsChangesPtr prepared) override; @@ -174,21 +251,54 @@ class RestCatalog : public ICatalog, public DB::WithContext bool flat_namespaces_, DB::ContextPtr context_); - void createNamespaceIfNotExists(const String & namespace_name, const String & location) const override; + void createNamespaceIfNotExists(const String & namespace_name, const String & location, const DB::ForwardedAuthTokenPtr & auth_token) const override; const std::filesystem::path base_url; const LoggerPtr log; - MultiVersion state{std::make_unique()}; + mutable MultiVersion state{std::make_unique()}; /// Parameters for OAuth (common for REST catalog). bool update_token_if_expired = false; std::string auth_scope; std::string oauth_server_uri; bool oauth_server_use_request_body; +<<<<<<< HEAD bool flat_namespaces = false; mutable MultiVersion access_token; +======= + /// Shared service or actor token; never store a user token here. + mutable MultiVersion access_token; + + TokenForwardingConfig token_forwarding; + + static constexpr size_t user_token_cache_max_entries = 1024; + mutable DB::CacheBase user_token_cache; + + /// Separate from `CatalogState`, which can be republished without an auth change. + /// Old requests retain their generation so their cache writes become unreachable after rotation. + std::atomic auth_generation{0}; + + /// Keep generation checks and token publication atomic with credential rotation. + /// Never hold this across a network request. + mutable std::mutex auth_publish_mutex; + + /// TTL for caching vended credentials per table (0 means no caching). + std::atomic vended_credentials_cache_ttl{std::chrono::seconds::zero()}; + + /// Sweep trigger threshold, not capacity! + static constexpr size_t credentials_cache_cleanup_threshold = 1000; + + static constexpr size_t credentials_cache_max_entries = 10000; + + static constexpr std::chrono::seconds credentials_expiry_safety_window{60}; + mutable std::mutex credentials_cache_mutex; + + mutable std::map credentials_cache + TSA_GUARDED_BY(credentials_cache_mutex); + +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) public: class AllowedNamespaces { @@ -226,10 +336,18 @@ class RestCatalog : public ICatalog, public DB::WithContext /// default arguments are therefore omitted (clang-tidy `google-default-arguments`). virtual DB::ReadWriteBufferFromHTTPPtr createReadBuffer( const CatalogState & catalog_state, + UInt64 generation, const std::string & endpoint, +<<<<<<< HEAD const Poco::URI::QueryParameters & params, const DB::HTTPHeaderEntries & headers, const std::optional & auth_headers) const; +======= + const DB::ForwardedAuthTokenPtr & auth_token, + const Poco::URI::QueryParameters & params = {}, + const DB::HTTPHeaderEntries & headers = {}, + const std::optional & auth_headers = std::nullopt) const; +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) Poco::URI::QueryParameters createParentNamespaceParams(const std::string & base_namespace) const; @@ -240,7 +358,8 @@ class RestCatalog : public ICatalog, public DB::WithContext const std::string & base_namespace, Namespaces & result, StopCondition stop_condition, - ExecuteFunc func) const; + ExecuteFunc func, + const DB::ForwardedAuthTokenPtr & auth_token) const; /// Whether this catalog has flat (single-level) namespaces, either because its type is always flat /// (BigLake, Databricks Delta Sharing, S3 Tables) or because of the `flat_namespaces` database @@ -250,15 +369,19 @@ class RestCatalog : public ICatalog, public DB::WithContext /// List the immediate child namespaces directly under `base_namespace` /// (single level, not recursive). An empty base lists the root namespaces. - Namespaces listChildNamespaces(const std::string & base_namespace) const; + Namespaces listChildNamespaces(const std::string & base_namespace, const DB::ForwardedAuthTokenPtr & auth_token) const; Namespaces parseNamespaces(DB::ReadBuffer & buf, const std::string & base_namespace, String & next_page_token) const; /// Non-recursive list of tables directly in `base_namespace` (not in sub-namespaces). /// `limit` is a soft cap on the number of returned names; 0 means no limit. - DB::Names listTablesInNamespace(const std::string & base_namespace, size_t limit = 0) const; + DB::Names listTablesInNamespace(const std::string & base_namespace, const DB::ForwardedAuthTokenPtr & auth_token, size_t limit = 0) const; +<<<<<<< HEAD CatalogTables listTablesInNamespaceDirect(const std::string & namespace_name) const override; +======= + DB::Names listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) DB::Names parseTables(DB::ReadBuffer & buf, const std::string & base_namespace, size_t limit, String & next_page_token) const; @@ -267,14 +390,39 @@ class RestCatalog : public ICatalog, public DB::WithContext const std::string & table_name, DB::ContextPtr context_, TableMetadata & result, + const DB::ForwardedAuthTokenPtr & auth_token, bool allow_credentials_cache = true) const; + bool tryGetTableMetadataImpl( + const std::string & namespace_name, + const std::string & table_name, + DB::ContextPtr context_, + TableMetadata & result, + const DB::ForwardedAuthTokenPtr & auth_token) const; + /// Load catalog config (special http handler) utilizing information from catalog_state and auth_headers. +<<<<<<< HEAD Config loadConfig(const CatalogState & catalog_state, const std::optional & auth_headers = std::nullopt); virtual DB::HTTPHeaderEntries getAuthHeaders( const CatalogState & catalog_state, bool update_token, bool * used_cached_oauth_token) const; +======= + Config loadConfig( + const CatalogState & catalog_state, + UInt64 generation, + const DB::ForwardedAuthTokenPtr & auth_token, + const std::optional & auth_headers = std::nullopt) const; + + virtual DB::HTTPHeaderEntries getAuthHeaders(const AuthContext & auth_context) const; + + void validateForwardedToken(const DB::ForwardedAuthTokenPtr & auth_token) const; + + String getForwardedToken( + const CatalogState & catalog_state, UInt64 generation, const DB::ForwardedAuthTokenPtr & auth_token, bool update_token) const; + + bool shouldRetryWithFreshToken(Poco::Net::HTTPResponse::HTTPStatus status) const; +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) void validateAuthHeaders(const DB::HTTPHeaderEntry & header) const; @@ -282,31 +430,48 @@ class RestCatalog : public ICatalog, public DB::WithContext virtual void sendRequest( const CatalogState & catalog_state, + UInt64 generation, const String & endpoint, Poco::JSON::Object::Ptr request_body, +<<<<<<< HEAD const String & method, bool ignore_result) const; +======= + const DB::ForwardedAuthTokenPtr & auth_token, + const String & method = Poco::Net::HTTPRequest::HTTP_POST, + bool ignore_result = false) const; +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) VendedStorageCredentials getCredentialsAndEndpoint(Poco::JSON::Object::Ptr object, const String & location) const; - std::optional tryGetCachedCredentials( - const std::string & namespace_name, const std::string & table_name) const; + String getCredentialsCachePrincipal(const DB::ForwardedAuthTokenPtr & auth_token) const; +<<<<<<< HEAD void cacheCredentials( const std::string & namespace_name, const std::string & table_name, const VendedStorageCredentials & parsed, const CatalogStateVersion & state_snapshot) const; +======= + std::optional tryGetCachedCredentials(const CredentialsCacheKey & key) const; + + void cacheCredentials(const CredentialsCacheKey & key, const VendedStorageCredentials & parsed) const; + + MultiVersion::Version publishServiceToken(AccessToken minted, UInt64 generation) const; + + AccessToken requestToken(Poco::URI url, const Poco::URI::QueryParameters & params, bool use_query_parameters = false) const; + + AccessToken exchangeUserToken( + const CatalogState & catalog_state, UInt64 generation, const DB::ForwardedAuthToken & auth_token, + const AccessToken * prepared_actor_token = nullptr) const; +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) AccessToken retrieveAccessToken(const std::string & client_id, const std::string & client_secret) const; + String getServicePrincipalToken(const CatalogState & catalog_state, UInt64 generation) const; + struct PreparedAuthChanges; - /// Hook for `prepareSettingsChanges`: validate `changes` and apply them to `new_state`, - /// building the new auth artifacts, without publishing anything. When the OAuth - /// credentials change, the eagerly fetched token goes into `new_access_token` and - /// `new_auth_headers`, so that wrong credentials fail the ALTER right here and the - /// config reload authenticates with the new token instead of the cached one. virtual void applySettingsChangesToState( const DB::SettingsChanges & changes, const CatalogState & old_state, @@ -360,10 +525,14 @@ class OneLakeCatalog : public RestCatalog /// OneLake keeps data in Azure Data Lake Storage, whatever the catalog reports. std::optional getStorageType() const override { return StorageType::Azure; } +<<<<<<< HEAD DB::HTTPHeaderEntries getAuthHeaders( const CatalogState & catalog_state, bool update_token, bool * used_cached_oauth_token) const override; +======= + DB::HTTPHeaderEntries getAuthHeaders(const AuthContext & auth_context) const override; +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) static void validateSettingsChanges(const DB::SettingsChanges & changes, AuthMode auth_mode); @@ -414,10 +583,14 @@ class BigLakeCatalog : public RestCatalog /// BigLake keeps data in Google Cloud Storage, which is accessed through the S3 API. std::optional getStorageType() const override { return StorageType::S3; } +<<<<<<< HEAD DB::HTTPHeaderEntries getAuthHeaders( const CatalogState & catalog_state, bool update_token, bool * used_cached_oauth_token) const override; +======= + DB::HTTPHeaderEntries getAuthHeaders(const AuthContext & auth_context) const override; +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) const std::string & getGoogleADCClientId() const { return google_adc_client_id; } const std::string & getGoogleADCClientSecret() const { return google_adc_client_secret; } diff --git a/src/Databases/DataLake/S3TablesCatalog.cpp b/src/Databases/DataLake/S3TablesCatalog.cpp index 2c8f3a7673404..2cd6d2607b2dd 100644 --- a/src/Databases/DataLake/S3TablesCatalog.cpp +++ b/src/Databases/DataLake/S3TablesCatalog.cpp @@ -123,7 +123,8 @@ S3TablesCatalog::S3TablesCatalog( /// The signer is fully initialised above, so the virtual `createReadBuffer` reached by /// `loadConfig` dispatches to this class's SigV4 implementation. CatalogState initial_state; - initial_state.config = loadConfig(initial_state); + initial_state.config = loadConfig(initial_state, /* generation */ 0, /* auth_token */ {}); + initial_state.config_loaded = true; if (initial_state.config.prefix.empty()) { @@ -135,12 +136,18 @@ S3TablesCatalog::S3TablesCatalog( state.set(std::make_unique(std::move(initial_state))); } +<<<<<<< HEAD /// S3 Tables only supports a single level of namespaces (no nesting), so we list the root /// namespaces directly with `listChildNamespaces("")` instead of the base class's recursive /// `getNamespacesRecursive`. CatalogTables S3TablesCatalog::getTables() const +======= +/// S3 Tables only supports a single level of namespaces (no nesting), +/// so we use flat listChildNamespaces() instead of the base class's getNamespacesRecursive(). +DB::Names S3TablesCatalog::getTables(const DB::ForwardedAuthTokenPtr & auth_token) const +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { - auto namespaces = listChildNamespaces(""); + auto namespaces = listChildNamespaces("", auth_token); auto & pool = getContext()->getIcebergCatalogThreadpool(); DB::ThreadPoolCallbackRunnerLocal runner(pool, DB::ThreadName::DATALAKE_REST_CATALOG); @@ -152,7 +159,11 @@ CatalogTables S3TablesCatalog::getTables() const runner.enqueueAndKeepTrack( [&, ns] { +<<<<<<< HEAD auto tables_in_ns = listTablesInNamespace(ns); +======= + auto tables_in_ns = RestCatalog::listTablesInNamespace(ns, auth_token); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) std::lock_guard lock(mutex); std::move(tables_in_ns.begin(), tables_in_ns.end(), std::back_inserter(tables)); }); @@ -214,8 +225,19 @@ bool S3TablesCatalog::tryGetTableMetadata( result.setEndpoint(endpoint); } +<<<<<<< HEAD if (auto props = result.getDataLakeSpecificProperties(); props && !props->iceberg_metadata_file_location.empty()) +======= + return true; +} + +ICatalog::CredentialsRefreshCallback S3TablesCatalog::getCredentialsConfigurationCallback( + const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) +{ + auto base_cb = RestCatalog::getCredentialsConfigurationCallback(storage_id, auth_token); + return [this, base_callback = std::move(base_cb)] () -> std::shared_ptr +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { const String & loc = props->iceberg_metadata_file_location; auto scheme_end = loc.find("://"); @@ -231,12 +253,17 @@ bool S3TablesCatalog::tryGetTableMetadata( return true; } -void S3TablesCatalog::dropTable(const String & namespace_name, const String & table_name, bool delete_data, bool if_exists) const +void S3TablesCatalog::dropTable( + const String & namespace_name, + const String & table_name, + bool delete_data, + bool if_exists, + const DB::ForwardedAuthTokenPtr & auth_token) const { /// https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-tables-delete.html if (!delete_data) { - if (if_exists && !existsTable(namespace_name, table_name)) + if (if_exists && !existsTable(namespace_name, table_name, auth_token)) return; throw DB::Exception( @@ -246,7 +273,7 @@ void S3TablesCatalog::dropTable(const String & namespace_name, const String & ta namespace_name, table_name); } - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); const std::string endpoint = (base_url / state_snapshot->config.prefix / "namespaces" / namespace_name / "tables" / table_name).string() + "?purgeRequested=True"; @@ -254,8 +281,15 @@ void S3TablesCatalog::dropTable(const String & namespace_name, const String & ta Poco::JSON::Object::Ptr request_body = nullptr; try { +<<<<<<< HEAD sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_DELETE, true); LOG_INFO(log, "S3 Tables: dropped table {}.{} (purgeRequested=True)", namespace_name, table_name); +======= + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogDropTable); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogDropTableMicroseconds); + sendRequest( + *state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token, Poco::Net::HTTPRequest::HTTP_DELETE, true); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } catch (const DB::HTTPException & ex) { @@ -268,8 +302,17 @@ void S3TablesCatalog::dropTable(const String & namespace_name, const String & ta } } +<<<<<<< HEAD namespace { +======= +DB::HTTPHeaderEntries S3TablesCatalog::getAuthHeaders(const AuthContext & auth_context) const +{ + DB::HTTPHeaderEntries all_signed; + signRequestWithAWSV4( + auth_context.method, auth_context.url, auth_context.extra_headers, auth_context.body, + *signer, region, "s3tables", all_signed); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) /// `signRequestWithAWSV4` returns the full set of headers that the AWS SDK kept on /// the signed request (input headers + signed auth). We only want the SigV4-specific diff --git a/src/Databases/DataLake/S3TablesCatalog.h b/src/Databases/DataLake/S3TablesCatalog.h index 9646af244b1df..60ba60b3b2c76 100644 --- a/src/Databases/DataLake/S3TablesCatalog.h +++ b/src/Databases/DataLake/S3TablesCatalog.h @@ -36,7 +36,11 @@ class S3TablesCatalog final : public RestCatalog DB::DatabaseDataLakeCatalogType getCatalogType() const override { return DB::DatabaseDataLakeCatalogType::S3_TABLES; } +<<<<<<< HEAD CatalogTables getTables() const override; +======= + DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) std::optional getStorageType() const override { return StorageType::S3; } @@ -48,6 +52,7 @@ class S3TablesCatalog final : public RestCatalog DB::ContextPtr context_, TableMetadata & result) const override; +<<<<<<< HEAD void dropTable(const String & namespace_name, const String & table_name, bool delete_data, bool if_exists) const override; protected: @@ -68,6 +73,22 @@ class S3TablesCatalog final : public RestCatalog Poco::JSON::Object::Ptr request_body, const String & method, bool ignore_result) const override; +======= + bool managesTableLocation() const override { return true; } + + void dropTable( + const String & namespace_name, + const String & table_name, + bool delete_data, + bool if_exists, + const DB::ForwardedAuthTokenPtr & auth_token) const override; + + ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback( + const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) override; + +protected: + DB::HTTPHeaderEntries getAuthHeaders(const AuthContext & auth_context) const override; +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) private: const String region; diff --git a/src/Databases/DataLake/UnityCatalog.cpp b/src/Databases/DataLake/UnityCatalog.cpp index 645cbe9d4d882..fb8f55ee979b5 100644 --- a/src/Databases/DataLake/UnityCatalog.cpp +++ b/src/Databases/DataLake/UnityCatalog.cpp @@ -106,7 +106,7 @@ std::pair UnityCatalog::postJSONRequest(const s return makeHTTPRequestAndReadJSON(base_url / route, context, bearer_token, {}, {}, Poco::Net::HTTPRequest::HTTP_POST, out_stream_callaback); } -bool UnityCatalog::empty() const +bool UnityCatalog::empty(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { auto all_schemas = getSchemas(""); for (const auto & schema : all_schemas) @@ -118,7 +118,11 @@ bool UnityCatalog::empty() const return true; } +<<<<<<< HEAD CatalogTables UnityCatalog::getTables() const +======= +DB::Names UnityCatalog::getTables(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { CatalogTables result; @@ -132,13 +136,17 @@ CatalogTables UnityCatalog::getTables() const return result; } -DataLake::ICatalog::Namespaces UnityCatalog::getNamespaces() const +DataLake::ICatalog::Namespaces UnityCatalog::getNamespaces(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { /// Unity schemas are flat — they cannot contain nested namespaces. return getSchemas(""); } +<<<<<<< HEAD CatalogTables UnityCatalog::listTablesInNamespaceDirect(const std::string & namespace_name) const +======= +DB::Names UnityCatalog::listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { return getTablesForSchema(namespace_name); } @@ -351,7 +359,7 @@ bool UnityCatalog::tryGetTableMetadata( } } -bool UnityCatalog::existsTable(const std::string & schema_name, const std::string & table_name) const +bool UnityCatalog::existsTable(const std::string & schema_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { if (!isNamespaceAllowed(schema_name)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Namespace {} is filtered by `namespaces` database parameter", schema_name); @@ -538,7 +546,13 @@ bool UnityCatalog::isNamespaceAllowed(const std::string & namespace_) const return allowed_namespaces.contains("*") || allowed_namespaces.contains(namespace_); } +<<<<<<< HEAD ICatalog::CredentialsRefreshCallback UnityCatalog::getCredentialsConfigurationCallback(const DB::StorageID & table_id) +======= +/// getCredentialsConfigurationCallback method is supported only for S3 storage +ICatalog::CredentialsRefreshCallback UnityCatalog::getCredentialsConfigurationCallback( + const DB::StorageID & table_id, const DB::ForwardedAuthTokenPtr & /*auth_token*/) +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { if (!table_id.hasUUID()) throw DB::Exception( diff --git a/src/Databases/DataLake/UnityCatalog.h b/src/Databases/DataLake/UnityCatalog.h index bf68293f72178..9b84d45f4abaa 100644 --- a/src/Databases/DataLake/UnityCatalog.h +++ b/src/Databases/DataLake/UnityCatalog.h @@ -27,13 +27,17 @@ class UnityCatalog final : public ICatalog, private DB::WithContext ~UnityCatalog() override = default; - bool empty() const override; + bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const override; +<<<<<<< HEAD CatalogTables getTables() const override; +======= + DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) - Namespaces getNamespaces() const override; + Namespaces getNamespaces(const DB::ForwardedAuthTokenPtr & auth_token) const override; - bool existsTable(const std::string & schema_name, const std::string & table_name) const override; + bool existsTable(const std::string & schema_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; void getTableMetadata( const std::string & namespace_name, @@ -70,8 +74,13 @@ class UnityCatalog final : public ICatalog, private DB::WithContext DataLake::ICatalog::Namespaces getSchemas(const std::string & base_prefix, size_t limit = 0) const; +<<<<<<< HEAD CatalogTables getTablesForSchema(const std::string & schema, size_t limit = 0) const; CatalogTables listTablesInNamespaceDirect(const std::string & namespace_name) const override; +======= + DB::Names getTablesForSchema(const std::string & schema, size_t limit = 0) const; + DB::Names listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) void getCredentials(const String & table_id, TableMetadata & metadata) const; Poco::JSON::Object::Ptr requestCredentials(const String & table_id, const String & operation) const; @@ -84,11 +93,16 @@ class UnityCatalog final : public ICatalog, private DB::WithContext const std::string & table_name, TableMetadata & result) const; +<<<<<<< HEAD ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback(const DB::StorageID & table_id) override; ICatalog::CredentialsRefreshCallback getWriteCredentialsConfigurationCallback(const DB::StorageID & table_id) override; ICatalog::CredentialsRefreshCallback getCredentialsCallbackForOperation(const String & unity_table_id, const String & operation); +======= + ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback( + const DB::StorageID & table_id, const DB::ForwardedAuthTokenPtr & auth_token) override; +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) }; } diff --git a/src/Databases/DataLake/tests/gtest_rest_catalog.cpp b/src/Databases/DataLake/tests/gtest_rest_catalog.cpp index 3b5bdef518657..f6c38993b9155 100644 --- a/src/Databases/DataLake/tests/gtest_rest_catalog.cpp +++ b/src/Databases/DataLake/tests/gtest_rest_catalog.cpp @@ -9,9 +9,10 @@ #include #include #include -#include +#include #include +<<<<<<< HEAD #include #include #include @@ -27,9 +28,13 @@ #include #include #include +======= +#include +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) #include using namespace DataLake; +using namespace RestCatalogTest; namespace ProfileEvents { @@ -42,10 +47,12 @@ namespace DB { namespace ErrorCodes { - extern const int LOGICAL_ERROR; extern const int BAD_ARGUMENTS; +<<<<<<< HEAD extern const int NOT_IMPLEMENTED; extern const int DATALAKE_DATABASE_ERROR; +======= +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } } @@ -70,42 +77,79 @@ enum class CatalogShape OAuthTokenRejected, }; +<<<<<<< HEAD void writeJSON(Poco::Net::HTTPServerResponse & response, const std::string & body, Poco::Net::HTTPResponse::HTTPStatus status = Poco::Net::HTTPResponse::HTTP_OK) { response.setStatus(status); response.setContentType("application/json"); response.setContentLength(body.size()); response.send() << body; +======= +std::string getParent(const std::string & query) +{ + Poco::URI uri; + uri.setRawQuery(query); + for (const auto & [key, value] : uri.getQueryParameters()) + if (key == "parent") + return value; + return {}; +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } -void writeError(Poco::Net::HTTPServerResponse & response, Poco::Net::HTTPResponse::HTTPStatus status, const std::string & body) +void installShape(ServerState & state, CatalogShape shape) { - response.setStatus(status); - response.setContentType("application/json"); - response.setContentLength(body.size()); - response.send() << body; + state.setRoute("/v1/namespaces", [shape](const RecordedRequest & request) + { + const auto parent = getParent(request.query); + if (parent.empty()) + { + if (shape == CatalogShape::NestedTableThenEmptySibling) + return json(R"({"namespaces":[["parent"],["empty_later"]]})"); + return json(R"({"namespaces":[["namespace"]]})"); + } + + if (shape == CatalogShape::NestedTableThenEmptySibling && parent == "parent") + return json(R"({"namespaces":[["leaf_with_table"]]})"); + return json(R"({"namespaces":[]})"); + }); + + state.setRoute("/v1/namespaces/namespace/tables", [shape](const RecordedRequest &) + { + if (shape == CatalogShape::TopLevelTable) + return json(R"({"identifiers":[{"name":"table_a"}]})"); + return json(R"({"identifiers":[]})"); + }); + + state.setStaticRoute("/v1/namespaces/parent/tables", R"({"identifiers":[]})"); + state.setStaticRoute("/v1/namespaces/empty_later/tables", R"({"identifiers":[]})"); + state.setStaticRoute("/v1/namespaces/parent%1Fleaf_with_table/tables", R"({"identifiers":[{"name":"table_a"}]})"); } -std::string getRawPath(const std::string & uri) +void installTokenEndpoint(ServerState & state) { - const auto query_pos = uri.find('?'); - if (query_pos == std::string::npos) - return uri; - return uri.substr(0, query_pos); + state.setStaticRoute("/v1/oauth/tokens", R"({"token_type":"Bearer","expires_in":3600,"access_token":"mock-access-token"})"); } -class RestCatalogRequestHandler final : public Poco::Net::HTTPRequestHandler +void installTableRoutes(ServerState & state) { +<<<<<<< HEAD public: RestCatalogRequestHandler(CatalogShape shape_, int token_expires_in_seconds_, std::atomic & token_requests_) : shape(shape_) , token_expires_in_seconds(token_expires_in_seconds_) , token_requests(token_requests_) +======= + state.setStaticRoute( + "/v1/namespaces/namespace/tables/table_a", R"({"metadata":{"table-uuid":"11111111-2222-3333-4444-555555555555"}})"); + state.setRoute("/v1/namespaces/namespace/tables/missing_table", [](const RecordedRequest &) +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { - } - - void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override + return respondWithStatus( + 404, R"({"error":{"message":"Table does not exist","type":"NoSuchTableException","code":404}})"); + }); + state.setRoute("/v1/namespaces/namespace/tables/unauthorized_table", [](const RecordedRequest &) { +<<<<<<< HEAD Poco::URI uri(request.getURI()); const auto path = getRawPath(request.getURI()); const auto params = uri.getQueryParameters(); @@ -318,6 +362,12 @@ class RestCatalogTestServer Poco::AutoPtr server_params; std::unique_ptr server; }; +======= + return respondWithStatus( + 401, R"({"error":{"message":"The access token has expired","type":"NotAuthorizedException","code":401}})"); + }); +} +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) void expectThrowsCode(std::function fn, int expected_code) { @@ -334,7 +384,9 @@ void expectThrowsCode(std::function fn, int expected_code) bool restCatalogEmpty(CatalogShape shape, bool flat_namespaces = false) { - RestCatalogTestServer server(shape); + TestServer server; + installShape(*server, shape); + auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -392,7 +444,7 @@ bool deltaSharingCatalogEmpty(CatalogShape shape) /* flat_namespaces */false, context); - return catalog.empty(); + return catalog.empty(/* auth_token */ {}); } } @@ -576,7 +628,8 @@ TEST(RestCatalog, TryGetTableMetadataAuthErrorPropagates) TEST(RestCatalog, ApplySettingsChangesWithoutAuthenticationRejected) { - RestCatalogTestServer server(CatalogShape::Empty); + TestServer server; + auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -599,7 +652,9 @@ TEST(RestCatalog, ApplySettingsChangesWithoutAuthenticationRejected) TEST(RestCatalog, ApplySettingsChangesCredentialMode) { - RestCatalogTestServer server(CatalogShape::Empty); + TestServer server; + installTokenEndpoint(*server); + auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -642,7 +697,8 @@ TEST(RestCatalog, ApplySettingsChangesCredentialMode) TEST(RestCatalog, ApplySettingsChangesAuthHeaderMode) { - RestCatalogTestServer server(CatalogShape::Empty); + TestServer server; + auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -673,7 +729,8 @@ TEST(RestCatalog, ApplySettingsChangesAuthHeaderMode) TEST(RestCatalog, OneLakeApplySettingsChangesBearerMode) { - RestCatalogTestServer server(CatalogShape::Empty); + TestServer server; + auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -914,7 +971,9 @@ TEST(RestCatalog, HorizonParseCredentialKeepsColonsInSecret) TEST(RestCatalog, HorizonCatalogAuthenticatesWithBarePAT) { - RestCatalogTestServer server(CatalogShape::TopLevelTable); + TestServer server; + installTableRoutes(*server); + auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -930,6 +989,7 @@ TEST(RestCatalog, HorizonCatalogAuthenticatesWithBarePAT) /* flat_namespaces */false, context); +<<<<<<< HEAD EXPECT_EQ(catalog.getCatalogType(), DB::DatabaseDataLakeCatalogType::ICEBERG_HORIZON); EXPECT_TRUE(catalog.getStateSnapshot()->client_id.empty()); EXPECT_EQ(catalog.getStateSnapshot()->client_secret, "horizon-pat-without-colon"); @@ -994,6 +1054,19 @@ TEST(RestCatalog, HorizonApplySettingsChangesBarePAT) DB::SettingsChanges mode_switch; mode_switch.emplace_back("auth_header", "Authorization: Bearer token"); expectThrowsCode([&] { catalog.applySettingsChanges(mode_switch); }, DB::ErrorCodes::BAD_ARGUMENTS); +======= + TableMetadata existing; + EXPECT_TRUE(catalog.tryGetTableMetadata("namespace", "table_a", context, existing)); + EXPECT_TRUE(catalog.existsTable("namespace", "table_a", /* auth_token */ {})); + + TableMetadata missing; + EXPECT_FALSE(catalog.tryGetTableMetadata("namespace", "missing_table", context, missing)); + EXPECT_FALSE(catalog.existsTable("namespace", "missing_table", /* auth_token */ {})); + + TableMetadata unauthorized; + EXPECT_THROW(catalog.tryGetTableMetadata("namespace", "unauthorized_table", context, unauthorized), DB::HTTPException); + EXPECT_THROW(catalog.existsTable("namespace", "unauthorized_table", /* auth_token */ {}), DB::HTTPException); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } #endif diff --git a/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp b/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp new file mode 100644 index 0000000000000..da4d69d6420f6 --- /dev/null +++ b/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp @@ -0,0 +1,379 @@ +#include "config.h" + +#if USE_AVRO + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include + +using namespace DataLake; +using namespace RestCatalogTest; + +namespace +{ + +constexpr auto CONFIG_PATH = "/v1/config"; +constexpr auto NAMESPACES_PATH = "/v1/namespaces"; +constexpr auto NS_TABLES_PATH = "/v1/namespaces/ns/tables"; +constexpr auto TABLE_PATH = "/v1/namespaces/ns/tables/t"; +constexpr auto CATALOG_TOKEN_PATH = "/v1/oauth/tokens"; +constexpr auto IDP_TOKEN_PATH = "/idp/token"; + +constexpr auto ALICE_TOKEN = "alice.jwt.token"; + +DB::ForwardedAuthTokenPtr makeToken() +{ + return DB::makeForwardedAuthToken(DB::TokenCredentials(ALICE_TOKEN), "alice"); +} + +DB::ContextMutablePtr makeQueryContext(const DB::ForwardedAuthTokenPtr & auth_token = {}) +{ + auto context = DB::Context::createCopy(getContext().context); + context->makeQueryContext(); + context->setForwardedAuthToken(auth_token); + return context; +} + +void installCatalogShape(ServerState & state) +{ + state.setRoute(NAMESPACES_PATH, [](const RecordedRequest & request) + { + if (request.query.find("parent=") != std::string::npos) + return json(R"({"namespaces":[]})"); + return json(R"({"namespaces":[["ns"]]})"); + }); + state.setStaticRoute(NS_TABLES_PATH, R"({"identifiers":[{"name":"t"}]})"); +} + +std::string loadTableResponse() +{ + const auto expires_at_ms + = std::chrono::duration_cast((std::chrono::system_clock::now() + std::chrono::hours(24)).time_since_epoch()) + .count(); + return fmt::format( + R"({{"metadata-location":"s3://bucket/t/metadata/v1.metadata.json",)" + R"("metadata":{{"table-uuid":"1e1c0e10-0000-4000-8000-000000000001","location":"s3://bucket/t","schemas":[],"current-schema-id":0}},)" + R"("config":{{"s3.access-key-id":"AKIA_VENDED","s3.secret-access-key":"secret","s3.session-token":"session",)" + R"("s3.session-token-expires-at-ms":{}}}}})", + expires_at_ms); +} + +void installTokenEndpoint(ServerState & state) +{ + state.setStaticRoute(IDP_TOKEN_PATH, R"({"access_token":"session_token","expires_in":3600})"); +} + +TokenForwardingConfig exchangeAt(const std::string & uri) +{ + return TokenForwardingConfig{ + .forward_user_token = true, + .token_exchange_uri = uri, + .subject_token_type = "urn:ietf:params:oauth:token-type:access_token", + .requested_token_type = "urn:ietf:params:oauth:token-type:access_token", + .forward_actor_token = false, + .user_token_cache_ttl = 300, + }; +} + +/// `DB::WithContext` retains only a weak pointer; callers must keep the context alive. +std::shared_ptr makeCatalog( + const TestServer & server, + const DB::ContextPtr & context, + const TokenForwardingConfig & forwarding) +{ + return std::make_shared( + "warehouse", + server.getUrl(), + "client:secret", + /* auth_scope */ "lakekeeper", + /* auth_header */ "", + /* oauth_server_uri */ "", + /* oauth_server_use_request_body */ true, + /* namespaces */ "*", + context, + forwarding); +} + +void loadTable(RestCatalog & catalog, const DB::ForwardedAuthTokenPtr & auth_token) +{ + auto query_context = makeQueryContext(auth_token); + TableMetadata metadata; + metadata.withLocation().withStorageCredentials(); + catalog.getTableMetadata("ns", "t", query_context, metadata); +} + +size_t countVendingRequests(const ServerState & state) +{ + size_t count = 0; + for (const auto & request : state.requestsTo(TABLE_PATH)) + if (request.header("X-Iceberg-Access-Delegation") == "vended-credentials") + ++count; + return count; +} + +std::map parseForm(const std::string & body) +{ + Poco::URI uri; + uri.setRawQuery(body); + const auto params = uri.getQueryParameters(); + return {params.begin(), params.end()}; +} + +} + +class RestCatalogTokenForwarding : public ::testing::Test +{ +protected: + RestCatalogTokenForwarding() + : previous(getContext().context->getAccessControl().isTokenForwardingEnabled()) + { + getContext().context->getAccessControl().setTokenForwardingEnabled(true); + } + + ~RestCatalogTokenForwarding() override + { + getContext().context->getAccessControl().setTokenForwardingEnabled(previous); + } + +private: + const bool previous; +}; + +TEST_F(RestCatalogTokenForwarding, PassesUserTokenToCatalog) +{ + TestServer server; + installCatalogShape(*server); + auto context = makeQueryContext(); + TokenForwardingConfig forwarding; + forwarding.forward_user_token = true; + auto catalog = makeCatalog(server, context, forwarding); + + ASSERT_EQ(catalog->getTables(makeToken()), DB::Names{"ns.t"}); + for (const auto * path : {CONFIG_PATH, NAMESPACES_PATH, NS_TABLES_PATH}) + { + const auto requests = server->requestsTo(path); + ASSERT_FALSE(requests.empty()); + for (const auto & request : requests) + EXPECT_EQ(request.header("Authorization"), "Bearer " + std::string(ALICE_TOKEN)); + } + EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); +} + +TEST_F(RestCatalogTokenForwarding, ExchangesAndCachesEachUserTokenSeparately) +{ + TestServer server; + installCatalogShape(*server); + server->setRoute(IDP_TOKEN_PATH, [](const RecordedRequest & request) + { + return json(fmt::format(R"({{"access_token":"{}_session","expires_in":3600}})", parseForm(request.body).at("subject_token"))); + }); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH)); + + for (const auto * token : {ALICE_TOKEN, "rotated.alice.token"}) + { + auto auth_token = DB::makeForwardedAuthToken(DB::TokenCredentials(token), "alice"); + server->clearRequests(); + ASSERT_EQ(catalog->getTables(auth_token), DB::Names{"ns.t"}); + ASSERT_EQ(catalog->getTables(auth_token), DB::Names{"ns.t"}); + + const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); + ASSERT_EQ(exchanges.size(), 1u); + const auto form = parseForm(exchanges.front().body); + EXPECT_EQ(form.at("grant_type"), "urn:ietf:params:oauth:grant-type:token-exchange"); + EXPECT_EQ(form.at("subject_token"), token); + EXPECT_EQ(form.at("subject_token_type"), "urn:ietf:params:oauth:token-type:access_token"); + for (const auto & request : server->requestsTo(NAMESPACES_PATH)) + EXPECT_EQ(request.header("Authorization"), "Bearer " + std::string(token) + "_session"); + EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); + } +} + +TEST_F(RestCatalogTokenForwarding, AlteringCatalogCredentialDropsCachedTokensAndCredentials) +{ + TestServer server; + installTokenEndpoint(*server); + server->setStaticRoute(TABLE_PATH, loadTableResponse()); + + auto alice = makeToken(); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH)); + catalog->setVendedCredentialsCacheTTL(std::chrono::seconds(300)); + + loadTable(*catalog, alice); + ASSERT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); + ASSERT_EQ(countVendingRequests(*server), 1u); + + loadTable(*catalog, alice); + ASSERT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); + ASSERT_EQ(countVendingRequests(*server), 1u); + + DB::SettingsChanges changes; + changes.emplace_back("catalog_credential", "client:rotated_secret"); + catalog->applySettingsChanges(changes, alice); + + loadTable(*catalog, alice); + EXPECT_EQ(countVendingRequests(*server), 2u); + + const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); + ASSERT_EQ(exchanges.size(), 3u); + EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); + EXPECT_EQ(parseForm(exchanges.back().body).at("client_secret"), "rotated_secret"); +} + +TEST_F(RestCatalogTokenForwarding, RejectedConfigReloadDoesNotPublishPreparedUserSession) +{ + TestServer server; + installCatalogShape(*server); + server->setRoute(IDP_TOKEN_PATH, [](const RecordedRequest & request) + { + return json(fmt::format(R"({{"access_token":"{}_session","expires_in":3600}})", parseForm(request.body).at("client_secret"))); + }); + auto alice = makeToken(); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH)); + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + server->clearRequests(); + server->setRoute(CONFIG_PATH, [](const RecordedRequest &) { return respondWithStatus(403); }); + + DB::SettingsChanges changes; + changes.emplace_back("catalog_credential", "client:rotated_secret"); + EXPECT_THROW(catalog->prepareSettingsChanges(changes, alice), DB::Exception); + ASSERT_EQ(server->countRequestsTo(CONFIG_PATH), 1u); + EXPECT_EQ(server->requestsTo(CONFIG_PATH).front().header("Authorization"), "Bearer rotated_secret_session"); + EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + for (const auto & request : server->requestsTo(NAMESPACES_PATH)) + EXPECT_EQ(request.header("Authorization"), "Bearer secret_session"); +} + +class ParkedRoute +{ +public: + RestCatalogTest::ServerState::Route handler(RestCatalogTest::ServerState::Route response) + { + return [this, response](const RecordedRequest & request) + { + { + std::unique_lock lock(mutex); + if (!arrived) + { + arrived = true; + cv.notify_all(); + cv.wait(lock, [this] { return released; }); + } + } + return response(request); + }; + } + + void waitUntilParked() + { + std::unique_lock lock(mutex); + cv.wait(lock, [this] { return arrived; }); + } + + void release() + { + { + std::lock_guard lock(mutex); + released = true; + } + cv.notify_all(); + } + +private: + std::mutex mutex; + std::condition_variable cv; + bool arrived = false; + bool released = false; +}; + +TEST_F(RestCatalogTokenForwarding, InFlightVendedCredentialsDoNotOutliveTheirGeneration) +{ + ParkedRoute parked; + TestServer server; + installTokenEndpoint(*server); + server->setRoute(TABLE_PATH, parked.handler([](const RecordedRequest &) { return json(loadTableResponse()); })); + + auto alice = makeToken(); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH)); + catalog->setVendedCredentialsCacheTTL(std::chrono::seconds(300)); + + std::thread in_flight([&] { loadTable(*catalog, alice); }); + SCOPE_EXIT({ + parked.release(); + if (in_flight.joinable()) + in_flight.join(); + }); + + parked.waitUntilParked(); + + DB::SettingsChanges changes; + changes.emplace_back("catalog_credential", "client:rotated_secret"); + catalog->applySettingsChanges(changes, alice); + + parked.release(); + in_flight.join(); + + const auto vends_before = countVendingRequests(*server); + + loadTable(*catalog, alice); + EXPECT_EQ(countVendingRequests(*server), vends_before + 1); +} + +TEST_F(RestCatalogTokenForwarding, ConfigLoadDoesNotRollBackAConcurrentCredentialChange) +{ + ParkedRoute parked; + TestServer server; + installCatalogShape(*server); + installTokenEndpoint(*server); + server->setRoute("/v1/config", parked.handler([](const RecordedRequest &) { return json(R"({"defaults":{},"overrides":{}})"); })); + + auto alice = makeToken(); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH)); + + std::thread in_flight([&] { catalog->getTables(alice); }); + SCOPE_EXIT({ + parked.release(); + if (in_flight.joinable()) + in_flight.join(); + }); + + parked.waitUntilParked(); + + DB::SettingsChanges changes; + changes.emplace_back("catalog_credential", "client:rotated_secret"); + catalog->applySettingsChanges(changes, alice); + + parked.release(); + in_flight.join(); + + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + + const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); + ASSERT_GE(exchanges.size(), 2u); + EXPECT_EQ(parseForm(exchanges.back().body).at("client_secret"), "rotated_secret"); +} + +#endif diff --git a/src/Databases/DataLake/tests/rest_catalog_test_server.h b/src/Databases/DataLake/tests/rest_catalog_test_server.h new file mode 100644 index 0000000000000..8c5dd2516dbd7 --- /dev/null +++ b/src/Databases/DataLake/tests/rest_catalog_test_server.h @@ -0,0 +1,199 @@ +#pragma once + +#include "config.h" + +#if USE_AVRO + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace RestCatalogTest +{ + +struct RecordedRequest +{ + std::string method; + std::string path; + std::string query; + std::string body; + std::map headers; + + std::string header(const std::string & name) const + { + auto it = headers.find(name); + return it == headers.end() ? std::string{} : it->second; + } +}; + +struct Response +{ + int status = 200; + std::string body; + std::string content_type = "application/json"; +}; + +inline Response json(const std::string & body) +{ + return Response{.body = body}; +} + +inline Response respondWithStatus(int status, const std::string & body = R"({"error":{"message":"denied"}})") +{ + return Response{.status = status, .body = body}; +} + +class ServerState +{ +public: + using Route = std::function; + + void setRoute(const std::string & path, Route route) + { + std::lock_guard lock(mutex); + routes[path] = std::move(route); + } + + void setStaticRoute(const std::string & path, const std::string & body) + { + setRoute(path, [body](const RecordedRequest &) { return json(body); }); + } + + std::vector requestsTo(const std::string & path) const + { + std::lock_guard lock(mutex); + std::vector result; + for (const auto & request : recorded) + if (request.path == path) + result.push_back(request); + return result; + } + + size_t countRequestsTo(const std::string & path) const { return requestsTo(path).size(); } + + void clearRequests() + { + std::lock_guard lock(mutex); + recorded.clear(); + } + + Response handle(RecordedRequest request) + { + Route route; + { + std::lock_guard lock(mutex); + recorded.push_back(request); + if (auto it = routes.find(request.path); it != routes.end()) + route = it->second; + } + + if (!route) + return Response{ + .status = 599, + .body = "unexpected request to fake Iceberg REST catalog: " + request.method + " " + request.path, + .content_type = "text/plain"}; + + return route(request); + } + +private: + mutable std::mutex mutex; + std::map routes; + std::vector recorded; +}; + +class RequestHandler final : public Poco::Net::HTTPRequestHandler +{ +public: + explicit RequestHandler(std::shared_ptr state_) : state(std::move(state_)) {} + + void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override + { + const std::string & raw_uri = request.getURI(); + const auto query_pos = raw_uri.find('?'); + + RecordedRequest recorded; + recorded.method = request.getMethod(); + /// Keep `%1F` in nested namespace paths encoded so it matches the route keys. + recorded.path = query_pos == std::string::npos ? raw_uri : raw_uri.substr(0, query_pos); + recorded.query = query_pos == std::string::npos ? std::string{} : raw_uri.substr(query_pos + 1); + Poco::StreamCopier::copyToString(request.stream(), recorded.body); + for (const auto & [name, value] : request) + recorded.headers[name] = value; + + const auto result = state->handle(std::move(recorded)); + + response.setStatus(static_cast(result.status)); + response.setContentType(result.content_type); + response.setContentLength(result.body.size()); + response.send() << result.body; + } + +private: + std::shared_ptr state; +}; + +class RequestHandlerFactory final : public Poco::Net::HTTPRequestHandlerFactory +{ +public: + explicit RequestHandlerFactory(std::shared_ptr state_) : state(std::move(state_)) {} + + Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest &) override + { + return new RequestHandler(state); + } + +private: + std::shared_ptr state; +}; + +class TestServer +{ +public: + TestServer() + : state(std::make_shared()) + , server_socket(Poco::Net::SocketAddress("127.0.0.1", 0)) + , server(new RequestHandlerFactory(state), server_socket, new Poco::Net::HTTPServerParams()) + { + /// Ephemeral ports can be reused; discard pooled sockets belonging to previous test servers. + DB::HTTPConnectionPools::instance().dropCache(); + + state->setStaticRoute("/v1/config", R"({"defaults":{},"overrides":{}})"); + server.start(); + } + + ~TestServer() + { + server.stop(); + DB::HTTPConnectionPools::instance().dropCache(); + } + + std::string getUrl() const { return "http://" + server_socket.address().toString(); } + + ServerState & operator*() const { return *state; } + ServerState * operator->() const { return state.get(); } + +private: + std::shared_ptr state; + Poco::Net::ServerSocket server_socket; + Poco::Net::HTTPServer server; +}; + +} + +#endif diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index 5527b29574d94..296e7ff824e0c 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -215,7 +215,7 @@ class IDatabase : public std::enable_shared_from_this virtual bool isDatalakeCatalog() const { return false; } - virtual void validateCreateTableEngine(const ASTStorage & /*storage*/) const {} + virtual void validateCreateTableEngine(const ASTStorage & /*storage*/, ContextPtr /*context*/) const {} /// True for databases such as `MySQL`/`PostgreSQL` whose table list lives on a remote service. /// This is distinct from `isExternal`, which classifies whether the engine supports ClickHouse internal table types. @@ -480,7 +480,7 @@ class IDatabase : public std::enable_shared_from_this return database_name; } - virtual void checkDatabase() const + virtual void checkDatabase(ContextPtr /*context*/) const { //No-op } diff --git a/src/IO/S3/Credentials.cpp b/src/IO/S3/Credentials.cpp index ae74bb8e43bae..00c364195081c 100644 --- a/src/IO/S3/Credentials.cpp +++ b/src/IO/S3/Credentials.cpp @@ -58,11 +58,13 @@ namespace S3 # include # include +# include # include # include # include # include +# include # include # include @@ -1163,7 +1165,30 @@ void AssumeRoleRequest::AddQueryStringParameters(Aws::Http::URI & uri) const uri.AddQueryStringParameter("ExternalId", external_id); } -AssumeRoleResult::AssumeRoleResult(Aws::AmazonWebServiceResult result) +AssumeRoleWithWebIdentityRequest::AssumeRoleWithWebIdentityRequest( + std::string role_arn_, std::string role_session_name_, std::string web_identity_token_) + : role_arn(std::move(role_arn_)) + , role_session_name(std::move(role_session_name_)) + , web_identity_token(std::move(web_identity_token_)) +{ +} + +Aws::Http::HeaderValueCollection AssumeRoleWithWebIdentityRequest::GetHeaders() const +{ + return {{Aws::Http::HeaderValuePair(Aws::Http::CONTENT_TYPE_HEADER, Aws::FORM_CONTENT_TYPE)}}; +} + +Aws::String AssumeRoleWithWebIdentityRequest::SerializePayload() const +{ + return fmt::format( + "Action=AssumeRoleWithWebIdentity&Version=2011-06-15&RoleArn={}&RoleSessionName={}&WebIdentityToken={}", + DB::formUrlEncode(role_arn), + DB::formUrlEncode(role_session_name), + DB::formUrlEncode(web_identity_token)); +} + +AssumeRoleResult::AssumeRoleResult( + Aws::AmazonWebServiceResult result, const char * result_node_name) { using namespace Aws::Utils::Xml; const auto & xml_document = result.GetPayload(); @@ -1175,10 +1200,10 @@ AssumeRoleResult::AssumeRoleResult(Aws::AmazonWebServiceResult client_) + : role_arn(std::move(role_arn_)) + , session_name(std::move(session_name_)) + , web_identity_token(std::move(web_identity_token_)) + , expiration_window_seconds(expiration_window_seconds_) + , client(std::move(client_)) + , logger(getLogger("AwsAuthSTSAssumeRoleWithWebIdentityCredentialsProvider")) +{ +} + +Aws::Auth::AWSCredentials AwsAuthSTSAssumeRoleWithWebIdentityCredentialsProvider::GetAWSCredentials() +{ + Aws::Utils::Threading::ReaderLockGuard guard(m_reloadLock); + if (!IsSetNeedRefresh() && !areCredentialsEmptyOrExpired(credentials, expiration_window_seconds)) + return credentials; + + guard.UpgradeToWriterLock(); + if (!IsSetNeedRefresh() && !areCredentialsEmptyOrExpired(credentials, expiration_window_seconds)) // double-checked lock to avoid refreshing twice + return credentials; + + Reload(); + return credentials; +} + +void AwsAuthSTSAssumeRoleWithWebIdentityCredentialsProvider::Reload() +{ + LOG_INFO(logger, "Credentials are empty or expired, attempting to renew with AssumeRoleWithWebIdentity for role {}", role_arn); + + AssumeRoleWithWebIdentityRequest request(role_arn, session_name, web_identity_token); + auto outcome = client->assumeRoleWithWebIdentity(request); + if (!outcome.IsSuccess()) + { + credentials = Aws::Auth::AWSCredentials{}; + last_error = outcome.GetError().GetMessage(); + LOG_WARNING(logger, "Failed to get credentials using AssumeRoleWithWebIdentity. Error: {}", last_error); + return; + } + + last_error.clear(); + const auto & result = outcome.GetResult(); + credentials.SetAWSAccessKeyId(result.getAccessKeyID()); + credentials.SetAWSSecretKey(result.getSecretAccessKey()); + credentials.SetSessionToken(result.getSessionToken()); + credentials.SetExpiration(result.getExpiration()); + + AWSCredentialsProvider::Reload(); + + LOG_TRACE(logger, "Successfully retrieved credentials for role {}", role_arn); +} + +std::string AwsAuthSTSAssumeRoleWithWebIdentityCredentialsProvider::getLastError() const +{ + Aws::Utils::Threading::ReaderLockGuard guard(m_reloadLock); + return last_error; +} +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) std::shared_ptr getCredentialsProvider( const DB::S3::PocoHTTPClientConfiguration & configuration, diff --git a/src/IO/S3/Credentials.h b/src/IO/S3/Credentials.h index 886bcf12aaffd..51a6a0f886145 100644 --- a/src/IO/S3/Credentials.h +++ b/src/IO/S3/Credentials.h @@ -266,11 +266,32 @@ class AssumeRoleRequest : public Aws::AmazonSerializableWebServiceRequest std::string role_session_name; std::string external_id; }; +class AssumeRoleWithWebIdentityRequest : public Aws::AmazonSerializableWebServiceRequest +{ +public: + AssumeRoleWithWebIdentityRequest(std::string role_arn_, std::string role_session_name_, std::string web_identity_token_); + + Aws::Http::HeaderValueCollection GetHeaders() const override; + + const char * GetServiceRequestName() const override { return "AssumeRoleWithWebIdentity"; } + + Aws::String SerializePayload() const override; + +private: + std::string role_arn; + std::string role_session_name; + std::string web_identity_token; +}; + class AssumeRoleResult { public: + AssumeRoleResult() = default; + /// NOLINTNEXTLINE - AssumeRoleResult(Aws::AmazonWebServiceResult result); + AssumeRoleResult( + Aws::AmazonWebServiceResult result, + const char * result_node_name = "AssumeRoleResult"); const std::string & getAccessKeyID() const { return access_key_id; } @@ -301,10 +322,13 @@ class AWSAssumeRoleClient : public Aws::Client::AWSXMLClient AssumeRoleOutcome assumeRole(const AssumeRoleRequest & request) const; + AssumeRoleOutcome assumeRoleWithWebIdentity(const AssumeRoleWithWebIdentityRequest & request) const; + const auto & getEndpoint() const { return endpoint; } private: Aws::Endpoint::AWSEndpoint endpoint; + Aws::Endpoint::AWSEndpoint web_identity_endpoint; }; class AwsAuthSTSAssumeRoleCredentialsProvider : public Aws::Auth::AWSCredentialsProvider @@ -353,6 +377,34 @@ class AwsAuthSTSAssumeRoleCredentialsProvider : public Aws::Auth::AWSCredentials LoggerPtr logger; }; +class AwsAuthSTSAssumeRoleWithWebIdentityCredentialsProvider : public Aws::Auth::AWSCredentialsProvider +{ +public: + AwsAuthSTSAssumeRoleWithWebIdentityCredentialsProvider( + std::string role_arn_, + std::string session_name_, + std::string web_identity_token_, + uint64_t expiration_window_seconds_, + std::shared_ptr client_); + + Aws::Auth::AWSCredentials GetAWSCredentials() override; + + std::string getLastError() const; + +protected: + void Reload() override; + +private: + std::string role_arn; + std::string session_name; + std::string web_identity_token; + uint64_t expiration_window_seconds; + std::shared_ptr client; + Aws::Auth::AWSCredentials credentials; + std::string last_error; + LoggerPtr logger; +}; + std::shared_ptr getCredentialsProvider( const DB::S3::PocoHTTPClientConfiguration & configuration, const Aws::Auth::AWSCredentials & credentials, diff --git a/src/IO/S3/tests/TestPocoHTTPServer.h b/src/IO/S3/tests/TestPocoHTTPServer.h index 9d8066a1a0c0e..787fff758bddc 100644 --- a/src/IO/S3/tests/TestPocoHTTPServer.h +++ b/src/IO/S3/tests/TestPocoHTTPServer.h @@ -16,6 +16,11 @@ #include #include #include +<<<<<<< HEAD +======= +#include +#include +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) #include /// Keep-alive is disabled so a handler thread exits right after sending the response: it does @@ -106,12 +111,14 @@ struct StsRequestInfo { Poco::Net::MessageHeader headers; Poco::URI::QueryParameters query_params; + std::string body; }; class MockStsRequestHandler : public Poco::Net::HTTPRequestHandler { public: - explicit MockStsRequestHandler(std::optional & last_request_info_, std::string role_access_key_, std::string role_secret_key_) + explicit MockStsRequestHandler( + std::optional & last_request_info_, std::string role_access_key_, std::string role_secret_key_) : last_request_info(last_request_info_) , role_access_key(std::move(role_access_key_)) , role_secret_key(std::move(role_secret_key_)) @@ -125,20 +132,24 @@ class MockStsRequestHandler : public Poco::Net::HTTPRequestHandler Poco::URI uri(request.getURI()); last_request_info->query_params = uri.getQueryParameters(); + Poco::StreamCopier::copyToString(request.stream(), last_request_info->body); + + const bool web_identity = last_request_info->body.find("Action=AssumeRoleWithWebIdentity") != std::string::npos; + const std::string_view action = web_identity ? "AssumeRoleWithWebIdentity" : "AssumeRole"; response.setStatus(Poco::Net::HTTPResponse::HTTP_OK); auto & out = response.send(); std::string result_xml = fmt::format(R"( - - +<{0}Response xmlns="https://sts.amazonaws.com/doc/2011-06-15/"> +<{0}Result> - {} - {} + {1} + {2} session_token - -)", role_access_key, role_secret_key); + +)", action, role_access_key, role_secret_key); out << result_xml; out.flush(); } @@ -159,7 +170,8 @@ class StsHTTPRequestHandlerFactory : public Poco::Net::HTTPRequestHandlerFactory return new MockStsRequestHandler(last_request_info, role_access_key, role_secret_key); } public: - explicit StsHTTPRequestHandlerFactory(std::optional & last_request_info_, std::string role_access_key_, std::string role_secret_key_) + explicit StsHTTPRequestHandlerFactory( + std::optional & last_request_info_, std::string role_access_key_, std::string role_secret_key_) : last_request_info(last_request_info_) , role_access_key(std::move(role_access_key_)) , role_secret_key(std::move(role_secret_key_)) @@ -218,4 +230,9 @@ class TestPocoHTTPStsServer { return last_request_info->query_params; } + + const std::string & getLastBody() const + { + return last_request_info->body; + } }; diff --git a/src/IO/S3/tests/gtest_sts_assume_role_with_web_identity.cpp b/src/IO/S3/tests/gtest_sts_assume_role_with_web_identity.cpp new file mode 100644 index 0000000000000..2d2fe1c0e1167 --- /dev/null +++ b/src/IO/S3/tests/gtest_sts_assume_role_with_web_identity.cpp @@ -0,0 +1,91 @@ +#include + +#include "config.h" + +#if USE_AWS_S3 + +#include +#include +#include +#include + +#include + +#include + +#include +#include + +namespace +{ + +constexpr std::string_view role_access_key = "role_access_key"; +constexpr std::string_view role_secret_key = "role_secret_key"; + +DB::S3::PocoHTTPClientConfiguration makeClientConfiguration(DB::RemoteHostFilter & remote_host_filter) +{ + return DB::S3::ClientFactory::instance().createClientConfiguration( + "eu-west-1", + remote_host_filter, + /* s3_max_redirects = */ 100, + DB::S3::PocoHTTPClientConfiguration::RetryStrategy{.max_retries = 0}, + /* s3_slow_all_threads_after_network_error = */ false, + /* s3_slow_all_threads_after_retryable_error = */ false, + /* enable_s3_requests_logging = */ false, + /* for_disk_s3 = */ false, + /* opt_disk_name = */ {}, + /* request_throttler = */ {}, + "http"); +} + +std::string formParameter(const std::string & body, const std::string & name) +{ + Poco::URI uri; + uri.setRawQuery(body); + for (const auto & [key, value] : uri.getQueryParameters()) + if (key == name) + return value; + return {}; +} + +} + +TEST(STSAssumeRoleWithWebIdentity, SendsTokenInTheBody) +{ + TestPocoHTTPStsServer sts_http(std::string{role_access_key}, std::string{role_secret_key}); + + DB::RemoteHostFilter remote_host_filter; + auto client_configuration = makeClientConfiguration(remote_host_filter); + + auto client = std::make_shared( + std::make_shared(), client_configuration, sts_http.getUrl()); + + const std::string token = "header.payload.signature"; + DB::S3::AwsAuthSTSAssumeRoleWithWebIdentityCredentialsProvider provider( + "arn:aws:iam::123456789012:role/data-lake-reader", "alice", token, /* expiration_window_seconds = */ 0, client); + + auto credentials = provider.GetAWSCredentials(); + + ASSERT_TRUE(sts_http.hasLastRequest()); + + const auto & body = sts_http.getLastBody(); + EXPECT_EQ(formParameter(body, "Action"), "AssumeRoleWithWebIdentity"); + EXPECT_EQ(formParameter(body, "Version"), "2011-06-15"); + EXPECT_EQ(formParameter(body, "RoleArn"), "arn:aws:iam::123456789012:role/data-lake-reader"); + EXPECT_EQ(formParameter(body, "RoleSessionName"), "alice"); + EXPECT_EQ(formParameter(body, "WebIdentityToken"), token); + + for (const auto & [key, value] : sts_http.getLastQueryParams()) + { + EXPECT_NE(key, "WebIdentityToken"); + EXPECT_EQ(value.find(token), std::string::npos); + } + + EXPECT_FALSE(sts_http.getLastRequestHeader().has("Authorization")); + + EXPECT_EQ(credentials.GetAWSAccessKeyId(), role_access_key); + EXPECT_EQ(credentials.GetAWSSecretKey(), role_secret_key); + EXPECT_EQ(credentials.GetSessionToken(), "session_token"); +} + +#endif diff --git a/src/Interpreters/Access/InterpreterExecuteAsQuery.cpp b/src/Interpreters/Access/InterpreterExecuteAsQuery.cpp index feb284043a18c..70abeaf2c4cb6 100644 --- a/src/Interpreters/Access/InterpreterExecuteAsQuery.cpp +++ b/src/Interpreters/Access/InterpreterExecuteAsQuery.cpp @@ -70,6 +70,9 @@ namespace context->setUser(context->getAccessControl().getID(target_user_name)); + /// `EXECUTE AS` changes the session identity; the original bearer token must not survive it. + context->setForwardedAuthToken(nullptr); + /// We need to update the client info to make currentUser() return `target_user_name`. context->setCurrentUserName(target_user_name); context->setInitialUserName(target_user_name); diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index a68b1e4662347..0482ab3cc4ad2 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -130,6 +130,7 @@ AsynchronousInsertQueue::InsertQuery::InsertQuery( const String & current_user_, const String & initial_user_, const String & authenticated_user_, + const ForwardedAuthTokenPtr & forwarded_auth_token_, const Settings & settings_, AsynchronousInsertQueueDataKind data_kind_) : query(query_->clone()) @@ -139,8 +140,10 @@ AsynchronousInsertQueue::InsertQuery::InsertQuery( , current_user(current_user_) , initial_user(initial_user_) , authenticated_user(authenticated_user_) + , forwarded_auth_token(forwarded_auth_token_) , settings(std::make_unique(settings_)) , data_kind(data_kind_) + , forwarded_auth_token_fingerprint(forwarded_auth_token ? forwarded_auth_token->fingerprint : String{}) { SipHash siphash; @@ -164,6 +167,9 @@ AsynchronousInsertQueue::InsertQuery::InsertQuery( siphash.update(identity_field); } + siphash.update(forwarded_auth_token_fingerprint.size()); + siphash.update(forwarded_auth_token_fingerprint); + setting_changes = settings->changes(); for (auto it = setting_changes.begin(); it != setting_changes.end();) { @@ -191,6 +197,8 @@ AsynchronousInsertQueue::InsertQuery::InsertQuery(const InsertQuery & other) current_user = other.current_user; initial_user = other.initial_user; authenticated_user = other.authenticated_user; + forwarded_auth_token = other.forwarded_auth_token; + forwarded_auth_token_fingerprint = other.forwarded_auth_token_fingerprint; settings = std::make_unique(*other.settings); data_kind = other.data_kind; hash = other.hash; @@ -209,6 +217,8 @@ AsynchronousInsertQueue::InsertQuery::operator=(const InsertQuery & other) current_user = other.current_user; initial_user = other.initial_user; authenticated_user = other.authenticated_user; + forwarded_auth_token = other.forwarded_auth_token; + forwarded_auth_token_fingerprint = other.forwarded_auth_token_fingerprint; settings = std::make_unique(*other.settings); data_kind = other.data_kind; hash = other.hash; @@ -591,6 +601,7 @@ AsynchronousInsertQueue::PushResult AsynchronousInsertQueue::pushDataChunk(ASTPt client_info.current_user, client_info.initial_user, client_info.authenticated_user, + query_context->getForwardedAuthToken(), settings, data_kind}; InsertDataPtr data_to_process; @@ -1109,6 +1120,7 @@ try insert_context->setCurrentUserName(key.current_user); insert_context->setInitialUserName(key.initial_user); insert_context->setAuthenticatedUserName(key.authenticated_user); + insert_context->setForwardedAuthToken(key.forwarded_auth_token); insert_context->setSettings(*key.settings); diff --git a/src/Interpreters/AsynchronousInsertQueue.h b/src/Interpreters/AsynchronousInsertQueue.h index 3530e230996ea..aaae79c13f7ab 100644 --- a/src/Interpreters/AsynchronousInsertQueue.h +++ b/src/Interpreters/AsynchronousInsertQueue.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -97,6 +98,7 @@ class AsynchronousInsertQueue : public WithContext String current_user; String initial_user; String authenticated_user; + ForwardedAuthTokenPtr forwarded_auth_token; std::unique_ptr settings; AsynchronousInsertQueueDataKind data_kind; @@ -109,6 +111,7 @@ class AsynchronousInsertQueue : public WithContext const String & current_user_, const String & initial_user_, const String & authenticated_user_, + const ForwardedAuthTokenPtr & forwarded_auth_token_, const Settings & settings_, AsynchronousInsertQueueDataKind data_kind_); @@ -118,7 +121,14 @@ class AsynchronousInsertQueue : public WithContext StorageID getStorageID() const; private: - auto toTupleCmp() const { return std::tie(data_kind, query_str_with_secrets, user_id, current_roles, current_user, initial_user, authenticated_user, setting_changes); } + auto toTupleCmp() const + { + return std::tie( + data_kind, query_str_with_secrets, user_id, current_roles, current_user, initial_user, + authenticated_user, forwarded_auth_token_fingerprint, setting_changes); + } + + String forwarded_auth_token_fingerprint; std::vector setting_changes; }; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 73e2cbe0ce341..6965f54fdf03d 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1382,6 +1382,7 @@ ContextData::ContextData() ContextData::ContextData(const ContextData &o) : shared(o.shared), client_info(o.client_info), + forwarded_auth_token(o.forwarded_auth_token), external_tables_initializer_callback(o.external_tables_initializer_callback), input_initializer_callback(o.input_initializer_callback), input_blocks_reader(o.input_blocks_reader), @@ -8210,6 +8211,11 @@ void Context::setClientInfo(const ClientInfo & client_info_) need_recalculate_access = true; } +void Context::setForwardedAuthToken(ForwardedAuthTokenPtr token) +{ + forwarded_auth_token = std::move(token); +} + void Context::setClientName(const String & client_name) { client_info.client_name = client_name; diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 2cde94446131c..c0ff2d0a0e5b0 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -366,6 +367,7 @@ class ContextData ContextSharedPart * shared{}; ClientInfo client_info; + ForwardedAuthTokenPtr forwarded_auth_token; ExternalTablesInitializer external_tables_initializer_callback; QueryPlanDeserializationCallback query_plan_deserialization_callback; @@ -1004,6 +1006,9 @@ class Context: public ContextData, public std::enable_shared_from_this /// Modify stored in the context information about the client executing a query. void setClientInfo(const ClientInfo & client_info_); + + const ForwardedAuthTokenPtr & getForwardedAuthToken() const { return forwarded_auth_token; } + void setForwardedAuthToken(ForwardedAuthTokenPtr token); void setClientName(const String & client_name); void setClientInterface(ClientInfo::Interface interface); void setClientVersion(UInt64 client_version_major, UInt64 client_version_minor, UInt64 client_version_patch, unsigned client_tcp_protocol_version); diff --git a/src/Interpreters/InterpreterCheckQuery.cpp b/src/Interpreters/InterpreterCheckQuery.cpp index 5a921a486249c..75a6289578316 100644 --- a/src/Interpreters/InterpreterCheckQuery.cpp +++ b/src/Interpreters/InterpreterCheckQuery.cpp @@ -449,7 +449,7 @@ BlockIO InterpreterCheckQuery::execute() LOG_DEBUG(log, "Checking database name = {} ", database_name); context->checkAccess(AccessType::CHECK, database_name); auto database = DatabaseCatalog::instance().getDatabase(database_name); - database->checkDatabase(); + database->checkDatabase(context); BlockIO res; return res; } diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index e882a80619028..6fad4b532a908 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -2051,7 +2051,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) "views, dictionaries, ATTACH, CLONE AS, and REPLACE TABLE are not allowed"); if (engine_user_specified) - database->validateCreateTableEngine(*create.storage); + database->validateCreateTableEngine(*create.storage, getContext()); if (datalake_unsupported_storage_clause && !ignore_unsupported_properties) { diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp index 8e3bed316abe8..8e5167e826d90 100644 --- a/src/Interpreters/Session.cpp +++ b/src/Interpreters/Session.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -410,8 +411,19 @@ void Session::authenticate(const Credentials & credentials_, const Poco::Net::So chassert(!auth_result.user_name.empty()); prepared_client_info->current_user = auth_result.user_name; prepared_client_info->authenticated_user = auth_result.user_name; +<<<<<<< HEAD prepared_client_info->current_address = Poco::Net::SocketAddress(address); prepared_client_info->connection_address = Poco::Net::SocketAddress(connection_address ? *connection_address : address); +======= + prepared_client_info->current_address = std::make_shared(address); + prepared_client_info->connection_address = std::make_shared(connection_address ? *connection_address : address); + + if (const auto * token_credentials = typeid_cast(&credentials_)) + { + if (global_context->getAccessControl().isTokenForwardingEnabled()) + forwarded_auth_token = makeForwardedAuthToken(*token_credentials, auth_result.user_name); + } +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } void Session::checkIfUserIsStillValid() const @@ -583,6 +595,7 @@ ContextMutablePtr Session::makeSessionContext() /// Copy prepared client info to the new session context. new_session_context->setClientInfo(*prepared_client_info); + new_session_context->setForwardedAuthToken(forwarded_auth_token); prepared_client_info.reset(); /// Set user information for the new context: current profiles, roles, access rights. @@ -654,6 +667,11 @@ ContextMutablePtr Session::makeSessionContext(const String & session_name_, std: max_sessions_for_user = max_session_for_user_field->safeGet(); } + /// Refresh reused named sessions only while they still represent the authenticated user. + /// An `EXECUTE AS` session must not regain the original token. + const bool runs_as_authenticated_user = new_session_context->getAccess()->getUserID() == user_id; + new_session_context->setForwardedAuthToken(runs_as_authenticated_user ? forwarded_auth_token : nullptr); + /// Session context is ready. session_context = std::move(new_session_context); named_session = new_named_session; @@ -728,6 +746,10 @@ ContextMutablePtr Session::makeQueryContextImpl(const ClientInfo * client_info_t else if (detached && session_context) query_context->setClientInfo(getClientInfo()); + /// A session context may have cleared its token for `EXECUTE AS`; do not restore it in the query copy. + if (!from_session_context) + query_context->setForwardedAuthToken(forwarded_auth_token); + /// Copy current user's name and address if it was authenticated after query_client_info was initialized. if (prepared_client_info && !prepared_client_info->current_user.empty()) { diff --git a/src/Interpreters/Session.h b/src/Interpreters/Session.h index 4a8bf55154f32..f25cb795e2b66 100644 --- a/src/Interpreters/Session.h +++ b/src/Interpreters/Session.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -129,6 +130,8 @@ class Session /// ClientInfo that will be copied to a session context when it's created. std::optional prepared_client_info; + ForwardedAuthTokenPtr forwarded_auth_token; + mutable UserPtr user; std::optional user_id; std::vector external_roles; diff --git a/src/Interpreters/tests/gtest_async_insert_key.cpp b/src/Interpreters/tests/gtest_async_insert_key.cpp index 327525b4db5c3..4822956750ae1 100644 --- a/src/Interpreters/tests/gtest_async_insert_key.cpp +++ b/src/Interpreters/tests/gtest_async_insert_key.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -37,10 +38,10 @@ TEST(AsyncInsertKey, SettingsChanges) auto kind = AsynchronousInsertQueueDataKind::Parsed; - AsynchronousInsertQueue::InsertQuery key1(query, {}, {}, {}, {}, {}, settings1, kind); - AsynchronousInsertQueue::InsertQuery key2(query, {}, {}, {}, {}, {}, settings2, kind); - AsynchronousInsertQueue::InsertQuery key3(query, {}, {}, {}, {}, {}, settings3, kind); - AsynchronousInsertQueue::InsertQuery key4(query, {}, {}, {}, {}, {}, settings4, kind); + AsynchronousInsertQueue::InsertQuery key1(query, {}, {}, {}, {}, {}, {}, settings1, kind); + AsynchronousInsertQueue::InsertQuery key2(query, {}, {}, {}, {}, {}, {}, settings2, kind); + AsynchronousInsertQueue::InsertQuery key3(query, {}, {}, {}, {}, {}, {}, settings3, kind); + AsynchronousInsertQueue::InsertQuery key4(query, {}, {}, {}, {}, {}, {}, settings4, kind); EXPECT_EQ(key1, key2); EXPECT_NE(key1, key3); @@ -62,7 +63,7 @@ TEST(AsyncInsertKey, IdentityHashIsNotAmbiguous) auto make_key = [&](const String & current_user, const String & initial_user, const String & authenticated_user) { return AsynchronousInsertQueue::InsertQuery( - query, {}, {}, current_user, initial_user, authenticated_user, settings, kind); + query, {}, {}, current_user, initial_user, authenticated_user, {}, settings, kind); }; /// The three identity fields are variable-length strings folded into the queue key hash, @@ -93,3 +94,32 @@ TEST(AsyncInsertKey, IdentityHashIsNotAmbiguous) EXPECT_NE(key_g.hash, key_h.hash); EXPECT_NE(key_g, key_h); } + +TEST(AsyncInsertKey, ForwardedTokenPartitionsBatches) +{ + String query_str = "INSERT INTO test (a) VALUES (1)"; + ParserInsertQuery parser(query_str.data() + query_str.size(), false); + ASTPtr query = parseQuery(parser, query_str, DBMS_DEFAULT_MAX_QUERY_SIZE, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS); + Settings settings; + + auto make_key = [&](const ForwardedAuthTokenPtr & token) + { + return AsynchronousInsertQueue::InsertQuery( + query, {}, {}, "alice", "alice", "alice", token, settings, AsynchronousInsertQueueDataKind::Parsed); + }; + auto token = makeForwardedAuthToken(TokenCredentials("alice-token"), "alice"); + auto same_token = makeForwardedAuthToken(TokenCredentials("alice-token"), "alice"); + auto rotated_token = makeForwardedAuthToken(TokenCredentials("alice-rotated-token"), "alice"); + + auto original = make_key(token); + auto same = make_key(same_token); + auto rotated = make_key(rotated_token); + auto no_token = make_key({}); + + EXPECT_EQ(original, same); + EXPECT_EQ(original.hash, same.hash); + EXPECT_NE(original, rotated); + EXPECT_NE(original.hash, rotated.hash); + EXPECT_NE(original, no_token); + EXPECT_NE(original.hash, no_token.hash); +} diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 965eebc773217..ff602cc25606e 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1746,8 +1746,9 @@ void TCPHandler::processTablesStatusRequest() } TablesStatusResponse response; - for (const QualifiedTableName & table_name: request.tables) + try { +<<<<<<< HEAD auto resolved_id = context_to_resolve_table_names->tryResolveStorageID({table_name.database, table_name.table}); StoragePtr table = DatabaseCatalog::instance().tryGetTable(resolved_id, context_to_resolve_table_names); if (!table) @@ -1756,15 +1757,39 @@ void TCPHandler::processTablesStatusRequest() TableStatus status; /// The initiator asks about this table by name, so a lazily loaded replica is loaded to report its delay. if (auto * replicated_table = castStorage(table, DeferredTable::Load).get()) +======= + for (const QualifiedTableName & table_name: request.tables) +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { - status.is_replicated = true; - status.absolute_delay = static_cast(replicated_table->getAbsoluteDelay()); - status.is_readonly = replicated_table->isTableReadOnly(); + auto resolved_id = context_to_resolve_table_names->tryResolveStorageID({table_name.database, table_name.table}); + StoragePtr table = DatabaseCatalog::instance().tryGetTable(resolved_id, context_to_resolve_table_names); + if (!table) + continue; + + TableStatus status; + if (auto * replicated_table = dynamic_cast(table.get())) + { + status.is_replicated = true; + status.absolute_delay = static_cast(replicated_table->getAbsoluteDelay()); + status.is_readonly = replicated_table->isTableReadOnly(); + } + else + status.is_replicated = false; + + response.table_states_by_id.emplace(table_name, std::move(status)); } - else - status.is_replicated = false; + } + catch (const Exception & e) + { + /// Before the first query is authenticated with the interserver secret, the connection is closed on any exception. + if (is_interserver_mode && !is_interserver_authenticated) + throw; - response.table_states_by_id.emplace(table_name, std::move(status)); + /// The request is read completely and nothing is written yet, so the exception can be sent + /// to the client (it expects either `TablesStatusResponse` or `Exception`) and the connection kept. + tryLogCurrentException(log, "Cannot get status of tables"); + sendException(e, send_exception_with_stack_trace); + return; } writeVarUInt(Protocol::Server::TablesStatusResponse, *out); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp index e8bebbef824f7..06b84243574e8 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp @@ -988,7 +988,7 @@ static bool writeConsolidatedManifestFile( { auto catalog_filename = path_resolver.resolveForCatalog(generated_metadata_info.path); const auto & [namespace_name, table_name] = DataLake::parseTableName(table_id.getTableName()); - if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot.snapshot)) + if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot.snapshot, context->getForwardedAuthToken())) { LOG_INFO(log, "Metadata commit conflict detected via catalog, cleaning up temporary files"); cleanup(); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ExpireSnapshotsExecute.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ExpireSnapshotsExecute.cpp index 022e731eea29a..4d1418fd30ad0 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ExpireSnapshotsExecute.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ExpireSnapshotsExecute.cpp @@ -974,7 +974,7 @@ ExpireSnapshotsResult expireSnapshots( { auto catalog_filename = persistent_table_components.path_resolver.resolveForCatalog(metadata_info.path); const auto & [namespace_name, parsed_table_name] = DataLake::parseTableName(table_name); - if (!catalog->updateMetadata(namespace_name, parsed_table_name, catalog_filename, nullptr)) + if (!catalog->updateMetadata(namespace_name, parsed_table_name, catalog_filename, nullptr, context->getForwardedAuthToken())) { throw Exception( ErrorCodes::LOGICAL_ERROR, diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index 84d522be5d69b..0d08fee840406 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -860,7 +860,7 @@ void IcebergMetadata::truncate(ContextPtr context, std::shared_ptrupdateMetadata(namespace_name, table_name, catalog_filename, new_snapshot)) + if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot, context->getForwardedAuthToken())) throw Exception(ErrorCodes::INCORRECT_DATA, "Failed to commit Iceberg truncate update to catalog."); } @@ -1152,7 +1152,11 @@ void IcebergMetadata::createInitial( /// validation, so a rejected CREATE leaves no trace in the catalog): a catalog /// that shares its storage view with the data (e.g. SeaweedFS) refuses to create /// a namespace over the plain directory those files would leave behind. +<<<<<<< HEAD catalog->createNamespaceIfNotExists(namespace_name, /* location */ ""); +======= + catalog->createNamespaceIfNotExists(namespace_name, namespace_location, local_context->getForwardedAuthToken()); +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } if (!catalog_writes_metadata) @@ -1208,7 +1212,9 @@ void IcebergMetadata::createInitial( /// Always ask for a conflict to be reported as `false`, so the files staged above are removed /// before `TABLE_ALREADY_EXISTS` is thrown, with or without `IF NOT EXISTS`. - if (!catalog->createTable(namespace_name, table_name, catalog_filename, metadata_content_object, compression_method, /* if_not_exists */ true)) + if (!catalog->createTable( + namespace_name, table_name, catalog_filename, metadata_content_object, compression_method, /* if_not_exists */ true, + local_context->getForwardedAuthToken())) { if (!catalog_writes_metadata) { @@ -2481,7 +2487,7 @@ std::optional IcebergMetadata::commitImport catalog_filename = blob_storage_type_name + "://" + blob_storage_namespace_name + "/" + catalog_filename; const auto & [namespace_name, table_name] = DataLake::parseTableName(table_id.getTableName()); - if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot)) + if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot, context->getForwardedAuthToken())) { cleanup(true); return {}; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp index c66f8881f2479..a336f415b1d5a 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp @@ -1959,7 +1959,7 @@ bool IcebergStorageSink::initializeMetadata() auto catalog_filename = resolver.resolveForCatalog(metadata_info.path); const auto & [namespace_name, table_name] = DataLake::parseTableName(table_id.getTableName()); - if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot)) + if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot, context->getForwardedAuthToken())) { cleanup(true); return false; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp index 8c11cd40620ee..25a7662b78853 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp @@ -655,8 +655,18 @@ static bool writeMetadataFiles( const auto & [namespace_name, table_name] = DataLake::parseTableName(table_id.getTableName()); if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot)) { +<<<<<<< HEAD cleanup(); return false; +======= + auto catalog_filename = path_resolver.resolveForCatalog(metadata_info.path); + const auto & [namespace_name, table_name] = DataLake::parseTableName(table_id.getTableName()); + if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot, context->getForwardedAuthToken())) + { + cleanup(); + return false; + } +>>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } } } @@ -1022,7 +1032,7 @@ void alter( metadata->getValue(Iceberg::f_last_column_id), getHighestFieldId(new_schema)); commit_attempted = true; - if (!catalog->updateSchema(namespace_name, table_name, catalog_filename, new_schema, previous_schema_id, new_last_column_id, metadata)) + if (!catalog->updateSchema(namespace_name, table_name, catalog_filename, new_schema, previous_schema_id, new_last_column_id, metadata, context->getForwardedAuthToken())) { ++i; continue; diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.cpp b/src/Storages/ObjectStorage/StorageObjectStorage.cpp index 5b8bbf5b22948..cb5f3c1abe101 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorage.cpp @@ -166,6 +166,7 @@ StorageObjectStorage::StorageObjectStorage( , is_table_function(is_table_function_) , log(getLogger(fmt::format("Storage{}({})", configuration->getEngineName(), table_id_.getFullTableName()))) , catalog(catalog_) + , catalog_auth_token(catalog_ && context ? context->getForwardedAuthToken() : DB::ForwardedAuthTokenPtr{}) , storage_id(table_id_) , background_operations_assignee(*this, table_id_, BackgroundJobsAssignee::Type::DataProcessing, Context::getGlobalContextInstance()) { @@ -1060,12 +1061,13 @@ void StorageObjectStorage::prepareForDrop(ContextPtr query_context) void StorageObjectStorage::drop() { - dropImpl(delete_data_on_drop.load(), catalog, configuration, storage_id, log); + dropImpl(delete_data_on_drop.load(), catalog, catalog_auth_token, configuration, storage_id, log); } void StorageObjectStorage::dropImpl( const std::optional & delete_data_on_drop, const std::shared_ptr & catalog, + const DB::ForwardedAuthTokenPtr & catalog_auth_token, const StorageObjectStorageConfigurationPtr & configuration, const StorageID & storage_id, const LoggerPtr & log) @@ -1086,7 +1088,7 @@ void StorageObjectStorage::dropImpl( if (catalog) { const auto [namespace_name, table_name] = DataLake::parseTableName(storage_id.getTableName()); - catalog->dropTable(namespace_name, table_name, delete_data, /* if_exists */ false); + catalog->dropTable(namespace_name, table_name, delete_data, /* if_exists */ false, catalog_auth_token); } configuration->drop(delete_data); } diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.h b/src/Storages/ObjectStorage/StorageObjectStorage.h index c67fce42f1df4..15c41e5f2e235 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.h +++ b/src/Storages/ObjectStorage/StorageObjectStorage.h @@ -1,4 +1,5 @@ #pragma once +#include #include #include #include @@ -122,6 +123,7 @@ class StorageObjectStorage : public IStorage, public IBackgroundOperation static void dropImpl( const std::optional & delete_data_on_drop, const std::shared_ptr & catalog, + const DB::ForwardedAuthTokenPtr & catalog_auth_token, const StorageObjectStorageConfigurationPtr & configuration, const StorageID & storage_id, const LoggerPtr & log); @@ -310,6 +312,7 @@ class StorageObjectStorage : public IStorage, public IBackgroundOperation LoggerPtr log; std::shared_ptr catalog; + DB::ForwardedAuthTokenPtr catalog_auth_token; StorageID storage_id; BackgroundJobsAssignee background_operations_assignee; diff --git a/tests/integration/test_datalake_glue_token_forwarding/__init__.py b/tests/integration/test_datalake_glue_token_forwarding/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_datalake_glue_token_forwarding/configs/token_forwarding.xml b/tests/integration/test_datalake_glue_token_forwarding/configs/token_forwarding.xml new file mode 100644 index 0000000000000..e6842a434aaec --- /dev/null +++ b/tests/integration/test_datalake_glue_token_forwarding/configs/token_forwarding.xml @@ -0,0 +1,25 @@ + + 1 + + sts + + + + jwt_static_key + HS256 + glue_token_forwarding_secret + false + true + + + + + + hs256 + default + + + + + + diff --git a/tests/integration/test_datalake_glue_token_forwarding/configs/users.xml b/tests/integration/test_datalake_glue_token_forwarding/configs/users.xml new file mode 100644 index 0000000000000..98522df4d705d --- /dev/null +++ b/tests/integration/test_datalake_glue_token_forwarding/configs/users.xml @@ -0,0 +1,10 @@ + + + + + + + 1 + + + diff --git a/tests/integration/test_datalake_glue_token_forwarding/s3_mocks/mock_sts.py b/tests/integration/test_datalake_glue_token_forwarding/s3_mocks/mock_sts.py new file mode 100644 index 0000000000000..fee74d5404a12 --- /dev/null +++ b/tests/integration/test_datalake_glue_token_forwarding/s3_mocks/mock_sts.py @@ -0,0 +1,73 @@ +import json +import sys +from datetime import datetime, timedelta, timezone +from urllib.parse import parse_qs + +from bottle import request, response, route, run + +recorded = [] + + +@route("/") +def ping(): + response.content_type = "text/plain" + response.set_header("Content-Length", 2) + return "OK" + + +@route("/_requests") +def list_requests(): + response.content_type = "application/json" + return json.dumps(recorded) + + +@route("/_reset") +def reset(): + recorded.clear() + response.content_type = "text/plain" + return "OK" + + +@route("/", method="POST") +def sts(): + body = request.body.read().decode() + params = {key: values[0] for key, values in parse_qs(body).items()} + + recorded.append( + { + "role_arn": params.get("RoleArn", ""), + "role_session_name": params.get("RoleSessionName", ""), + "web_identity_token": params.get("WebIdentityToken", ""), + } + ) + + if params.get("RoleSessionName") == "rejected": + response.status = 403 + response.content_type = "text/xml" + return """ + + + Sender + InvalidIdentityToken + Incorrect token audience + + + """ + + expiration = datetime.now(timezone.utc) + timedelta(hours=1) + + return f""" + + + + testing + testing + session-for-{params.get("RoleSessionName", "")} + {expiration.strftime("%Y-%m-%dT%H:%M:%SZ")} + + + + """ + + +run(host="0.0.0.0", port=int(sys.argv[1])) diff --git a/tests/integration/test_datalake_glue_token_forwarding/test.py b/tests/integration/test_datalake_glue_token_forwarding/test.py new file mode 100644 index 0000000000000..3178eabcb7aab --- /dev/null +++ b/tests/integration/test_datalake_glue_token_forwarding/test.py @@ -0,0 +1,155 @@ +import json +import logging +import os +import uuid + +import jwt +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.mock_servers import start_mock_servers + +SECRET = "glue_token_forwarding_secret" +BASE_URL = "http://glue:3000" +ROLE_ARN = "arn:aws:iam::123456789012:role/data-lake-reader" +STS_CONTAINER = "sts" + +DATABASE_SETTINGS = { + "catalog_type": "glue", + "warehouse": "test", + "region": "us-east-1", + "aws_role_arn": ROLE_ARN, + "aws_sts_endpoint": f"http://{STS_CONTAINER}:80", + "oauth_forward_user_token": "1", +} + + +def make_token(user): + return jwt.encode({"sub": user}, SECRET, algorithm="HS256") + + +@pytest.fixture(scope="module") +def started_cluster(): + cluster = ClickHouseCluster(__file__) + try: + cluster.add_instance( + "node1", + main_configs=["configs/token_forwarding.xml"], + user_configs=["configs/users.xml"], + stay_alive=True, + with_glue_catalog=True, + ) + + sts = cluster.add_instance( + name=STS_CONTAINER, + hostname=STS_CONTAINER, + image="altinityinfra/python-bottle", + tag="latest", + stay_alive=True, + ) + sts.stop_clickhouse(kill=True) + + logging.info("Starting cluster...") + cluster.start() + start_mock_servers( + cluster, + os.path.join(os.path.dirname(__file__), "s3_mocks"), + [("mock_sts.py", STS_CONTAINER, "80")], + ) + + node = cluster.instances["node1"] + node.query("CREATE ROLE IF NOT EXISTS token_users") + node.query("GRANT SHOW, SELECT ON *.* TO token_users") + + yield cluster + finally: + cluster.shutdown() + + +def sts_requests(started_cluster): + output = started_cluster.exec_in_container( + started_cluster.get_container_id(STS_CONTAINER), + [ + "python3", + "-c", + "import urllib.request;" + "print(urllib.request.urlopen('http://localhost:80/_requests').read().decode())", + ], + ) + return json.loads(output) + + +@pytest.fixture(autouse=True) +def clean_sts_log(started_cluster): + started_cluster.exec_in_container( + started_cluster.get_container_id(STS_CONTAINER), + [ + "python3", + "-c", + "import urllib.request; urllib.request.urlopen('http://localhost:80/_reset').read()", + ], + ) + + +def create_database_query(name, **overrides): + settings = {**DATABASE_SETTINGS, **overrides} + return ( + f"DROP DATABASE IF EXISTS {name}; " + f"CREATE DATABASE {name} ENGINE = DataLakeCatalog('{BASE_URL}') " + f"SETTINGS {','.join(k + '=' + repr(v) for k, v in settings.items())}" + ) + + +def create_database(node, name): + node.query(create_database_query(name), settings={"allow_database_glue_catalog": 1}) + + +def query_with_token(node, token, sql): + response = node.http_request( + "", method="POST", data=sql, headers={"Authorization": f"Bearer {token}"} + ) + response.raise_for_status() + + +def test_user_tokens_are_exchanged_into_separate_sts_sessions(started_cluster): + node = started_cluster.instances["node1"] + db = f"glue_{uuid.uuid4().hex[:8]}" + create_database(node, db) + + token = make_token("alice") + query_with_token(node, token, f"SHOW TABLES FROM {db}") + + query_with_token(node, make_token("bob"), f"SHOW TABLES FROM {db}") + sessions = sts_requests(started_cluster) + assert sorted(request["role_session_name"] for request in sessions) == ["alice", "bob"] + assert {request["web_identity_token"] for request in sessions} == {token, make_token("bob")} + assert all(request["role_arn"] == ROLE_ARN for request in sessions) + + +def test_rejected_token_does_not_fall_back(started_cluster): + node = started_cluster.instances["node1"] + db = f"glue_{uuid.uuid4().hex[:8]}" + create_database(node, db) + + response = node.http_request( + "", + method="POST", + data=f"SHOW TABLES FROM {db}", + headers={"Authorization": f"Bearer {make_token('rejected')}"}, + ) + assert response.status_code != 200 + assert "Could not assume role" in response.text, response.text + assert "InvalidIdentityToken" in response.text, response.text + + assert len(sts_requests(started_cluster)) == 1 + + +def test_sts_endpoint_checked_against_remote_host_filter(started_cluster): + node = started_cluster.instances["node1"] + error = node.query_and_get_error( + create_database_query( + f"glue_{uuid.uuid4().hex[:8]}", aws_sts_endpoint="http://not_allowed:80" + ), + settings={"allow_database_glue_catalog": 1}, + ) + assert "UNACCEPTABLE_URL" in error, error diff --git a/tests/integration/test_datalake_token_forwarding/__init__.py b/tests/integration/test_datalake_token_forwarding/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_datalake_token_forwarding/configs/token_forwarding.xml b/tests/integration/test_datalake_token_forwarding/configs/token_forwarding.xml new file mode 100644 index 0000000000000..9b148d647c292 --- /dev/null +++ b/tests/integration/test_datalake_token_forwarding/configs/token_forwarding.xml @@ -0,0 +1,22 @@ + + 1 + + + jwt_static_key + HS256 + datalake_token_forwarding_secret + false + true + + + + + + hs256 + default + + + + + + diff --git a/tests/integration/test_datalake_token_forwarding/configs/users.xml b/tests/integration/test_datalake_token_forwarding/configs/users.xml new file mode 100644 index 0000000000000..752bc965edb25 --- /dev/null +++ b/tests/integration/test_datalake_token_forwarding/configs/users.xml @@ -0,0 +1,15 @@ + + + + + + + 1 + + + passworduser_password + default + ::/0 + + + diff --git a/tests/integration/test_datalake_token_forwarding/test.py b/tests/integration/test_datalake_token_forwarding/test.py new file mode 100644 index 0000000000000..35279ce2821f4 --- /dev/null +++ b/tests/integration/test_datalake_token_forwarding/test.py @@ -0,0 +1,226 @@ +import logging +import uuid + +import jwt +import pytest +import requests + +from helpers.cluster import ClickHouseCluster +from helpers.config_cluster import minio_access_key, minio_secret_key + +SECRET = "datalake_token_forwarding_secret" +BASE_URL = "http://rest:8181/v1" +CATALOG_NAME = "demo" +DATABASE_SETTINGS = { + "catalog_type": "rest", + "warehouse": "demo", + "storage_endpoint": "http://minio1:9001/warehouse-rest", + "catalog_credential": "service:principal", + "oauth_forward_user_token": 1, +} + + +def make_token(user): + return jwt.encode({"sub": user}, SECRET, algorithm="HS256") + + +@pytest.fixture(scope="module") +def started_cluster(): + cluster = ClickHouseCluster(__file__) + try: + cluster.add_instance( + "node1", + main_configs=["configs/token_forwarding.xml"], + user_configs=["configs/users.xml"], + stay_alive=True, + with_iceberg_catalog=True, + extra_parameters={ + "docker_compose_file_name": "docker_compose_iceberg_rest_catalog.yml" + }, + ) + logging.info("Starting cluster...") + cluster.start() + + node = cluster.instances["node1"] + node.query("CREATE ROLE IF NOT EXISTS token_users") + node.query("GRANT CHECK, DROP TABLE, INSERT, SELECT, SHOW ON *.* TO token_users") + node.query("GRANT S3 ON *.* TO token_users") + + yield cluster + finally: + cluster.shutdown() + + +def catalog_local_url(started_cluster): + return f"http://localhost:{started_cluster.iceberg_rest_catalog_port}/v1" + + +def create_namespace(started_cluster, namespace): + response = requests.post( + f"{catalog_local_url(started_cluster)}/namespaces", + json={"namespace": [namespace], "properties": {}}, + timeout=30, + ) + assert response.status_code in (200, 409), response.text + + +def query_with_token(node, token, sql, **kwargs): + response = node.http_request( + "", + method="POST", + data=sql, + headers={"Authorization": f"Bearer {token}"}, + **kwargs, + ) + response.raise_for_status() + return response.text + + +def create_database(node, storage_credentials=False): + arguments = f"'{BASE_URL}'" + if storage_credentials: + arguments += f", '{minio_access_key}', '{minio_secret_key}'" + node.query(f"DROP DATABASE IF EXISTS {CATALOG_NAME}") + node.query( + f"SET allow_experimental_database_iceberg=true;" + f"CREATE DATABASE {CATALOG_NAME} ENGINE = DataLakeCatalog({arguments}) " + f"SETTINGS {','.join(k + '=' + repr(v) for k, v in DATABASE_SETTINGS.items())}" + ) + + +def create_table_in_catalog(started_cluster, namespace, table): + response = requests.post( + f"{catalog_local_url(started_cluster)}/namespaces/{namespace}/tables", + json={ + "name": table, + "location": f"s3://warehouse-rest/{table}", + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [{"id": 1, "name": "x", "required": False, "type": "string"}], + }, + }, + timeout=30, + ) + assert response.status_code in (200, 409), response.text + + +def catalog_tables(started_cluster, namespace): + response = requests.get( + f"{catalog_local_url(started_cluster)}/namespaces/{namespace}/tables", timeout=30 + ) + response.raise_for_status() + return {identifier["name"] for identifier in response.json()["identifiers"]} + + +def profile_event(node, query_id, event): + node.query("SYSTEM FLUSH LOGS") + return int( + node.query( + f"SELECT sum(ProfileEvents['{event}']) FROM system.query_log " + f"WHERE query_id = '{query_id}' AND type = 'QueryFinish'" + ).strip() + ) + + +def test_password_user_is_denied(started_cluster): + node = started_cluster.instances["node1"] + create_database(node) + + response = node.http_request( + "", + method="POST", + data=f"SELECT * FROM {CATALOG_NAME}.`nonexistent.table`", + params={"user": "passworduser", "password": "passworduser_password"}, + ) + assert "CATALOG_USER_TOKEN_NOT_AVAILABLE" in response.text, response.text + + +def test_native_protocol_forwards_jwt(started_cluster): + node = started_cluster.instances["node1"] + create_database(node) + + node.exec_in_container( + ["clickhouse", "client", "--jwt", make_token("alice"), "--query", f"CHECK DATABASE {CATALOG_NAME}"] + ) + + +def test_no_forwarding_without_the_server_setting(started_cluster): + node = started_cluster.instances["node1"] + + create_database(node) + + node.replace_in_config( + "/etc/clickhouse-server/config.d/token_forwarding.xml", + "1", + "0", + ) + node.query("SYSTEM RELOAD CONFIG") + try: + response = node.http_request( + "", + method="POST", + data=f"SELECT * FROM {CATALOG_NAME}.`nonexistent.table`", + headers={"Authorization": f"Bearer {make_token('alice')}"}, + ) + assert "CATALOG_USER_TOKEN_NOT_AVAILABLE" in response.text, response.text + finally: + node.replace_in_config( + "/etc/clickhouse-server/config.d/token_forwarding.xml", + "0", + "1", + ) + node.query("SYSTEM RELOAD CONFIG") + + +def write_fixture(started_cluster, node): + namespace = f"ns_{uuid.uuid4().hex[:8]}" + table = f"t_{uuid.uuid4().hex[:8]}" + create_namespace(started_cluster, namespace) + create_table_in_catalog(started_cluster, namespace, table) + create_database(node, storage_credentials=True) + return namespace, table + + +def test_async_insert_retains_the_querying_user_token(started_cluster): + node = started_cluster.instances["node1"] + namespace, table = write_fixture(started_cluster, node) + token = make_token("writer") + query_id = f"insert-{uuid.uuid4()}" + query_with_token( + node, + token, + f"INSERT INTO {CATALOG_NAME}.`{namespace}.{table}` VALUES ('written by the token user')", + params={ + "query_id": query_id, + "allow_insert_into_iceberg": 1, + "write_full_path_in_iceberg_metadata": 1, + "async_insert": 1, + "wait_for_async_insert": 0, + "async_insert_use_adaptive_busy_timeout": 0, + "async_insert_busy_timeout_ms": 60000, + }, + ) + + node.query("SYSTEM FLUSH ASYNC INSERT QUEUE") + assert profile_event(node, query_id, "AsyncInsertQuery") == 1 + assert ( + query_with_token(node, token, f"SELECT x FROM {CATALOG_NAME}.`{namespace}.{table}`").strip() + == "written by the token user" + ) + + +def test_drop_table_reaches_the_catalog_as_the_querying_user(started_cluster): + node = started_cluster.instances["node1"] + namespace, table = write_fixture(started_cluster, node) + + query_id = f"drop-{uuid.uuid4()}" + query_with_token( + node, + make_token("dropper"), + f"DROP TABLE {CATALOG_NAME}.`{namespace}.{table}`", + params={"query_id": query_id}, + ) + + assert table not in catalog_tables(started_cluster, namespace) + assert profile_event(node, query_id, "DataLakeRestCatalogAuthTokenRetrieve") == 0 From 0cb944a9db1f102503ca57538c19a59dec84414c Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Thu, 22 Oct 2026 14:17:52 +0200 Subject: [PATCH 2/2] Resolve conflicts in cherry-pick of #2329 Kept antalya-26.8 behaviour where the cherry-pick only carried pre-existing antalya-26.6 code, and translated the PR's changes to antalya-26.8's shape: - Catalog listing API: antalya-26.8 returns `CatalogTables` from `getTables` and `listTablesInNamespaceDirect`; the PR's `auth_token` parameter is added to those signatures in every catalog. - `RestCatalog`: merged the `flat_namespaces` constructor argument with the PR's `TokenForwardingConfig`; `createReadBuffer` and `sendRequest` keep antalya-26.8's virtual, default-free signatures with the PR's `generation` and `auth_token` added, so call sites pass all arguments explicitly. `getAuthHeaders` takes the PR's `AuthContext`; the OneLake refresh-token branch is kept on top of it. `sendRequest` passes empty `extra_headers` because antalya-26.8 has no such local there. - Vended credentials cache: kept antalya-26.8's mutex-guarded TTL and the `state_snapshot` staleness check in `cacheCredentials`, keyed by the PR's `CredentialsCacheKey` and bounded by `credentials_cache_max_entries`. - `requestToken` / `retrieveAccessToken`: took the PR's refactor and kept antalya-26.8's Horizon rule of omitting an empty `client_id`; the antalya-26.8-only OneLake refresh-token function is unchanged. `HorizonCatalog` (antalya-26.8 only) uses the new `loadConfig` signature. - `S3TablesCatalog`: antalya-26.8 signs requests by overriding `createReadBuffer` / `sendRequest` and has no `getAuthHeaders` or `getCredentialsConfigurationCallback` override, so the PR's versions of those overrides are not applied; the existing overrides gain the new parameters. - `UnityCatalog`, `GlueCatalog`, `DatabaseDataLake`: token arguments added to antalya-26.8 code paths (`getWriteCredentialsConfigurationCallback` kept, credentials-refresh lambda, `CatalogTables` listings, `createNamespaceIfNotExists` with an empty location). - `AlterDropPartitionExecutor` (antalya-26.8 only) passes the forwarded token to the new `updateMetadata` signature. - `CATALOG_USER_TOKEN_NOT_AVAILABLE` renumbered to 784 after the codes antalya-26.8 already uses. - `gtest_rest_catalog.cpp`: kept antalya-26.8's in-file test server (it backs tests that exist only on antalya-26.8) and applied the PR's `auth_token` arguments to the catalog calls. - `TCPHandler::processTablesStatusRequest`: the PR's `try`/`catch` around the loop, with antalya-26.8's `castStorage(..., DeferredTable::Load)`. Source-PR: #2329 (https://github.com/Altinity/ClickHouse/pull/2329) --- .../engines/database-engines/datalake.mdx | 11 +- src/Access/AccessControl.h | 3 - src/Common/ErrorCodes.cpp | 6 +- src/Core/ServerSettings.cpp | 19 +- src/Databases/DataLake/DatabaseDataLake.cpp | 66 +--- .../DataLake/DatabaseDataLakeSettings.cpp | 6 - src/Databases/DataLake/GlueCatalog.cpp | 43 +-- src/Databases/DataLake/GlueCatalog.h | 17 +- src/Databases/DataLake/HiveCatalog.cpp | 12 +- src/Databases/DataLake/HiveCatalog.h | 12 +- src/Databases/DataLake/ICatalog.cpp | 11 +- src/Databases/DataLake/ICatalog.h | 23 +- src/Databases/DataLake/PaimonRestCatalog.cpp | 12 +- src/Databases/DataLake/PaimonRestCatalog.h | 12 +- src/Databases/DataLake/RestCatalog.cpp | 354 +++++------------- src/Databases/DataLake/RestCatalog.h | 87 +---- src/Databases/DataLake/S3TablesCatalog.cpp | 46 +-- src/Databases/DataLake/S3TablesCatalog.h | 34 +- src/Databases/DataLake/UnityCatalog.cpp | 17 +- src/Databases/DataLake/UnityCatalog.h | 21 +- .../DataLake/tests/gtest_rest_catalog.cpp | 137 ++----- .../gtest_rest_catalog_token_forwarding.cpp | 21 +- src/IO/S3/Credentials.cpp | 30 +- src/IO/S3/tests/TestPocoHTTPServer.h | 4 - src/Interpreters/Session.cpp | 5 - src/Server/TCPHandler.cpp | 14 +- .../Iceberg/AlterDropPartitionExecutor.cpp | 3 +- .../DataLakes/Iceberg/IcebergMetadata.cpp | 6 +- .../DataLakes/Iceberg/Mutations.cpp | 12 +- 29 files changed, 232 insertions(+), 812 deletions(-) diff --git a/docs/reference/engines/database-engines/datalake.mdx b/docs/reference/engines/database-engines/datalake.mdx index 6be5f84d242b6..e91ef95cfcc7e 100644 --- a/docs/reference/engines/database-engines/datalake.mdx +++ b/docs/reference/engines/database-engines/datalake.mdx @@ -55,25 +55,16 @@ The following settings are supported: | `storage_endpoint` | Endpoint URL for the underlying storage | | `oauth_server_uri` | URI of the OAuth2 authorization server for authentication | | `vended_credentials` | Boolean indicating whether to use vended credentials from the catalog (supports AWS S3 and Azure ADLS Gen2) | -<<<<<<< HEAD:docs/reference/engines/database-engines/datalake.mdx | `aws_access_key_id` | AWS access key ID for S3/Glue access (if not using vended credentials) | | `aws_secret_access_key` | AWS secret access key for S3/Glue access (if not using vended credentials) | | `aws_role_arn` | ARN of the IAM role to assume for AWS/Glue access. When set, ClickHouse uses AWS STS `AssumeRole` with base credentials from `aws_access_key_id` and `aws_secret_access_key` when both are provided, or from the default AWS credential chain otherwise (the role must trust the identity the server runs under). | | `aws_role_session_name` | Session name used for the AWS STS `AssumeRole` call. Optional; defaults to `ClickHouseSession`. | | `aws_external_id` | External ID passed to AWS STS `AssumeRole`, matching the `sts:ExternalId` condition on the role's trust policy. Use this when the role is owned by a third party, such as ClickHouse Cloud. | | `region` | AWS region for the service (e.g., `us-east-1`) | +| `aws_sts_endpoint` | Custom AWS STS endpoint for Glue role assumption | | `dlf_access_key_id` | Access key ID for DLF access | | `dlf_access_key_secret` | Access key Secret for DLF access | | `force_add_bucket` | When constructing object-storage URLs from the catalog-provided table location and `storage_endpoint`, prepend the bucket/container name even if the endpoint already contains it. Default: `false`. Set to `true` for catalogs that hand back paths without the bucket and require it to be added at the URL-construction step (Polaris-style paths). | -======= -| `vended_credentials_cache_ttl` | Maximum cache entry lifetime (in seconds) for vended credentials (REST catalogs only). Default `300`; `0` disables caching. | -| `aws_access_key_id` | AWS access key ID for S3/Glue access (if not using vended credentials) | -| `aws_secret_access_key` | AWS secret access key for S3/Glue access (if not using vended credentials) | -| `region` | AWS region for the service (e.g., `us-east-1`) | -| `aws_sts_endpoint` | Custom AWS STS endpoint for Glue role assumption | -| `dlf_access_key_id` | Access key ID for DLF access | -| `dlf_access_key_secret` | Access key Secret for DLF access | ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake):docs/en/engines/database-engines/datalake.md | `namespaces` | Comma-separated list of namespaces, implemented for catalog types: `rest`, `glue` and `unity` | | `oauth_forward_user_token` | Authenticate to the catalog as the user running the query instead of as the shared service identity. Iceberg REST and Glue only. See [Forwarding the user's identity to the catalog](#user-token-forwarding) | | `oauth_token_exchange_uri` | Empty (the default) forwards the user's token unchanged; non-empty performs an RFC 8693 token exchange at this URL first | diff --git a/src/Access/AccessControl.h b/src/Access/AccessControl.h index ab5a542ade5f4..38ad65607c9e2 100644 --- a/src/Access/AccessControl.h +++ b/src/Access/AccessControl.h @@ -329,11 +329,8 @@ class AccessControl : public MultipleAccessStorage std::atomic_bool enable_read_write_grants = false; std::atomic_bool allow_impersonate_user = false; std::atomic_bool enable_token_auth = true; -<<<<<<< HEAD std::atomic_bool readonly_restricts_set_role = false; -======= std::atomic_bool enable_token_forwarding = false; ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) }; } diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 503d148aa651f..a7ff3faa9e859 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -658,16 +658,12 @@ M(776, RESOURCE_LIMIT_EXCEEDED) \ M(777, MEMORY_RESERVATION_KILLED) \ M(778, MEMORY_RESERVATION_FAILED) \ -<<<<<<< HEAD M(779, UNSUPPORTED_MEDIA_TYPE) \ M(780, SW_SERVER_NO_WORKERS_AVAILABLE) \ M(781, AI_PROVIDER_RESPONSE_TRUNCATED) \ M(782, AI_PROVIDER_RESPONSE_INCOMPLETE) \ M(783, CATALOG_NAMESPACE_DISABLED) \ -======= - M(779, CATALOG_NAMESPACE_DISABLED) \ - M(780, CATALOG_USER_TOKEN_NOT_AVAILABLE) \ ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + M(784, CATALOG_USER_TOKEN_NOT_AVAILABLE) \ \ M(900, DISTRIBUTED_CACHE_ERROR) \ M(901, CANNOT_USE_DISTRIBUTED_CACHE) \ diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index e1fc1b0b901b0..306841da8c0e4 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -987,23 +987,18 @@ When disabled: - `TokenAccessStorage` (token user directory) is not added. - Authentication via tokens (`--jwt` option in clickhouse-client or `Authorization: Bearer` HTTP header) is rejected. -<<<<<<< HEAD Default value: `true` (token authentication is enabled). )", 0) \ -======= - Default value: `true` (token authentication is enabled). - )", 0) \ DECLARE(Bool, enable_token_forwarding, false, R"( - Retain authenticated bearer tokens for forwarding through the `DataLakeCatalog` - setting `oauth_forward_user_token`. Supports Iceberg REST and AWS STS for Glue. - When disabled, tokens are not retained in sessions for forwarding. +Retain authenticated bearer tokens for forwarding through the `DataLakeCatalog` +setting `oauth_forward_user_token`. Supports Iceberg REST and AWS STS for Glue. +When disabled, tokens are not retained in sessions for forwarding. - Database creators choose the endpoints that receive users' tokens. Grant `CREATE DATABASE` - only to trusted users and restrict `remote_url_allow_hosts`. +Database creators choose the endpoints that receive users' tokens. Grant `CREATE DATABASE` +only to trusted users and restrict `remote_url_allow_hosts`. - Default value: `false`. - )", 0) \ ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) +Default value: `false`. +)", 0) \ DECLARE(UInt64, concurrent_threads_soft_limit_num, 0, R"( The maximum number of query processing threads, excluding threads for retrieving data from remote servers, allowed to run all queries. This is not a hard limit. In case if the limit is reached the query will still get at least one thread to run. Query can upscale to desired number of threads during execution if more threads become available. diff --git a/src/Databases/DataLake/DatabaseDataLake.cpp b/src/Databases/DataLake/DatabaseDataLake.cpp index 5860d0f876959..a7b7b16c33833 100644 --- a/src/Databases/DataLake/DatabaseDataLake.cpp +++ b/src/Databases/DataLake/DatabaseDataLake.cpp @@ -402,9 +402,16 @@ void DatabaseDataLake::initialize() const settings[DatabaseDataLakeSetting::oauth_server_uri].value, settings[DatabaseDataLakeSetting::oauth_server_use_request_body].value, settings[DatabaseDataLakeSetting::namespaces].value, -<<<<<<< HEAD settings[DatabaseDataLakeSetting::flat_namespaces].value, - Context::getGlobalContextInstance()); + Context::getGlobalContextInstance(), + DataLake::TokenForwardingConfig{ + .forward_user_token = settings[DatabaseDataLakeSetting::oauth_forward_user_token].value, + .token_exchange_uri = settings[DatabaseDataLakeSetting::oauth_token_exchange_uri].value, + .subject_token_type = settings[DatabaseDataLakeSetting::oauth_subject_token_type].value, + .requested_token_type = settings[DatabaseDataLakeSetting::oauth_requested_token_type].value, + .forward_actor_token = settings[DatabaseDataLakeSetting::oauth_forward_actor_token].value, + .user_token_cache_ttl = settings[DatabaseDataLakeSetting::oauth_user_token_cache_ttl].value, + }); break; } case DB::DatabaseDataLakeCatalogType::ICEBERG_DELTA_SHARING: @@ -440,17 +447,6 @@ void DatabaseDataLake::initialize() const settings[DatabaseDataLakeSetting::namespaces].value, settings[DatabaseDataLakeSetting::flat_namespaces].value, Context::getGlobalContextInstance()); -======= - Context::getGlobalContextInstance(), - DataLake::TokenForwardingConfig{ - .forward_user_token = settings[DatabaseDataLakeSetting::oauth_forward_user_token].value, - .token_exchange_uri = settings[DatabaseDataLakeSetting::oauth_token_exchange_uri].value, - .subject_token_type = settings[DatabaseDataLakeSetting::oauth_subject_token_type].value, - .requested_token_type = settings[DatabaseDataLakeSetting::oauth_requested_token_type].value, - .forward_actor_token = settings[DatabaseDataLakeSetting::oauth_forward_actor_token].value, - .user_token_cache_ttl = settings[DatabaseDataLakeSetting::oauth_user_token_cache_ttl].value, - }); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) break; } case DB::DatabaseDataLakeCatalogType::ICEBERG_ONELAKE: @@ -1097,7 +1093,7 @@ StoragePtr DatabaseDataLake::tryGetTableImpl(const String & name, ContextPtr con return std::nullopt; if (!with_vended_credentials && !catalog_manages_provider_chain) return std::nullopt; - return catalog->getCredentialsConfigurationCallback(storage_id); + return catalog->getCredentialsConfigurationCallback(storage_id, DataLake::getForwardedAuthToken(context_)); }; const auto catalog_uuid = table_metadata.getTableUUID(); @@ -1111,15 +1107,7 @@ StoragePtr DatabaseDataLake::tryGetTableImpl(const String & name, ContextPtr con auto storage_cluster = std::make_shared( cluster_name, configuration, -<<<<<<< HEAD configuration->createObjectStorage(context_copy, /* is_readonly */ false, get_credentials_refresh_callback(StorageID(getDatabaseName(), name, table_uuid))), -======= - configuration->createObjectStorage( - context_copy, - /* is_readonly */ false, - catalog->getCredentialsConfigurationCallback( - StorageID(getDatabaseName(), name, table_uuid), DataLake::getForwardedAuthToken(context_))), ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) StorageID(getDatabaseName(), name, table_uuid), /* columns */columns, /* constraints */ConstraintsDescription{}, @@ -1368,7 +1356,6 @@ void DatabaseDataLake::createTable( const auto compression_method_str = context_->getSettingsRef()[Setting::iceberg_metadata_compression_method].value; const auto compression_method = chooseCompressionMethod(compression_method_str, compression_method_str); -<<<<<<< HEAD if (compression_method != CompressionMethod::None) { /// A catalog that writes the first metadata file itself reads the codec from this property. @@ -1377,15 +1364,8 @@ void DatabaseDataLake::createTable( metadata_content->set("properties", properties); } - catalog->createNamespaceIfNotExists(namespace_name, /* location */ ""); -======= - /// The namespace's default location is the namespace base, not this first table's directory. - String namespace_location = location; - if (const String table_suffix = "/" + table_name; namespace_location.ends_with(table_suffix)) - namespace_location.resize(namespace_location.size() - table_suffix.size()); const auto auth_token = DataLake::getForwardedAuthToken(context_); - catalog->createNamespaceIfNotExists(namespace_name, namespace_location, auth_token); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + catalog->createNamespaceIfNotExists(namespace_name, /* location */ "", auth_token); const bool created = catalog->createTable( namespace_name, table_name, /* metadata_path */ "", metadata_content, compression_method, create.if_not_exists, auth_token); @@ -1457,11 +1437,7 @@ DatabaseTablesIteratorPtr DatabaseDataLake::getTablesIteratorImpl( throw Exception(ErrorCodes::DATALAKE_DATABASE_ERROR, "Injected catalog listing failure"); }); -<<<<<<< HEAD - catalog_tables = getCatalog()->getTables(toCatalogTableNameFilter(tables_filter)); -======= - iceberg_tables = getCatalog()->getTables(toCatalogTableNameFilter(tables_filter), DataLake::getForwardedAuthToken(context_)); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + catalog_tables = getCatalog()->getTables(toCatalogTableNameFilter(tables_filter), DataLake::getForwardedAuthToken(context_)); } catch (...) { @@ -1610,11 +1586,7 @@ std::vector DatabaseDataLake::getLightweightTablesItera throw Exception(ErrorCodes::DATALAKE_DATABASE_ERROR, "Injected catalog listing failure"); }); -<<<<<<< HEAD - catalog_tables = getCatalog()->getTables(toCatalogTableNameFilter(tables_filter)); -======= - iceberg_tables = getCatalog()->getTables(toCatalogTableNameFilter(tables_filter), DataLake::getForwardedAuthToken(context_)); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + catalog_tables = getCatalog()->getTables(toCatalogTableNameFilter(tables_filter), DataLake::getForwardedAuthToken(context_)); } catch (...) { @@ -1637,11 +1609,7 @@ std::vector DatabaseDataLake::getLightweightTablesItera return result; } -<<<<<<< HEAD -VectorWithMemoryTracking DatabaseDataLake::getAllTableNames(ContextPtr /*context*/) const -======= -Strings DatabaseDataLake::getAllTableNames(ContextPtr context_) const ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) +VectorWithMemoryTracking DatabaseDataLake::getAllTableNames(ContextPtr context_) const { VectorWithMemoryTracking result; @@ -1650,8 +1618,7 @@ Strings DatabaseDataLake::getAllTableNames(ContextPtr context_) const /// must not fail even when the catalog is temporarily unreachable. try { -<<<<<<< HEAD - DataLake::CatalogTables tables = getCatalog()->getTables(); + DataLake::CatalogTables tables = getCatalog()->getTables(DataLake::getForwardedAuthToken(context_)); result.reserve(tables.size()); for (auto & table : tables) { @@ -1660,9 +1627,6 @@ Strings DatabaseDataLake::getAllTableNames(ContextPtr context_) const continue; result.push_back(std::move(table.name)); } -======= - result = getCatalog()->getTables(DataLake::getForwardedAuthToken(context_)); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } catch (...) { diff --git a/src/Databases/DataLake/DatabaseDataLakeSettings.cpp b/src/Databases/DataLake/DatabaseDataLakeSettings.cpp index d74830fc84c31..5b0495f4bbe05 100644 --- a/src/Databases/DataLake/DatabaseDataLakeSettings.cpp +++ b/src/Databases/DataLake/DatabaseDataLakeSettings.cpp @@ -36,16 +36,10 @@ namespace ErrorCodes DECLARE(String, aws_access_key_id, "", "AWS access key id used to connect to the Glue catalog, and forwarded as a static S3 storage credential for non-Glue DataLakeCatalog table reads when vended_credentials = false", 0) \ DECLARE(String, aws_secret_access_key, "", "AWS secret access key used to connect to the Glue catalog, and forwarded as a static S3 storage credential for non-Glue DataLakeCatalog table reads when vended_credentials = false", 0) \ DECLARE(String, region, "", "Region for Glue catalog", 0) \ -<<<<<<< HEAD DECLARE(String, aws_role_arn, "", "ARN of the IAM role to assume for AWS/Glue access. When set, ClickHouse uses AWS STS `AssumeRole` with base credentials from `aws_access_key_id` and `aws_secret_access_key` when both are provided, or from the default AWS credential chain otherwise (the role must trust the identity the server runs under)", 0) \ DECLARE(String, aws_role_session_name, "", "Session name used for the AWS STS `AssumeRole` call. Optional; defaults to `ClickHouseSession`", 0) \ DECLARE(String, aws_external_id, "", "External ID passed to AWS STS `AssumeRole`, matching the `sts:ExternalId` condition on the role's trust policy. Use this when the role is owned by a third party, such as ClickHouse Cloud", 0) \ -======= - DECLARE(String, aws_role_arn, "", "Role arn for AWS connection for Glue catalog", 0) \ - DECLARE(String, aws_role_session_name, "", "Role session name for AWS connection for Glue catalog", 0) \ - DECLARE(String, aws_external_id, "", "External id for the AWS STS AssumeRole trust policy for Glue catalog", 0) \ DECLARE(String, aws_sts_endpoint, "", "Custom AWS STS endpoint for Glue catalog", 0) \ ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) DECLARE(String, storage_endpoint, "", "Object storage endpoint", 0) \ DECLARE(String, default_base_location, "", "Base URI under which CREATE TABLE places new tables. Used only when the catalog does not report `default-base-location`", 0) \ DECLARE(S3UriStyle, storage_uri_style, S3UriStyle::AUTO, "URL style used when constructing object storage URLs from catalog-provided table locations. Use 'virtual_hosted' when the object storage server requires the bucket in the hostname (e.g. https://bucket.endpoint.com/path/)", 0) \ diff --git a/src/Databases/DataLake/GlueCatalog.cpp b/src/Databases/DataLake/GlueCatalog.cpp index 73583d8b8c2d7..120978371eca7 100644 --- a/src/Databases/DataLake/GlueCatalog.cpp +++ b/src/Databases/DataLake/GlueCatalog.cpp @@ -215,7 +215,6 @@ namespace DataLake namespace { -<<<<<<< HEAD /// A Glue table is readable only if it is an Iceberg table. `table_type` is available in /// both the bulk `GetTables` listing and `tryGetTableMetadata`, so the two stay consistent. bool isReadableGlueTable(const Aws::Glue::Model::Table & table) @@ -224,7 +223,8 @@ bool isReadableGlueTable(const Aws::Glue::Model::Table & table) auto it = parameters.find("table_type"); const std::string table_type = it != parameters.end() ? it->second : ""; return Poco::toUpper(table_type) == "ICEBERG"; -======= +} + /// STS `RoleSessionName`: 2-64 characters matching `[\w+=,.@-]*`. /// If the principal does not fit as is, a hash of the full principal is appended to keep names of different principals distinct. std::string makeRoleSessionName(const std::string & principal) @@ -245,7 +245,6 @@ std::string makeRoleSessionName(const std::string & principal) std::string readable = sanitized.empty() ? "ClickHouseUser" : sanitized.substr(0, max_length - hash_suffix_length); return readable + "-" + getHexUIntLowercase(sipHash64(principal)); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } } @@ -478,11 +477,7 @@ DataLake::ICatalog::Namespaces GlueCatalog::getDatabases(const AuthenticatedClie return result; } -<<<<<<< HEAD -CatalogTables GlueCatalog::getTablesForDatabase(const std::string & db_name, size_t limit) const -======= -DB::Names GlueCatalog::getTablesForDatabase(const AuthenticatedClient & client, const std::string & db_name, size_t limit) const ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) +CatalogTables GlueCatalog::getTablesForDatabase(const AuthenticatedClient & client, const std::string & db_name, size_t limit) const { LOG_TEST(log, "Getting tables for database '{}' with limit {}", db_name, limit); CatalogTables result; @@ -545,18 +540,11 @@ DB::Names GlueCatalog::getTablesForDatabase(const AuthenticatedClient & client, return result; } -<<<<<<< HEAD -CatalogTables GlueCatalog::getTables() const -{ - auto databases = getDatabases(""); - CatalogTables result; -======= -DB::Names GlueCatalog::getTables(const DB::ForwardedAuthTokenPtr & auth_token) const +CatalogTables GlueCatalog::getTables(const DB::ForwardedAuthTokenPtr & auth_token) const { auto client = getClient(auth_token); auto databases = getDatabases(client, ""); - DB::Names result; ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + CatalogTables result; for (const auto & database : databases) { auto tables_in_database = getTablesForDatabase(client, database); @@ -571,27 +559,14 @@ DataLake::ICatalog::Namespaces GlueCatalog::getNamespaces(const DB::ForwardedAut return getDatabases(getClient(auth_token), ""); } -<<<<<<< HEAD -CatalogTables GlueCatalog::listTablesInNamespaceDirect(const std::string & namespace_name) const -======= -DB::Names GlueCatalog::listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & auth_token) const ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) +CatalogTables GlueCatalog::listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & auth_token) const { return getTablesForDatabase(getClient(auth_token), namespace_name); } bool GlueCatalog::existsTable(const std::string & database_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const { -<<<<<<< HEAD - if (!isNamespaceAllowed(database_name)) - throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Namespace {} is filtered by `namespaces` database parameter", database_name); - - TableMetadata metadata; - return tryGetTableMetadata(database_name, table_name, getContext(), metadata); -} -======= auto client = getClient(auth_token); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) if (!isNamespaceAllowed(database_name)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Namespace {} is filtered by `namespaces` database parameter", database_name); @@ -866,16 +841,12 @@ String GlueCatalog::getActualTimestampType( const TableMetadata & table_metadata, const String & glue_column_type) const { -<<<<<<< HEAD auto table_specific_properties = table_metadata.getDataLakeSpecificProperties(); if (!table_specific_properties.has_value()) throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Failed to read table metadata, reason why table is unreadable: {}", table_metadata.getReasonWhyTableIsUnreadable()); - auto metadata_object = getOrFetchMetadataObject(table_specific_properties->iceberg_metadata_file_location, table_metadata); + auto metadata_object = getOrFetchMetadataObject(client, table_specific_properties->iceberg_metadata_file_location, table_metadata); return resolveTimestampTypeFromMetadata(metadata_object, column_name, glue_column_type); -======= - return resolveTimestampTypeFromMetadata(getIcebergMetadataObject(client, table_metadata), column_name, glue_column_type); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } Poco::JSON::Object::Ptr GlueCatalog::getIcebergMetadataObject(const AuthenticatedClient & client, const TableMetadata & table_metadata) const diff --git a/src/Databases/DataLake/GlueCatalog.h b/src/Databases/DataLake/GlueCatalog.h index 6594e1ef4ec25..1e41548d3a4b9 100644 --- a/src/Databases/DataLake/GlueCatalog.h +++ b/src/Databases/DataLake/GlueCatalog.h @@ -43,11 +43,7 @@ class GlueCatalog final : public ICatalog, private DB::WithContext bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const override; -<<<<<<< HEAD - CatalogTables getTables() const override; -======= - DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + CatalogTables getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; Namespaces getNamespaces(const DB::ForwardedAuthTokenPtr & auth_token) const override; @@ -148,17 +144,10 @@ class GlueCatalog final : public ICatalog, private DB::WithContext bool isNamespaceAllowed(const std::string & namespace_) const; -<<<<<<< HEAD - DataLake::ICatalog::Namespaces getDatabases(const std::string & prefix, size_t limit = 0) const; - CatalogTables getTablesForDatabase(const std::string & db_name, size_t limit = 0) const; - CatalogTables listTablesInNamespaceDirect(const std::string & namespace_name) const override; - void setCredentials(TableMetadata & metadata) const; -======= DataLake::ICatalog::Namespaces getDatabases(const AuthenticatedClient & client, const std::string & prefix, size_t limit = 0) const; - DB::Names getTablesForDatabase(const AuthenticatedClient & client, const std::string & db_name, size_t limit = 0) const; - DB::Names listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; + CatalogTables getTablesForDatabase(const AuthenticatedClient & client, const std::string & db_name, size_t limit = 0) const; + CatalogTables listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; void setCredentials(const AuthenticatedClient & client, TableMetadata & metadata) const; ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) /// The Glue catalog does not store detailed information about the types of timestamp columns, such as whether the column is timestamp or timestamptz. /// This method allows to clarify the actual type of the timestamp column. diff --git a/src/Databases/DataLake/HiveCatalog.cpp b/src/Databases/DataLake/HiveCatalog.cpp index 1c1e48c7184d3..5eadc6a71b41c 100644 --- a/src/Databases/DataLake/HiveCatalog.cpp +++ b/src/Databases/DataLake/HiveCatalog.cpp @@ -154,11 +154,7 @@ bool HiveCatalog::empty(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const return result.empty(); } -<<<<<<< HEAD -CatalogTables HiveCatalog::getTables() const -======= -DB::Names HiveCatalog::getTables(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) +CatalogTables HiveCatalog::getTables(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { CatalogTables result; DB::Names databases; @@ -184,11 +180,7 @@ DataLake::ICatalog::Namespaces HiveCatalog::getNamespaces(const DB::ForwardedAut return databases; } -<<<<<<< HEAD -CatalogTables HiveCatalog::listTablesInNamespaceDirect(const std::string & namespace_name) const -======= -DB::Names HiveCatalog::listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) +CatalogTables HiveCatalog::listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { DB::Names current_tables; /// Retry transient `TTransportException` (same wrapper as `getTables`) rather diff --git a/src/Databases/DataLake/HiveCatalog.h b/src/Databases/DataLake/HiveCatalog.h index 6c47b262b20e2..705494fc00874 100644 --- a/src/Databases/DataLake/HiveCatalog.h +++ b/src/Databases/DataLake/HiveCatalog.h @@ -34,11 +34,7 @@ class HiveCatalog final : public ICatalog, private DB::WithContext bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const override; -<<<<<<< HEAD - CatalogTables getTables() const override; -======= - DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + CatalogTables getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; Namespaces getNamespaces(const DB::ForwardedAuthTokenPtr & auth_token) const override; @@ -66,11 +62,7 @@ class HiveCatalog final : public ICatalog, private DB::WithContext template void executeWithRetry(Func && func) const; -<<<<<<< HEAD - CatalogTables listTablesInNamespaceDirect(const std::string & namespace_name) const override; -======= - DB::Names listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + CatalogTables listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; String base_url; diff --git a/src/Databases/DataLake/ICatalog.cpp b/src/Databases/DataLake/ICatalog.cpp index 2f1b909ab27b4..af738dd36c922 100644 --- a/src/Databases/DataLake/ICatalog.cpp +++ b/src/Databases/DataLake/ICatalog.cpp @@ -409,9 +409,6 @@ DB::SettingsChanges CatalogSettings::allChanged() const return changes; } -<<<<<<< HEAD -CatalogTables ICatalog::getTables(const TableNameFilter & filter) const -======= void ICatalog::validateForwardedToken( const DB::ContextPtr & context, const DB::ForwardedAuthTokenPtr & auth_token, const std::string & catalog_description) const { @@ -434,8 +431,7 @@ void ICatalog::validateForwardedToken( catalog_description); } -DB::Names ICatalog::getTables(const TableNameFilter & filter, const DB::ForwardedAuthTokenPtr & auth_token) const ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) +CatalogTables ICatalog::getTables(const TableNameFilter & filter, const DB::ForwardedAuthTokenPtr & auth_token) const { switch (filter.kind) { @@ -469,13 +465,8 @@ DB::Names ICatalog::getTables(const TableNameFilter & filter, const DB::Forwarde if (fixed_prefix.empty()) return getTables(auth_token); -<<<<<<< HEAD CatalogTables result; - for (const auto & namespace_name : getNamespaces()) -======= - DB::Names result; for (const auto & namespace_name : getNamespaces(auth_token)) ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { const std::string namespace_prefix = namespace_name + "."; if (!startsWith(namespace_prefix, fixed_prefix) && !startsWith(fixed_prefix, namespace_prefix)) diff --git a/src/Databases/DataLake/ICatalog.h b/src/Databases/DataLake/ICatalog.h index e7662e475aead..752a073faf1f2 100644 --- a/src/Databases/DataLake/ICatalog.h +++ b/src/Databases/DataLake/ICatalog.h @@ -235,30 +235,18 @@ class ICatalog /// Does catalog have any tables? virtual bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const = 0; -<<<<<<< HEAD /// Fetch the list of tables (names contain full namespaces). Each entry carries an /// `is_readable` flag so listings can drop unreadable tables without a metadata fetch. - virtual CatalogTables getTables() const = 0; -======= - /// Fetch tables' names list. - /// Contains full namespaces in names. - virtual DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const = 0; ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + virtual CatalogTables getTables(const DB::ForwardedAuthTokenPtr & auth_token) const = 0; /// Enumerate every namespace as a full dot-separated path (hierarchical catalogs /// return every nested level; flat catalogs their single-level names). virtual Namespaces getNamespaces(const DB::ForwardedAuthTokenPtr & auth_token) const = 0; -<<<<<<< HEAD /// Fetch the list of tables restricted by the `name` predicate (see `TableNameFilter`); /// each entry carries an `is_readable` flag like getTables(). Default impl prunes /// namespaces via `getNamespaces()`. - virtual CatalogTables getTables(const TableNameFilter & filter) const; -======= - /// Fetch fully-qualified table names, restricted by the `name` predicate (see - /// `TableNameFilter`). Default impl prunes namespaces via `getNamespaces()`. - virtual DB::Names getTables(const TableNameFilter & filter, const DB::ForwardedAuthTokenPtr & auth_token) const; ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + virtual CatalogTables getTables(const TableNameFilter & filter, const DB::ForwardedAuthTokenPtr & auth_token) const; /// Check that a table exists in a given namespace. virtual bool existsTable( @@ -390,13 +378,8 @@ class ICatalog const std::string & catalog_description) const; /// List tables directly in `namespace_name` (non-recursive), as fully-qualified -<<<<<<< HEAD /// `namespace.table` entries carrying an `is_readable` flag like getTables(). - virtual CatalogTables listTablesInNamespaceDirect(const std::string & namespace_name) const = 0; -======= - /// `namespace.table` names. - virtual DB::Names listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & auth_token) const = 0; ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + virtual CatalogTables listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & auth_token) const = 0; /// Name of the warehouse, /// which is sometimes also called "catalog name". diff --git a/src/Databases/DataLake/PaimonRestCatalog.cpp b/src/Databases/DataLake/PaimonRestCatalog.cpp index c2404c0c1f56e..15aa6cb5e968a 100644 --- a/src/Databases/DataLake/PaimonRestCatalog.cpp +++ b/src/Databases/DataLake/PaimonRestCatalog.cpp @@ -431,11 +431,7 @@ bool PaimonRestCatalog::empty(const DB::ForwardedAuthTokenPtr & /*auth_token*/) return tables.empty(); } -<<<<<<< HEAD -CatalogTables PaimonRestCatalog::getTables() const -======= -DB::Names PaimonRestCatalog::getTables(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) +CatalogTables PaimonRestCatalog::getTables(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { DB::Strings databases; DB::Names tables; @@ -458,11 +454,7 @@ DataLake::ICatalog::Namespaces PaimonRestCatalog::getNamespaces(const DB::Forwar return databases; } -<<<<<<< HEAD -CatalogTables PaimonRestCatalog::listTablesInNamespaceDirect(const std::string & namespace_name) const -======= -DB::Names PaimonRestCatalog::listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) +CatalogTables PaimonRestCatalog::listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { DB::Names tables; forEachTables(namespace_name, tables, {}); diff --git a/src/Databases/DataLake/PaimonRestCatalog.h b/src/Databases/DataLake/PaimonRestCatalog.h index e43969809ca7a..def8846f9272b 100644 --- a/src/Databases/DataLake/PaimonRestCatalog.h +++ b/src/Databases/DataLake/PaimonRestCatalog.h @@ -84,11 +84,7 @@ class PaimonRestCatalog final : public ICatalog, private DB::WithContext bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const override; -<<<<<<< HEAD - CatalogTables getTables() const override; -======= - DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + CatalogTables getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; Namespaces getNamespaces(const DB::ForwardedAuthTokenPtr & auth_token) const override; @@ -138,11 +134,7 @@ class PaimonRestCatalog final : public ICatalog, private DB::WithContext void forEachTables(const String & database, DB::Names & tables, StopCondition stop_condition = {}, ExecuteFunc execute_func = {}) const; -<<<<<<< HEAD - CatalogTables listTablesInNamespaceDirect(const std::string & namespace_name) const override; -======= - DB::Names listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + CatalogTables listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; Poco::JSON::Object::Ptr requestRest( const String & endpoint, diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index 37f8137bc1430..5125f53e1fe88 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -5,11 +5,7 @@ #include #include #include -<<<<<<< HEAD -======= #include -#include ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) #include #include #include @@ -104,17 +100,14 @@ namespace ProfileEvents { extern const Event DataLakeRestCatalogCredentialsVended; extern const Event DataLakeRestCatalogCredentialsCacheHits; -<<<<<<< HEAD extern const Event OneLakeAccessTokenRequests; extern const Event OneLakeAccessTokenRequestFailures; extern const Event OneLakeAccessTokenRequestMicroseconds; extern const Event OneLakeAccessTokenExpirations; -======= extern const Event DataLakeRestCatalogTokenExchange; extern const Event DataLakeRestCatalogTokenExchangeMicroseconds; extern const Event DataLakeRestCatalogTokenExchangeFailures; extern const Event DataLakeRestCatalogUserTokenCacheHits; ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) extern const Event DataLakeRestCatalogLoadConfig; extern const Event DataLakeRestCatalogLoadConfigMicroseconds; extern const Event DataLakeRestCatalogGetNamespaces; @@ -509,13 +502,9 @@ RestCatalog::RestCatalog( const std::string & oauth_server_uri_, bool oauth_server_use_request_body_, const std::string & namespaces_, -<<<<<<< HEAD bool flat_namespaces_, - DB::ContextPtr context_) -======= DB::ContextPtr context_, const TokenForwardingConfig & token_forwarding_) ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) : ICatalog(warehouse_) , DB::WithContext(context_) , base_url(correctAPIURI(base_url_)) @@ -523,15 +512,12 @@ RestCatalog::RestCatalog( , auth_scope(auth_scope_) , oauth_server_uri(oauth_server_uri_) , oauth_server_use_request_body(oauth_server_use_request_body_) -<<<<<<< HEAD , flat_namespaces(flat_namespaces_) -======= , token_forwarding(token_forwarding_) , user_token_cache( CurrentMetrics::end(), CurrentMetrics::DataLakeCatalogUserTokenCacheEntries, user_token_cache_max_entries) ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) , allowed_namespaces(namespaces_) { CatalogState initial_state; @@ -545,21 +531,17 @@ RestCatalog::RestCatalog( initial_state.auth_header = parseAuthHeader(auth_header_); validateAuthHeaders(initial_state.auth_header.value()); } -<<<<<<< HEAD - /// `loadConfig` reaches the virtual `createReadBuffer`, but no catalog constructed through this - /// ctor (`OneLakeCatalog`, `BigLakeCatalog`) overrides it, so the base implementation is the intended - /// target. `S3TablesCatalog` overrides it and uses the separate ctor, calling `loadConfig` afterwards. - /// NOLINTNEXTLINE(clang-analyzer-optin.cplusplus.VirtualCall) - initial_state.config = loadConfig(initial_state); -======= /// Defer `/v1/config` until a query supplies the user token needed to authenticate it. if (!token_forwarding.forward_user_token) { + /// `loadConfig` reaches the virtual `createReadBuffer`, but no catalog constructed through this + /// ctor (`OneLakeCatalog`, `BigLakeCatalog`) overrides it, so the base implementation is the intended + /// target. `S3TablesCatalog` overrides it and uses the separate ctor, calling `loadConfig` afterwards. + /// NOLINTNEXTLINE(clang-analyzer-optin.cplusplus.VirtualCall) initial_state.config = loadConfig(initial_state, /* generation */ 0, /* auth_token */ {}); initial_state.config_loaded = true; } ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) state.set(std::make_unique(std::move(initial_state))); } @@ -579,14 +561,11 @@ RestCatalog::RestCatalog( , auth_scope(auth_scope_) , oauth_server_uri(oauth_server_uri_) , oauth_server_use_request_body(oauth_server_use_request_body_) -<<<<<<< HEAD , flat_namespaces(flat_namespaces_) -======= , user_token_cache( CurrentMetrics::end(), CurrentMetrics::DataLakeCatalogUserTokenCacheEntries, user_token_cache_max_entries) ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) , allowed_namespaces(namespaces_) { } @@ -629,11 +608,7 @@ RestCatalog::Config RestCatalog::loadConfig( { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogLoadConfig); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogLoadConfigMicroseconds); -<<<<<<< HEAD - auto buf = createReadBuffer(catalog_state, CONFIG_ENDPOINT, params, /* headers */ {}, auth_headers); -======= - auto buf = createReadBuffer(catalog_state, generation, CONFIG_ENDPOINT, auth_token, params, /* headers */{}, auth_headers); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + auto buf = createReadBuffer(catalog_state, generation, CONFIG_ENDPOINT, auth_token, params, /* headers */ {}, auth_headers); readJSONObjectPossiblyInvalid(json_str, *buf); } @@ -679,14 +654,7 @@ void RestCatalog::validateAuthHeaders(const DB::HTTPHeaderEntry & header) const getContext()->getGlobalContext()->getHTTPHeaderFilter().checkAndNormalizeHeaders(header_to_check); } -<<<<<<< HEAD -DB::HTTPHeaderEntries RestCatalog::getAuthHeaders( - const CatalogState & catalog_state, - bool update_token, - bool * used_cached_oauth_token) const -======= DB::HTTPHeaderEntries RestCatalog::getAuthHeaders(const AuthContext & auth_context) const ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { fiu_do_on(DB::FailPoints::check_database_datalake_negative, { @@ -705,9 +673,6 @@ DB::HTTPHeaderEntries RestCatalog::getAuthHeaders(const AuthContext & auth_conte return DB::HTTPHeaderEntries{catalog_state.auth_header.value()}; } -<<<<<<< HEAD - /// Option 2: user provided grant_type and client credentials for OAuthClientCredentialsRequest. -======= if (token_forwarding.forward_user_token) { DB::HTTPHeaderEntries headers; @@ -718,8 +683,7 @@ DB::HTTPHeaderEntries RestCatalog::getAuthHeaders(const AuthContext & auth_conte return headers; } - /// We would make OAuthClientCredentialsRequest ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + /// Option 2: user provided grant_type and client credentials for OAuthClientCredentialsRequest. /// https://github.com/apache/iceberg/blob/3badfe0c1fcf0c0adfc7aa4a10f0b50365c48cf9/open-api/rest-catalog-open-api.yaml#L3498C5-L3498C34 /// Horizon accepts a secret-only credential (PAT/JWT as `client_secret` with empty `client_id`). if (!catalog_state.client_id.empty() || !catalog_state.client_secret.empty()) @@ -996,22 +960,6 @@ void RestCatalog::commitSettingsChanges(ICatalog::PreparedSettingsChangesPtr pre if (!prepared_auth || !prepared_auth->new_state) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Settings changes to commit were not prepared by this catalog"); -<<<<<<< HEAD - state.set(std::move(prepared_auth->new_state)); - if (prepared_auth->new_access_token) - access_token.set(std::move(prepared_auth->new_access_token)); - - std::lock_guard lock(credentials_cache_mutex); - credentials_cache.clear(); -} - -void RestCatalog::setVendedCredentialsCacheTTL(std::chrono::seconds ttl) -{ - std::lock_guard lock(credentials_cache_mutex); - if (ttl != vended_credentials_cache_ttl) - { - vended_credentials_cache_ttl = ttl; -======= { std::lock_guard lock(auth_publish_mutex); state.set(std::move(prepared_auth->new_state)); @@ -1025,7 +973,16 @@ void RestCatalog::setVendedCredentialsCacheTTL(std::chrono::seconds ttl) user_token_cache.clear(); { std::lock_guard lock(credentials_cache_mutex); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + credentials_cache.clear(); + } +} + +void RestCatalog::setVendedCredentialsCacheTTL(std::chrono::seconds ttl) +{ + std::lock_guard lock(credentials_cache_mutex); + if (ttl != vended_credentials_cache_ttl) + { + vended_credentials_cache_ttl = ttl; credentials_cache.clear(); } } @@ -1066,27 +1023,18 @@ void RestCatalog::applySettingsChangesToState( } } -<<<<<<< HEAD -DB::HTTPHeaderEntries OneLakeCatalog::getAuthHeaders( - const CatalogState & catalog_state, - bool update_token, - bool * used_cached_oauth_token) const +DB::HTTPHeaderEntries OneLakeCatalog::getAuthHeaders(const AuthContext & auth_context) const { DB::HTTPHeaderEntries headers; - if (!catalog_state.refresh_token.empty()) + if (!auth_context.catalog_state.refresh_token.empty()) { - const auto token = getValidAccessToken(catalog_state, update_token); + const auto token = getValidAccessToken(auth_context.catalog_state, auth_context.update_token); headers.emplace_back("Authorization", "Bearer " + token.token); } else { - headers = RestCatalog::getAuthHeaders(catalog_state, update_token, used_cached_oauth_token); + headers = RestCatalog::getAuthHeaders(auth_context); } -======= -DB::HTTPHeaderEntries OneLakeCatalog::getAuthHeaders(const AuthContext & auth_context) const -{ - auto headers = RestCatalog::getAuthHeaders(auth_context); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) headers.emplace_back("User-Agent", fmt::format("ClickHouse/{}{} OneLake-Catalog", VERSION_STRING, VERSION_OFFICIAL)); return headers; } @@ -1229,7 +1177,8 @@ HorizonCatalog::HorizonCatalog( "or `auth_header` (Authorization: Bearer )"); } - initial_state.config = loadConfig(initial_state); + initial_state.config = loadConfig(initial_state, /* generation */ 0, /* auth_token */ {}); + initial_state.config_loaded = true; state.set(std::make_unique(std::move(initial_state))); } @@ -1321,47 +1270,13 @@ AccessToken RestCatalog::requestToken(Poco::URI url, const Poco::URI::QueryParam { String body; -<<<<<<< HEAD - if (oauth_server_uri.empty() && !oauth_server_use_request_body) - { - url = Poco::URI(base_url / oauth_tokens_endpoint); - - Poco::URI::QueryParameters params = { - {"grant_type", "client_credentials"}, - {"scope", auth_scope}, - {"client_secret", client_secret}, - }; - /// Snowflake Horizon OAuth uses secret-only credentials (PAT/JWT); omit empty client_id. - if (!client_id.empty()) - params.emplace_back("client_id", client_id); -======= if (use_query_parameters) ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) url.setQueryParameters(params); else { -<<<<<<< HEAD - String encoded_auth_scope; - String encoded_client_secret; - Poco::URI::encode(auth_scope, auth_scope, encoded_auth_scope); - Poco::URI::encode(client_secret, client_secret, encoded_client_secret); - - body = fmt::format( - "grant_type=client_credentials&scope={}&client_secret={}", - encoded_auth_scope, encoded_client_secret); - if (!client_id.empty()) - { - String encoded_client_id; - Poco::URI::encode(client_id, client_id, encoded_client_id); - body += "&client_id=" + encoded_client_id; - } - body_size = body.size(); - out_stream_callback = [&](std::ostream & os) -======= DB::WriteBufferFromOwnString wb; bool first = true; for (const auto & [name, value] : params) ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { if (!first) wb << "&"; @@ -1391,34 +1306,46 @@ AccessToken RestCatalog::requestToken(Poco::URI url, const Poco::URI::QueryParam std::string json_str; Poco::StreamCopier::copyToString(rs, json_str); -<<<<<<< HEAD - /// The body of a failed response is an OAuth error object, safe to show. + /// The body of a failed client credentials request is an OAuth error object, safe to show. /// The URL is omitted: its query string can carry `client_secret`. - if (response.getStatus() != Poco::Net::HTTPResponse::HTTP_OK) + const bool has_subject_token = std::ranges::any_of(params, [](const auto & param) { return param.first == "subject_token"; }); + if (response.getStatus() != Poco::Net::HTTPResponse::HTTP_OK && !has_subject_token) throw DB::Exception( DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "OAuth token request failed with status {} ({}): {}", static_cast(response.getStatus()), response.getReason(), json_str); - Poco::JSON::Parser parser; - Poco::Dynamic::Var res_json = parser.parse(json_str); - const Poco::JSON::Object::Ptr & object = res_json.extract(); + /// OAuth error bodies of a token exchange may echo the subject token, so exclude them from exceptions. + const auto describe_endpoint = [&url, &response] + { + return fmt::format( + "OAuth token endpoint {}://{}:{}{} returned HTTP {}", + url.getScheme(), url.getHost(), url.getPort(), url.getPath(), + static_cast(response.getStatus())); + }; - if (!object->has("access_token")) + Poco::JSON::Object::Ptr object; + try + { + object = Poco::JSON::Parser().parse(json_str).extract(); + } + catch (const Poco::Exception &) + { + object = nullptr; + } + if (!object) throw DB::Exception( - DB::ErrorCodes::DATALAKE_DATABASE_ERROR, - "OAuth token response has no `access_token` field: {}", - json_str); + DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "{} with a body that is not a JSON object", describe_endpoint()); AccessToken token; - token.token = object->getValue("access_token"); + if (!object->has("access_token")) + throw DB::Exception( + DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "{} with no `access_token` field", describe_endpoint()); + token.token = object->get("access_token").extract(); if (object->has("expires_in")) { - Int64 expires_in = object->getValue("expires_in"); - /// Use 90% of the token lifetime as the validity window so that short-lived tokens - /// (e.g. expires_in=300) still get a sensible buffer instead of going non-positive. - token.expires_at = std::chrono::system_clock::now() + std::chrono::seconds(expires_in * 9 / 10); + token.expires_at = tokenExpiresAt(object->getValue("expires_in")); } return token; @@ -1520,44 +1447,14 @@ AccessToken OneLakeCatalog::retrieveAccessTokenViaRefreshToken(const CatalogStat Poco::JSON::Parser parser; Poco::Dynamic::Var res_json = parser.parse(json_str); const Poco::JSON::Object::Ptr & object = res_json.extract(); -======= - /// OAuth error bodies may echo the subject token, so exclude them from exceptions. - const auto describe_endpoint = [&url, &response] - { - return fmt::format( - "OAuth token endpoint {}://{}:{}{} returned HTTP {}", - url.getScheme(), url.getHost(), url.getPort(), url.getPath(), - static_cast(response.getStatus())); - }; - - Poco::JSON::Object::Ptr object; - try - { - object = Poco::JSON::Parser().parse(json_str).extract(); - } - catch (const Poco::Exception &) - { - object = nullptr; - } - if (!object) - throw DB::Exception( - DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "{} with a body that is not a JSON object", describe_endpoint()); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) AccessToken token; - if (!object->has("access_token")) - throw DB::Exception( - DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "{} with no `access_token` field", describe_endpoint()); token.token = object->get("access_token").extract(); if (object->has("expires_in")) { -<<<<<<< HEAD Int64 expires_in = object->getValue("expires_in"); token.expires_at = std::chrono::system_clock::now() + std::chrono::seconds(expires_in * 9 / 10); -======= - token.expires_at = tokenExpiresAt(object->getValue("expires_in")); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } /// The response also carries a rotated `refresh_token`. It is deliberately ignored: @@ -1572,12 +1469,14 @@ AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, cons { static constexpr auto oauth_tokens_endpoint = "oauth/tokens"; - const Poco::URI::QueryParameters params = { + Poco::URI::QueryParameters params = { {"grant_type", "client_credentials"}, {"scope", auth_scope}, - {"client_id", client_id}, {"client_secret", client_secret}, }; + /// Snowflake Horizon OAuth uses secret-only credentials (PAT/JWT); omit empty client_id. + if (!client_id.empty()) + params.emplace_back("client_id", client_id); const auto url = oauth_server_uri.empty() ? Poco::URI(base_url / oauth_tokens_endpoint) : Poco::URI(oauth_server_uri); ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogAuthTokenRetrieve); @@ -1628,14 +1527,7 @@ BigLakeCatalog::BigLakeCatalog( state.set(std::make_unique(std::move(initial_state))); } -<<<<<<< HEAD -DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders( - const CatalogState & catalog_state, - bool update_token, - bool * used_cached_oauth_token) const -======= DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders(const AuthContext & auth_context) const ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { /// Google Cloud OAuth2 for BigLake. /// Uses GCP metadata service or Application Default Credentials to get access token. @@ -1674,11 +1566,7 @@ DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders(const AuthContext & auth_co return headers; } -<<<<<<< HEAD - return RestCatalog::getAuthHeaders(catalog_state, update_token, used_cached_oauth_token); -======= return RestCatalog::getAuthHeaders(auth_context); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } AccessToken BigLakeCatalog::retrieveGoogleCloudAccessTokenFromRefreshToken() const @@ -1842,9 +1730,6 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( auto create_buffer = [&](bool update_token, bool & used_cached_oauth_token) { -<<<<<<< HEAD - auto result_headers = auth_headers ? *auth_headers : getAuthHeaders(catalog_state, update_token, &used_cached_oauth_token); -======= AuthContext auth_context{ .catalog_state = catalog_state, .generation = generation, @@ -1857,7 +1742,6 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( .used_cached_oauth_token = &used_cached_oauth_token, }; auto result_headers = auth_headers ? *auth_headers : getAuthHeaders(auth_context); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) std::move(headers.begin(), headers.end(), std::back_inserter(result_headers)); return DB::BuilderRWBufferFromHTTP(url) @@ -1928,11 +1812,7 @@ bool RestCatalog::empty(const DB::ForwardedAuthTokenPtr & auth_token) const return !found_table; } -<<<<<<< HEAD -CatalogTables RestCatalog::getTables() const -======= -DB::Names RestCatalog::getTables(const DB::ForwardedAuthTokenPtr & auth_token) const ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) +CatalogTables RestCatalog::getTables(const DB::ForwardedAuthTokenPtr & auth_token) const { loadConfigIfNeeded(auth_token); @@ -1992,22 +1872,17 @@ RestCatalog::Namespaces RestCatalog::getNamespaces(const DB::ForwardedAuthTokenP return namespaces; } -<<<<<<< HEAD -CatalogTables RestCatalog::listTablesInNamespaceDirect(const std::string & namespace_name) const +CatalogTables RestCatalog::listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & auth_token) const { + loadConfigIfNeeded(auth_token); + /// A REST catalog is Iceberg-only, so every listed table is readable. CatalogTables result; - auto names = listTablesInNamespace(namespace_name); + auto names = listTablesInNamespace(namespace_name, auth_token); result.reserve(names.size()); for (auto & name : names) result.push_back(CatalogTable{.name = std::move(name)}); return result; -======= -DB::Names RestCatalog::listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & auth_token) const -{ - loadConfigIfNeeded(auth_token); - return listTablesInNamespace(namespace_name, auth_token); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } void RestCatalog::getNamespacesRecursive( @@ -2047,16 +1922,10 @@ void RestCatalog::getNamespacesRecursive( } } -<<<<<<< HEAD if (!hasFlatNamespaces()) -======= - if (allowed_namespaces.isNamespaceAllowed(current_namespace, /*nested*/ true)) - getNamespacesRecursive(current_namespace, result, stop_condition, func, auth_token); - else ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { if (allowed_namespaces.isNamespaceAllowed(current_namespace, /*nested*/ true)) - getNamespacesRecursive(current_namespace, result, stop_condition, func); + getNamespacesRecursive(current_namespace, result, stop_condition, func, auth_token); else { LOG_DEBUG(log, "Nested namespaces in namespace {} are filtered", current_namespace); @@ -2081,7 +1950,6 @@ Poco::URI::QueryParameters RestCatalog::createParentNamespaceParams(const std::s return {{"parent", parent_param}}; } -<<<<<<< HEAD bool RestCatalog::hasFlatNamespaces() const { /// Catalogs whose namespaces are single-level and which ignore or reject the `parent` filter when @@ -2096,10 +1964,7 @@ bool RestCatalog::hasFlatNamespaces() const || type == DB::DatabaseDataLakeCatalogType::S3_TABLES; } -RestCatalog::Namespaces RestCatalog::listChildNamespaces(const std::string & base_namespace) const -======= RestCatalog::Namespaces RestCatalog::listChildNamespaces(const std::string & base_namespace, const DB::ForwardedAuthTokenPtr & auth_token) const ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { const auto state_snapshot = getStateSnapshot(); @@ -2130,11 +1995,8 @@ RestCatalog::Namespaces RestCatalog::listChildNamespaces(const std::string & bas ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetNamespaces); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetNamespacesMicroseconds); auto buf = createReadBuffer( -<<<<<<< HEAD - *state_snapshot, state_snapshot->config.prefix / NAMESPACES_ENDPOINT, params, /* headers */ {}, /* auth_headers */ std::nullopt); -======= - *state_snapshot, state_snapshot.generation, state_snapshot->config.prefix / NAMESPACES_ENDPOINT, auth_token, params); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + *state_snapshot, state_snapshot.generation, state_snapshot->config.prefix / NAMESPACES_ENDPOINT, auth_token, params, + /* headers */ {}, /* auth_headers */ std::nullopt); String next_page_token; auto page_namespaces = parseNamespaces(*buf, base_namespace, next_page_token); LOG_DEBUG( @@ -2299,12 +2161,9 @@ DB::Names RestCatalog::listTablesInNamespace(const std::string & base_namespace, ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetTables); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetTablesMicroseconds); -<<<<<<< HEAD auto buf = createReadBuffer( - *state_snapshot, state_snapshot->config.prefix / endpoint, params, /* headers */ {}, /* auth_headers */ std::nullopt); -======= - auto buf = createReadBuffer(*state_snapshot, state_snapshot.generation, state_snapshot->config.prefix / endpoint, auth_token, params); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + *state_snapshot, state_snapshot.generation, state_snapshot->config.prefix / endpoint, auth_token, params, + /* headers */ {}, /* auth_headers */ std::nullopt); /// Pass through the remaining limit so that single-page short-circuiting still works /// when the caller is in `empty()` (limit=1) and the first page already contains a row. @@ -2570,11 +2429,8 @@ bool RestCatalog::getTableMetadataImpl( ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetTableMetadata); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetTableMetadataMicroseconds); auto buf = createReadBuffer( -<<<<<<< HEAD - *state_snapshot, state_snapshot->config.prefix / endpoint, /* params */ {}, headers, /* auth_headers */ std::nullopt); -======= - *state_snapshot, state_snapshot.generation, state_snapshot->config.prefix / endpoint, auth_token, /* params */{}, headers); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + *state_snapshot, state_snapshot.generation, state_snapshot->config.prefix / endpoint, auth_token, /* params */ {}, headers, + /* auth_headers */ std::nullopt); if (buf->eof()) { @@ -2653,11 +2509,7 @@ bool RestCatalog::getTableMetadataImpl( if (parsed.credentials) { result.setStorageCredentials(parsed.credentials); -<<<<<<< HEAD - cacheCredentials(namespace_name, table_name, parsed, state_snapshot); -======= - cacheCredentials(credentials_key, parsed); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + cacheCredentials(credentials_key, parsed, state_snapshot.state); } if (!parsed.endpoint.empty()) result.setEndpoint(parsed.endpoint); @@ -2711,22 +2563,18 @@ void RestCatalog::sendRequest( auto create_buffer = [&](bool update_token, bool & used_cached_oauth_token) { -<<<<<<< HEAD - DB::HTTPHeaderEntries headers = getAuthHeaders(catalog_state, update_token, &used_cached_oauth_token); -======= AuthContext auth_context{ .catalog_state = catalog_state, .generation = generation, .update_token = update_token, .method = method, .url = url, - .extra_headers = extra_headers, + .extra_headers = {}, .body = body_str, .auth_token = auth_token, .used_cached_oauth_token = &used_cached_oauth_token, }; DB::HTTPHeaderEntries headers = getAuthHeaders(auth_context); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) headers.emplace_back("Content-Type", "application/json"); headers.emplace_back("X-Iceberg-Access-Delegation", "vended-credentials"); return DB::BuilderRWBufferFromHTTP(url) @@ -2814,11 +2662,9 @@ void RestCatalog::createNamespaceIfNotExists(const String & namespace_name, cons { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogCreateNamespace); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogCreateNamespaceMicroseconds); -<<<<<<< HEAD - sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_POST, /* ignore_result */ false); -======= - sendRequest(*state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + sendRequest( + *state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token, + Poco::Net::HTTPRequest::HTTP_POST, /* ignore_result */ false); } catch (const DB::HTTPException & e) { @@ -2887,11 +2733,9 @@ bool RestCatalog::createTable( { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogCreateTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogCreateTableMicroseconds); -<<<<<<< HEAD - sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_POST, /* ignore_result */ false); -======= - sendRequest(*state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + sendRequest( + *state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token, + Poco::Net::HTTPRequest::HTTP_POST, /* ignore_result */ false); } catch (const DB::HTTPException & ex) { @@ -2923,11 +2767,9 @@ bool RestCatalog::updateMetadata(const String & namespace_name, const String & t { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUpdateTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogUpdateTableMicroseconds); -<<<<<<< HEAD - sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_POST, /* ignore_result */ false); -======= - sendRequest(*state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + sendRequest( + *state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token, + Poco::Net::HTTPRequest::HTTP_POST, /* ignore_result */ false); } catch (const DB::HTTPException & ex) { @@ -2971,11 +2813,9 @@ bool RestCatalog::updateSchema( try { -<<<<<<< HEAD - sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_POST, /* ignore_result */ false); -======= - sendRequest(*state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + sendRequest( + *state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token, + Poco::Net::HTTPRequest::HTTP_POST, /* ignore_result */ false); } catch (const DB::HTTPException & ex) { @@ -3169,12 +3009,7 @@ std::optional RestCatalog::tryGetCachedCredentials(con if (vended_credentials_cache_ttl <= std::chrono::seconds::zero()) return std::nullopt; -<<<<<<< HEAD - auto it = credentials_cache.find({namespace_name, table_name}); -======= - std::lock_guard lock(credentials_cache_mutex); auto it = credentials_cache.find(key); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) if (it == credentials_cache.end()) return std::nullopt; if (std::chrono::system_clock::now() >= it->second.expires_at.value()) @@ -3186,15 +3021,10 @@ std::optional RestCatalog::tryGetCachedCredentials(con return it->second; } -<<<<<<< HEAD void RestCatalog::cacheCredentials( - const std::string & namespace_name, - const std::string & table_name, + const CredentialsCacheKey & key, const VendedStorageCredentials & parsed, const CatalogStateVersion & state_snapshot) const -======= -void RestCatalog::cacheCredentials(const CredentialsCacheKey & key, const VendedStorageCredentials & parsed) const ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { if (!parsed.credentials || parsed.credentials->isEmpty()) return; @@ -3217,10 +3047,6 @@ void RestCatalog::cacheCredentials(const CredentialsCacheKey & key, const Vended if (credentials_cache.size() >= credentials_cache_cleanup_threshold) std::erase_if(credentials_cache, [&now](const auto & entry) { return now >= entry.second.expires_at.value(); }); -<<<<<<< HEAD - credentials_cache[{namespace_name, table_name}] - = VendedStorageCredentials{parsed.credentials, parsed.endpoint, refresh_after, parsed.table_uuid, parsed.location}; -======= while (credentials_cache.size() >= credentials_cache_max_entries) { @@ -3232,8 +3058,7 @@ void RestCatalog::cacheCredentials(const CredentialsCacheKey & key, const Vended } credentials_cache[key] - = VendedStorageCredentials{parsed.credentials, parsed.endpoint, refresh_after, parsed.table_uuid}; ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + = VendedStorageCredentials{parsed.credentials, parsed.endpoint, refresh_after, parsed.table_uuid, parsed.location}; } ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCallback( @@ -3256,11 +3081,8 @@ ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCal ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetCredentials); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetCredentialsMicroseconds); auto buf = createReadBuffer( -<<<<<<< HEAD - *state_snapshot, state_snapshot->config.prefix / endpoint, /* params */ {}, headers, /* auth_headers */ std::nullopt); -======= - *state_snapshot, state_snapshot.generation, state_snapshot->config.prefix / endpoint, auth_token, /* params */{}, headers); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + *state_snapshot, state_snapshot.generation, state_snapshot->config.prefix / endpoint, auth_token, /* params */ {}, headers, + /* auth_headers */ std::nullopt); if (buf->eof()) { @@ -3305,17 +3127,11 @@ ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCal } auto parsed = getCredentialsAndEndpoint(config_object, location); -<<<<<<< HEAD parsed.table_uuid = table_uuid; parsed.location = location; - cacheCredentials(namespace_name, table_name, parsed, state_snapshot); -======= - if (metadata_object) - parsed.table_uuid = parseTableUuid(metadata_object); - /// Refresh the per-table cache so subsequent queries reuse these freshly vended credentials. cacheCredentials( - {state_snapshot.generation, getCredentialsCachePrincipal(auth_token), namespace_name, table_name}, parsed); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + {state_snapshot.generation, getCredentialsCachePrincipal(auth_token), namespace_name, table_name}, parsed, + state_snapshot.state); return parsed.credentials; }; } diff --git a/src/Databases/DataLake/RestCatalog.h b/src/Databases/DataLake/RestCatalog.h index b6a1c44ecc3cb..d98db51a7effa 100644 --- a/src/Databases/DataLake/RestCatalog.h +++ b/src/Databases/DataLake/RestCatalog.h @@ -84,23 +84,15 @@ class RestCatalog : public ICatalog, public DB::WithContext const std::string & oauth_server_uri_, bool oauth_server_use_request_body_, const std::string & namespaces_, -<<<<<<< HEAD bool flat_namespaces_, - DB::ContextPtr context_); -======= DB::ContextPtr context_, const TokenForwardingConfig & token_forwarding_ = {}); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) ~RestCatalog() override = default; bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const override; -<<<<<<< HEAD - CatalogTables getTables() const override; -======= - DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + CatalogTables getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; Namespaces getNamespaces(const DB::ForwardedAuthTokenPtr & auth_token) const override; @@ -263,11 +255,7 @@ class RestCatalog : public ICatalog, public DB::WithContext std::string auth_scope; std::string oauth_server_uri; bool oauth_server_use_request_body; -<<<<<<< HEAD bool flat_namespaces = false; - mutable MultiVersion access_token; - -======= /// Shared service or actor token; never store a user token here. mutable MultiVersion access_token; @@ -284,21 +272,6 @@ class RestCatalog : public ICatalog, public DB::WithContext /// Never hold this across a network request. mutable std::mutex auth_publish_mutex; - /// TTL for caching vended credentials per table (0 means no caching). - std::atomic vended_credentials_cache_ttl{std::chrono::seconds::zero()}; - - /// Sweep trigger threshold, not capacity! - static constexpr size_t credentials_cache_cleanup_threshold = 1000; - - static constexpr size_t credentials_cache_max_entries = 10000; - - static constexpr std::chrono::seconds credentials_expiry_safety_window{60}; - mutable std::mutex credentials_cache_mutex; - - mutable std::map credentials_cache - TSA_GUARDED_BY(credentials_cache_mutex); - ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) public: class AllowedNamespaces { @@ -321,11 +294,13 @@ class RestCatalog : public ICatalog, public DB::WithContext static constexpr size_t credentials_cache_cleanup_threshold = 1000; + static constexpr size_t credentials_cache_max_entries = 10000; + static constexpr std::chrono::seconds credentials_expiry_safety_window{60}; mutable std::mutex credentials_cache_mutex; std::chrono::seconds vended_credentials_cache_ttl TSA_GUARDED_BY(credentials_cache_mutex){std::chrono::seconds::zero()}; - mutable std::map, VendedStorageCredentials> credentials_cache + mutable std::map credentials_cache TSA_GUARDED_BY(credentials_cache_mutex); Poco::Net::HTTPBasicCredentials credentials{}; @@ -338,16 +313,10 @@ class RestCatalog : public ICatalog, public DB::WithContext const CatalogState & catalog_state, UInt64 generation, const std::string & endpoint, -<<<<<<< HEAD + const DB::ForwardedAuthTokenPtr & auth_token, const Poco::URI::QueryParameters & params, const DB::HTTPHeaderEntries & headers, const std::optional & auth_headers) const; -======= - const DB::ForwardedAuthTokenPtr & auth_token, - const Poco::URI::QueryParameters & params = {}, - const DB::HTTPHeaderEntries & headers = {}, - const std::optional & auth_headers = std::nullopt) const; ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) Poco::URI::QueryParameters createParentNamespaceParams(const std::string & base_namespace) const; @@ -377,11 +346,7 @@ class RestCatalog : public ICatalog, public DB::WithContext /// `limit` is a soft cap on the number of returned names; 0 means no limit. DB::Names listTablesInNamespace(const std::string & base_namespace, const DB::ForwardedAuthTokenPtr & auth_token, size_t limit = 0) const; -<<<<<<< HEAD - CatalogTables listTablesInNamespaceDirect(const std::string & namespace_name) const override; -======= - DB::Names listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + CatalogTables listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; DB::Names parseTables(DB::ReadBuffer & buf, const std::string & base_namespace, size_t limit, String & next_page_token) const; @@ -401,13 +366,6 @@ class RestCatalog : public ICatalog, public DB::WithContext const DB::ForwardedAuthTokenPtr & auth_token) const; /// Load catalog config (special http handler) utilizing information from catalog_state and auth_headers. -<<<<<<< HEAD - Config loadConfig(const CatalogState & catalog_state, const std::optional & auth_headers = std::nullopt); - virtual DB::HTTPHeaderEntries getAuthHeaders( - const CatalogState & catalog_state, - bool update_token, - bool * used_cached_oauth_token) const; -======= Config loadConfig( const CatalogState & catalog_state, UInt64 generation, @@ -422,7 +380,6 @@ class RestCatalog : public ICatalog, public DB::WithContext const CatalogState & catalog_state, UInt64 generation, const DB::ForwardedAuthTokenPtr & auth_token, bool update_token) const; bool shouldRetryWithFreshToken(Poco::Net::HTTPResponse::HTTPStatus status) const; ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) void validateAuthHeaders(const DB::HTTPHeaderEntry & header) const; @@ -433,29 +390,20 @@ class RestCatalog : public ICatalog, public DB::WithContext UInt64 generation, const String & endpoint, Poco::JSON::Object::Ptr request_body, -<<<<<<< HEAD + const DB::ForwardedAuthTokenPtr & auth_token, const String & method, bool ignore_result) const; -======= - const DB::ForwardedAuthTokenPtr & auth_token, - const String & method = Poco::Net::HTTPRequest::HTTP_POST, - bool ignore_result = false) const; ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) VendedStorageCredentials getCredentialsAndEndpoint(Poco::JSON::Object::Ptr object, const String & location) const; String getCredentialsCachePrincipal(const DB::ForwardedAuthTokenPtr & auth_token) const; -<<<<<<< HEAD + std::optional tryGetCachedCredentials(const CredentialsCacheKey & key) const; + void cacheCredentials( - const std::string & namespace_name, - const std::string & table_name, + const CredentialsCacheKey & key, const VendedStorageCredentials & parsed, const CatalogStateVersion & state_snapshot) const; -======= - std::optional tryGetCachedCredentials(const CredentialsCacheKey & key) const; - - void cacheCredentials(const CredentialsCacheKey & key, const VendedStorageCredentials & parsed) const; MultiVersion::Version publishServiceToken(AccessToken minted, UInt64 generation) const; @@ -464,7 +412,6 @@ class RestCatalog : public ICatalog, public DB::WithContext AccessToken exchangeUserToken( const CatalogState & catalog_state, UInt64 generation, const DB::ForwardedAuthToken & auth_token, const AccessToken * prepared_actor_token = nullptr) const; ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) AccessToken retrieveAccessToken(const std::string & client_id, const std::string & client_secret) const; @@ -525,14 +472,7 @@ class OneLakeCatalog : public RestCatalog /// OneLake keeps data in Azure Data Lake Storage, whatever the catalog reports. std::optional getStorageType() const override { return StorageType::Azure; } -<<<<<<< HEAD - DB::HTTPHeaderEntries getAuthHeaders( - const CatalogState & catalog_state, - bool update_token, - bool * used_cached_oauth_token) const override; -======= DB::HTTPHeaderEntries getAuthHeaders(const AuthContext & auth_context) const override; ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) static void validateSettingsChanges(const DB::SettingsChanges & changes, AuthMode auth_mode); @@ -583,14 +523,7 @@ class BigLakeCatalog : public RestCatalog /// BigLake keeps data in Google Cloud Storage, which is accessed through the S3 API. std::optional getStorageType() const override { return StorageType::S3; } -<<<<<<< HEAD - DB::HTTPHeaderEntries getAuthHeaders( - const CatalogState & catalog_state, - bool update_token, - bool * used_cached_oauth_token) const override; -======= DB::HTTPHeaderEntries getAuthHeaders(const AuthContext & auth_context) const override; ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) const std::string & getGoogleADCClientId() const { return google_adc_client_id; } const std::string & getGoogleADCClientSecret() const { return google_adc_client_secret; } diff --git a/src/Databases/DataLake/S3TablesCatalog.cpp b/src/Databases/DataLake/S3TablesCatalog.cpp index 2cd6d2607b2dd..259070a7a5c40 100644 --- a/src/Databases/DataLake/S3TablesCatalog.cpp +++ b/src/Databases/DataLake/S3TablesCatalog.cpp @@ -136,16 +136,10 @@ S3TablesCatalog::S3TablesCatalog( state.set(std::make_unique(std::move(initial_state))); } -<<<<<<< HEAD /// S3 Tables only supports a single level of namespaces (no nesting), so we list the root /// namespaces directly with `listChildNamespaces("")` instead of the base class's recursive /// `getNamespacesRecursive`. -CatalogTables S3TablesCatalog::getTables() const -======= -/// S3 Tables only supports a single level of namespaces (no nesting), -/// so we use flat listChildNamespaces() instead of the base class's getNamespacesRecursive(). -DB::Names S3TablesCatalog::getTables(const DB::ForwardedAuthTokenPtr & auth_token) const ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) +CatalogTables S3TablesCatalog::getTables(const DB::ForwardedAuthTokenPtr & auth_token) const { auto namespaces = listChildNamespaces("", auth_token); @@ -159,11 +153,7 @@ DB::Names S3TablesCatalog::getTables(const DB::ForwardedAuthTokenPtr & auth_toke runner.enqueueAndKeepTrack( [&, ns] { -<<<<<<< HEAD - auto tables_in_ns = listTablesInNamespace(ns); -======= - auto tables_in_ns = RestCatalog::listTablesInNamespace(ns, auth_token); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + auto tables_in_ns = listTablesInNamespace(ns, auth_token); std::lock_guard lock(mutex); std::move(tables_in_ns.begin(), tables_in_ns.end(), std::back_inserter(tables)); }); @@ -225,19 +215,8 @@ bool S3TablesCatalog::tryGetTableMetadata( result.setEndpoint(endpoint); } -<<<<<<< HEAD if (auto props = result.getDataLakeSpecificProperties(); props && !props->iceberg_metadata_file_location.empty()) -======= - return true; -} - -ICatalog::CredentialsRefreshCallback S3TablesCatalog::getCredentialsConfigurationCallback( - const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) -{ - auto base_cb = RestCatalog::getCredentialsConfigurationCallback(storage_id, auth_token); - return [this, base_callback = std::move(base_cb)] () -> std::shared_ptr ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { const String & loc = props->iceberg_metadata_file_location; auto scheme_end = loc.find("://"); @@ -281,15 +260,9 @@ void S3TablesCatalog::dropTable( Poco::JSON::Object::Ptr request_body = nullptr; try { -<<<<<<< HEAD - sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_DELETE, true); - LOG_INFO(log, "S3 Tables: dropped table {}.{} (purgeRequested=True)", namespace_name, table_name); -======= - ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogDropTable); - auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogDropTableMicroseconds); sendRequest( *state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token, Poco::Net::HTTPRequest::HTTP_DELETE, true); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + LOG_INFO(log, "S3 Tables: dropped table {}.{} (purgeRequested=True)", namespace_name, table_name); } catch (const DB::HTTPException & ex) { @@ -302,17 +275,8 @@ void S3TablesCatalog::dropTable( } } -<<<<<<< HEAD namespace { -======= -DB::HTTPHeaderEntries S3TablesCatalog::getAuthHeaders(const AuthContext & auth_context) const -{ - DB::HTTPHeaderEntries all_signed; - signRequestWithAWSV4( - auth_context.method, auth_context.url, auth_context.extra_headers, auth_context.body, - *signer, region, "s3tables", all_signed); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) /// `signRequestWithAWSV4` returns the full set of headers that the AWS SDK kept on /// the signed request (input headers + signed auth). We only want the SigV4-specific @@ -343,7 +307,9 @@ DB::HTTPHeaderEntries extractSigV4AuthHeaders(DB::HTTPHeaderEntries && all_signe DB::ReadWriteBufferFromHTTPPtr S3TablesCatalog::createReadBuffer( const CatalogState & /* catalog_state */, + UInt64 /* generation */, const std::string & endpoint, + const DB::ForwardedAuthTokenPtr & /* auth_token */, const Poco::URI::QueryParameters & params, const DB::HTTPHeaderEntries & headers, const std::optional & /* auth_headers */) const @@ -379,8 +345,10 @@ DB::ReadWriteBufferFromHTTPPtr S3TablesCatalog::createReadBuffer( void S3TablesCatalog::sendRequest( const CatalogState & /* catalog_state */, + UInt64 /* generation */, const String & endpoint, Poco::JSON::Object::Ptr request_body, + const DB::ForwardedAuthTokenPtr & /* auth_token */, const String & method, bool ignore_result) const { diff --git a/src/Databases/DataLake/S3TablesCatalog.h b/src/Databases/DataLake/S3TablesCatalog.h index 60ba60b3b2c76..7d8f6ea414068 100644 --- a/src/Databases/DataLake/S3TablesCatalog.h +++ b/src/Databases/DataLake/S3TablesCatalog.h @@ -36,11 +36,7 @@ class S3TablesCatalog final : public RestCatalog DB::DatabaseDataLakeCatalogType getCatalogType() const override { return DB::DatabaseDataLakeCatalogType::S3_TABLES; } -<<<<<<< HEAD - CatalogTables getTables() const override; -======= - DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + CatalogTables getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; std::optional getStorageType() const override { return StorageType::S3; } @@ -52,8 +48,12 @@ class S3TablesCatalog final : public RestCatalog DB::ContextPtr context_, TableMetadata & result) const override; -<<<<<<< HEAD - void dropTable(const String & namespace_name, const String & table_name, bool delete_data, bool if_exists) const override; + void dropTable( + const String & namespace_name, + const String & table_name, + bool delete_data, + bool if_exists, + const DB::ForwardedAuthTokenPtr & auth_token) const override; protected: /// Override the network primitives instead of `getAuthHeaders` so the SigV4 signer has @@ -62,33 +62,21 @@ class S3TablesCatalog final : public RestCatalog /// SigV4 signer rather than from the catalog state snapshot. DB::ReadWriteBufferFromHTTPPtr createReadBuffer( const CatalogState & catalog_state, + UInt64 generation, const std::string & endpoint, + const DB::ForwardedAuthTokenPtr & auth_token, const Poco::URI::QueryParameters & params, const DB::HTTPHeaderEntries & headers, const std::optional & auth_headers) const override; void sendRequest( const CatalogState & catalog_state, + UInt64 generation, const String & endpoint, Poco::JSON::Object::Ptr request_body, + const DB::ForwardedAuthTokenPtr & auth_token, const String & method, bool ignore_result) const override; -======= - bool managesTableLocation() const override { return true; } - - void dropTable( - const String & namespace_name, - const String & table_name, - bool delete_data, - bool if_exists, - const DB::ForwardedAuthTokenPtr & auth_token) const override; - - ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback( - const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) override; - -protected: - DB::HTTPHeaderEntries getAuthHeaders(const AuthContext & auth_context) const override; ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) private: const String region; diff --git a/src/Databases/DataLake/UnityCatalog.cpp b/src/Databases/DataLake/UnityCatalog.cpp index fb8f55ee979b5..fa66e39cb7637 100644 --- a/src/Databases/DataLake/UnityCatalog.cpp +++ b/src/Databases/DataLake/UnityCatalog.cpp @@ -118,11 +118,7 @@ bool UnityCatalog::empty(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const return true; } -<<<<<<< HEAD -CatalogTables UnityCatalog::getTables() const -======= -DB::Names UnityCatalog::getTables(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) +CatalogTables UnityCatalog::getTables(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { CatalogTables result; @@ -142,11 +138,7 @@ DataLake::ICatalog::Namespaces UnityCatalog::getNamespaces(const DB::ForwardedAu return getSchemas(""); } -<<<<<<< HEAD -CatalogTables UnityCatalog::listTablesInNamespaceDirect(const std::string & namespace_name) const -======= -DB::Names UnityCatalog::listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) +CatalogTables UnityCatalog::listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { return getTablesForSchema(namespace_name); } @@ -546,13 +538,8 @@ bool UnityCatalog::isNamespaceAllowed(const std::string & namespace_) const return allowed_namespaces.contains("*") || allowed_namespaces.contains(namespace_); } -<<<<<<< HEAD -ICatalog::CredentialsRefreshCallback UnityCatalog::getCredentialsConfigurationCallback(const DB::StorageID & table_id) -======= -/// getCredentialsConfigurationCallback method is supported only for S3 storage ICatalog::CredentialsRefreshCallback UnityCatalog::getCredentialsConfigurationCallback( const DB::StorageID & table_id, const DB::ForwardedAuthTokenPtr & /*auth_token*/) ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { if (!table_id.hasUUID()) throw DB::Exception( diff --git a/src/Databases/DataLake/UnityCatalog.h b/src/Databases/DataLake/UnityCatalog.h index 9b84d45f4abaa..f740173a8b12a 100644 --- a/src/Databases/DataLake/UnityCatalog.h +++ b/src/Databases/DataLake/UnityCatalog.h @@ -29,11 +29,7 @@ class UnityCatalog final : public ICatalog, private DB::WithContext bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const override; -<<<<<<< HEAD - CatalogTables getTables() const override; -======= - DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + CatalogTables getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; Namespaces getNamespaces(const DB::ForwardedAuthTokenPtr & auth_token) const override; @@ -74,13 +70,8 @@ class UnityCatalog final : public ICatalog, private DB::WithContext DataLake::ICatalog::Namespaces getSchemas(const std::string & base_prefix, size_t limit = 0) const; -<<<<<<< HEAD CatalogTables getTablesForSchema(const std::string & schema, size_t limit = 0) const; - CatalogTables listTablesInNamespaceDirect(const std::string & namespace_name) const override; -======= - DB::Names getTablesForSchema(const std::string & schema, size_t limit = 0) const; - DB::Names listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + CatalogTables listTablesInNamespaceDirect(const std::string & namespace_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; void getCredentials(const String & table_id, TableMetadata & metadata) const; Poco::JSON::Object::Ptr requestCredentials(const String & table_id, const String & operation) const; @@ -93,16 +84,12 @@ class UnityCatalog final : public ICatalog, private DB::WithContext const std::string & table_name, TableMetadata & result) const; -<<<<<<< HEAD - ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback(const DB::StorageID & table_id) override; + ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback( + const DB::StorageID & table_id, const DB::ForwardedAuthTokenPtr & auth_token) override; ICatalog::CredentialsRefreshCallback getWriteCredentialsConfigurationCallback(const DB::StorageID & table_id) override; ICatalog::CredentialsRefreshCallback getCredentialsCallbackForOperation(const String & unity_table_id, const String & operation); -======= - ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback( - const DB::StorageID & table_id, const DB::ForwardedAuthTokenPtr & auth_token) override; ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) }; } diff --git a/src/Databases/DataLake/tests/gtest_rest_catalog.cpp b/src/Databases/DataLake/tests/gtest_rest_catalog.cpp index f6c38993b9155..fff9c5bd83e66 100644 --- a/src/Databases/DataLake/tests/gtest_rest_catalog.cpp +++ b/src/Databases/DataLake/tests/gtest_rest_catalog.cpp @@ -9,10 +9,9 @@ #include #include #include -#include +#include #include -<<<<<<< HEAD #include #include #include @@ -28,13 +27,9 @@ #include #include #include -======= -#include ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) #include using namespace DataLake; -using namespace RestCatalogTest; namespace ProfileEvents { @@ -47,12 +42,10 @@ namespace DB { namespace ErrorCodes { + extern const int LOGICAL_ERROR; extern const int BAD_ARGUMENTS; -<<<<<<< HEAD extern const int NOT_IMPLEMENTED; extern const int DATALAKE_DATABASE_ERROR; -======= ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } } @@ -77,79 +70,42 @@ enum class CatalogShape OAuthTokenRejected, }; -<<<<<<< HEAD void writeJSON(Poco::Net::HTTPServerResponse & response, const std::string & body, Poco::Net::HTTPResponse::HTTPStatus status = Poco::Net::HTTPResponse::HTTP_OK) { response.setStatus(status); response.setContentType("application/json"); response.setContentLength(body.size()); response.send() << body; -======= -std::string getParent(const std::string & query) -{ - Poco::URI uri; - uri.setRawQuery(query); - for (const auto & [key, value] : uri.getQueryParameters()) - if (key == "parent") - return value; - return {}; ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } -void installShape(ServerState & state, CatalogShape shape) +void writeError(Poco::Net::HTTPServerResponse & response, Poco::Net::HTTPResponse::HTTPStatus status, const std::string & body) { - state.setRoute("/v1/namespaces", [shape](const RecordedRequest & request) - { - const auto parent = getParent(request.query); - if (parent.empty()) - { - if (shape == CatalogShape::NestedTableThenEmptySibling) - return json(R"({"namespaces":[["parent"],["empty_later"]]})"); - return json(R"({"namespaces":[["namespace"]]})"); - } - - if (shape == CatalogShape::NestedTableThenEmptySibling && parent == "parent") - return json(R"({"namespaces":[["leaf_with_table"]]})"); - return json(R"({"namespaces":[]})"); - }); - - state.setRoute("/v1/namespaces/namespace/tables", [shape](const RecordedRequest &) - { - if (shape == CatalogShape::TopLevelTable) - return json(R"({"identifiers":[{"name":"table_a"}]})"); - return json(R"({"identifiers":[]})"); - }); - - state.setStaticRoute("/v1/namespaces/parent/tables", R"({"identifiers":[]})"); - state.setStaticRoute("/v1/namespaces/empty_later/tables", R"({"identifiers":[]})"); - state.setStaticRoute("/v1/namespaces/parent%1Fleaf_with_table/tables", R"({"identifiers":[{"name":"table_a"}]})"); + response.setStatus(status); + response.setContentType("application/json"); + response.setContentLength(body.size()); + response.send() << body; } -void installTokenEndpoint(ServerState & state) +std::string getRawPath(const std::string & uri) { - state.setStaticRoute("/v1/oauth/tokens", R"({"token_type":"Bearer","expires_in":3600,"access_token":"mock-access-token"})"); + const auto query_pos = uri.find('?'); + if (query_pos == std::string::npos) + return uri; + return uri.substr(0, query_pos); } -void installTableRoutes(ServerState & state) +class RestCatalogRequestHandler final : public Poco::Net::HTTPRequestHandler { -<<<<<<< HEAD public: RestCatalogRequestHandler(CatalogShape shape_, int token_expires_in_seconds_, std::atomic & token_requests_) : shape(shape_) , token_expires_in_seconds(token_expires_in_seconds_) , token_requests(token_requests_) -======= - state.setStaticRoute( - "/v1/namespaces/namespace/tables/table_a", R"({"metadata":{"table-uuid":"11111111-2222-3333-4444-555555555555"}})"); - state.setRoute("/v1/namespaces/namespace/tables/missing_table", [](const RecordedRequest &) ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { - return respondWithStatus( - 404, R"({"error":{"message":"Table does not exist","type":"NoSuchTableException","code":404}})"); - }); - state.setRoute("/v1/namespaces/namespace/tables/unauthorized_table", [](const RecordedRequest &) + } + + void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override { -<<<<<<< HEAD Poco::URI uri(request.getURI()); const auto path = getRawPath(request.getURI()); const auto params = uri.getQueryParameters(); @@ -362,12 +318,6 @@ class RestCatalogTestServer Poco::AutoPtr server_params; std::unique_ptr server; }; -======= - return respondWithStatus( - 401, R"({"error":{"message":"The access token has expired","type":"NotAuthorizedException","code":401}})"); - }); -} ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) void expectThrowsCode(std::function fn, int expected_code) { @@ -384,9 +334,7 @@ void expectThrowsCode(std::function fn, int expected_code) bool restCatalogEmpty(CatalogShape shape, bool flat_namespaces = false) { - TestServer server; - installShape(*server, shape); - + RestCatalogTestServer server(shape); auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -402,7 +350,7 @@ bool restCatalogEmpty(CatalogShape shape, bool flat_namespaces = false) flat_namespaces, context); - return catalog.empty(); + return catalog.empty(/* auth_token */ {}); } DataLake::ICatalog::Namespaces restCatalogNamespaces(CatalogShape shape, bool flat_namespaces) @@ -423,7 +371,7 @@ DataLake::ICatalog::Namespaces restCatalogNamespaces(CatalogShape shape, bool fl flat_namespaces, context); - return catalog.getNamespaces(); + return catalog.getNamespaces(/* auth_token */ {}); } bool deltaSharingCatalogEmpty(CatalogShape shape) @@ -551,8 +499,8 @@ TEST(RestCatalog, OneLakeFlatNamespacesSettingSkipsSubNamespaceListing) /* flat_namespaces */true, context); - EXPECT_FALSE(catalog.empty()); - EXPECT_EQ(catalog.getNamespaces(), ICatalog::Namespaces{"namespace"}); + EXPECT_FALSE(catalog.empty(/* auth_token */ {})); + EXPECT_EQ(catalog.getNamespaces(/* auth_token */ {}), ICatalog::Namespaces{"namespace"}); } TEST(RestCatalog, TryGetTableMetadataDistinguishesMissingTableFromOtherErrors) @@ -576,15 +524,15 @@ TEST(RestCatalog, TryGetTableMetadataDistinguishesMissingTableFromOtherErrors) auto existing = TableMetadata().withLocation(); EXPECT_TRUE(catalog.tryGetTableMetadata("namespace", "table_a", context, existing)); EXPECT_EQ(existing.getLocation(), "s3://bucket/table_a"); - EXPECT_TRUE(catalog.existsTable("namespace", "table_a")); + EXPECT_TRUE(catalog.existsTable("namespace", "table_a", /* auth_token */ {})); TableMetadata missing; EXPECT_FALSE(catalog.tryGetTableMetadata("namespace", "missing_table", context, missing)); - EXPECT_FALSE(catalog.existsTable("namespace", "missing_table")); + EXPECT_FALSE(catalog.existsTable("namespace", "missing_table", /* auth_token */ {})); TableMetadata unauthorized; EXPECT_THROW(catalog.tryGetTableMetadata("namespace", "unauthorized_table", context, unauthorized), DB::HTTPException); - EXPECT_THROW(catalog.existsTable("namespace", "unauthorized_table"), DB::HTTPException); + EXPECT_THROW(catalog.existsTable("namespace", "unauthorized_table", /* auth_token */ {}), DB::HTTPException); } TEST(RestCatalog, TryGetTableMetadataAuthErrorPropagates) @@ -623,13 +571,12 @@ TEST(RestCatalog, TryGetTableMetadataAuthErrorPropagates) EXPECT_NE(e.displayText().find("the token is expired"), std::string::npos); } - EXPECT_THROW(catalog.existsTable("namespace", "expired_token_table"), DB::HTTPException); + EXPECT_THROW(catalog.existsTable("namespace", "expired_token_table", /* auth_token */ {}), DB::HTTPException); } TEST(RestCatalog, ApplySettingsChangesWithoutAuthenticationRejected) { - TestServer server; - + RestCatalogTestServer server(CatalogShape::Empty); auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -652,9 +599,7 @@ TEST(RestCatalog, ApplySettingsChangesWithoutAuthenticationRejected) TEST(RestCatalog, ApplySettingsChangesCredentialMode) { - TestServer server; - installTokenEndpoint(*server); - + RestCatalogTestServer server(CatalogShape::Empty); auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -697,8 +642,7 @@ TEST(RestCatalog, ApplySettingsChangesCredentialMode) TEST(RestCatalog, ApplySettingsChangesAuthHeaderMode) { - TestServer server; - + RestCatalogTestServer server(CatalogShape::Empty); auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -729,8 +673,7 @@ TEST(RestCatalog, ApplySettingsChangesAuthHeaderMode) TEST(RestCatalog, OneLakeApplySettingsChangesBearerMode) { - TestServer server; - + RestCatalogTestServer server(CatalogShape::Empty); auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -859,7 +802,7 @@ TEST(RestCatalog, OneLakeRefreshTokenTransparentRenewal) const auto token_requests_before = ProfileEvents::global_counters[ProfileEvents::OneLakeAccessTokenRequests]; const auto expirations_before = ProfileEvents::global_counters[ProfileEvents::OneLakeAccessTokenExpirations]; - EXPECT_TRUE(catalog.empty()); + EXPECT_TRUE(catalog.empty(/* auth_token */ {})); EXPECT_GT(server.tokenRequests(), requests_after_construction); EXPECT_GT(ProfileEvents::global_counters[ProfileEvents::OneLakeAccessTokenRequests], token_requests_before); @@ -971,9 +914,7 @@ TEST(RestCatalog, HorizonParseCredentialKeepsColonsInSecret) TEST(RestCatalog, HorizonCatalogAuthenticatesWithBarePAT) { - TestServer server; - installTableRoutes(*server); - + RestCatalogTestServer server(CatalogShape::TopLevelTable); auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -989,11 +930,10 @@ TEST(RestCatalog, HorizonCatalogAuthenticatesWithBarePAT) /* flat_namespaces */false, context); -<<<<<<< HEAD EXPECT_EQ(catalog.getCatalogType(), DB::DatabaseDataLakeCatalogType::ICEBERG_HORIZON); EXPECT_TRUE(catalog.getStateSnapshot()->client_id.empty()); EXPECT_EQ(catalog.getStateSnapshot()->client_secret, "horizon-pat-without-colon"); - EXPECT_FALSE(catalog.empty()); + EXPECT_FALSE(catalog.empty(/* auth_token */ {})); TableMetadata metadata; metadata.withLocation(); @@ -1054,19 +994,6 @@ TEST(RestCatalog, HorizonApplySettingsChangesBarePAT) DB::SettingsChanges mode_switch; mode_switch.emplace_back("auth_header", "Authorization: Bearer token"); expectThrowsCode([&] { catalog.applySettingsChanges(mode_switch); }, DB::ErrorCodes::BAD_ARGUMENTS); -======= - TableMetadata existing; - EXPECT_TRUE(catalog.tryGetTableMetadata("namespace", "table_a", context, existing)); - EXPECT_TRUE(catalog.existsTable("namespace", "table_a", /* auth_token */ {})); - - TableMetadata missing; - EXPECT_FALSE(catalog.tryGetTableMetadata("namespace", "missing_table", context, missing)); - EXPECT_FALSE(catalog.existsTable("namespace", "missing_table", /* auth_token */ {})); - - TableMetadata unauthorized; - EXPECT_THROW(catalog.tryGetTableMetadata("namespace", "unauthorized_table", context, unauthorized), DB::HTTPException); - EXPECT_THROW(catalog.existsTable("namespace", "unauthorized_table", /* auth_token */ {}), DB::HTTPException); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } #endif diff --git a/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp b/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp index da4d69d6420f6..68529ba9378bf 100644 --- a/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp +++ b/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp @@ -108,10 +108,19 @@ std::shared_ptr makeCatalog( /* oauth_server_uri */ "", /* oauth_server_use_request_body */ true, /* namespaces */ "*", + /* flat_namespaces */ false, context, forwarding); } +DB::Names tableNames(const CatalogTables & tables) +{ + DB::Names names; + for (const auto & table : tables) + names.push_back(table.name); + return names; +} + void loadTable(RestCatalog & catalog, const DB::ForwardedAuthTokenPtr & auth_token) { auto query_context = makeQueryContext(auth_token); @@ -166,7 +175,7 @@ TEST_F(RestCatalogTokenForwarding, PassesUserTokenToCatalog) forwarding.forward_user_token = true; auto catalog = makeCatalog(server, context, forwarding); - ASSERT_EQ(catalog->getTables(makeToken()), DB::Names{"ns.t"}); + ASSERT_EQ(tableNames(catalog->getTables(makeToken())), DB::Names{"ns.t"}); for (const auto * path : {CONFIG_PATH, NAMESPACES_PATH, NS_TABLES_PATH}) { const auto requests = server->requestsTo(path); @@ -192,8 +201,8 @@ TEST_F(RestCatalogTokenForwarding, ExchangesAndCachesEachUserTokenSeparately) { auto auth_token = DB::makeForwardedAuthToken(DB::TokenCredentials(token), "alice"); server->clearRequests(); - ASSERT_EQ(catalog->getTables(auth_token), DB::Names{"ns.t"}); - ASSERT_EQ(catalog->getTables(auth_token), DB::Names{"ns.t"}); + ASSERT_EQ(tableNames(catalog->getTables(auth_token)), DB::Names{"ns.t"}); + ASSERT_EQ(tableNames(catalog->getTables(auth_token)), DB::Names{"ns.t"}); const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); ASSERT_EQ(exchanges.size(), 1u); @@ -250,7 +259,7 @@ TEST_F(RestCatalogTokenForwarding, RejectedConfigReloadDoesNotPublishPreparedUse auto alice = makeToken(); auto context = makeQueryContext(); auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH)); - ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + ASSERT_EQ(tableNames(catalog->getTables(alice)), DB::Names{"ns.t"}); server->clearRequests(); server->setRoute(CONFIG_PATH, [](const RecordedRequest &) { return respondWithStatus(403); }); @@ -260,7 +269,7 @@ TEST_F(RestCatalogTokenForwarding, RejectedConfigReloadDoesNotPublishPreparedUse ASSERT_EQ(server->countRequestsTo(CONFIG_PATH), 1u); EXPECT_EQ(server->requestsTo(CONFIG_PATH).front().header("Authorization"), "Bearer rotated_secret_session"); EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); - ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + ASSERT_EQ(tableNames(catalog->getTables(alice)), DB::Names{"ns.t"}); for (const auto & request : server->requestsTo(NAMESPACES_PATH)) EXPECT_EQ(request.header("Authorization"), "Bearer secret_session"); } @@ -369,7 +378,7 @@ TEST_F(RestCatalogTokenForwarding, ConfigLoadDoesNotRollBackAConcurrentCredentia parked.release(); in_flight.join(); - ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + ASSERT_EQ(tableNames(catalog->getTables(alice)), DB::Names{"ns.t"}); const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); ASSERT_GE(exchanges.size(), 2u); diff --git a/src/IO/S3/Credentials.cpp b/src/IO/S3/Credentials.cpp index 00c364195081c..c5b67eaf236d6 100644 --- a/src/IO/S3/Credentials.cpp +++ b/src/IO/S3/Credentials.cpp @@ -1386,21 +1386,6 @@ void AwsAuthSTSAssumeRoleCredentialsProvider::Reload() LOG_TRACE(logger, "Successfully retrieved credentials"); } -<<<<<<< HEAD -/// Actionable tail of the "server-managed credentials refused" messages. In ClickHouse Cloud the -/// setting is fixed by a profile constraint and cannot be enabled per query, so the Cloud build points -/// to the supported access method (an IAM role) instead of naming a setting the user cannot change. -#if CLICKHOUSE_CLOUD -#define S3_SERVER_CREDENTIALS_HINT \ - "Grant access through an IAM role passed as extra_credentials(role_arn = '...'), or provide explicit " \ - "credentials (use NOSIGN for public buckets). See " \ - "https://clickhouse.com/docs/products/cloud/guides/data-sources/accessing-s3-data-securely" -#else -#define S3_SERVER_CREDENTIALS_HINT \ - "Provide explicit credentials, use NOSIGN for public buckets, grant access through an IAM role passed " \ - "as extra_credentials(role_arn = '...'), or enable the setting `s3_allow_server_credentials_in_user_queries`." -#endif -======= AwsAuthSTSAssumeRoleWithWebIdentityCredentialsProvider::AwsAuthSTSAssumeRoleWithWebIdentityCredentialsProvider( std::string role_arn_, std::string session_name_, @@ -1461,7 +1446,20 @@ std::string AwsAuthSTSAssumeRoleWithWebIdentityCredentialsProvider::getLastError Aws::Utils::Threading::ReaderLockGuard guard(m_reloadLock); return last_error; } ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + +/// Actionable tail of the "server-managed credentials refused" messages. In ClickHouse Cloud the +/// setting is fixed by a profile constraint and cannot be enabled per query, so the Cloud build points +/// to the supported access method (an IAM role) instead of naming a setting the user cannot change. +#if CLICKHOUSE_CLOUD +#define S3_SERVER_CREDENTIALS_HINT \ + "Grant access through an IAM role passed as extra_credentials(role_arn = '...'), or provide explicit " \ + "credentials (use NOSIGN for public buckets). See " \ + "https://clickhouse.com/docs/products/cloud/guides/data-sources/accessing-s3-data-securely" +#else +#define S3_SERVER_CREDENTIALS_HINT \ + "Provide explicit credentials, use NOSIGN for public buckets, grant access through an IAM role passed " \ + "as extra_credentials(role_arn = '...'), or enable the setting `s3_allow_server_credentials_in_user_queries`." +#endif std::shared_ptr getCredentialsProvider( const DB::S3::PocoHTTPClientConfiguration & configuration, diff --git a/src/IO/S3/tests/TestPocoHTTPServer.h b/src/IO/S3/tests/TestPocoHTTPServer.h index 787fff758bddc..51b5568261168 100644 --- a/src/IO/S3/tests/TestPocoHTTPServer.h +++ b/src/IO/S3/tests/TestPocoHTTPServer.h @@ -16,11 +16,7 @@ #include #include #include -<<<<<<< HEAD -======= #include -#include ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) #include /// Keep-alive is disabled so a handler thread exits right after sending the response: it does diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp index 8e5167e826d90..0045f6bf7516d 100644 --- a/src/Interpreters/Session.cpp +++ b/src/Interpreters/Session.cpp @@ -411,19 +411,14 @@ void Session::authenticate(const Credentials & credentials_, const Poco::Net::So chassert(!auth_result.user_name.empty()); prepared_client_info->current_user = auth_result.user_name; prepared_client_info->authenticated_user = auth_result.user_name; -<<<<<<< HEAD prepared_client_info->current_address = Poco::Net::SocketAddress(address); prepared_client_info->connection_address = Poco::Net::SocketAddress(connection_address ? *connection_address : address); -======= - prepared_client_info->current_address = std::make_shared(address); - prepared_client_info->connection_address = std::make_shared(connection_address ? *connection_address : address); if (const auto * token_credentials = typeid_cast(&credentials_)) { if (global_context->getAccessControl().isTokenForwardingEnabled()) forwarded_auth_token = makeForwardedAuthToken(*token_credentials, auth_result.user_name); } ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } void Session::checkIfUserIsStillValid() const diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index ff602cc25606e..a091e58f7f938 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1748,18 +1748,7 @@ void TCPHandler::processTablesStatusRequest() TablesStatusResponse response; try { -<<<<<<< HEAD - auto resolved_id = context_to_resolve_table_names->tryResolveStorageID({table_name.database, table_name.table}); - StoragePtr table = DatabaseCatalog::instance().tryGetTable(resolved_id, context_to_resolve_table_names); - if (!table) - continue; - - TableStatus status; - /// The initiator asks about this table by name, so a lazily loaded replica is loaded to report its delay. - if (auto * replicated_table = castStorage(table, DeferredTable::Load).get()) -======= for (const QualifiedTableName & table_name: request.tables) ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) { auto resolved_id = context_to_resolve_table_names->tryResolveStorageID({table_name.database, table_name.table}); StoragePtr table = DatabaseCatalog::instance().tryGetTable(resolved_id, context_to_resolve_table_names); @@ -1767,7 +1756,8 @@ void TCPHandler::processTablesStatusRequest() continue; TableStatus status; - if (auto * replicated_table = dynamic_cast(table.get())) + /// The initiator asks about this table by name, so a lazily loaded replica is loaded to report its delay. + if (auto * replicated_table = castStorage(table, DeferredTable::Load).get()) { status.is_replicated = true; status.absolute_delay = static_cast(replicated_table->getAbsoluteDelay()); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.cpp index 6a006e5e95650..a736235f214cd 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/AlterDropPartitionExecutor.cpp @@ -936,7 +936,8 @@ bool AlterDropPartitionExecutor::commitMetadataJSON( /// Assume readers can reach the snapshot until the catalog says it rejected the commit. published = true; if (!catalog->updateMetadata( - namespace_name, table_name, components.path_resolver.resolveForCatalog(metadata_info.path), manifest_list_result.new_snapshot)) + namespace_name, table_name, components.path_resolver.resolveForCatalog(metadata_info.path), manifest_list_result.new_snapshot, + context->getForwardedAuthToken())) { if (!catalogCommittedSnapshot(new_snapshot_id)) { diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index 0d08fee840406..223bffc5124b1 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -1152,11 +1152,7 @@ void IcebergMetadata::createInitial( /// validation, so a rejected CREATE leaves no trace in the catalog): a catalog /// that shares its storage view with the data (e.g. SeaweedFS) refuses to create /// a namespace over the plain directory those files would leave behind. -<<<<<<< HEAD - catalog->createNamespaceIfNotExists(namespace_name, /* location */ ""); -======= - catalog->createNamespaceIfNotExists(namespace_name, namespace_location, local_context->getForwardedAuthToken()); ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) + catalog->createNamespaceIfNotExists(namespace_name, /* location */ "", local_context->getForwardedAuthToken()); } if (!catalog_writes_metadata) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp index 25a7662b78853..9ea399c062116 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp @@ -653,20 +653,10 @@ static bool writeMetadataFiles( { auto catalog_filename = path_resolver.resolveForCatalog(metadata_info.path); const auto & [namespace_name, table_name] = DataLake::parseTableName(table_id.getTableName()); - if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot)) + if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot, context->getForwardedAuthToken())) { -<<<<<<< HEAD cleanup(); return false; -======= - auto catalog_filename = path_resolver.resolveForCatalog(metadata_info.path); - const auto & [namespace_name, table_name] = DataLake::parseTableName(table_id.getTableName()); - if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot, context->getForwardedAuthToken())) - { - cleanup(); - return false; - } ->>>>>>> 893adc4f4879 (Merge pull request #2329 from Altinity/feature/antalya-26.6/aouth-forward-to-datalake) } } }