From 54370092e65db2a0b01483907625125334fcacb2 Mon Sep 17 00:00:00 2001 From: Nicholas Jiang Date: Wed, 22 Jul 2026 10:15:24 +0800 Subject: [PATCH] feat(rest): support rest catalog with database/table operations and snapshot listing Add a REST catalog implementation selected via the "metastore=rest" option: - RestHttpClient: blocking libcurl client with exponential-backoff retries (429/503 retried for all methods, transport errors only for idempotent ones, honoring a Retry-After header in both the delta-seconds and the HTTP-date form when it yields a positive delay, with a locale-independent date parse; a server closing the connection without responding is not retried) and per-request debug logging. - RestApi: HTTP + JSON layer with "/v1/config" option merging (overrides > client options > defaults), "header." options as request headers, paged listing and error-to-Status mapping that prefers the code of the parsed error body over the http status, redacts sensitive server messages, never echoes response bodies into error messages (they may carry credentials), carries the request id (x-request-id, falling back to any request-id header) and attaches a RestErrorDetail with the mapped code so callers can distinguish e.g. authentication failures programmatically. - RestCatalog: database/table operations and snapshot listing, "table-default." option defaults, system/branch table checks and schema conversion with the computed highestFieldId, rejecting duplicated field ids at any nesting level. Branch identifiers are passed through to the server so it resolves the branch and returns the branch's own schema (the default branch "main" resolves to the bare table), snapshots of a branch are listed under the branch object name, and the "sys" database serves the local global system tables like FileSystemCatalog. - Bear token authentication provider. - CatalogOptions: catalog-level option keys (metastore, uri, token, token.provider) and the "table-default." option prefix in a dedicated public header, keeping them separate from the table-level CoreOptions. - CatalogUtils: system database and system/branch table checks shared by FileSystemCatalog and RestCatalog; FileSystemCatalog's create/drop/rename checks now use them instead of inline messages. - SpecialFieldIds::SYSTEM_FIELD_ID_START names the reserved system field id bound, which is excluded from the highest field id. The pieces overlapping with the object store file systems are unified under common/utils: UrlUtils carries both URL encoding flavors (form-urlencoded for the REST api, RFC 3986 for S3), the generic HTTP client moved there from common/fs, and the rest client reuses its curl global-init guard, ScopeGuard cleanup and header-line parsing. libcurl detection in CMake is a single find_package guarded by PAIMON_ENABLE_S3 OR PAIMON_ENABLE_REST. libcurl is used from the system rather than bundled, so the new PAIMON_ENABLE_REST option defaults to OFF like the other optional components with external requirements and is documented alongside them. Co-Authored-By: Claude Fable 5 --- CMakeLists.txt | 11 +- ci/scripts/build_paimon.sh | 1 + docs/source/building.rst | 1 + include/paimon/catalog_options.h | 46 + include/paimon/utils/special_field_ids.h | 4 + src/paimon/CMakeLists.txt | 55 +- src/paimon/common/catalog_options.cpp | 27 + .../common/{fs => utils}/http_client.cpp | 44 +- src/paimon/common/{fs => utils}/http_client.h | 8 + src/paimon/common/utils/http_client_test.cpp | 47 + src/paimon/common/utils/url_utils.cpp | 134 +++ src/paimon/common/utils/url_utils.h | 54 + src/paimon/common/utils/url_utils_test.cpp | 81 ++ src/paimon/core/catalog/catalog.cpp | 21 + src/paimon/core/catalog/catalog_utils.cpp | 56 + src/paimon/core/catalog/catalog_utils.h | 43 + .../core/catalog/file_system_catalog.cpp | 54 +- .../core/catalog/file_system_catalog_test.cpp | 33 +- src/paimon/core/schema/table_schema.cpp | 63 ++ src/paimon/core/schema/table_schema.h | 8 + src/paimon/core/schema/table_schema_test.cpp | 41 + src/paimon/core/snapshot.cpp | 23 + src/paimon/core/snapshot.h | 3 + src/paimon/core/snapshot_test.cpp | 33 + src/paimon/fs/s3/s3_file_system.cpp | 55 +- src/paimon/fs/s3/s3_file_system.h | 2 +- src/paimon/rest/mock_rest_server.cpp | 215 ++++ src/paimon/rest/mock_rest_server.h | 92 ++ src/paimon/rest/resource_paths.cpp | 64 ++ src/paimon/rest/resource_paths.h | 45 + src/paimon/rest/rest_api.cpp | 294 ++++++ src/paimon/rest/rest_api.h | 139 +++ src/paimon/rest/rest_auth.cpp | 56 + src/paimon/rest/rest_auth.h | 66 ++ src/paimon/rest/rest_catalog.cpp | 431 ++++++++ src/paimon/rest/rest_catalog.h | 101 ++ src/paimon/rest/rest_catalog_test.cpp | 997 ++++++++++++++++++ src/paimon/rest/rest_http_client.cpp | 362 +++++++ src/paimon/rest/rest_http_client.h | 125 +++ src/paimon/rest/rest_http_client_test.cpp | 421 ++++++++ src/paimon/rest/rest_messages.cpp | 373 +++++++ src/paimon/rest/rest_messages.h | 374 +++++++ src/paimon/rest/rest_messages_test.cpp | 293 +++++ src/paimon/rest/rest_util.cpp | 92 ++ src/paimon/rest/rest_util.h | 54 + 45 files changed, 5407 insertions(+), 135 deletions(-) create mode 100644 include/paimon/catalog_options.h create mode 100644 src/paimon/common/catalog_options.cpp rename src/paimon/common/{fs => utils}/http_client.cpp (90%) rename src/paimon/common/{fs => utils}/http_client.h (80%) create mode 100644 src/paimon/common/utils/http_client_test.cpp create mode 100644 src/paimon/common/utils/url_utils.cpp create mode 100644 src/paimon/common/utils/url_utils.h create mode 100644 src/paimon/common/utils/url_utils_test.cpp create mode 100644 src/paimon/core/catalog/catalog_utils.cpp create mode 100644 src/paimon/core/catalog/catalog_utils.h create mode 100644 src/paimon/rest/mock_rest_server.cpp create mode 100644 src/paimon/rest/mock_rest_server.h create mode 100644 src/paimon/rest/resource_paths.cpp create mode 100644 src/paimon/rest/resource_paths.h create mode 100644 src/paimon/rest/rest_api.cpp create mode 100644 src/paimon/rest/rest_api.h create mode 100644 src/paimon/rest/rest_auth.cpp create mode 100644 src/paimon/rest/rest_auth.h create mode 100644 src/paimon/rest/rest_catalog.cpp create mode 100644 src/paimon/rest/rest_catalog.h create mode 100644 src/paimon/rest/rest_catalog_test.cpp create mode 100644 src/paimon/rest/rest_http_client.cpp create mode 100644 src/paimon/rest/rest_http_client.h create mode 100644 src/paimon/rest/rest_http_client_test.cpp create mode 100644 src/paimon/rest/rest_messages.cpp create mode 100644 src/paimon/rest/rest_messages.h create mode 100644 src/paimon/rest/rest_messages_test.cpp create mode 100644 src/paimon/rest/rest_util.cpp create mode 100644 src/paimon/rest/rest_util.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 1d7d00002..3dd22f37a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -66,9 +66,17 @@ option(PAIMON_ENABLE_LUMINA "Whether to enable lumina vector index" OFF) option(PAIMON_ENABLE_LUCENE "Whether to enable lucene index" OFF) option(PAIMON_ENABLE_TANTIVY "Whether to enable tantivy-fulltext global index (Rust FFI, experimental)" OFF) +option(PAIMON_ENABLE_REST "Whether to enable the rest catalog (requires libcurl)" OFF) if(PAIMON_ENABLE_ORC) add_definitions(-DPAIMON_ENABLE_ORC) endif() +if(PAIMON_ENABLE_REST) + add_definitions(-DPAIMON_ENABLE_REST) +endif() +# libcurl backs the HTTP client shared by the S3 file system and the rest catalog. +if(PAIMON_ENABLE_S3 OR PAIMON_ENABLE_REST) + find_package(CURL REQUIRED) +endif() if(PAIMON_ENABLE_AVRO) add_definitions(-DPAIMON_ENABLE_AVRO) endif() @@ -487,9 +495,6 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/PaimonConfig.cmake" config_summary_message() -if(PAIMON_ENABLE_S3) - find_package(CURL REQUIRED) -endif() add_subdirectory(src/paimon) add_subdirectory(src/paimon/fs/local) add_subdirectory(src/paimon/fs/jindo) diff --git a/ci/scripts/build_paimon.sh b/ci/scripts/build_paimon.sh index 339d6b163..872fc0f90 100755 --- a/ci/scripts/build_paimon.sh +++ b/ci/scripts/build_paimon.sh @@ -130,6 +130,7 @@ CMAKE_ARGS=( "-DPAIMON_ENABLE_LUMINA=${ENABLE_LUMINA}" "-DPAIMON_ENABLE_LUCENE=ON" "-DPAIMON_ENABLE_TANTIVY=${ENABLE_TANTIVY}" + "-DPAIMON_ENABLE_REST=ON" "-DPAIMON_LINT_GIT_TARGET_COMMIT=${lint_git_target_commit}" ) diff --git a/docs/source/building.rst b/docs/source/building.rst index 66f932414..87e39ae16 100644 --- a/docs/source/building.rst +++ b/docs/source/building.rst @@ -145,6 +145,7 @@ boolean flags to ``cmake``. * ``-DPAIMON_ENABLE_AVRO=ON``: Apache Avro libraries and Paimon integration * ``-DPAIMON_ENABLE_JINDO=ON``: Support for Alibaba Jindo filesystems * ``-DPAIMON_ENABLE_LUMINA=ON``: Support for Lumina vector index, lumina is only supported on gcc9 or higher. +* ``-DPAIMON_ENABLE_REST=ON``: Support for the REST catalog (``metastore=rest``), requires the libcurl development package. Third-party dependency source ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/include/paimon/catalog_options.h b/include/paimon/catalog_options.h new file mode 100644 index 000000000..49b61b8ac --- /dev/null +++ b/include/paimon/catalog_options.h @@ -0,0 +1,46 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "paimon/visibility.h" + +namespace paimon { + +/// Catalog-level configuration option keys; table-level keys live in `Options`. +struct PAIMON_EXPORT CatalogOptions { + /// "metastore" - Metastore of the paimon catalog. + /// Supported values are "filesystem" (default) and "rest". + static const char METASTORE[]; + + /// "uri" - Server url of the REST catalog. Only used when METASTORE is "rest". + static const char URI[]; + + /// "token" - Token of the "bear" token provider of the REST catalog. + static const char TOKEN[]; + + /// "token.provider" - Authentication provider of the REST catalog. + /// Currently only "bear" is supported ("bear" is the protocol's historical + /// spelling of "bearer", do not "fix" it). + static const char TOKEN_PROVIDER[]; + + /// "table-default." - Prefix of the catalog options that provide table option + /// defaults: "table-default.=" applies "=" to a created + /// table when the caller left "" unset. + static const char TABLE_DEFAULT_OPTION_PREFIX[]; +}; + +} // namespace paimon diff --git a/include/paimon/utils/special_field_ids.h b/include/paimon/utils/special_field_ids.h index c5c796150..def91f390 100644 --- a/include/paimon/utils/special_field_ids.h +++ b/include/paimon/utils/special_field_ids.h @@ -39,6 +39,10 @@ class SpecialFieldIds { /// Special field ID reserved for index score. Value: CPP_FIELD_ID_END - 1 static const int32_t INDEX_SCORE = CPP_FIELD_ID_END - 1; + + /// The lowest field ID reserved for system fields; IDs at or above this bound are + /// excluded when computing the highest field ID of a schema. Value: INT32_MAX / 2 + static const int32_t SYSTEM_FIELD_ID_START = std::numeric_limits::max() / 2; }; } // namespace paimon diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 409c58a50..31c22b283 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -13,6 +13,7 @@ # limitations under the License. set(PAIMON_COMMON_SRCS + common/catalog_options.cpp common/compression/block_compression_factory.cpp common/compression/block_compressor.cpp common/compression/block_decompressor.cpp @@ -176,12 +177,18 @@ set(PAIMON_COMMON_SRCS common/utils/roaring_bitmap64.cpp common/utils/row_range_index.cpp common/utils/status.cpp - common/utils/string_utils.cpp) + common/utils/string_utils.cpp + common/utils/url_utils.cpp) +# The shared HTTP client is used by both the object store file systems and the +# rest catalog. +set(PAIMON_CURL_LINK_LIBS) +if(PAIMON_ENABLE_S3 OR PAIMON_ENABLE_REST) + list(APPEND PAIMON_COMMON_SRCS common/utils/http_client.cpp) + set(PAIMON_CURL_LINK_LIBS CURL::libcurl) +endif() if(PAIMON_ENABLE_S3) - list(APPEND PAIMON_COMMON_SRCS common/fs/http_client.cpp - common/fs/object_store_file_system.cpp) - set(PAIMON_OBJECT_STORE_LINK_LIBS CURL::libcurl) + list(APPEND PAIMON_COMMON_SRCS common/fs/object_store_file_system.cpp) endif() set(PAIMON_CORE_SRCS @@ -224,6 +231,7 @@ set(PAIMON_CORE_SRCS core/casting/timestamp_to_timestamp_cast_executor.cpp core/casting/casting_utils.cpp core/catalog/catalog.cpp + core/catalog/catalog_utils.cpp core/catalog/file_system_catalog.cpp core/catalog/identifier.cpp core/core_options.cpp @@ -403,6 +411,18 @@ set(PAIMON_CORE_SRCS core/utils/special_field_ids.cpp core/utils/tag_manager.cpp) +if(PAIMON_ENABLE_REST) + list(APPEND + PAIMON_CORE_SRCS + rest/rest_http_client.cpp + rest/resource_paths.cpp + rest/rest_api.cpp + rest/rest_auth.cpp + rest/rest_catalog.cpp + rest/rest_messages.cpp + rest/rest_util.cpp) +endif() + add_paimon_lib(paimon SOURCES ${PAIMON_COMMON_SRCS} @@ -416,7 +436,7 @@ add_paimon_lib(paimon xxhash Threads::Threads RapidJSON - ${PAIMON_OBJECT_STORE_LINK_LIBS} + ${PAIMON_CURL_LINK_LIBS} STATIC_LINK_LIBS arrow tbb @@ -426,7 +446,7 @@ add_paimon_lib(paimon xxhash Threads::Threads RapidJSON - ${PAIMON_OBJECT_STORE_LINK_LIBS} + ${PAIMON_CURL_LINK_LIBS} SHARED_LINK_FLAGS ${PAIMON_VERSION_SCRIPT_FLAGS}) @@ -434,6 +454,13 @@ add_subdirectory(common/file_index) add_subdirectory(common/global_index) if(PAIMON_BUILD_TESTS) + # The shared HTTP client is compiled only for the components that need libcurl, + # so its test follows the same gate. + set(PAIMON_COMMON_HTTP_CLIENT_TEST_SRCS) + if(PAIMON_ENABLE_S3 OR PAIMON_ENABLE_REST) + set(PAIMON_COMMON_HTTP_CLIENT_TEST_SRCS common/utils/http_client_test.cpp) + endif() + add_paimon_test(memory_test SOURCES common/memory/memory_pool_test.cpp @@ -594,6 +621,8 @@ if(PAIMON_BUILD_TESTS) common/utils/status_test.cpp common/utils/stream_utils_test.cpp common/utils/string_utils_test.cpp + common/utils/url_utils_test.cpp + ${PAIMON_COMMON_HTTP_CLIENT_TEST_SRCS} common/utils/range_test.cpp common/utils/uuid_test.cpp common/utils/decimal_utils_test.cpp @@ -870,4 +899,18 @@ if(PAIMON_BUILD_TESTS) EXTRA_INCLUDES ${JINDOSDK_INCLUDE_DIR}) + if(PAIMON_ENABLE_REST) + add_paimon_test(rest_test + SOURCES + rest/rest_http_client_test.cpp + rest/mock_rest_server.cpp + rest/rest_catalog_test.cpp + rest/rest_messages_test.cpp + STATIC_LINK_LIBS + paimon_shared + test_utils_static + ${TEST_STATIC_LINK_LIBS} + ${GTEST_LINK_TOOLCHAIN}) + endif() + endif() diff --git a/src/paimon/common/catalog_options.cpp b/src/paimon/common/catalog_options.cpp new file mode 100644 index 000000000..6e89e80a8 --- /dev/null +++ b/src/paimon/common/catalog_options.cpp @@ -0,0 +1,27 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/catalog_options.h" + +namespace paimon { + +const char CatalogOptions::METASTORE[] = "metastore"; +const char CatalogOptions::URI[] = "uri"; +const char CatalogOptions::TOKEN[] = "token"; +const char CatalogOptions::TOKEN_PROVIDER[] = "token.provider"; +const char CatalogOptions::TABLE_DEFAULT_OPTION_PREFIX[] = "table-default."; + +} // namespace paimon diff --git a/src/paimon/common/fs/http_client.cpp b/src/paimon/common/utils/http_client.cpp similarity index 90% rename from src/paimon/common/fs/http_client.cpp rename to src/paimon/common/utils/http_client.cpp index afd816ce8..e9a39acf1 100644 --- a/src/paimon/common/fs/http_client.cpp +++ b/src/paimon/common/utils/http_client.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "paimon/common/fs/http_client.h" +#include "paimon/common/utils/http_client.h" #include @@ -44,11 +44,6 @@ class CurlGlobalGuard { } }; -std::shared_ptr GetCurlGlobalGuard() { - static auto guard = std::make_shared(); - return guard; -} - void TrimHttpWhitespace(std::string* value) { constexpr char kHttpWhitespace[] = " \t\r\n"; size_t begin = value->find_first_not_of(kHttpWhitespace); @@ -92,16 +87,7 @@ size_t WriteCallback(char* data, size_t size, size_t count, void* user_data) { size_t HeaderCallback(char* data, size_t size, size_t count, void* user_data) { auto* context = static_cast(user_data); size_t bytes = size * count; - std::string line(data, bytes); - size_t colon = line.find(':'); - if (colon != std::string::npos) { - std::string name = line.substr(0, colon); - TrimHttpWhitespace(&name); - name = StringUtils::ToLowerCase(name); - std::string value = line.substr(colon + 1); - TrimHttpWhitespace(&value); - context->response.headers[name] = std::move(value); - } + ParseHttpHeaderLine(data, bytes, &context->response.headers); return bytes; } @@ -116,9 +102,31 @@ bool IsRetryable(CURLcode code, int64_t status_code) { } // namespace +std::shared_ptr EnsureCurlGlobalInit() { + static auto guard = std::make_shared(); + return guard; +} + +void ParseHttpHeaderLine(const char* data, size_t size, HttpHeaders* headers) { + std::string line(data, size); + size_t colon = line.find(':'); + if (colon == std::string::npos) { + return; + } + std::string name = line.substr(0, colon); + TrimHttpWhitespace(&name); + if (name.empty()) { + return; + } + name = StringUtils::ToLowerCase(name); + std::string value = line.substr(colon + 1); + TrimHttpWhitespace(&value); + (*headers)[name] = std::move(value); +} + class CurlHttpClient::Impl { public: - Impl() : guard_(GetCurlGlobalGuard()) {} + Impl() : guard_(EnsureCurlGlobalInit()) {} ~Impl() { for (CURL* handle : handles_) { @@ -143,7 +151,7 @@ class CurlHttpClient::Impl { } private: - std::shared_ptr guard_; + std::shared_ptr guard_; mutable std::mutex mutex_; mutable std::vector handles_; }; diff --git a/src/paimon/common/fs/http_client.h b/src/paimon/common/utils/http_client.h similarity index 80% rename from src/paimon/common/fs/http_client.h rename to src/paimon/common/utils/http_client.h index 586972223..0a11e39cd 100644 --- a/src/paimon/common/fs/http_client.h +++ b/src/paimon/common/utils/http_client.h @@ -31,6 +31,14 @@ enum class HttpMethod { HEAD, GET }; using HttpHeaders = std::map; using HttpBodyConsumer = std::function; +/// Ensures libcurl's global state is initialized; the returned guard keeps it alive. +PAIMON_EXPORT std::shared_ptr EnsureCurlGlobalInit(); + +/// Parses one raw HTTP header line into `headers`, lower-casing the name and trimming +/// HTTP whitespace around the name and value; lines without a ':' or with an empty +/// name are ignored. +PAIMON_EXPORT void ParseHttpHeaderLine(const char* data, size_t size, HttpHeaders* headers); + struct HttpRequest { HttpMethod method = HttpMethod::GET; std::string url; diff --git a/src/paimon/common/utils/http_client_test.cpp b/src/paimon/common/utils/http_client_test.cpp new file mode 100644 index 000000000..5bf3ce232 --- /dev/null +++ b/src/paimon/common/utils/http_client_test.cpp @@ -0,0 +1,47 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/utils/http_client.h" + +#include + +#include "gtest/gtest.h" + +namespace paimon::test { + +TEST(HttpClientUtilTest, ParseHttpHeaderLine) { + HttpHeaders headers; + const std::string line = "Content-Type: application/json\r\n"; + ParseHttpHeaderLine(line.data(), line.size(), &headers); + ASSERT_EQ((HttpHeaders{{"content-type", "application/json"}}), headers); + + const std::string overwrite = "content-type: text/PLAIN \r\n"; + ParseHttpHeaderLine(overwrite.data(), overwrite.size(), &headers); + ASSERT_EQ((HttpHeaders{{"content-type", "text/PLAIN"}}), headers); + + // Lines without a colon (like the status line) and empty names are ignored. + const std::string status_line = "HTTP/1.1 200 OK\r\n"; + ParseHttpHeaderLine(status_line.data(), status_line.size(), &headers); + const std::string empty_name = ": value\r\n"; + ParseHttpHeaderLine(empty_name.data(), empty_name.size(), &headers); + ASSERT_EQ(1, headers.size()); + + const std::string empty_value = "x-empty:\r\n"; + ParseHttpHeaderLine(empty_value.data(), empty_value.size(), &headers); + ASSERT_EQ("", headers.at("x-empty")); +} + +} // namespace paimon::test diff --git a/src/paimon/common/utils/url_utils.cpp b/src/paimon/common/utils/url_utils.cpp new file mode 100644 index 000000000..33cb00836 --- /dev/null +++ b/src/paimon/common/utils/url_utils.cpp @@ -0,0 +1,134 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/utils/url_utils.h" + +#include + +namespace paimon { + +namespace { + +bool IsAlphaNumeric(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9'); +} + +/// The characters `EncodeString` keeps unencoded. +bool IsFormUnreservedChar(char c) { + return IsAlphaNumeric(c) || c == '.' || c == '-' || c == '*' || c == '_'; +} + +/// The unreserved characters of RFC 3986. +bool IsRfc3986UnreservedChar(char c) { + return IsAlphaNumeric(c) || c == '-' || c == '.' || c == '_' || c == '~'; +} + +void AppendPercentEncoded(char c, std::string* out) { + char buf[4]; + std::snprintf(buf, sizeof(buf), "%%%02X", static_cast(c)); + out->append(buf); +} + +int32_t HexValue(char c) { + if (c >= '0' && c <= '9') { + return c - '0'; + } + if (c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + if (c >= 'A' && c <= 'F') { + return c - 'A' + 10; + } + return -1; +} + +} // namespace + +std::string UrlUtils::EncodeString(const std::string& input) { + std::string encoded; + encoded.reserve(input.size()); + for (char c : input) { + if (IsFormUnreservedChar(c)) { + encoded.push_back(c); + } else if (c == ' ') { + encoded.push_back('+'); + } else { + AppendPercentEncoded(c, &encoded); + } + } + return encoded; +} + +std::string UrlUtils::DecodeString(const std::string& input) { + std::string decoded; + decoded.reserve(input.size()); + for (size_t i = 0; i < input.size(); i++) { + char c = input[i]; + if (c == '+') { + decoded.push_back(' '); + } else if (c == '%' && i + 2 < input.size()) { + int32_t high = HexValue(input[i + 1]); + int32_t low = HexValue(input[i + 2]); + if (high >= 0 && low >= 0) { + decoded.push_back(static_cast((high << 4) | low)); + i += 2; + } else { + decoded.push_back(c); + } + } else { + decoded.push_back(c); + } + } + return decoded; +} + +std::string UrlUtils::PercentEncode(std::string_view value, bool preserve_slash) { + std::string encoded; + encoded.reserve(value.size()); + for (char c : value) { + if (IsRfc3986UnreservedChar(c) || (preserve_slash && c == '/')) { + encoded.push_back(c); + } else { + AppendPercentEncoded(c, &encoded); + } + } + return encoded; +} + +Result UrlUtils::PercentDecode(std::string_view value) { + std::string decoded; + decoded.reserve(value.size()); + for (size_t i = 0; i < value.size(); i++) { + char c = value[i]; + if (c == '%') { + if (i + 2 >= value.size()) { + return Status::IOError("invalid percent encoding in URL component"); + } + int32_t high = HexValue(value[i + 1]); + int32_t low = HexValue(value[i + 2]); + if (high < 0 || low < 0) { + return Status::IOError("invalid percent encoding in URL component"); + } + decoded.push_back(static_cast((high << 4) | low)); + i += 2; + } else { + decoded.push_back(c); + } + } + return decoded; +} + +} // namespace paimon diff --git a/src/paimon/common/utils/url_utils.h b/src/paimon/common/utils/url_utils.h new file mode 100644 index 000000000..481acdf1e --- /dev/null +++ b/src/paimon/common/utils/url_utils.h @@ -0,0 +1,54 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/result.h" +#include "paimon/visibility.h" + +namespace paimon { + +/// URL encoding and decoding shared by the REST catalog and the object store file +/// systems. +class PAIMON_EXPORT UrlUtils { + public: + UrlUtils() = delete; + ~UrlUtils() = delete; + + /// URL-encode a string in the `application/x-www-form-urlencoded` flavor (UTF-8): + /// alphanumeric characters and ".", "-", "*", "_" are kept, a space becomes "+", all + /// other bytes become percent-encoded "%XX". This is the flavor the REST catalog + /// server expects; note it escapes "~", unlike RFC 3986. + static std::string EncodeString(const std::string& input); + + /// Decodes `EncodeString` output ('+' back to space, "%XX" to the byte); malformed + /// escape sequences are kept as-is instead of failing. + static std::string DecodeString(const std::string& input); + + /// Percent-encode a URL component with RFC 3986 rules: alphanumeric characters and + /// "-", ".", "_", "~" are kept, every other byte (including a space) becomes "%XX". + /// With `preserve_slash`, '/' is kept as-is so an object key keeps its path shape. + static std::string PercentEncode(std::string_view value, bool preserve_slash = false); + + /// Strict inverse of `PercentEncode`: "%XX" becomes the byte, '+' is kept as-is, and + /// a '%' not followed by two hex digits is an error. + static Result PercentDecode(std::string_view value); +}; + +} // namespace paimon diff --git a/src/paimon/common/utils/url_utils_test.cpp b/src/paimon/common/utils/url_utils_test.cpp new file mode 100644 index 000000000..e85860e45 --- /dev/null +++ b/src/paimon/common/utils/url_utils_test.cpp @@ -0,0 +1,81 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/utils/url_utils.h" + +#include "gtest/gtest.h" +#include "paimon/status.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +TEST(UrlUtilsTest, EncodeString) { + ASSERT_EQ("abcDEF012.-*_", UrlUtils::EncodeString("abcDEF012.-*_")); + ASSERT_EQ("a+b", UrlUtils::EncodeString("a b")); + ASSERT_EQ("a%2Fb", UrlUtils::EncodeString("a/b")); + ASSERT_EQ("a%3Db%26c", UrlUtils::EncodeString("a=b&c")); + // '~' is the character distinguishing the two encoding flavors: EncodeString + // (form-urlencoded) escapes it while PercentEncode (RFC 3986) keeps it. + ASSERT_EQ("%7E", UrlUtils::EncodeString("~")); + ASSERT_EQ("a%2Bb", UrlUtils::EncodeString("a+b")); + ASSERT_EQ("%E4%B8%AD", UrlUtils::EncodeString("中")); +} + +TEST(UrlUtilsTest, DecodeString) { + ASSERT_EQ("a/b", UrlUtils::DecodeString("a%2Fb")); + ASSERT_EQ("a b", UrlUtils::DecodeString("a+b")); + ASSERT_EQ("中", UrlUtils::DecodeString("%E4%B8%AD")); + ASSERT_EQ("a/", UrlUtils::DecodeString("a%2F")); + // Malformed escapes are kept as-is. + ASSERT_EQ("a%", UrlUtils::DecodeString("a%")); + ASSERT_EQ("%2", UrlUtils::DecodeString("%2")); + ASSERT_EQ("%ZZ", UrlUtils::DecodeString("%ZZ")); + ASSERT_EQ("100%", UrlUtils::DecodeString("100%")); +} + +TEST(UrlUtilsTest, PercentEncode) { + ASSERT_EQ("abcDEF012-._~", UrlUtils::PercentEncode("abcDEF012-._~")); + ASSERT_EQ("a%20b", UrlUtils::PercentEncode("a b")); + ASSERT_EQ("a%2Ab", UrlUtils::PercentEncode("a*b")); + ASSERT_EQ("a%2Fb", UrlUtils::PercentEncode("a/b")); + ASSERT_EQ("a/b%3Dc", UrlUtils::PercentEncode("a/b=c", /*preserve_slash=*/true)); + ASSERT_EQ("%E4%B8%AD", UrlUtils::PercentEncode("中")); +} + +TEST(UrlUtilsTest, PercentDecode) { + ASSERT_OK_AND_ASSIGN(std::string decoded, UrlUtils::PercentDecode("a%2Fb%20c")); + ASSERT_EQ("a/b c", decoded); + // '+' is form encoding only; percent decoding keeps it. + ASSERT_OK_AND_ASSIGN(decoded, UrlUtils::PercentDecode("a+b")); + ASSERT_EQ("a+b", decoded); + ASSERT_OK_AND_ASSIGN(decoded, UrlUtils::PercentDecode("%e4%b8%ad")); + ASSERT_EQ("中", decoded); + // Malformed escapes are an error. + ASSERT_NOK(UrlUtils::PercentDecode("a%").status()); + ASSERT_NOK(UrlUtils::PercentDecode("%2").status()); + ASSERT_NOK(UrlUtils::PercentDecode("%ZZ").status()); + ASSERT_NOK(UrlUtils::PercentDecode("100%").status()); +} + +TEST(UrlUtilsTest, RoundTrip) { + const std::string input = "db 1/table$branch_b1=中%"; + ASSERT_EQ(input, UrlUtils::DecodeString(UrlUtils::EncodeString(input))); + ASSERT_OK_AND_ASSIGN(std::string decoded, + UrlUtils::PercentDecode(UrlUtils::PercentEncode(input))); + ASSERT_EQ(input, decoded); +} + +} // namespace paimon::test diff --git a/src/paimon/core/catalog/catalog.cpp b/src/paimon/core/catalog/catalog.cpp index 80f93cb29..5b704d866 100644 --- a/src/paimon/core/catalog/catalog.cpp +++ b/src/paimon/core/catalog/catalog.cpp @@ -18,8 +18,13 @@ #include +#include "paimon/catalog_options.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/core/catalog/file_system_catalog.h" #include "paimon/core/core_options.h" +#ifdef PAIMON_ENABLE_REST +#include "paimon/rest/rest_catalog.h" +#endif namespace paimon { @@ -31,6 +36,22 @@ const char Catalog::DB_LOCATION_PROP[] = "location"; Result> Catalog::Create(const std::string& root_path, const std::map& options, const std::shared_ptr& file_system) { + std::string metastore = "filesystem"; + auto metastore_iter = options.find(CatalogOptions::METASTORE); + if (metastore_iter != options.end()) { + metastore = StringUtils::ToLowerCase(metastore_iter->second); + } + if (metastore == "rest") { +#ifdef PAIMON_ENABLE_REST + return RestCatalog::Create(root_path, options, file_system); +#else + return Status::NotImplemented( + "the rest catalog requires building paimon with PAIMON_ENABLE_REST=ON"); +#endif + } + if (metastore != "filesystem") { + return Status::Invalid("unsupported metastore: ", metastore); + } PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options, file_system)); return std::make_unique(core_options.GetFileSystem(), root_path, options); } diff --git a/src/paimon/core/catalog/catalog_utils.cpp b/src/paimon/core/catalog/catalog_utils.cpp new file mode 100644 index 000000000..99352115e --- /dev/null +++ b/src/paimon/core/catalog/catalog_utils.cpp @@ -0,0 +1,56 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/catalog/catalog_utils.h" + +#include + +#include "fmt/format.h" +#include "paimon/catalog/catalog.h" +#include "paimon/result.h" + +namespace paimon { + +bool CatalogUtils::IsSystemDatabase(const std::string& db_name) { + return db_name == Catalog::SYSTEM_DATABASE_NAME; +} + +Status CatalogUtils::CheckNotSystemTable(const Identifier& identifier, const std::string& action) { + if (IsSystemDatabase(identifier.GetDatabaseName())) { + return Status::Invalid( + fmt::format("Cannot '{}' for system table '{}', please use data table.", action, + identifier.ToString())); + } + PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); + if (is_system_table) { + return Status::Invalid( + fmt::format("Cannot '{}' for system table '{}', please use data table.", action, + identifier.ToString())); + } + return Status::OK(); +} + +Status CatalogUtils::CheckNotBranch(const Identifier& identifier, const std::string& action) { + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + if (branch) { + return Status::Invalid(fmt::format( + "Cannot '{}' for branch table '{}', please modify the table with the default branch.", + action, identifier.ToString())); + } + return Status::OK(); +} + +} // namespace paimon diff --git a/src/paimon/core/catalog/catalog_utils.h b/src/paimon/core/catalog/catalog_utils.h new file mode 100644 index 000000000..f2602e1ec --- /dev/null +++ b/src/paimon/core/catalog/catalog_utils.h @@ -0,0 +1,43 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/status.h" + +namespace paimon { + +/// Checks shared by the catalog implementations. +class CatalogUtils { + public: + CatalogUtils() = delete; + ~CatalogUtils() = delete; + + /// Returns whether `db_name` is the reserved system database "sys". + static bool IsSystemDatabase(const std::string& db_name); + + /// Fails when `identifier` denotes a system table or a table inside the system + /// database. + static Status CheckNotSystemTable(const Identifier& identifier, const std::string& action); + + /// Fails when `identifier` carries a branch. + static Status CheckNotBranch(const Identifier& identifier, const std::string& action); +}; + +} // namespace paimon diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index ec142736f..ca6d3d886 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -29,6 +29,7 @@ #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" +#include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/core_options.h" #include "paimon/core/snapshot.h" #include "paimon/core/table/system/global_system_tables.h" @@ -134,12 +135,7 @@ Status FileSystemCatalog::CreateTable(const Identifier& identifier, ArrowSchema* const std::vector& primary_keys, const std::map& options, bool ignore_if_exists) { - PAIMON_ASSIGN_OR_RAISE(bool is_system_table, IsSystemTable(identifier)); - if (is_system_table) { - return Status::Invalid( - fmt::format("Cannot create table for system table {}, please use data table.", - identifier.ToString())); - } + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(identifier, "create table")); PAIMON_ASSIGN_OR_RAISE(bool db_exist, DatabaseExists(identifier.GetDatabaseName())); if (!db_exist) { return Status::Invalid( @@ -197,7 +193,7 @@ const std::map& FileSystemCatalog::GetOptions() const } bool FileSystemCatalog::IsSystemDatabase(const std::string& db_name) { - return db_name == SYSTEM_DATABASE_NAME; + return CatalogUtils::IsSystemDatabase(db_name); } Result FileSystemCatalog::IsSpecifiedSystemTable(const Identifier& identifier) { @@ -425,10 +421,7 @@ Status FileSystemCatalog::DropTableImpl(const Identifier& identifier, } Status FileSystemCatalog::DropTable(const Identifier& identifier, bool ignore_if_not_exists) { - PAIMON_ASSIGN_OR_RAISE(bool is_system_table, IsSystemTable(identifier)); - if (is_system_table) { - return Status::Invalid(fmt::format("Cannot drop system table {}.", identifier.ToString())); - } + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(identifier, "drop table")); PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); PAIMON_ASSIGN_OR_RAISE(bool exist, fs_->Exists(table_path)); if (!exist) { @@ -488,12 +481,8 @@ Status FileSystemCatalog::DropTable(const Identifier& identifier, bool ignore_if Status FileSystemCatalog::RenameTable(const Identifier& from_table, const Identifier& to_table, bool ignore_if_not_exists) { - PAIMON_ASSIGN_OR_RAISE(bool is_from_system_table, IsSystemTable(from_table)); - PAIMON_ASSIGN_OR_RAISE(bool is_to_system_table, IsSystemTable(to_table)); - if (is_from_system_table || is_to_system_table) { - return Status::Invalid(fmt::format("Cannot rename system table {} or {}.", - from_table.ToString(), to_table.ToString())); - } + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(from_table, "rename table")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(to_table, "rename table")); if (from_table.GetDatabaseName() != to_table.GetDatabaseName()) { return Status::Invalid( @@ -521,24 +510,6 @@ Status FileSystemCatalog::RenameTable(const Identifier& from_table, const Identi return Status::OK(); } -namespace { -SnapshotInfo::CommitKind ConvertCommitKind(Snapshot::CommitKind internal) { - if (internal == Snapshot::CommitKind::Append()) { - return SnapshotInfo::CommitKind::APPEND; - } - if (internal == Snapshot::CommitKind::Compact()) { - return SnapshotInfo::CommitKind::COMPACT; - } - if (internal == Snapshot::CommitKind::Overwrite()) { - return SnapshotInfo::CommitKind::OVERWRITE; - } - if (internal == Snapshot::CommitKind::Analyze()) { - return SnapshotInfo::CommitKind::ANALYZE; - } - return SnapshotInfo::CommitKind::UNKNOWN; -} -} // namespace - Result> FileSystemCatalog::ListSnapshots( const Identifier& identifier, const std::string& branch) const { PAIMON_ASSIGN_OR_RAISE(bool exists, TableExists(identifier)); @@ -553,20 +524,9 @@ Result> FileSystemCatalog::ListSnapshots( std::vector result; result.reserve(snapshots.size()); - for (const auto& snap : snapshots) { - SnapshotInfo info; - info.snapshot_id = snap.Id(); - info.schema_id = snap.SchemaId(); - info.commit_user = snap.CommitUser(); - info.commit_kind = ConvertCommitKind(snap.GetCommitKind()); - info.time_millis = snap.TimeMillis(); - info.total_record_count = snap.TotalRecordCount(); - info.delta_record_count = snap.DeltaRecordCount(); - info.watermark = snap.Watermark(); - result.push_back(std::move(info)); + result.push_back(snap.ToSnapshotInfo()); } - return result; } diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index a48ff6ce1..a0eaa63f7 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -114,7 +114,7 @@ TEST(FileSystemCatalogTest, TestCreateSystemDatabaseAndTable) { ASSERT_TRUE(arrow::ExportSchema(typed_schema, &schema).ok()); ASSERT_NOK_WITH_MSG( catalog.CreateTable(Identifier("db1", "ta$ble"), &schema, {"f1"}, {}, options, false), - "Cannot create table for system table"); + "Cannot 'create table' for system table"); ArrowSchemaRelease(&schema); } } @@ -206,11 +206,12 @@ TEST(FileSystemCatalogTest, TestOptionsSystemTableCatalog) { ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &system_create_schema).ok()); ASSERT_NOK_WITH_MSG( catalog.CreateTable(options_identifier, &system_create_schema, {}, {}, options, false), - "Cannot create table for system table"); + "Cannot 'create table' for system table"); ArrowSchemaRelease(&system_create_schema); - ASSERT_NOK_WITH_MSG(catalog.DropTable(options_identifier, false), "Cannot drop system table"); + ASSERT_NOK_WITH_MSG(catalog.DropTable(options_identifier, false), + "Cannot 'drop table' for system table"); ASSERT_NOK_WITH_MSG(catalog.RenameTable(options_identifier, Identifier("db1", "tbl2"), false), - "Cannot rename system table"); + "Cannot 'rename table' for system table"); } TEST(FileSystemCatalogTest, TestAuditLogAndBinlogSystemTableCatalog) { @@ -277,11 +278,12 @@ TEST(FileSystemCatalogTest, TestAuditLogAndBinlogSystemTableCatalog) { ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &system_create_schema).ok()); ASSERT_NOK_WITH_MSG( catalog.CreateTable(audit_log_identifier, &system_create_schema, {}, {}, options, false), - "Cannot create table for system table"); + "Cannot 'create table' for system table"); ArrowSchemaRelease(&system_create_schema); - ASSERT_NOK_WITH_MSG(catalog.DropTable(binlog_identifier, false), "Cannot drop system table"); + ASSERT_NOK_WITH_MSG(catalog.DropTable(binlog_identifier, false), + "Cannot 'drop table' for system table"); ASSERT_NOK_WITH_MSG(catalog.RenameTable(audit_log_identifier, Identifier("db1", "tbl2"), false), - "Cannot rename system table"); + "Cannot 'rename table' for system table"); } TEST(FileSystemCatalogTest, TestMetadataSystemTableCatalog) { @@ -420,11 +422,12 @@ TEST(FileSystemCatalogTest, TestMetadataSystemTableCatalog) { ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &system_create_schema).ok()); ASSERT_NOK_WITH_MSG( catalog.CreateTable(snapshots_identifier, &system_create_schema, {}, {}, options, false), - "Cannot create table for system table"); + "Cannot 'create table' for system table"); ArrowSchemaRelease(&system_create_schema); - ASSERT_NOK_WITH_MSG(catalog.DropTable(snapshots_identifier, false), "Cannot drop system table"); + ASSERT_NOK_WITH_MSG(catalog.DropTable(snapshots_identifier, false), + "Cannot 'drop table' for system table"); ASSERT_NOK_WITH_MSG(catalog.RenameTable(snapshots_identifier, Identifier("db1", "tbl2"), false), - "Cannot rename system table"); + "Cannot 'rename table' for system table"); } TEST(FileSystemCatalogTest, TestCreateTableWithBlob) { @@ -785,10 +788,10 @@ TEST(FileSystemCatalogTest, TestDropTable) { ASSERT_FALSE(exist); /// Test 4: Drop system table. - ASSERT_NOK_WITH_MSG( - catalog.DropTable(Identifier("test_db", "tbl$system"), - /*ignore_if_not_exists=*/false), - "Cannot drop system table Identifier{database='test_db', table='tbl$system'}."); + ASSERT_NOK_WITH_MSG(catalog.DropTable(Identifier("test_db", "tbl$system"), + /*ignore_if_not_exists=*/false), + "Cannot 'drop table' for system table 'Identifier{database='test_db', " + "table='tbl$system'}', please use data table."); ArrowSchemaRelease(&schema); } @@ -854,7 +857,7 @@ TEST(FileSystemCatalogTest, TestRenameTable) { ASSERT_NOK_WITH_MSG(catalog.RenameTable(Identifier("test_db", "tbl$system"), Identifier("test_db", "new_system_tbl"), /*ignore_if_not_exists=*/false), - "Cannot rename system table"); + "Cannot 'rename table' for system table"); ArrowSchemaRelease(&schema1); ArrowSchemaRelease(&schema2); diff --git a/src/paimon/core/schema/table_schema.cpp b/src/paimon/core/schema/table_schema.cpp index b991d59f1..1fe2f1bfb 100644 --- a/src/paimon/core/schema/table_schema.cpp +++ b/src/paimon/core/schema/table_schema.cpp @@ -36,6 +36,7 @@ #include "paimon/core/schema/arrow_schema_validator.h" #include "paimon/defs.h" #include "paimon/status.h" +#include "paimon/utils/special_field_ids.h" #include "rapidjson/allocators.h" #include "rapidjson/document.h" #include "rapidjson/rapidjson.h" @@ -234,6 +235,68 @@ Result> TableSchema::CreateFromJson(const std::stri table_schema.options_, table_schema.comment_, table_schema.time_millis_); } +namespace { + +Status CollectFieldIds(const rapidjson::Value& fields, std::set* seen_ids, + int32_t* max_id); + +Status CollectTypeFieldIds(const rapidjson::Value& type, std::set* seen_ids, + int32_t* max_id) { + if (!type.IsObject()) { + return Status::OK(); + } + // ROW type carries nested "fields", ARRAY/MULTISET carry "element" and MAP carries + // "key"/"value". + if (type.HasMember("fields") && type["fields"].IsArray()) { + PAIMON_RETURN_NOT_OK(CollectFieldIds(type["fields"], seen_ids, max_id)); + } + if (type.HasMember("element")) { + PAIMON_RETURN_NOT_OK(CollectTypeFieldIds(type["element"], seen_ids, max_id)); + } + if (type.HasMember("key")) { + PAIMON_RETURN_NOT_OK(CollectTypeFieldIds(type["key"], seen_ids, max_id)); + } + if (type.HasMember("value")) { + PAIMON_RETURN_NOT_OK(CollectTypeFieldIds(type["value"], seen_ids, max_id)); + } + return Status::OK(); +} + +Status CollectFieldIds(const rapidjson::Value& fields, std::set* seen_ids, + int32_t* max_id) { + for (const auto& field : fields.GetArray()) { + if (!field.IsObject()) { + return Status::Invalid("Broken schema, a field must be an object."); + } + if (!field.HasMember("id") || !field["id"].IsInt()) { + return Status::Invalid("Broken schema, a field misses an integer id."); + } + int32_t id = field["id"].GetInt(); + if (!seen_ids->insert(id).second) { + return Status::Invalid(fmt::format("Broken schema, field id {} is duplicated.", id)); + } + if (id < SpecialFieldIds::SYSTEM_FIELD_ID_START) { + *max_id = std::max(*max_id, id); + } + if (field.HasMember("type")) { + PAIMON_RETURN_NOT_OK(CollectTypeFieldIds(field["type"], seen_ids, max_id)); + } + } + return Status::OK(); +} + +} // namespace + +Result TableSchema::ComputeHighestFieldId(const rapidjson::Value& fields) { + if (!fields.IsArray()) { + return Status::Invalid("Broken schema, 'fields' must be an array."); + } + int32_t max_id = -1; + std::set seen_ids; + PAIMON_RETURN_NOT_OK(CollectFieldIds(fields, &seen_ids, &max_id)); + return max_id; +} + Result> TableSchema::InitSchema( int64_t schema_id, const std::vector& fields, int32_t highest_field_id, const std::vector& partition_keys, const std::vector& primary_keys, diff --git a/src/paimon/core/schema/table_schema.h b/src/paimon/core/schema/table_schema.h index abbb54113..df9cb6fee 100644 --- a/src/paimon/core/schema/table_schema.h +++ b/src/paimon/core/schema/table_schema.h @@ -51,6 +51,14 @@ class TableSchema : public DataSchema, public Jsonizable { static Result> CreateFromJson(const std::string& json_str); + /// Computes the highest non-system field id (ids at or above + /// `SpecialFieldIds::SYSTEM_FIELD_ID_START` are excluded) across all nesting + /// levels of a "fields" JSON array, or -1 when there is none. A non-object field, + /// a duplicated, missing or non-integer field id fails the computation. Used when + /// the schema source (e.g. a rest catalog server) does not report the + /// "highestFieldId" itself. + static Result ComputeHighestFieldId(const rapidjson::Value& fields); + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const noexcept(false) override; diff --git a/src/paimon/core/schema/table_schema_test.cpp b/src/paimon/core/schema/table_schema_test.cpp index 4f486fa19..63e302c53 100644 --- a/src/paimon/core/schema/table_schema_test.cpp +++ b/src/paimon/core/schema/table_schema_test.cpp @@ -27,6 +27,8 @@ #include "paimon/fs/local/local_file_system.h" #include "paimon/status.h" #include "paimon/testing/utils/testharness.h" +#include "paimon/utils/special_field_ids.h" +#include "rapidjson/document.h" namespace paimon::test { @@ -148,6 +150,45 @@ TEST_F(TableSchemaTest, TestGetFieldTypeForVariant) { ASSERT_EQ(variant_type, FieldType::VARIANT); } +TEST_F(TableSchemaTest, TestComputeHighestFieldId) { + auto compute = [](const std::string& fields_json) -> Result { + rapidjson::Document doc; + doc.Parse(fields_json.c_str()); + EXPECT_FALSE(doc.HasParseError()) << fields_json; + return TableSchema::ComputeHighestFieldId(doc); + }; + // ids inside nested rows are included + ASSERT_OK_AND_ASSIGN(int32_t highest, compute(R"([ + {"id": 0, "name": "f0", "type": "INT"}, + {"id": 1, "name": "s", "type": {"type": "ROW", + "fields": [{"id": 5, "name": "inner", "type": "INT"}]}} + ])")); + ASSERT_EQ(5, highest); + // system field ids are reserved and excluded from the highest field id + ASSERT_OK_AND_ASSIGN(highest, + compute(R"([{"id": 3, "name": "f0", "type": "INT"}, + {"id": )" + + std::to_string(SpecialFieldIds::SEQUENCE_NUMBER) + + R"(, "name": "_SEQUENCE_NUMBER", "type": "BIGINT"}])")); + ASSERT_EQ(3, highest); + // no fields at all yields -1 + ASSERT_OK_AND_ASSIGN(highest, compute("[]")); + ASSERT_EQ(-1, highest); + // broken schemas fail instead of being silently skipped + ASSERT_NOK_WITH_MSG(compute(R"([{"id": 0, "name": "a", "type": "INT"}, + {"id": 0, "name": "b", "type": "INT"}])") + .status(), + "duplicated"); + ASSERT_NOK_WITH_MSG(compute(R"([{"name": "a", "type": "INT"}])").status(), "integer id"); + ASSERT_NOK_WITH_MSG(compute(R"([{"id": "0", "name": "a", "type": "INT"}])").status(), + "integer id"); + ASSERT_NOK_WITH_MSG(compute("[1]").status(), "must be an object"); + rapidjson::Document not_an_array; + not_an_array.Parse("{}"); + ASSERT_NOK_WITH_MSG(TableSchema::ComputeHighestFieldId(not_an_array).status(), + "must be an array"); +} + TEST_F(TableSchemaTest, TestInvalidCreate) { // partial fields have field id arrow::FieldVector fields = MakeArrowField(3); diff --git a/src/paimon/core/snapshot.cpp b/src/paimon/core/snapshot.cpp index 8e50a6464..f27846a12 100644 --- a/src/paimon/core/snapshot.cpp +++ b/src/paimon/core/snapshot.cpp @@ -313,4 +313,27 @@ Result Snapshot::FromPath(const std::shared_ptr& fs, return snapshot; } +SnapshotInfo Snapshot::ToSnapshotInfo() const { + SnapshotInfo info; + info.snapshot_id = Id(); + info.schema_id = SchemaId(); + info.commit_user = CommitUser(); + if (commit_kind_ == CommitKind::Append()) { + info.commit_kind = SnapshotInfo::CommitKind::APPEND; + } else if (commit_kind_ == CommitKind::Compact()) { + info.commit_kind = SnapshotInfo::CommitKind::COMPACT; + } else if (commit_kind_ == CommitKind::Overwrite()) { + info.commit_kind = SnapshotInfo::CommitKind::OVERWRITE; + } else if (commit_kind_ == CommitKind::Analyze()) { + info.commit_kind = SnapshotInfo::CommitKind::ANALYZE; + } else { + info.commit_kind = SnapshotInfo::CommitKind::UNKNOWN; + } + info.time_millis = TimeMillis(); + info.total_record_count = TotalRecordCount(); + info.delta_record_count = DeltaRecordCount(); + info.watermark = Watermark(); + return info; +} + } // namespace paimon diff --git a/src/paimon/core/snapshot.h b/src/paimon/core/snapshot.h index 2ef09e377..6520bf3ca 100644 --- a/src/paimon/core/snapshot.h +++ b/src/paimon/core/snapshot.h @@ -25,6 +25,7 @@ #include "paimon/common/utils/jsonizable.h" #include "paimon/result.h" +#include "paimon/snapshot/snapshot_info.h" #include "paimon/type_fwd.h" #include "rapidjson/allocators.h" #include "rapidjson/document.h" @@ -129,6 +130,8 @@ class Snapshot : public Jsonizable { bool operator==(const Snapshot& other) const; bool TEST_Equal(const Snapshot& other) const; + SnapshotInfo ToSnapshotInfo() const; + public: static constexpr int64_t FIRST_SNAPSHOT_ID = 1; static constexpr int32_t TABLE_STORE_02_VERSION = 1; diff --git a/src/paimon/core/snapshot_test.cpp b/src/paimon/core/snapshot_test.cpp index a6388e1cb..27ac93ee4 100644 --- a/src/paimon/core/snapshot_test.cpp +++ b/src/paimon/core/snapshot_test.cpp @@ -327,6 +327,39 @@ TEST_F(SnapshotTest, TestSnapshotInfoCommitKindToString) { ASSERT_EQ("UNKNOWN", SnapshotInfo::CommitKindToString(SnapshotInfo::CommitKind::UNKNOWN)); } +TEST_F(SnapshotTest, TestToSnapshotInfo) { + auto make_snapshot = [](const Snapshot::CommitKind& kind) { + return Snapshot( + /*id=*/10, /*schema_id=*/15, /*base_manifest_list=*/"bml", + /*base_manifest_list_size=*/10, + /*delta_manifest_list=*/"dml", /*delta_manifest_list_size=*/20, + /*changelog_manifest_list=*/std::nullopt, /*changelog_manifest_list_size=*/std::nullopt, + /*index_manifest=*/std::nullopt, /*commit_user=*/"user1", /*commit_identifier=*/20, + kind, /*time_millis=*/1234, /*total_record_count=*/35, /*delta_record_count=*/40, + /*changelog_record_count=*/std::nullopt, /*watermark=*/50, + /*statistics=*/std::nullopt, /*properties=*/std::nullopt, + /*next_row_id=*/std::nullopt); + }; + SnapshotInfo info = make_snapshot(Snapshot::CommitKind::Append()).ToSnapshotInfo(); + ASSERT_EQ(10, info.snapshot_id); + ASSERT_EQ(15, info.schema_id); + ASSERT_EQ("user1", info.commit_user); + ASSERT_EQ(SnapshotInfo::CommitKind::APPEND, info.commit_kind); + ASSERT_EQ(1234, info.time_millis); + ASSERT_EQ(35, info.total_record_count.value()); + ASSERT_EQ(40, info.delta_record_count.value()); + ASSERT_EQ(50, info.watermark.value()); + // every commit kind maps to its SnapshotInfo counterpart, anything else to UNKNOWN + ASSERT_EQ(SnapshotInfo::CommitKind::COMPACT, + make_snapshot(Snapshot::CommitKind::Compact()).ToSnapshotInfo().commit_kind); + ASSERT_EQ(SnapshotInfo::CommitKind::OVERWRITE, + make_snapshot(Snapshot::CommitKind::Overwrite()).ToSnapshotInfo().commit_kind); + ASSERT_EQ(SnapshotInfo::CommitKind::ANALYZE, + make_snapshot(Snapshot::CommitKind::Analyze()).ToSnapshotInfo().commit_kind); + ASSERT_EQ(SnapshotInfo::CommitKind::UNKNOWN, + make_snapshot(Snapshot::CommitKind::Unknown()).ToSnapshotInfo().commit_kind); +} + TEST_F(SnapshotTest, TestChangelogManifestListSerialization) { // Test with changelog_manifest_list set to a non-null value { diff --git a/src/paimon/fs/s3/s3_file_system.cpp b/src/paimon/fs/s3/s3_file_system.cpp index c3f9d376a..948e2c6e8 100644 --- a/src/paimon/fs/s3/s3_file_system.cpp +++ b/src/paimon/fs/s3/s3_file_system.cpp @@ -42,52 +42,22 @@ #include #include "fmt/format.h" -#include "paimon/common/fs/http_client.h" +#include "paimon/common/utils/http_client.h" #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/common/utils/string_utils.h" +#include "paimon/common/utils/url_utils.h" #include "paimon/executor.h" namespace paimon::s3 { namespace { -Result PercentEncode(std::string_view value, bool preserve_slash) { - if (value.size() > std::numeric_limits::max()) { - return Status::IOError("S3 URL component is too large to encode"); - } - char* encoded = curl_easy_escape(nullptr, value.data(), static_cast(value.size())); - if (encoded == nullptr) { - return Status::IOError("failed to URL encode S3 component"); - } - ScopeGuard free_encoded([encoded] { curl_free(encoded); }); - std::string result(encoded); - if (preserve_slash) { - result = StringUtils::Replace(result, "%2F", "/"); - } - return result; -} - Result PercentDecode(std::string_view value, const std::string& field) { - for (size_t position = 0; position < value.size(); ++position) { - if (value[position] == '%' && - (position + 2 >= value.size() || - !std::isxdigit(static_cast(value[position + 1])) || - !std::isxdigit(static_cast(value[position + 2])))) { - return Status::IOError(fmt::format("invalid URL encoding in S3 {}", field)); - } - } - if (value.size() > std::numeric_limits::max()) { - return Status::IOError(fmt::format("S3 {} is too large to URL decode", field)); + Result decoded = UrlUtils::PercentDecode(value); + if (!decoded.ok()) { + return Status::IOError(fmt::format("invalid URL encoding in S3 {}", field)); } - int decoded_size = 0; - char* decoded = - curl_easy_unescape(nullptr, value.data(), static_cast(value.size()), &decoded_size); - if (decoded == nullptr) { - return Status::IOError(fmt::format("failed to URL decode S3 {}", field)); - } - ScopeGuard free_decoded([decoded] { curl_free(decoded); }); - std::string result(decoded, decoded_size); - return result; + return decoded; } Result ParseNonNegativeInt64(const std::string& value, const std::string& field) { @@ -704,13 +674,10 @@ class S3ObjectStoreClient : public ObjectStoreClient, int32_t max_keys) const override { std::string query = "list-type=2&delimiter=%2F&encoding-type=url"; if (!path.key.empty()) { - PAIMON_ASSIGN_OR_RAISE(std::string encoded_prefix, PercentEncode(path.key, false)); - query += "&prefix=" + encoded_prefix; + query += "&prefix=" + UrlUtils::PercentEncode(path.key); } if (!continuation_token.empty()) { - PAIMON_ASSIGN_OR_RAISE(std::string encoded_token, - PercentEncode(continuation_token, false)); - query += "&continuation-token=" + encoded_token; + query += "&continuation-token=" + UrlUtils::PercentEncode(continuation_token); } if (max_keys > 0) { query += "&max-keys=" + std::to_string(max_keys); @@ -847,13 +814,11 @@ class S3ObjectStoreClient : public ObjectStoreClient, : endpoint_.scheme != "http" || IsIpAddressAuthority(endpoint_.authority) || !IsVirtualHostableS3Bucket(object.bucket, true)); if (use_path_style) { - PAIMON_ASSIGN_OR_RAISE(std::string encoded_bucket, PercentEncode(object.bucket, false)); - request_path += encoded_bucket + "/"; + request_path += UrlUtils::PercentEncode(object.bucket) + "/"; } else { authority = object.bucket + "." + authority; } - PAIMON_ASSIGN_OR_RAISE(std::string encoded_key, PercentEncode(object.key, true)); - request_path += encoded_key; + request_path += UrlUtils::PercentEncode(object.key, /*preserve_slash=*/true); if (!query.empty()) { request_path += "?" + query; } diff --git a/src/paimon/fs/s3/s3_file_system.h b/src/paimon/fs/s3/s3_file_system.h index 66c796f89..bca948f5e 100644 --- a/src/paimon/fs/s3/s3_file_system.h +++ b/src/paimon/fs/s3/s3_file_system.h @@ -20,8 +20,8 @@ #include #include -#include "paimon/common/fs/http_client.h" #include "paimon/common/fs/object_store_file_system.h" +#include "paimon/common/utils/http_client.h" namespace paimon::s3 { diff --git a/src/paimon/rest/mock_rest_server.cpp b/src/paimon/rest/mock_rest_server.cpp new file mode 100644 index 000000000..a793de0ef --- /dev/null +++ b/src/paimon/rest/mock_rest_server.cpp @@ -0,0 +1,215 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/mock_rest_server.h" + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/common/utils/url_utils.h" + +namespace paimon { + +namespace { + +bool ReceiveAll(int32_t fd, std::string* buffer, size_t min_size) { + char chunk[4096]; + while (buffer->size() < min_size) { + ssize_t received = ::recv(fd, chunk, sizeof(chunk), 0); + if (received <= 0) { + return false; + } + buffer->append(chunk, static_cast(received)); + } + return true; +} + +std::map ParseQuery(const std::string& query) { + std::map params; + for (const std::string& pair : StringUtils::Split(query, "&", /*ignore_empty=*/true)) { + size_t eq = pair.find('='); + if (eq == std::string::npos) { + params[UrlUtils::DecodeString(pair)] = ""; + } else { + params[UrlUtils::DecodeString(pair.substr(0, eq))] = + UrlUtils::DecodeString(pair.substr(eq + 1)); + } + } + return params; +} + +} // namespace + +MockRestServer::MockRestServer(Handler handler, int32_t listen_fd, int32_t port) + : handler_(std::move(handler)), listen_fd_(listen_fd), port_(port) { + accept_thread_ = std::thread([this] { AcceptLoop(); }); +} + +Result> MockRestServer::Start(Handler handler) { + int32_t listen_fd = ::socket(AF_INET, SOCK_STREAM, 0); + if (listen_fd < 0) { + return Status::IOError("mock rest server: failed to create socket: ", std::strerror(errno)); + } + int32_t reuse = 1; + ::setsockopt(listen_fd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)); + struct sockaddr_in address; + std::memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + address.sin_port = 0; + if (::bind(listen_fd, reinterpret_cast(&address), sizeof(address)) < 0) { + ::close(listen_fd); + return Status::IOError("mock rest server: failed to bind: ", std::strerror(errno)); + } + if (::listen(listen_fd, 16) < 0) { + ::close(listen_fd); + return Status::IOError("mock rest server: failed to listen: ", std::strerror(errno)); + } + socklen_t address_len = sizeof(address); + if (::getsockname(listen_fd, reinterpret_cast(&address), &address_len) < 0) { + ::close(listen_fd); + return Status::IOError("mock rest server: failed to get port: ", std::strerror(errno)); + } + int32_t port = ntohs(address.sin_port); + return std::unique_ptr(new MockRestServer(std::move(handler), listen_fd, port)); +} + +MockRestServer::~MockRestServer() { + Stop(); +} + +void MockRestServer::Stop() { + if (stopped_.exchange(true)) { + return; + } + ::shutdown(listen_fd_, SHUT_RDWR); + ::close(listen_fd_); + if (accept_thread_.joinable()) { + accept_thread_.join(); + } +} + +std::string MockRestServer::GetBaseUri() const { + return fmt::format("http://127.0.0.1:{}", port_); +} + +void MockRestServer::AcceptLoop() { + while (!stopped_.load()) { + int32_t connection_fd = ::accept(listen_fd_, nullptr, nullptr); + if (connection_fd < 0) { + if (stopped_.load()) { + return; + } + continue; + } + // The connection is handled on the accept thread; the socket timeouts keep a + // wedged peer from blocking `Stop()` indefinitely. + struct timeval timeout; + timeout.tv_sec = 30; + timeout.tv_usec = 0; + ::setsockopt(connection_fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)); + ::setsockopt(connection_fd, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(timeout)); + HandleConnection(connection_fd); + ::close(connection_fd); + } +} + +void MockRestServer::HandleConnection(int32_t connection_fd) { + std::string buffer; + size_t header_end; + while ((header_end = buffer.find("\r\n\r\n")) == std::string::npos) { + if (!ReceiveAll(connection_fd, &buffer, buffer.size() + 1)) { + return; + } + } + + Request request; + std::string header_part = buffer.substr(0, header_end); + std::vector lines = StringUtils::Split(header_part, "\r\n", + /*ignore_empty=*/true); + if (lines.empty()) { + return; + } + std::vector request_line = StringUtils::Split(lines[0], " ", + /*ignore_empty=*/true); + if (request_line.size() < 2) { + return; + } + request.method = request_line[0]; + std::string target = request_line[1]; + size_t question = target.find('?'); + if (question == std::string::npos) { + request.path = UrlUtils::DecodeString(target); + } else { + request.path = UrlUtils::DecodeString(target.substr(0, question)); + request.query_params = ParseQuery(target.substr(question + 1)); + } + size_t content_length = 0; + for (size_t i = 1; i < lines.size(); i++) { + size_t colon = lines[i].find(':'); + if (colon == std::string::npos) { + continue; + } + std::string name = lines[i].substr(0, colon); + std::string value = lines[i].substr(colon + 1); + StringUtils::Trim(&name); + StringUtils::Trim(&value); + request.headers[StringUtils::ToLowerCase(name)] = value; + } + auto length_iter = request.headers.find("content-length"); + if (length_iter != request.headers.end()) { + content_length = + static_cast(std::strtoul(length_iter->second.c_str(), nullptr, 10)); + } + size_t body_begin = header_end + 4; + if (!ReceiveAll(connection_fd, &buffer, body_begin + content_length)) { + return; + } + request.body = buffer.substr(body_begin, content_length); + + Response response = handler_(request); + if (response.close_without_response) { + return; + } + std::string extra_headers; + for (const auto& [name, value] : response.headers) { + extra_headers += fmt::format("{}: {}\r\n", name, value); + } + std::string payload = fmt::format( + "HTTP/1.1 {} MOCK\r\nContent-Type: {}\r\nContent-Length: {}\r\n{}Connection: " + "close\r\n\r\n{}", + response.code, response.content_type, response.body.size() + response.missing_body_bytes, + extra_headers, response.body); + size_t sent = 0; + while (sent < payload.size()) { + ssize_t written = ::send(connection_fd, payload.data() + sent, payload.size() - sent, 0); + if (written <= 0) { + return; + } + sent += static_cast(written); + } +} + +} // namespace paimon diff --git a/src/paimon/rest/mock_rest_server.h b/src/paimon/rest/mock_rest_server.h new file mode 100644 index 000000000..5fd09f108 --- /dev/null +++ b/src/paimon/rest/mock_rest_server.h @@ -0,0 +1,92 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +/// A minimal blocking HTTP/1.1 server for REST catalog unit tests. It listens on a +/// random port of 127.0.0.1, parses one request per connection and answers it with the +/// response returned by the registered handler. Test only, not production code. +class MockRestServer { + public: + struct Request { + std::string method; + /// Url-decoded path, e.g. "/v1/config". + std::string path; + /// Url-decoded query parameters. + std::map query_params; + /// Headers with lower-cased names. + std::map headers; + std::string body; + }; + + struct Response { + int32_t code = 200; + std::string body; + std::string content_type = "application/json"; + /// Extra response headers, e.g. "Retry-After". + std::map headers; + /// Close the connection without sending any response; the client observes + /// CURLE_GOT_NOTHING, a transport error that is never retried. + bool close_without_response = false; + /// Advertise this many bytes beyond the actual body in `Content-Length` and + /// close the connection without sending them; the client observes the response + /// being cut off mid-body (CURLE_PARTIAL_FILE), a retriable transport error. + size_t missing_body_bytes = 0; + }; + + using Handler = std::function; + + /// Starts the server with `handler` invoked for every request (on the server + /// thread). + static Result> Start(Handler handler); + + ~MockRestServer(); + + void Stop(); + + int32_t GetPort() const { + return port_; + } + + /// "http://127.0.0.1:" + std::string GetBaseUri() const; + + private: + MockRestServer(Handler handler, int32_t listen_fd, int32_t port); + + void AcceptLoop(); + void HandleConnection(int32_t connection_fd); + + Handler handler_; + int32_t listen_fd_ = -1; + int32_t port_ = 0; + std::atomic stopped_{false}; + std::thread accept_thread_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/resource_paths.cpp b/src/paimon/rest/resource_paths.cpp new file mode 100644 index 000000000..484482378 --- /dev/null +++ b/src/paimon/rest/resource_paths.cpp @@ -0,0 +1,64 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/resource_paths.h" + +#include "paimon/common/utils/url_utils.h" + +namespace paimon { + +namespace { +constexpr const char kV1[] = "/v1"; +} // namespace + +ResourcePaths::ResourcePaths(const std::string& prefix) { + base_ = kV1; + if (!prefix.empty()) { + base_ += "/" + UrlUtils::EncodeString(prefix); + } +} + +std::string ResourcePaths::Config() { + return std::string(kV1) + "/config"; +} + +std::string ResourcePaths::Databases() const { + return base_ + "/databases"; +} + +std::string ResourcePaths::Database(const std::string& database_name) const { + return Databases() + "/" + UrlUtils::EncodeString(database_name); +} + +std::string ResourcePaths::Tables(const std::string& database_name) const { + return Database(database_name) + "/tables"; +} + +std::string ResourcePaths::Table(const std::string& database_name, + const std::string& table_name) const { + return Tables(database_name) + "/" + UrlUtils::EncodeString(table_name); +} + +std::string ResourcePaths::RenameTable() const { + return base_ + "/tables/rename"; +} + +std::string ResourcePaths::Snapshots(const std::string& database_name, + const std::string& table_name) const { + return Table(database_name, table_name) + "/snapshots"; +} + +} // namespace paimon diff --git a/src/paimon/rest/resource_paths.h b/src/paimon/rest/resource_paths.h new file mode 100644 index 000000000..ea36bf480 --- /dev/null +++ b/src/paimon/rest/resource_paths.h @@ -0,0 +1,45 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +namespace paimon { + +/// Builds resource paths of the REST catalog server. All path segments are +/// url-encoded; `prefix` (usually pushed down by the server through `/v1/config`) may +/// be empty, in which case it is skipped. +class ResourcePaths { + public: + explicit ResourcePaths(const std::string& prefix); + + /// "/v1/config", the only path that does not carry the prefix. + static std::string Config(); + + std::string Databases() const; + std::string Database(const std::string& database_name) const; + std::string Tables(const std::string& database_name) const; + std::string Table(const std::string& database_name, const std::string& table_name) const; + std::string RenameTable() const; + std::string Snapshots(const std::string& database_name, const std::string& table_name) const; + + private: + /// "/v1" or "/v1/{encoded prefix}". + std::string base_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_api.cpp b/src/paimon/rest/rest_api.cpp new file mode 100644 index 000000000..727f26d26 --- /dev/null +++ b/src/paimon/rest/rest_api.cpp @@ -0,0 +1,294 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_api.h" + +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/catalog_options.h" +#include "paimon/common/utils/rapidjson_util.h" +#include "paimon/common/utils/url_utils.h" +#include "paimon/logging.h" +#include "paimon/rest/rest_util.h" + +namespace paimon { + +namespace { +// The `PAIMON_ASSIGN_OR_RAISE` macro cannot take a declaration type containing a comma. +using StringMap = std::map; + +// Parses a response body without echoing the body (or the parse failure carrying it) +// into the error message: a successful response may contain credentials (e.g. a token +// response), so only the request path is reported. +template +Status ParseResponseBody(const std::string& body, const std::string& path, ResponseT* entity) { + if (!RapidJsonUtil::FromJsonString(body, entity).ok()) { + return Status::Invalid( + fmt::format("failed to deserialize the response of {} from the rest server", path)); + } + return Status::OK(); +} +} // namespace + +RestApi::RestApi(std::unique_ptr client, + std::unique_ptr auth_provider, + const std::map& base_headers, + const std::map& options, const ResourcePaths& paths) + : client_(std::move(client)), + auth_provider_(std::move(auth_provider)), + base_headers_(base_headers), + options_(options), + resource_paths_(paths) {} + +Result> RestApi::Create(const std::map& options, + const std::string& warehouse, bool config_required, + const RestHttpClient::Config& http_config) { + auto uri_iter = options.find(CatalogOptions::URI); + if (uri_iter == options.end() || uri_iter->second.empty()) { + return Status::Invalid(fmt::format("option '{}' must be configured for the rest catalog", + CatalogOptions::URI)); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr client, + RestHttpClient::Create(uri_iter->second, http_config)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr auth_provider, + AuthProvider::Create(options)); + + std::map merged_options = options; + std::map base_headers = + RestUtil::ExtractPrefixMap(options, kHeaderOptionPrefix); + if (config_required) { + std::map query_params; + if (!warehouse.empty()) { + query_params[kQueryParamWarehouse] = warehouse; + } + RestAuthParameter auth_parameter; + auth_parameter.method = "GET"; + auth_parameter.resource_path = ResourcePaths::Config(); + for (const auto& [key, value] : query_params) { + auth_parameter.parameters[key] = UrlUtils::EncodeString(value); + } + PAIMON_ASSIGN_OR_RAISE(StringMap headers, + auth_provider->MergeAuthHeader(base_headers, auth_parameter)); + PAIMON_ASSIGN_OR_RAISE( + RestHttpClient::Response response, + client->Execute("GET", ResourcePaths::Config(), query_params, headers, "")); + if (!response.IsSuccessful()) { + return ErrorToStatus(response); + } + ConfigResponse config; + PAIMON_RETURN_NOT_OK(ParseResponseBody(response.body, ResourcePaths::Config(), &config)); + merged_options = config.Merge(options); + for (const auto& [key, value] : + RestUtil::ExtractPrefixMap(merged_options, kHeaderOptionPrefix)) { + base_headers[key] = value; + } + std::shared_ptr logger = Logger::GetLogger("RestApi"); + PAIMON_LOG_DEBUG(logger, + "merged %zu client options with the /v1/config response into %zu options", + options.size(), merged_options.size()); + } + std::string prefix; + auto prefix_iter = merged_options.find(kOptionUrlPrefix); + if (prefix_iter != merged_options.end()) { + prefix = prefix_iter->second; + } + return std::unique_ptr(new RestApi(std::move(client), std::move(auth_provider), + base_headers, merged_options, + ResourcePaths(prefix))); +} + +Status RestApi::ErrorToStatus(const RestHttpClient::Response& response) { + // The code of the parsed error body takes precedence over the http status, which + // a gateway may have rewritten. + int64_t code = response.code; + std::string message; + std::string resource_info; + if (!response.body.empty()) { + ErrorResponse error; + if (RapidJsonUtil::FromJsonString(response.body, &error).ok()) { + // The message may embed secrets (e.g. "password=..."), so it is redacted. + message = RestUtil::RedactText(error.GetMessage()); + if (!error.GetResourceType().empty()) { + resource_info = fmt::format(" (resource type: {}, resource name: {})", + error.GetResourceType(), error.GetResourceName()); + } + if (error.GetCode() != 0) { + code = error.GetCode(); + } + } + } + if (message.empty()) { + message = fmt::format("rest server returned http status {}", response.code); + } + message += resource_info; + // Carry the request id through for server-side tracing. + auto request_id_iter = response.headers.find("x-request-id"); + if (request_id_iter == response.headers.end()) { + // Fall back to any header carrying a request id (e.g. x-amz-request-id). + request_id_iter = + std::find_if(response.headers.begin(), response.headers.end(), [](const auto& header) { + return header.first.find("request-id") != std::string::npos; + }); + } + if (request_id_iter != response.headers.end() && !request_id_iter->second.empty() && + request_id_iter->second != "unknown") { + message += fmt::format(" requestId:{}", request_id_iter->second); + } + Status status; + switch (code) { + case HttpStatus::kBadRequest: + status = Status::Invalid(message); + break; + case HttpStatus::kUnauthorized: + status = Status::IOError("not authorized: ", message); + break; + case HttpStatus::kForbidden: + status = Status::IOError("forbidden: ", message); + break; + case HttpStatus::kNotFound: + status = Status::NotExist(message); + break; + case HttpStatus::kConflict: + status = Status::Exist(message); + break; + case HttpStatus::kInternalServerError: + status = Status::IOError("server error: ", message); + break; + case HttpStatus::kNotImplemented: + status = Status::NotImplemented(message); + break; + case HttpStatus::kServiceUnavailable: + status = Status::IOError("service unavailable: ", message); + break; + default: + status = + Status::IOError(fmt::format("rest request failed with code {}: {}", code, message)); + break; + } + return status.WithDetail(std::make_shared(code)); +} + +Result RestApi::Execute( + const std::string& method, const std::string& path, + const std::map& query_params, const std::string& body) const { + RestAuthParameter auth_parameter; + auth_parameter.method = method; + auth_parameter.resource_path = path; + auth_parameter.data = body; + for (const auto& [key, value] : query_params) { + auth_parameter.parameters[key] = UrlUtils::EncodeString(value); + } + StringMap request_headers = base_headers_; + if (!body.empty()) { + // Set before merging the authentication headers, so a provider that signs the + // request headers covers the content type. + request_headers["Content-Type"] = "application/json"; + } + PAIMON_ASSIGN_OR_RAISE(StringMap headers, + auth_provider_->MergeAuthHeader(request_headers, auth_parameter)); + PAIMON_ASSIGN_OR_RAISE(RestHttpClient::Response response, + client_->Execute(method, path, query_params, headers, body)); + if (!response.IsSuccessful()) { + return ErrorToStatus(response); + } + return response; +} + +template +Result RestApi::GetEntity(const std::string& path, + const std::map& query_params) const { + PAIMON_ASSIGN_OR_RAISE(RestHttpClient::Response response, + Execute("GET", path, query_params, "")); + ResponseT entity; + PAIMON_RETURN_NOT_OK(ParseResponseBody(response.body, path, &entity)); + return entity; +} + +template +Result> RestApi::ListAllPages( + const std::string& path) const { + std::vector items; + std::map query_params; + while (true) { + PAIMON_ASSIGN_OR_RAISE(ResponseT response, GetEntity(path, query_params)); + const auto& data = response.Data(); + items.insert(items.end(), data.begin(), data.end()); + const std::optional& next_page_token = response.NextPageToken(); + if (!next_page_token || next_page_token.value().empty() || data.empty()) { + return items; + } + query_params[kQueryParamPageToken] = next_page_token.value(); + } +} + +Result> RestApi::ListDatabases() const { + return ListAllPages(resource_paths_.Databases()); +} + +Status RestApi::CreateDatabase(const std::string& name, + const std::map& options) const { + CreateDatabaseRequest request(name, options); + PAIMON_ASSIGN_OR_RAISE(std::string body, request.ToJsonString()); + return Execute("POST", resource_paths_.Databases(), {}, body).status(); +} + +Result RestApi::GetDatabase(const std::string& name) const { + return GetEntity(resource_paths_.Database(name), {}); +} + +Status RestApi::DropDatabase(const std::string& name) const { + return Execute("DELETE", resource_paths_.Database(name), {}, "").status(); +} + +Result> RestApi::ListTables(const std::string& database_name) const { + return ListAllPages(resource_paths_.Tables(database_name)); +} + +Result RestApi::GetTable(const Identifier& identifier) const { + return GetEntity( + resource_paths_.Table(identifier.GetDatabaseName(), identifier.GetTableName()), {}); +} + +Status RestApi::CreateTable(const Identifier& identifier, const std::string& schema_json) const { + CreateTableRequest request(identifier.GetDatabaseName(), identifier.GetTableName(), + schema_json); + PAIMON_ASSIGN_OR_RAISE(std::string body, request.ToJsonString()); + return Execute("POST", resource_paths_.Tables(identifier.GetDatabaseName()), {}, body).status(); +} + +Status RestApi::DropTable(const Identifier& identifier) const { + return Execute("DELETE", + resource_paths_.Table(identifier.GetDatabaseName(), identifier.GetTableName()), + {}, "") + .status(); +} + +Status RestApi::RenameTable(const Identifier& from_table, const Identifier& to_table) const { + RenameTableRequest request(from_table.GetDatabaseName(), from_table.GetTableName(), + to_table.GetDatabaseName(), to_table.GetTableName()); + PAIMON_ASSIGN_OR_RAISE(std::string body, request.ToJsonString()); + return Execute("POST", resource_paths_.RenameTable(), {}, body).status(); +} + +Result> RestApi::ListSnapshots(const Identifier& identifier) const { + return ListAllPages( + resource_paths_.Snapshots(identifier.GetDatabaseName(), identifier.GetTableName())); +} + +} // namespace paimon diff --git a/src/paimon/rest/rest_api.h b/src/paimon/rest/rest_api.h new file mode 100644 index 000000000..c4c5f805e --- /dev/null +++ b/src/paimon/rest/rest_api.h @@ -0,0 +1,139 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/core/snapshot.h" +#include "paimon/rest/resource_paths.h" +#include "paimon/rest/rest_auth.h" +#include "paimon/rest/rest_http_client.h" +#include "paimon/rest/rest_messages.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +/// Carries the code of a failed rest request (the code of the parsed error body, +/// falling back to the http status) so callers can distinguish e.g. authentication +/// failures (401/403) from other IO errors programmatically. +class RestErrorDetail : public StatusDetail { + public: + static constexpr const char* kTypeId = "rest-error-detail"; + + explicit RestErrorDetail(int64_t code) : code_(code) {} + + const char* type_id() const override { + return kTypeId; + } + + std::string ToString() const override { + return "rest error code " + std::to_string(code_); + } + + int64_t GetCode() const { + return code_; + } + + private: + int64_t code_; +}; + +/// The client of the REST catalog server. This layer only talks HTTP + JSON and never +/// touches the file system. +class RestApi { + public: + static constexpr const char* kQueryParamPageToken = "pageToken"; + static constexpr const char* kQueryParamWarehouse = "warehouse"; + /// "prefix" - option key of the url path prefix inserted after "/v1", usually + /// pushed down by "/v1/config". + static constexpr const char* kOptionUrlPrefix = "prefix"; + /// Options with this prefix are sent as http headers (with the prefix stripped). + static constexpr const char* kHeaderOptionPrefix = "header."; + + /// Creates the api client. + /// + /// @param options Client side options; `CatalogOptions::URI` and + /// `CatalogOptions::TOKEN_PROVIDER` are required. + /// @param warehouse Warehouse sent as query parameter of "/v1/config"; may be empty. + /// @param config_required When true, fetch "/v1/config" and merge the server + /// defaults/overrides into `options` with the precedence + /// overrides > client options > defaults. + /// @param http_config Transport level settings, mainly overridable for tests. + static Result> Create( + const std::map& options, const std::string& warehouse, + bool config_required, const RestHttpClient::Config& http_config = RestHttpClient::Config()); + + /// Options merged with the server side config. + const std::map& GetMergedOptions() const { + return options_; + } + + Result> ListDatabases() const; + Status CreateDatabase(const std::string& name, + const std::map& options) const; + Result GetDatabase(const std::string& name) const; + Status DropDatabase(const std::string& name) const; + + Result> ListTables(const std::string& database_name) const; + Result GetTable(const Identifier& identifier) const; + /// `schema_json` uses the schema JSON layout of the protocol + /// (fields/partitionKeys/primaryKeys/options/comment). + Status CreateTable(const Identifier& identifier, const std::string& schema_json) const; + Status DropTable(const Identifier& identifier) const; + Status RenameTable(const Identifier& from_table, const Identifier& to_table) const; + + Result> ListSnapshots(const Identifier& identifier) const; + + /// Maps a non-successful http response to a status: 404 becomes `NotExist`, 409 + /// becomes `Exist`, 400 becomes `Invalid`, 501 becomes `NotImplemented` and the + /// other codes become `IOError`. The status carries a `RestErrorDetail` with the + /// mapped code. + static Status ErrorToStatus(const RestHttpClient::Response& response); + + private: + RestApi(std::unique_ptr client, std::unique_ptr auth_provider, + const std::map& base_headers, + const std::map& options, const ResourcePaths& paths); + + /// Executes one request with authentication headers and returns the response when it + /// is successful, otherwise the mapped error status. + Result Execute(const std::string& method, const std::string& path, + const std::map& query_params, + const std::string& body) const; + + template + Result GetEntity(const std::string& path, + const std::map& query_params) const; + + /// Fetches all pages of a paged listing api. + template + Result> ListAllPages(const std::string& path) const; + + std::unique_ptr client_; + std::unique_ptr auth_provider_; + /// Base http headers extracted from the "header." options. + std::map base_headers_; + std::map options_; + ResourcePaths resource_paths_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_auth.cpp b/src/paimon/rest/rest_auth.cpp new file mode 100644 index 000000000..ea53f4a30 --- /dev/null +++ b/src/paimon/rest/rest_auth.cpp @@ -0,0 +1,56 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_auth.h" + +#include "fmt/format.h" +#include "paimon/catalog_options.h" +#include "paimon/common/utils/string_utils.h" + +namespace paimon { + +Result> BearTokenAuthProvider::MergeAuthHeader( + const std::map& base_header, + const RestAuthParameter& parameter) const { + std::map headers = base_header; + headers["Authorization"] = "Bearer " + token_; + return headers; +} + +Result> AuthProvider::Create( + const std::map& options) { + auto provider_iter = options.find(CatalogOptions::TOKEN_PROVIDER); + if (provider_iter == options.end() || provider_iter->second.empty()) { + return Status::Invalid(fmt::format("option '{}' must be configured for the rest catalog", + CatalogOptions::TOKEN_PROVIDER)); + } + // Matched leniently in lower case; other clients may match the provider name + // case-sensitively, so only the exact "bear" spelling is portable. + std::string provider = StringUtils::ToLowerCase(provider_iter->second); + if (provider == "bear") { + auto token_iter = options.find(CatalogOptions::TOKEN); + if (token_iter == options.end() || token_iter->second.empty()) { + return Status::Invalid( + fmt::format("option '{}' must be configured for the bear token provider", + CatalogOptions::TOKEN)); + } + return std::make_unique(token_iter->second); + } + return Status::NotImplemented( + fmt::format("unsupported token provider: {}, only 'bear' is supported for now", provider)); +} + +} // namespace paimon diff --git a/src/paimon/rest/rest_auth.h b/src/paimon/rest/rest_auth.h new file mode 100644 index 000000000..1cdb67402 --- /dev/null +++ b/src/paimon/rest/rest_auth.h @@ -0,0 +1,66 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +/// Input of one request signature. +struct RestAuthParameter { + std::string method; + std::string resource_path; + /// Query parameters with url-encoded values. + std::map parameters; + /// Request body, empty when the request carries none. + std::string data; +}; + +/// Generates authentication headers for REST catalog requests. +class AuthProvider { + public: + virtual ~AuthProvider() = default; + + /// Returns `base_header` merged with the authentication headers of this provider. + virtual Result> MergeAuthHeader( + const std::map& base_header, + const RestAuthParameter& parameter) const = 0; + + /// Creates the provider configured by `CatalogOptions::TOKEN_PROVIDER`. + static Result> Create( + const std::map& options); +}; + +/// Adds `Authorization: Bearer `. +class BearTokenAuthProvider : public AuthProvider { + public: + explicit BearTokenAuthProvider(const std::string& token) : token_(token) {} + + Result> MergeAuthHeader( + const std::map& base_header, + const RestAuthParameter& parameter) const override; + + private: + std::string token_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp new file mode 100644 index 000000000..dae8d4634 --- /dev/null +++ b/src/paimon/rest/rest_catalog.cpp @@ -0,0 +1,431 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_catalog.h" + +#include +#include + +#include "arrow/c/bridge.h" +#include "fmt/format.h" +#include "paimon/catalog/table.h" +#include "paimon/catalog_options.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/rapidjson_util.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/catalog/catalog_utils.h" +#include "paimon/core/core_options.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/system/global_system_tables.h" +#include "paimon/core/table/system/system_table.h" +#include "paimon/core/table/system/system_table_schema.h" +#include "paimon/defs.h" +#include "paimon/fs/file_system.h" +#include "paimon/rest/rest_util.h" +#include "rapidjson/document.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +namespace { + +constexpr const char kPathOption[] = "path"; + +/// Maps the default branch "main" (case-insensitive) to "no branch": the default +/// branch is addressed as the bare table. +std::optional NormalizeBranch(std::optional branch) { + if (branch && StringUtils::ToLowerCase(branch.value()) == Identifier::kDefaultMainBranch) { + return std::nullopt; + } + return branch; +} + +/// Builds the identifier sent to the rest server: the system table suffix is stripped +/// while the branch stays in the object name, so the server resolves the branch itself +/// and returns the branch's own schema. The returned table path is the data table +/// root in either case; the branch subdirectory is derived downstream from the +/// branch option (see `ToTableSchema`). +Result ToLoadIdentifier(const Identifier& identifier) { + PAIMON_ASSIGN_OR_RAISE(std::string data_table_name, identifier.GetDataTableName()); + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + branch = NormalizeBranch(std::move(branch)); + std::string object_name = std::move(data_table_name); + if (branch) { + object_name.append(Identifier::kSystemTableSplitter); + object_name.append(Identifier::kSystemBranchPrefix); + object_name.append(branch.value()); + } + return Identifier(identifier.GetDatabaseName(), object_name); +} + +} // namespace + +RestCatalog::RestCatalog(std::unique_ptr api, const std::shared_ptr& fs, + const std::string& warehouse) + : api_(std::move(api)), + fs_(fs), + warehouse_(warehouse), + table_default_options_(RestUtil::ExtractPrefixMap( + api_->GetMergedOptions(), CatalogOptions::TABLE_DEFAULT_OPTION_PREFIX)), + logger_(Logger::GetLogger("RestCatalog")) {} + +Result> RestCatalog::Create( + const std::string& warehouse, const std::map& options, + const std::shared_ptr& file_system, const RestHttpClient::Config& http_config) { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr api, + RestApi::Create(options, warehouse, /*config_required=*/true, http_config)); + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, + CoreOptions::FromMap(api->GetMergedOptions(), file_system)); + return std::unique_ptr( + new RestCatalog(std::move(api), core_options.GetFileSystem(), warehouse)); +} + +const std::map& RestCatalog::GetOptions() const { + return api_->GetMergedOptions(); +} + +Status RestCatalog::CreateDatabase(const std::string& name, + const std::map& options, + bool ignore_if_exists) { + if (CatalogUtils::IsSystemDatabase(name)) { + return Status::Invalid(fmt::format("Cannot create database for system database {}.", name)); + } + Status status = api_->CreateDatabase(name, options); + if (status.IsExist() && ignore_if_exists) { + return Status::OK(); + } + return status; +} + +Result> RestCatalog::ListDatabases() const { + return api_->ListDatabases(); +} + +Result RestCatalog::DatabaseExists(const std::string& db_name) const { + if (CatalogUtils::IsSystemDatabase(db_name)) { + return true; + } + Result response = api_->GetDatabase(db_name); + if (response.ok()) { + return true; + } + if (response.status().IsNotExist()) { + return false; + } + return response.status(); +} + +Status RestCatalog::DropDatabase(const std::string& name, bool ignore_if_not_exists, bool cascade) { + if (CatalogUtils::IsSystemDatabase(name)) { + return Status::Invalid(fmt::format("Cannot drop system database {}.", name)); + } + if (!cascade) { + Result> tables = ListTables(name); + if (!tables.ok()) { + if (tables.status().IsNotExist() && ignore_if_not_exists) { + return Status::OK(); + } + return tables.status(); + } + if (!tables.value().empty()) { + return Status::Invalid( + fmt::format("Cannot drop non-empty database {}. Use cascade=true to force.", name)); + } + } + Status status = api_->DropDatabase(name); + if (status.IsNotExist() && ignore_if_not_exists) { + return Status::OK(); + } + return status; +} + +std::string RestCatalog::GetDatabaseLocation(const std::string& db_name) const { + Result response = api_->GetDatabase(db_name); + if (!response.ok()) { + PAIMON_LOG_WARN(logger_, "failed to get location of database %s: %s", db_name.c_str(), + response.status().ToString().c_str()); + return ""; + } + return response.value().GetLocation(); +} + +Result> RestCatalog::ListTables(const std::string& db_name) const { + if (CatalogUtils::IsSystemDatabase(db_name)) { + return GlobalSystemTableLoader::GetSupportedTableNames(api_->GetMergedOptions()); + } + return api_->ListTables(db_name); +} + +Status RestCatalog::CreateTable(const Identifier& identifier, ArrowSchema* c_schema, + const std::vector& partition_keys, + const std::vector& primary_keys, + const std::map& options, + bool ignore_if_exists) { + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(identifier, "create table")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(identifier, "create table")); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr schema, + arrow::ImportSchema(c_schema)); + std::map effective_options = options; + for (const auto& [key, value] : table_default_options_) { + effective_options.emplace(key, value); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_schema, + TableSchema::Create(TableSchema::FIRST_SCHEMA_ID, schema, partition_keys, + primary_keys, effective_options)); + std::string schema_json; + try { + rapidjson::Document doc; + doc.SetObject(); + rapidjson::Document::AllocatorType& allocator = doc.GetAllocator(); + doc.AddMember(rapidjson::StringRef("fields"), + RapidJsonUtil::SerializeValue(table_schema->Fields(), &allocator).Move(), + allocator); + doc.AddMember(rapidjson::StringRef("partitionKeys"), + RapidJsonUtil::SerializeValue(partition_keys, &allocator).Move(), allocator); + doc.AddMember(rapidjson::StringRef("primaryKeys"), + RapidJsonUtil::SerializeValue(primary_keys, &allocator).Move(), allocator); + doc.AddMember(rapidjson::StringRef("options"), + RapidJsonUtil::SerializeValue(effective_options, &allocator).Move(), + allocator); + schema_json = RestUtil::JsonToString(doc); + } catch (const std::exception& e) { + return Status::SerializationError("failed to serialize create table schema: ", e.what()); + } + Status status = api_->CreateTable(identifier, schema_json); + if (status.IsExist() && ignore_if_exists) { + return Status::OK(); + } + return status; +} + +Result RestCatalog::TableExists(const Identifier& identifier) const { + if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { + return GlobalSystemTableLoader::IsSupported(identifier.GetTableName(), + api_->GetMergedOptions()); + } + PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); + if (is_system_table) { + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, + identifier.GetSystemTableName()); + if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { + return false; + } + } + PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); + Result response = api_->GetTable(load_identifier); + if (response.ok()) { + return true; + } + if (response.status().IsNotExist()) { + return false; + } + return response.status(); +} + +Result RestCatalog::GetTableLocation(const Identifier& identifier) const { + PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); + PAIMON_ASSIGN_OR_RAISE(GetTableResponse response, api_->GetTable(load_identifier)); + return response.GetPath(); +} + +Status RestCatalog::DropTable(const Identifier& identifier, bool ignore_if_not_exists) { + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(identifier, "drop table")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(identifier, "drop table")); + Status status = api_->DropTable(identifier); + if (status.IsNotExist() && ignore_if_not_exists) { + return Status::OK(); + } + return status; +} + +Status RestCatalog::RenameTable(const Identifier& from_table, const Identifier& to_table, + bool ignore_if_not_exists) { + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(from_table, "rename table")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(to_table, "rename table")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(from_table, "rename table")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(to_table, "rename table")); + Status status = api_->RenameTable(from_table, to_table); + if (status.IsNotExist() && ignore_if_not_exists) { + return Status::OK(); + } + return status; +} + +Result> RestCatalog::ToTableSchema( + const GetTableResponse& response, const std::optional& branch) { + std::string table_schema_json; + try { + rapidjson::Document src; + src.Parse(response.GetSchemaJson().c_str()); + if (src.HasParseError() || !src.IsObject() || !src.HasMember("fields") || + !src["fields"].IsArray()) { + return Status::Invalid("invalid table schema json from the rest server"); + } + rapidjson::Document out; + out.SetObject(); + rapidjson::Document::AllocatorType& allocator = out.GetAllocator(); + out.AddMember(rapidjson::StringRef("version"), TableSchema::CURRENT_VERSION, allocator); + out.AddMember(rapidjson::StringRef("id"), response.GetSchemaId(), allocator); + PAIMON_ASSIGN_OR_RAISE(int32_t highest_field_id, + TableSchema::ComputeHighestFieldId(src["fields"])); + rapidjson::Value fields(rapidjson::kArrayType); + fields.CopyFrom(src["fields"], allocator); + out.AddMember(rapidjson::StringRef("highestFieldId"), highest_field_id, allocator); + out.AddMember(rapidjson::StringRef("fields"), fields.Move(), allocator); + for (const char* key : {"partitionKeys", "primaryKeys"}) { + rapidjson::Value keys(rapidjson::kArrayType); + if (src.HasMember(key) && src[key].IsArray()) { + keys.CopyFrom(src[key], allocator); + } + out.AddMember(rapidjson::StringRef(key), keys.Move(), allocator); + } + std::map options_map; + if (src.HasMember("options") && src["options"].IsObject()) { + options_map = + RapidJsonUtil::DeserializeValue>(src["options"]); + } + options_map[kPathOption] = response.GetPath(); + response.GetAuditFields().PutAuditOptionsTo(&options_map); + if (branch) { + options_map[Options::BRANCH] = branch.value(); + } + out.AddMember(rapidjson::StringRef("options"), + RapidJsonUtil::SerializeValue(options_map, &allocator).Move(), allocator); + if (src.HasMember("comment") && src["comment"].IsString()) { + rapidjson::Value comment; + comment.CopyFrom(src["comment"], allocator); + out.AddMember(rapidjson::StringRef("comment"), comment.Move(), allocator); + } + // The server's audit time is used instead of the current time so the + // conversion stays deterministic; 0 (the epoch) when the server did not report + // an "updatedAt". + out.AddMember(rapidjson::StringRef("timeMillis"), + response.GetAuditFields().updated_at.value_or(0), allocator); + table_schema_json = RestUtil::JsonToString(out); + } catch (const std::exception& e) { + return Status::Invalid("failed to convert rest table schema: ", e.what()); + } + return TableSchema::CreateFromJson(table_schema_json); +} + +Result> RestCatalog::LoadDataTableSchema( + const Identifier& data_identifier, const std::optional& branch, + std::string* table_path) const { + PAIMON_ASSIGN_OR_RAISE(GetTableResponse response, api_->GetTable(data_identifier)); + if (table_path != nullptr) { + *table_path = response.GetPath(); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr schema, ToTableSchema(response, branch)); + return std::shared_ptr(std::move(schema)); +} + +Result> RestCatalog::LoadTableSchema(const Identifier& identifier) const { + // Serve the global system tables of the "sys" database locally, like + // FileSystemCatalog. + if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { + PAIMON_ASSIGN_OR_RAISE(bool supported, + GlobalSystemTableLoader::IsSupported(identifier.GetTableName(), + api_->GetMergedOptions())); + if (!supported) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + GlobalSystemTableContext context; + context.catalog = this; + context.fs = fs_; + context.warehouse = warehouse_; + context.catalog_options = api_->GetMergedOptions(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + GlobalSystemTableLoader::Load(identifier.GetTableName(), context)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, + system_table->ArrowSchema()); + return std::make_shared(std::move(arrow_schema)); + } + PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + branch = NormalizeBranch(std::move(branch)); + PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); + if (is_system_table) { + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, + identifier.GetSystemTableName()); + if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + std::string table_path; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr latest_schema, + LoadDataTableSchema(load_identifier, branch, &table_path)); + std::map dynamic_options; + if (branch) { + dynamic_options[Options::BRANCH] = branch.value(); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + SystemTableLoader::Load(system_table_name.value(), fs_, table_path, + latest_schema, dynamic_options)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, + system_table->ArrowSchema()); + return std::make_shared(std::move(arrow_schema)); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + LoadDataTableSchema(load_identifier, branch, nullptr)); + return std::static_pointer_cast(schema); +} + +Result> RestCatalog::GetTable(const Identifier& identifier) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, LoadTableSchema(identifier)); + return std::make_shared(schema, identifier.GetDatabaseName(), identifier.GetTableName()); +} + +std::string RestCatalog::GetRootPath() const { + return warehouse_; +} + +std::shared_ptr RestCatalog::GetFileSystem() const { + return fs_; +} + +Result> RestCatalog::ListSnapshots(const Identifier& identifier, + const std::string& branch) const { + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(identifier, "list snapshots")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(identifier, "list snapshots")); + // The default branch "main" is addressed as the bare table; any other branch is + // sent under its branch object name so the server resolves the branch itself. + std::optional normalized_branch = + NormalizeBranch(branch.empty() ? std::nullopt : std::make_optional(branch)); + std::string object_name = identifier.GetTableName(); + if (normalized_branch) { + object_name.append(Identifier::kSystemTableSplitter); + object_name.append(Identifier::kSystemBranchPrefix); + object_name.append(normalized_branch.value()); + } + Identifier load_identifier(identifier.GetDatabaseName(), object_name); + // The Catalog interface has no pagination, so all pages are fetched. The server + // does not guarantee an ordering across pages while the ListSnapshots contract + // requires ascending ids, hence the sort. + PAIMON_ASSIGN_OR_RAISE(std::vector snapshots, api_->ListSnapshots(load_identifier)); + std::sort(snapshots.begin(), snapshots.end(), + [](const Snapshot& a, const Snapshot& b) { return a.Id() < b.Id(); }); + std::vector result; + result.reserve(snapshots.size()); + for (const auto& snapshot : snapshots) { + result.push_back(snapshot.ToSnapshotInfo()); + } + return result; +} + +} // namespace paimon diff --git a/src/paimon/rest/rest_catalog.h b/src/paimon/rest/rest_catalog.h new file mode 100644 index 000000000..94c67ee53 --- /dev/null +++ b/src/paimon/rest/rest_catalog.h @@ -0,0 +1,101 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "paimon/catalog/catalog.h" +#include "paimon/logging.h" +#include "paimon/rest/rest_api.h" +#include "paimon/result.h" +#include "paimon/status.h" + +struct ArrowSchema; + +namespace paimon { +class FileSystem; +class TableSchema; + +/// A catalog backed by a REST catalog server. Metadata operations are delegated to +/// `RestApi`; table data is accessed through the file system configured by the +/// (server merged) options. +class RestCatalog : public Catalog { + public: + /// Creates the catalog: fetches and merges "/v1/config" from the server configured + /// by `CatalogOptions::URI`, then builds the file system from the merged options. + /// + /// @param warehouse The warehouse identifier sent to the server; may be empty. + static Result> Create( + const std::string& warehouse, const std::map& options, + const std::shared_ptr& file_system, + const RestHttpClient::Config& http_config = RestHttpClient::Config()); + + Status CreateDatabase(const std::string& name, + const std::map& options, + bool ignore_if_exists) override; + Status CreateTable(const Identifier& identifier, ArrowSchema* c_schema, + const std::vector& partition_keys, + const std::vector& primary_keys, + const std::map& options, + bool ignore_if_exists) override; + Status DropDatabase(const std::string& name, bool ignore_if_not_exists, bool cascade) override; + Status DropTable(const Identifier& identifier, bool ignore_if_not_exists) override; + Status RenameTable(const Identifier& from_table, const Identifier& to_table, + bool ignore_if_not_exists) override; + Result> ListDatabases() const override; + Result> ListTables(const std::string& db_name) const override; + Result DatabaseExists(const std::string& db_name) const override; + Result TableExists(const Identifier& identifier) const override; + std::string GetDatabaseLocation(const std::string& db_name) const override; + Result GetTableLocation(const Identifier& identifier) const override; + Result> LoadTableSchema(const Identifier& identifier) const override; + std::string GetRootPath() const override; + std::shared_ptr GetFileSystem() const override; + Result> GetTable(const Identifier& identifier) const override; + Result> ListSnapshots(const Identifier& identifier, + const std::string& branch) const override; + + /// Options merged with the server side config. + const std::map& GetOptions() const override; + + private: + RestCatalog(std::unique_ptr api, const std::shared_ptr& fs, + const std::string& warehouse); + + /// Loads the table from the server and converts the response to a `TableSchema` + /// (options are enriched with the table path, audit info and branch). + Result> LoadDataTableSchema( + const Identifier& data_identifier, const std::optional& branch, + std::string* table_path) const; + + static Result> ToTableSchema( + const GetTableResponse& response, const std::optional& branch); + + std::unique_ptr api_; + std::shared_ptr fs_; + std::string warehouse_; + /// The "table-default." options of the merged config, applied to `CreateTable` + /// options when absent. + std::map table_default_options_; + std::shared_ptr logger_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp new file mode 100644 index 000000000..239750e5a --- /dev/null +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -0,0 +1,997 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_catalog.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "fmt/format.h" +#include "gtest/gtest.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/table.h" +#include "paimon/catalog_options.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/defs.h" +#include "paimon/rest/mock_rest_server.h" +#include "paimon/rest/rest_api.h" +#include "paimon/schema/schema.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +namespace { + +constexpr const char kToken[] = "test-token"; +constexpr const char kPrefix[] = "paimon"; +constexpr const char kWarehouse[] = "wh1"; + +/// The in-memory catalog state behind the mock rest server. +struct MockCatalogState { + struct TableData { + std::string schema_json; + int64_t schema_id = 0; + std::string path; + }; + // database name -> table name -> table + std::map> databases; + // headers of the last request, with lower-cased names + std::map last_headers; + // when set, every request except "/v1/config" fails with this http code + std::optional force_error_code; + // guards all fields above: the handler runs on the mock server's accept thread + // while tests seed and inspect the state from the test thread + std::mutex mutex; +}; + +MockRestServer::Response JsonResponse(int32_t code, const std::string& body) { + MockRestServer::Response response; + response.code = code; + response.body = body; + return response; +} + +MockRestServer::Response MockError(int32_t code, const std::string& resource_type, + const std::string& resource_name, const std::string& message) { + ErrorResponse error(resource_type, resource_name, message, code); + return JsonResponse(code, error.ToJsonString().value()); +} + +std::string SnapshotJson(int64_t id) { + return fmt::format( + R"({{"version":3,"id":{},"schemaId":0,"baseManifestList":"bml","deltaManifestList":"dml",)" + R"("commitUser":"user1","commitIdentifier":1,"commitKind":"APPEND","timeMillis":100,)" + R"("totalRecordCount":10,"deltaRecordCount":1}})", + id); +} + +std::string TableResponseJson(const std::string& name, const MockCatalogState::TableData& table) { + return fmt::format( + R"({{"id":"1","name":"{}","path":"{}","isExternal":false,"schemaId":{},"schema":{},)" + R"("owner":"owner1","updatedAt":123}})", + name, table.path, table.schema_id, table.schema_json); +} + +/// Serves `names` one item per page to exercise the pagination loop of the client. +std::pair, std::optional> PageOf( + const std::vector& names, const MockRestServer::Request& request) { + size_t index = 0; + auto token_iter = request.query_params.find(RestApi::kQueryParamPageToken); + if (token_iter != request.query_params.end()) { + // The exception-free parse keeps a malformed token from terminating the test + // binary: the handler runs on the mock server's accept thread. + index = static_cast( + StringUtils::StringToValue(token_iter->second).value_or(0)); + } + std::vector page; + std::optional next_page_token; + if (index < names.size()) { + page.push_back(names[index]); + if (index + 1 < names.size()) { + next_page_token = std::to_string(index + 1); + } + } + return {page, next_page_token}; +} + +/// Implements the subset of the rest catalog protocol used by `RestCatalog` on top of +/// `MockCatalogState`. +MockRestServer::Response HandleCatalogRequest(MockCatalogState* state, + const MockRestServer::Request& request) { + std::lock_guard lock(state->mutex); + state->last_headers = request.headers; + auto auth_iter = request.headers.find("authorization"); + if (auth_iter == request.headers.end() || + auth_iter->second != std::string("Bearer ") + kToken) { + return MockError(401, "", "", "invalid token"); + } + if (request.path == "/v1/config") { + auto warehouse_iter = request.query_params.find("warehouse"); + if (warehouse_iter == request.query_params.end() || warehouse_iter->second != kWarehouse) { + return MockError(400, "", "", "unexpected warehouse"); + } + ConfigResponse config( + {{RestApi::kOptionUrlPrefix, kPrefix}, + {"header.x-server-header", "from-config"}, + {"table-default.write-only", "true"}, + {"table-default.bucket", "8"}}, + {{"server-override", "from-server"}, {"header.x-shared-header", "from-config"}}); + return JsonResponse(200, config.ToJsonString().value()); + } + if (state->force_error_code) { + return MockError(state->force_error_code.value(), "", "", "injected failure"); + } + const std::string base = std::string("/v1/") + kPrefix; + if (request.path.rfind(base, 0) != 0) { + return MockError(404, "", "", "unknown path " + request.path); + } + std::string rest = request.path.substr(base.size()); + + if (rest == "/databases") { + if (request.method == "GET") { + std::vector names; + for (const auto& [name, tables] : state->databases) { + names.push_back(name); + } + auto [page, next_page_token] = PageOf(names, request); + ListDatabasesResponse response(page, next_page_token); + return JsonResponse(200, response.ToJsonString().value()); + } + if (request.method == "POST") { + CreateDatabaseRequest create_request("", {}); + if (!RapidJsonUtil::FromJsonString(request.body, &create_request).ok()) { + return MockError(400, "", "", "bad create database request"); + } + if (state->databases.count(create_request.GetName()) > 0) { + return MockError(409, ErrorResponse::kResourceTypeDatabase, + create_request.GetName(), "database already exists"); + } + state->databases[create_request.GetName()] = {}; + return JsonResponse(200, ""); + } + return MockError(400, "", "", "unsupported method"); + } + + if (rest == "/tables/rename" && request.method == "POST") { + RenameTableRequest rename_request("", "", "", ""); + if (!RapidJsonUtil::FromJsonString(request.body, &rename_request).ok()) { + return MockError(400, "", "", "bad rename table request"); + } + auto db_iter = state->databases.find(rename_request.GetSourceDatabase()); + if (db_iter == state->databases.end() || + db_iter->second.count(rename_request.GetSourceTable()) == 0) { + return MockError(404, ErrorResponse::kResourceTypeTable, + rename_request.GetSourceTable(), "table not found"); + } + auto& dest_tables = state->databases[rename_request.GetDestinationDatabase()]; + if (dest_tables.count(rename_request.GetDestinationTable()) > 0) { + return MockError(409, ErrorResponse::kResourceTypeTable, + rename_request.GetDestinationTable(), "table already exists"); + } + dest_tables[rename_request.GetDestinationTable()] = + db_iter->second[rename_request.GetSourceTable()]; + db_iter->second.erase(rename_request.GetSourceTable()); + return JsonResponse(200, ""); + } + + const std::string databases_prefix = "/databases/"; + if (rest.rfind(databases_prefix, 0) != 0) { + return MockError(404, "", "", "unknown path " + request.path); + } + std::string remainder = rest.substr(databases_prefix.size()); + size_t tables_pos = remainder.find("/tables"); + + if (tables_pos == std::string::npos) { + const std::string& db_name = remainder; + auto db_iter = state->databases.find(db_name); + if (request.method == "GET") { + if (db_iter == state->databases.end()) { + return MockError(404, ErrorResponse::kResourceTypeDatabase, db_name, + "database not found"); + } + std::string body = fmt::format( + R"({{"id":"1","name":"{}","location":"{}/{}.db","options":{{"dbk":"dbv"}}}})", + db_name, kWarehouse, db_name); + return JsonResponse(200, body); + } + if (request.method == "DELETE") { + if (db_iter == state->databases.end()) { + return MockError(404, ErrorResponse::kResourceTypeDatabase, db_name, + "database not found"); + } + state->databases.erase(db_iter); + return JsonResponse(200, ""); + } + return MockError(400, "", "", "unsupported method"); + } + + std::string db_name = remainder.substr(0, tables_pos); + auto db_iter = state->databases.find(db_name); + if (db_iter == state->databases.end()) { + return MockError(404, ErrorResponse::kResourceTypeDatabase, db_name, "database not found"); + } + auto& tables = db_iter->second; + std::string table_part = remainder.substr(tables_pos + std::strlen("/tables")); + + if (table_part.empty()) { + if (request.method == "GET") { + std::vector names; + for (const auto& [name, table] : tables) { + names.push_back(name); + } + auto [page, next_page_token] = PageOf(names, request); + ListTablesResponse response(page, next_page_token); + return JsonResponse(200, response.ToJsonString().value()); + } + if (request.method == "POST") { + CreateTableRequest create_request("", "", ""); + if (!RapidJsonUtil::FromJsonString(request.body, &create_request).ok()) { + return MockError(400, "", "", "bad create table request"); + } + if (tables.count(create_request.GetTable()) > 0) { + return MockError(409, ErrorResponse::kResourceTypeTable, create_request.GetTable(), + "table already exists"); + } + MockCatalogState::TableData table; + table.schema_json = create_request.GetSchemaJson(); + table.schema_id = 0; + table.path = fmt::format("{}/{}.db/{}", kWarehouse, db_name, create_request.GetTable()); + tables[create_request.GetTable()] = table; + return JsonResponse(200, ""); + } + return MockError(400, "", "", "unsupported method"); + } + + // "/{table}" or "/{table}/snapshots" + std::string table_name = table_part.substr(1); + bool list_snapshots = false; + const std::string snapshots_suffix = "/snapshots"; + if (table_name.size() > snapshots_suffix.size() && + table_name.compare(table_name.size() - snapshots_suffix.size(), snapshots_suffix.size(), + snapshots_suffix) == 0) { + table_name = table_name.substr(0, table_name.size() - snapshots_suffix.size()); + list_snapshots = true; + } + auto table_iter = tables.find(table_name); + if (table_iter == tables.end()) { + return MockError(404, ErrorResponse::kResourceTypeTable, table_name, "table not found"); + } + if (list_snapshots) { + // two pages, out of order to exercise pagination and sorting + auto token_iter = request.query_params.find(RestApi::kQueryParamPageToken); + if (token_iter == request.query_params.end()) { + return JsonResponse( + 200, fmt::format(R"({{"snapshots":[{}],"nextPageToken":"1"}})", SnapshotJson(2))); + } + return JsonResponse(200, fmt::format(R"({{"snapshots":[{}]}})", SnapshotJson(1))); + } + if (request.method == "GET") { + return JsonResponse(200, TableResponseJson(table_name, table_iter->second)); + } + if (request.method == "DELETE") { + tables.erase(table_iter); + return JsonResponse(200, ""); + } + return MockError(400, "", "", "unsupported method"); +} + +} // namespace + +class RestCatalogTest : public ::testing::Test { + protected: + void SetUp() override { + state_ = std::make_shared(); + ASSERT_OK_AND_ASSIGN( + server_, MockRestServer::Start([state = state_](const MockRestServer::Request& req) { + return HandleCatalogRequest(state.get(), req); + })); + options_ = { + {CatalogOptions::METASTORE, "rest"}, + {CatalogOptions::URI, server_->GetBaseUri()}, + {CatalogOptions::TOKEN_PROVIDER, "bear"}, + {CatalogOptions::TOKEN, kToken}, + {Options::FILE_SYSTEM, "local"}, + // mock_format is linked statically into the test binary, so its factory is + // registered in the binary's own registry even when the real format plugin + // dylibs register into a different one (macOS two-level namespace) + {Options::FILE_FORMAT, "mock_format"}, + {Options::MANIFEST_FORMAT, "mock_format"}, + {"header.x-client-header", "from-client"}, + {"header.x-shared-header", "from-client"}, + }; + } + + void TearDown() override { + if (server_) { + server_->Stop(); + } + } + + Result> CreateRestCatalog() { + return RestCatalog::Create(kWarehouse, options_, nullptr); + } + + Status CreateSampleTable(Catalog* catalog, const Identifier& identifier, + bool ignore_if_exists = false) { + std::shared_ptr schema = + arrow::schema({arrow::field("f0", arrow::int32(), /*nullable=*/false), + arrow::field("f1", arrow::utf8())}); + struct ArrowSchema c_schema; + if (!arrow::ExportSchema(*schema, &c_schema).ok()) { + return Status::Invalid("failed to export arrow schema"); + } + Status status = + catalog->CreateTable(identifier, &c_schema, /*partition_keys=*/{}, + /*primary_keys=*/{"f0"}, {{"bucket", "2"}}, ignore_if_exists); + // CreateTable takes ownership of the exported schema only once it reaches + // arrow::ImportSchema, which an identifier rejected by its checks never does + if (c_schema.release != nullptr) { + c_schema.release(&c_schema); + } + return status; + } + + /// Seeds `schema_json` as table `table_name` of "db1" behind the mock server and + /// expects loading the table to fail with an Invalid status carrying + /// `expected_message`. + void ExpectBrokenSchemaRejected(Catalog* catalog, const std::string& table_name, + const std::string& schema_json, + const std::string& expected_message) { + MockCatalogState::TableData table_data; + table_data.schema_json = schema_json; + table_data.path = "wh1/db1.db/" + table_name; + { + std::lock_guard lock(state_->mutex); + state_->databases["db1"][table_name] = table_data; + } + Status status = catalog->GetTable(Identifier("db1", table_name)).status(); + ASSERT_TRUE(status.IsInvalid()) << status.ToString(); + ASSERT_NOK_WITH_MSG(status, expected_message); + } + + std::shared_ptr state_; + std::unique_ptr server_; + std::map options_; +}; + +TEST_F(RestCatalogTest, CreateMergesServerConfig) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + const std::map& merged = catalog->GetOptions(); + ASSERT_EQ(kPrefix, merged.at(RestApi::kOptionUrlPrefix)); + ASSERT_EQ("from-server", merged.at("server-override")); + ASSERT_EQ(kWarehouse, catalog->GetRootPath()); + ASSERT_NE(nullptr, catalog->GetFileSystem()); +} + +TEST_F(RestCatalogTest, CatalogFactoryMetastoreDispatch) { + // "metastore=rest" selects the rest catalog + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(kWarehouse, options_)); + ASSERT_OK_AND_ASSIGN(std::vector databases, catalog->ListDatabases()); + ASSERT_TRUE(databases.empty()); + // an unknown metastore is rejected + options_[CatalogOptions::METASTORE] = "something-else"; + Status status = Catalog::Create(kWarehouse, options_).status(); + ASSERT_TRUE(status.IsInvalid()) << status.ToString(); + ASSERT_NOK_WITH_MSG(status, "unsupported metastore"); +} + +TEST_F(RestCatalogTest, CreateWithWrongTokenFails) { + options_[CatalogOptions::TOKEN] = "wrong-token"; + ASSERT_NOK_WITH_MSG(CreateRestCatalog().status(), "not authorized"); +} + +TEST_F(RestCatalogTest, CreateRejectsInvalidOptions) { + // all rejected by client side validation, before any request reaches the server + const std::map valid_options = options_; + + options_.erase(CatalogOptions::URI); + ASSERT_NOK_WITH_MSG(CreateRestCatalog().status(), "'uri' must be configured"); + + options_ = valid_options; + options_.erase(CatalogOptions::TOKEN_PROVIDER); + ASSERT_NOK_WITH_MSG(CreateRestCatalog().status(), "'token.provider' must be configured"); + + options_ = valid_options; + options_[CatalogOptions::TOKEN_PROVIDER] = "dlf"; + Status unsupported_provider = CreateRestCatalog().status(); + ASSERT_TRUE(unsupported_provider.IsNotImplemented()) << unsupported_provider.ToString(); + ASSERT_NOK_WITH_MSG(unsupported_provider, "unsupported token provider"); + + options_ = valid_options; + options_.erase(CatalogOptions::TOKEN); + ASSERT_NOK_WITH_MSG(CreateRestCatalog().status(), "bear token provider"); +} + +TEST_F(RestCatalogTest, DatabaseOperations) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(catalog->CreateDatabase("db2", {}, /*ignore_if_exists=*/false)); + Status duplicated = catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false); + ASSERT_TRUE(duplicated.IsExist()) << duplicated.ToString(); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/true)); + + // the mock server returns one database per page + ASSERT_OK_AND_ASSIGN(std::vector databases, catalog->ListDatabases()); + ASSERT_EQ((std::vector{"db1", "db2"}), databases); + + ASSERT_OK_AND_ASSIGN(bool exists, catalog->DatabaseExists("db1")); + ASSERT_TRUE(exists); + ASSERT_OK_AND_ASSIGN(exists, catalog->DatabaseExists("db3")); + ASSERT_FALSE(exists); + + ASSERT_EQ("wh1/db1.db", catalog->GetDatabaseLocation("db1")); + ASSERT_EQ("", catalog->GetDatabaseLocation("db3")); + + ASSERT_OK(catalog->DropDatabase("db2", /*ignore_if_not_exists=*/false, /*cascade=*/false)); + ASSERT_OK(catalog->DropDatabase("db2", /*ignore_if_not_exists=*/true, /*cascade=*/false)); + Status missing = catalog->DropDatabase("db2", /*ignore_if_not_exists=*/false, + /*cascade=*/false); + ASSERT_TRUE(missing.IsNotExist()) << missing.ToString(); + + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); + ASSERT_NOK_WITH_MSG( + catalog->DropDatabase("db1", /*ignore_if_not_exists=*/false, /*cascade=*/false), + "non-empty database"); + // cascade drop skips the emptiness check + ASSERT_OK(catalog->DropDatabase("db1", /*ignore_if_not_exists=*/false, /*cascade=*/true)); + ASSERT_OK_AND_ASSIGN(exists, catalog->DatabaseExists("db1")); + ASSERT_FALSE(exists); +} + +TEST_F(RestCatalogTest, TableOperations) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + Identifier identifier("db1", "t1"); + + Status missing_db = CreateSampleTable(catalog.get(), Identifier("db_missing", "t1")); + ASSERT_TRUE(missing_db.IsNotExist()) << missing_db.ToString(); + + ASSERT_OK(CreateSampleTable(catalog.get(), identifier)); + Status duplicated = CreateSampleTable(catalog.get(), identifier); + ASSERT_TRUE(duplicated.IsExist()) << duplicated.ToString(); + ASSERT_OK(CreateSampleTable(catalog.get(), identifier, /*ignore_if_exists=*/true)); + + ASSERT_OK_AND_ASSIGN(std::vector tables, catalog->ListTables("db1")); + ASSERT_EQ((std::vector{"t1"}), tables); + Status list_missing = catalog->ListTables("db_missing").status(); + ASSERT_TRUE(list_missing.IsNotExist()) << list_missing.ToString(); + + ASSERT_OK_AND_ASSIGN(bool exists, catalog->TableExists(identifier)); + ASSERT_TRUE(exists); + ASSERT_OK_AND_ASSIGN(exists, catalog->TableExists(Identifier("db1", "t2"))); + ASSERT_FALSE(exists); + + ASSERT_OK_AND_ASSIGN(std::string location, catalog->GetTableLocation(identifier)); + ASSERT_EQ("wh1/db1.db/t1", location); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(identifier)); + ASSERT_EQ("t1", table->Name()); + std::shared_ptr schema = + std::dynamic_pointer_cast(table->LatestSchema()); + ASSERT_NE(nullptr, schema); + ASSERT_EQ((std::vector{"f0", "f1"}), schema->FieldNames()); + ASSERT_EQ((std::vector{"f0"}), schema->PrimaryKeys()); + ASSERT_EQ(0, schema->Id()); + // options are enriched with the table path and audit info from the server + ASSERT_EQ("wh1/db1.db/t1", schema->Options().at("path")); + ASSERT_EQ("owner1", schema->Options().at("owner")); + + // "table-default." options of the merged config apply only where the caller left the + // option unset: "write-only" is taken from the config, "bucket" keeps the value passed + // to CreateTable instead of the configured "table-default.bucket" of 8 + ASSERT_EQ("true", schema->Options().at("write-only")); + ASSERT_EQ("2", schema->Options().at("bucket")); + + // timeMillis is backed by the server's audit "updatedAt" instead of the current + // time, keeping the conversion deterministic + std::shared_ptr table_schema = + std::dynamic_pointer_cast(table->LatestSchema()); + ASSERT_NE(nullptr, table_schema); + ASSERT_EQ(123, table_schema->TimeMillis()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr loaded_schema, + catalog->LoadTableSchema(identifier)); + ASSERT_EQ((std::vector{"f0", "f1"}), loaded_schema->FieldNames()); + Status schema_missing = catalog->LoadTableSchema(Identifier("db1", "t2")).status(); + ASSERT_TRUE(schema_missing.IsNotExist()) << schema_missing.ToString(); + + // a schema comment of the server response is carried into the table schema + MockCatalogState::TableData commented; + commented.schema_json = R"({"fields": [{"id": 0, "name": "f0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {},)" + R"( "comment": "a table comment"})"; + commented.path = "wh1/db1.db/commented"; + { + std::lock_guard lock(state_->mutex); + state_->databases["db1"]["commented"] = commented; + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr
commented_table, + catalog->GetTable(Identifier("db1", "commented"))); + ASSERT_EQ("a table comment", commented_table->LatestSchema()->Comment().value_or("")); + + ASSERT_OK(catalog->RenameTable(identifier, Identifier("db1", "t2"), + /*ignore_if_not_exists=*/false)); + ASSERT_OK_AND_ASSIGN(exists, catalog->TableExists(Identifier("db1", "t2"))); + ASSERT_TRUE(exists); + ASSERT_OK(catalog->RenameTable(identifier, Identifier("db1", "t3"), + /*ignore_if_not_exists=*/true)); + Status rename_missing = catalog->RenameTable(identifier, Identifier("db1", "t3"), + /*ignore_if_not_exists=*/false); + ASSERT_TRUE(rename_missing.IsNotExist()) << rename_missing.ToString(); + + ASSERT_OK(catalog->DropTable(Identifier("db1", "t2"), /*ignore_if_not_exists=*/false)); + ASSERT_OK(catalog->DropTable(Identifier("db1", "t2"), /*ignore_if_not_exists=*/true)); + Status drop_missing = catalog->DropTable(Identifier("db1", "t2"), + /*ignore_if_not_exists=*/false); + ASSERT_TRUE(drop_missing.IsNotExist()) << drop_missing.ToString(); +} + +TEST_F(RestCatalogTest, ClientAndServerHeadersAreSent) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK_AND_ASSIGN(std::vector databases, catalog->ListDatabases()); + // "header." options from both the client and the merged server config are sent as + // http headers on every request + { + std::lock_guard lock(state_->mutex); + ASSERT_EQ("from-client", state_->last_headers.at("x-client-header")); + ASSERT_EQ("from-config", state_->last_headers.at("x-server-header")); + // when the client and the server config set the same "header." option, the + // merged config wins (overrides > client options > defaults) + ASSERT_EQ("from-config", state_->last_headers.at("x-shared-header")); + ASSERT_EQ(std::string("Bearer ") + kToken, state_->last_headers.at("authorization")); + } + // a request carrying a body declares the json content type (set before the auth + // headers are merged, so a signing auth provider covers it) + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + { + std::lock_guard lock(state_->mutex); + ASSERT_EQ("application/json", state_->last_headers.at("content-type")); + } +} + +TEST_F(RestCatalogTest, SystemTableSchema) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); + + // the "options" system table has a static schema and needs no file system access + Identifier system_identifier("db1", "t1$options"); + ASSERT_OK_AND_ASSIGN(bool exists, catalog->TableExists(system_identifier)); + ASSERT_TRUE(exists); + ASSERT_OK_AND_ASSIGN(exists, catalog->TableExists(Identifier("db1", "t9$options"))); + ASSERT_FALSE(exists); + ASSERT_OK_AND_ASSIGN(exists, catalog->TableExists(Identifier("db1", "t1$unsupported"))); + ASSERT_FALSE(exists); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, + catalog->LoadTableSchema(system_identifier)); + ASSERT_EQ((std::vector{"key", "value"}), schema->FieldNames()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(system_identifier)); + ASSERT_EQ("t1$options", table->Name()); + ASSERT_EQ((std::vector{"key", "value"}), table->LatestSchema()->FieldNames()); + + Status unsupported = catalog->LoadTableSchema(Identifier("db1", "t1$unsupported")).status(); + ASSERT_TRUE(unsupported.IsNotExist()) << unsupported.ToString(); +} + +TEST_F(RestCatalogTest, BranchTableLoadsBranchSchemaFromServer) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); + + // the branch is seeded under its object name with a schema diverged from the main + // table: the identifier sent to the server keeps the branch, and the server + // resolves it. The path it reports is the data table root (not the branch + // subdirectory): downstream readers derive "/branch/branch-" from the + // branch option, so a branch path here would be applied twice + MockCatalogState::TableData branch_data; + branch_data.schema_json = R"({"fields": [{"id": 0, "name": "b0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {}})"; + branch_data.schema_id = 3; + branch_data.path = "wh1/db1.db/t1"; + { + std::lock_guard lock(state_->mutex); + state_->databases["db1"]["t1$branch_b1"] = branch_data; + } + + Identifier branch_identifier("db1", "t1$branch_b1"); + ASSERT_OK_AND_ASSIGN(bool exists, catalog->TableExists(branch_identifier)); + ASSERT_TRUE(exists); + // a branch the server does not know is missing instead of silently falling back to + // the main table + ASSERT_OK_AND_ASSIGN(exists, catalog->TableExists(Identifier("db1", "t1$branch_missing"))); + ASSERT_FALSE(exists); + + ASSERT_OK_AND_ASSIGN(std::string location, catalog->GetTableLocation(branch_identifier)); + ASSERT_EQ("wh1/db1.db/t1", location); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(branch_identifier)); + ASSERT_EQ("t1$branch_b1", table->Name()); + std::shared_ptr schema = + std::dynamic_pointer_cast(table->LatestSchema()); + ASSERT_NE(nullptr, schema); + // the branch's own schema and schema id, not the main table's + ASSERT_EQ((std::vector{"b0"}), schema->FieldNames()); + ASSERT_EQ(3, schema->Id()); + ASSERT_EQ("b1", schema->Options().at(Options::BRANCH)); + + // the default branch is addressed as the bare table: "t1$branch_MAIN" resolves to + // "t1" (case-insensitively) and carries no branch option + ASSERT_OK_AND_ASSIGN(std::shared_ptr
main_table, + catalog->GetTable(Identifier("db1", "t1$branch_MAIN"))); + std::shared_ptr main_schema = + std::dynamic_pointer_cast(main_table->LatestSchema()); + ASSERT_NE(nullptr, main_schema); + ASSERT_EQ((std::vector{"f0", "f1"}), main_schema->FieldNames()); + ASSERT_EQ(0, main_schema->Options().count(Options::BRANCH)); + + // a system table on a branch resolves against the branch's data table: the system + // suffix is stripped while the branch stays in the identifier sent to the server + ASSERT_OK_AND_ASSIGN(std::shared_ptr options_schema, + catalog->LoadTableSchema(Identifier("db1", "t1$branch_b1$options"))); + ASSERT_EQ((std::vector{"key", "value"}), options_schema->FieldNames()); + // a missing branch fails through the system table path too instead of silently + // resolving against the main table + Status missing_branch = + catalog->LoadTableSchema(Identifier("db1", "t1$branch_missing$options")).status(); + ASSERT_TRUE(missing_branch.IsNotExist()) << missing_branch.ToString(); + + // writing operations reject branch tables + ASSERT_NOK_WITH_MSG(catalog->DropTable(branch_identifier, /*ignore_if_not_exists=*/false), + "branch table"); + ASSERT_NOK_WITH_MSG(catalog->RenameTable(branch_identifier, Identifier("db1", "t2"), + /*ignore_if_not_exists=*/false), + "branch table"); + ASSERT_NOK_WITH_MSG(CreateSampleTable(catalog.get(), Identifier("db1", "t2$branch_b1")), + "branch table"); +} + +TEST_F(RestCatalogTest, NestedSchemaHighestFieldId) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + // inject a table with nested types directly; ids inside nested rows must be + // included when computing highestFieldId + MockCatalogState::TableData table_data; + table_data.schema_json = R"({ + "fields": [ + {"id": 0, "name": "f0", "type": "INT NOT NULL"}, + {"id": 1, "name": "s", "type": {"type": "ROW", + "fields": [{"id": 3, "name": "inner", "type": "INT"}]}}, + {"id": 2, "name": "arr", "type": {"type": "ARRAY", + "element": {"type": "ROW", + "fields": [{"id": 7, "name": "deep", "type": "BIGINT"}]}}}, + {"id": 4, "name": "m", "type": {"type": "MAP", + "key": {"type": "ROW NOT NULL", + "fields": [{"id": 8, "name": "k", "type": "INT NOT NULL"}]}, + "value": {"type": "ROW", + "fields": [{"id": 9, "name": "v", "type": "INT"}]}}} + ], + "partitionKeys": [], + "primaryKeys": [], + "options": {} + })"; + table_data.schema_id = 5; + table_data.path = "wh1/db1.db/nested"; + { + std::lock_guard lock(state_->mutex); + state_->databases["db1"]["nested"] = table_data; + } + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("db1", "nested"))); + std::shared_ptr schema = + std::dynamic_pointer_cast(table->LatestSchema()); + ASSERT_NE(nullptr, schema); + ASSERT_EQ(5, schema->Id()); + // 9 lives inside the value row of the map: ROW, ARRAY element and MAP key/value + // must all be traversed + ASSERT_EQ(9, schema->HighestFieldId()); + ASSERT_EQ((std::vector{"f0", "s", "arr", "m"}), schema->FieldNames()); +} + +TEST_F(RestCatalogTest, ListTablesPaged) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t2"))); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t3"))); + // the mock server returns one table per page + ASSERT_OK_AND_ASSIGN(std::vector tables, catalog->ListTables("db1")); + ASSERT_EQ((std::vector{"t1", "t2", "t3"}), tables); +} + +TEST_F(RestCatalogTest, BrokenSchemaRejected) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + // a broken schema with duplicated field ids from the server must be rejected + ExpectBrokenSchemaRejected( + catalog.get(), "duplicate", + R"({"fields": [{"id": 0, "name": "f0", "type": "INT NOT NULL"},)" + R"( {"id": 0, "name": "f1", "type": "STRING"}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {}})", + "duplicated"); + // an id inside a nested row colliding with an outer field id is a duplicate too + ExpectBrokenSchemaRejected( + catalog.get(), "duplicate_nested", + R"({"fields": [{"id": 0, "name": "f0", "type": "INT NOT NULL"},)" + R"( {"id": 1, "name": "s", "type": {"type": "ROW",)" + R"( "fields": [{"id": 0, "name": "inner", "type": "INT"}]}}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {}})", + "duplicated"); + // a field without an integer id must fail the conversion instead of being + // silently skipped when computing highestFieldId + ExpectBrokenSchemaRejected(catalog.get(), "no_id", + R"({"fields": [{"name": "f0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {}})", + "integer id"); + ExpectBrokenSchemaRejected( + catalog.get(), "string_id", + R"({"fields": [{"id": "0", "name": "f0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {}})", + "integer id"); + // a field that is not an object fails too instead of being silently skipped + ExpectBrokenSchemaRejected(catalog.get(), "non_object", + R"({"fields": [1],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {}})", + "must be an object"); +} + +TEST_F(RestCatalogTest, ServerErrorIsPropagatedNotMappedToAbsent) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + { + std::lock_guard lock(state_->mutex); + state_->force_error_code = 500; + } + // a server failure surfaces as an error instead of "does not exist" + Status db_status = catalog->DatabaseExists("db1").status(); + ASSERT_NOK_WITH_MSG(db_status, "server error"); + Status table_status = catalog->TableExists(Identifier("db1", "t1")).status(); + ASSERT_NOK_WITH_MSG(table_status, "server error"); +} + +TEST_F(RestCatalogTest, SystemTableChecks) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_NOK_WITH_MSG(catalog->CreateDatabase("sys", {}, false), "system database"); + ASSERT_NOK_WITH_MSG(catalog->DropDatabase("sys", false, false), "system database"); + ASSERT_NOK_WITH_MSG(catalog->DropTable(Identifier("sys", "t"), false), "system table"); + ASSERT_NOK_WITH_MSG( + catalog->RenameTable(Identifier("db1", "t1$snapshots"), Identifier("db1", "t2"), false), + "system table"); +} + +TEST_F(RestCatalogTest, SystemDatabase) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + // the "sys" database and its global system tables are resolved locally without + // contacting the server, like in FileSystemCatalog + ASSERT_OK_AND_ASSIGN(bool exists, catalog->DatabaseExists("sys")); + ASSERT_TRUE(exists); + + ASSERT_OK_AND_ASSIGN(std::vector sys_tables, catalog->ListTables("sys")); + ASSERT_TRUE(std::find(sys_tables.begin(), sys_tables.end(), "tables") != sys_tables.end()); + + ASSERT_OK_AND_ASSIGN(exists, catalog->TableExists(Identifier("sys", "tables"))); + ASSERT_TRUE(exists); + ASSERT_OK_AND_ASSIGN(exists, catalog->TableExists(Identifier("sys", "unsupported"))); + ASSERT_FALSE(exists); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, + catalog->LoadTableSchema(Identifier("sys", "tables"))); + ASSERT_FALSE(schema->FieldNames().empty()); + Status missing = catalog->LoadTableSchema(Identifier("sys", "unsupported")).status(); + ASSERT_TRUE(missing.IsNotExist()) << missing.ToString(); +} + +TEST_F(RestCatalogTest, ListSnapshots) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + Identifier identifier("db1", "t1"); + ASSERT_OK(CreateSampleTable(catalog.get(), identifier)); + + ASSERT_OK_AND_ASSIGN(std::vector snapshots, + catalog->ListSnapshots(identifier, "")); + ASSERT_EQ(2, snapshots.size()); + // fetched via two pages and sorted by snapshot id + ASSERT_EQ(1, snapshots[0].snapshot_id); + ASSERT_EQ(2, snapshots[1].snapshot_id); + ASSERT_EQ("user1", snapshots[0].commit_user); + ASSERT_EQ(SnapshotInfo::CommitKind::APPEND, snapshots[0].commit_kind); + + // the default branch is addressed as the bare table, so passing it explicitly + // (case-insensitively) equals the branch-less call + ASSERT_OK_AND_ASSIGN(std::vector main_snapshots, + catalog->ListSnapshots(identifier, "MAIN")); + ASSERT_EQ(2, main_snapshots.size()); + + Status missing = catalog->ListSnapshots(Identifier("db1", "t9"), "").status(); + ASSERT_TRUE(missing.IsNotExist()) << missing.ToString(); + + // a non-main branch is sent under its branch object name, so the server resolves + // the branch and lists its own snapshots; the reported path is the data table + // root, the branch subdirectory is derived from the branch option downstream + MockCatalogState::TableData branch_data; + branch_data.schema_json = R"({"fields": [{"id": 0, "name": "f0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {}})"; + branch_data.path = "wh1/db1.db/t1"; + { + std::lock_guard lock(state_->mutex); + state_->databases["db1"]["t1$branch_b1"] = branch_data; + } + ASSERT_OK_AND_ASSIGN(std::vector branch_snapshots, + catalog->ListSnapshots(identifier, "b1")); + ASSERT_EQ(2, branch_snapshots.size()); + // a branch the server does not know is missing instead of silently falling back + // to the main table + Status missing_branch = catalog->ListSnapshots(identifier, "b_missing").status(); + ASSERT_TRUE(missing_branch.IsNotExist()) << missing_branch.ToString(); + + // a branch must be passed as the branch argument, not encoded in the identifier + ASSERT_NOK_WITH_MSG(catalog->ListSnapshots(Identifier("db1", "t1$branch_b1"), "").status(), + "branch table"); +} + +TEST(RestApiErrorTest, ErrorToStatus) { + RestHttpClient::Response response; + response.code = 404; + response.body = R"({"message": "no table", "resourceType": "TABLE", "resourceName": "t1"})"; + response.headers["x-request-id"] = "req-123"; + Status status = RestApi::ErrorToStatus(response); + ASSERT_TRUE(status.IsNotExist()); + ASSERT_TRUE(status.ToString().find("requestId:req-123") != std::string::npos) + << status.ToString(); + + response.code = 409; + Status exist_status = RestApi::ErrorToStatus(response); + ASSERT_TRUE(exist_status.IsExist()); + + response.code = 501; + response.body = ""; + ASSERT_TRUE(RestApi::ErrorToStatus(response).IsNotImplemented()); + + response.code = 500; + response.body = "not-a-json"; + ASSERT_NOK_WITH_MSG(RestApi::ErrorToStatus(response), "server error"); + + // the code of the error body wins over the http status when they disagree (e.g. a + // gateway rewriting the status) + response.code = 500; + response.body = R"({"message": "gone", "code": 404})"; + ASSERT_TRUE(RestApi::ErrorToStatus(response).IsNotExist()); + + // an empty server message still yields a status naming the http code, and the + // resource info is kept + response.code = 404; + response.body = R"({"resourceType": "TABLE", "resourceName": "t1"})"; + Status empty_message = RestApi::ErrorToStatus(response); + ASSERT_TRUE(empty_message.ToString().find("http status 404") != std::string::npos) + << empty_message.ToString(); + ASSERT_TRUE(empty_message.ToString().find("resource name: t1") != std::string::npos) + << empty_message.ToString(); + + // server messages that may embed secrets are redacted as a whole + response.code = 400; + response.body = R"({"message": "bad option password=abc123", "code": 400})"; + Status redacted = RestApi::ErrorToStatus(response); + ASSERT_TRUE(redacted.IsInvalid()) << redacted.ToString(); + ASSERT_TRUE(redacted.ToString().find("abc123") == std::string::npos) << redacted.ToString(); + ASSERT_TRUE(redacted.ToString().find("******") != std::string::npos) << redacted.ToString(); + + // any header carrying a request id is used when x-request-id is absent + response.code = 404; + response.body = ""; + response.headers.clear(); + response.headers["x-amz-request-id"] = "amz-1"; + Status fallback = RestApi::ErrorToStatus(response); + ASSERT_TRUE(fallback.ToString().find("requestId:amz-1") != std::string::npos) + << fallback.ToString(); + + // the "unknown" placeholder is not a real request id + response.headers.clear(); + response.headers["x-request-id"] = "unknown"; + Status unknown_id = RestApi::ErrorToStatus(response); + ASSERT_TRUE(unknown_id.ToString().find("requestId") == std::string::npos) + << unknown_id.ToString(); + + // 401/403 map to IOError; the mapped code is carried as a status detail so + // callers can distinguish them + response.headers.clear(); + response.code = 401; + Status not_authorized = RestApi::ErrorToStatus(response); + ASSERT_NOK_WITH_MSG(not_authorized, "not authorized"); + ASSERT_NE(nullptr, not_authorized.detail()); + ASSERT_EQ(std::string(RestErrorDetail::kTypeId), not_authorized.detail()->type_id()); + ASSERT_EQ(401, std::static_pointer_cast(not_authorized.detail())->GetCode()); + response.code = 403; + Status forbidden = RestApi::ErrorToStatus(response); + ASSERT_NOK_WITH_MSG(forbidden, "forbidden"); + ASSERT_EQ(403, std::static_pointer_cast(forbidden.detail())->GetCode()); + + // 503 and the codes without an own mapping (e.g. 429) become IOError with a + // message naming the code + response.code = 503; + ASSERT_NOK_WITH_MSG(RestApi::ErrorToStatus(response), "service unavailable"); + response.code = 429; + ASSERT_NOK_WITH_MSG(RestApi::ErrorToStatus(response), "rest request failed with code 429"); + response.code = 418; + ASSERT_NOK_WITH_MSG(RestApi::ErrorToStatus(response), "rest request failed with code 418"); +} + +TEST(RestApiErrorTest, MalformedSuccessBodyFails) { + // a 200 response whose body is not the expected json must fail, not crash or + // return partial data + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([](const MockRestServer::Request& request) { + MockRestServer::Response response; + response.body = "not-a-json"; + return response; + })); + std::map options = { + {CatalogOptions::URI, server->GetBaseUri()}, + {CatalogOptions::TOKEN_PROVIDER, "bear"}, + {CatalogOptions::TOKEN, kToken}, + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr api, + RestApi::Create(options, "", /*config_required=*/false)); + Status list_status = api->ListDatabases().status(); + ASSERT_NOK(list_status); + // the body is not echoed into the error: a successful response may contain + // credentials + ASSERT_EQ(std::string::npos, list_status.ToString().find("not-a-json")) + << list_status.ToString(); + ASSERT_NOK(api->GetTable(Identifier("db1", "t1")).status()); + + // the same garbage on "/v1/config" fails the creation with the config fetch + Status config_status = RestApi::Create(options, "", /*config_required=*/true).status(); + ASSERT_NOK(config_status); + ASSERT_EQ(std::string::npos, config_status.ToString().find("not-a-json")) + << config_status.ToString(); +} + +TEST(RestApiErrorTest, PagedListingStopsOnEmptyPageWithToken) { + // a server that keeps returning a page token with no data must not loop forever + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + request_count++; + MockRestServer::Response response; + response.body = R"({"databases":[],"nextPageToken":"more"})"; + return response; + })); + std::map options = { + {CatalogOptions::URI, server->GetBaseUri()}, + {CatalogOptions::TOKEN_PROVIDER, "bear"}, + {CatalogOptions::TOKEN, kToken}, + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr api, + RestApi::Create(options, "", /*config_required=*/false)); + ASSERT_OK_AND_ASSIGN(std::vector databases, api->ListDatabases()); + ASSERT_TRUE(databases.empty()); + ASSERT_EQ(1, request_count.load()); +} + +} // namespace paimon::test diff --git a/src/paimon/rest/rest_http_client.cpp b/src/paimon/rest/rest_http_client.cpp new file mode 100644 index 000000000..185b40228 --- /dev/null +++ b/src/paimon/rest/rest_http_client.cpp @@ -0,0 +1,362 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_http_client.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "curl/curl.h" +#include "fmt/format.h" +#include "paimon/common/utils/http_client.h" +#include "paimon/common/utils/scope_guard.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/common/utils/url_utils.h" + +namespace paimon { + +namespace { + +size_t WriteBodyCallback(char* data, size_t size, size_t nmemb, void* user_data) { + auto* body = static_cast(user_data); + body->append(data, size * nmemb); + return size * nmemb; +} + +size_t WriteHeaderCallback(char* data, size_t size, size_t nmemb, void* user_data) { + auto* headers = static_cast(user_data); + size_t total = size * nmemb; + // A status line starts the header block of the next response of the connection + // (e.g. after a followed redirect): the headers of the previous response are + // dropped so that only the final response's headers are reported. + if (total >= 5 && std::strncmp(data, "HTTP/", 5) == 0) { + headers->clear(); + } + ParseHttpHeaderLine(data, total, headers); + return total; +} + +// Of the supported methods (see `Execute`), only GET and DELETE are idempotent. +bool IsIdempotent(const std::string& method) { + return method == "GET" || method == "DELETE"; +} + +bool IsRetriableCode(int64_t code) { + return code == HttpStatus::kTooManyRequests || code == HttpStatus::kServiceUnavailable; +} + +// Parses the RFC 1123 form of an http date, e.g. "Wed, 21 Oct 2015 07:28:00 GMT", +// into unix epoch seconds. The names are matched against fixed English tables +// independently of the process locale. +std::optional ParseHttpDateSeconds(const std::string& value) { + // ", :: GMT" + std::vector parts = StringUtils::Split(value, " ", /*ignore_empty=*/true); + if (parts.size() != 6 || parts[5] != "GMT" || parts[0].size() != 4 || parts[0][3] != ',') { + return std::nullopt; + } + static constexpr const char* kMonths[] = {"Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}; + int64_t month = 0; + for (size_t i = 0; i < 12; i++) { + if (parts[2] == kMonths[i]) { + month = static_cast(i) + 1; + break; + } + } + std::optional day = StringUtils::StringToValue(parts[1]); + std::optional year = StringUtils::StringToValue(parts[3]); + std::vector time_parts = StringUtils::Split(parts[4], ":", + /*ignore_empty=*/false); + if (month == 0 || !day || !year || time_parts.size() != 3) { + return std::nullopt; + } + std::optional hour = StringUtils::StringToValue(time_parts[0]); + std::optional minute = StringUtils::StringToValue(time_parts[1]); + std::optional second = StringUtils::StringToValue(time_parts[2]); + if (!hour || !minute || !second || day.value() < 1 || day.value() > 31 || year.value() < 1970 || + year.value() > 9999 || hour.value() > 23 || minute.value() > 59 || second.value() > 60) { + return std::nullopt; + } + // Days between 1970-01-01 and the given civil date ("days from civil" algorithm). + int64_t y = year.value() - (month <= 2 ? 1 : 0); + int64_t era = y / 400; + int64_t yoe = y - era * 400; + int64_t doy = (153 * (month + (month > 2 ? -3 : 9)) + 2) / 5 + day.value() - 1; + int64_t doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + int64_t days = era * 146097 + doe - 719468; + return days * 86400 + hour.value() * 3600 + minute.value() * 60 + second.value(); +} + +// Returns false for the transport failure kinds that do not become healthy by +// retrying: name resolution and connect failures, timeouts, TLS failures, and the +// server closing the connection without a response (CURLE_GOT_NOTHING). +// +// A response truncated mid-body (CURLE_PARTIAL_FILE) stays retriable: the request +// demonstrably reached a live server, so retrying the idempotent methods is safe +// and likely to succeed. +bool IsRetriableTransportError(CURLcode code) { + switch (code) { + case CURLE_COULDNT_RESOLVE_PROXY: + case CURLE_COULDNT_RESOLVE_HOST: + case CURLE_COULDNT_CONNECT: + case CURLE_OPERATION_TIMEDOUT: + case CURLE_GOT_NOTHING: + case CURLE_SSL_CONNECT_ERROR: + case CURLE_SSL_CERTPROBLEM: + case CURLE_SSL_CIPHER: + case CURLE_PEER_FAILED_VERIFICATION: + case CURLE_SSL_CACERT_BADFILE: + case CURLE_SSL_ISSUER_ERROR: + return false; + default: + return true; + } +} + +} // namespace + +RestHttpClient::RestHttpClient(const std::string& base_uri, const Config& config) + : curl_guard_(EnsureCurlGlobalInit()), + base_uri_(base_uri), + config_(config), + logger_(Logger::GetLogger("RestHttpClient")) {} + +Result> RestHttpClient::Create(const std::string& base_uri) { + return Create(base_uri, Config()); +} + +Result> RestHttpClient::Create(const std::string& base_uri, + const Config& config) { + if (base_uri.empty()) { + return Status::Invalid("uri of the http client is empty"); + } + return std::unique_ptr(new RestHttpClient(NormalizeUri(base_uri), config)); +} + +std::string RestHttpClient::NormalizeUri(const std::string& uri) { + std::string normalized = uri; + StringUtils::Trim(&normalized); + while (!normalized.empty() && normalized.back() == '/') { + normalized.pop_back(); + } + if (normalized.rfind("http://", 0) != 0 && normalized.rfind("https://", 0) != 0) { + normalized = "http://" + normalized; + } + return normalized; +} + +std::string RestHttpClient::BuildQueryString( + const std::map& query_params) { + std::string query; + for (const auto& [key, value] : query_params) { + if (!query.empty()) { + query.push_back('&'); + } + query.append(UrlUtils::EncodeString(key)); + query.push_back('='); + query.append(UrlUtils::EncodeString(value)); + } + return query; +} + +Result RestHttpClient::ExecuteOnce( + const std::string& method, const std::string& url, + const std::map& headers, const std::string& body, + bool* transport_retriable) const { + CURL* curl = curl_easy_init(); + if (curl == nullptr) { + return Status::IOError("failed to create curl handle"); + } + ScopeGuard cleanup_curl([curl] { curl_easy_cleanup(curl); }); + Response response; + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, + static_cast(config_.connect_timeout_ms)); // NOLINT(runtime/int) + curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, + static_cast(config_.request_timeout_ms)); // NOLINT(runtime/int) + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteBodyCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response.body); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, WriteHeaderCallback); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, &response.headers); + // Follow redirects transparently, restricted to http(s) targets. + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 50L); +#if CURL_AT_LEAST_VERSION(7, 85, 0) + curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS_STR, "http,https"); +#else + curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); +#endif + + if (method == "POST") { + curl_easy_setopt(curl, CURLOPT_POST, 1L); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, + static_cast(body.size())); // NOLINT(runtime/int) + } else if (method == "DELETE") { + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); + if (!body.empty()) { + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, + static_cast(body.size())); // NOLINT(runtime/int) + } + } + + struct curl_slist* header_list = nullptr; + ScopeGuard free_headers([&header_list] { curl_slist_free_all(header_list); }); + auto append_header = [&header_list](const std::string& line) -> bool { + struct curl_slist* updated_list = curl_slist_append(header_list, line.c_str()); + if (updated_list == nullptr) { + return false; + } + header_list = updated_list; + return true; + }; + // Disable the automatic "Expect: 100-continue" behavior of libcurl on POST. + bool headers_ok = append_header("Expect:"); + for (const auto& [name, value] : headers) { + headers_ok = headers_ok && append_header(name + ": " + value); + } + if (!headers_ok) { + return Status::IOError("failed to create http headers"); + } + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, header_list); + + CURLcode curl_code = curl_easy_perform(curl); + if (curl_code != CURLE_OK) { + *transport_retriable = IsRetriableTransportError(curl_code); + // Do not include the response body or full url in errors, they may carry + // sensitive information such as credentials. + return Status::IOError( + fmt::format("http {} request failed: {}", method, curl_easy_strerror(curl_code))); + } + // curl writes a `long` through the pointer, so the type cannot be narrowed here. + // NOLINTNEXTLINE(google-runtime-int) + long http_code = 0; // NOLINT(runtime/int) + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + response.code = http_code; + return response; +} + +int64_t RestHttpClient::ComputeRetryDelayMs(const Config& config, int32_t execution_count, + const Response* response, int64_t now_epoch_seconds) { + if (response != nullptr) { + auto iter = response->headers.find("retry-after"); + if (iter != response->headers.end()) { + std::optional retry_after_ms; + std::optional retry_after_seconds = + StringUtils::StringToValue(iter->second); + if (retry_after_seconds) { + // Clamped so that a misbehaving server cannot overflow the conversion. + constexpr int64_t kMaxSeconds = std::numeric_limits::max() / 1000; + retry_after_ms = + std::clamp(retry_after_seconds.value(), -kMaxSeconds, kMaxSeconds) * 1000; + } else { + std::optional date_seconds = ParseHttpDateSeconds(iter->second); + if (date_seconds) { + retry_after_ms = (date_seconds.value() - now_epoch_seconds) * 1000; + } + } + // Non-positive values fall through to the exponential backoff. + if (retry_after_ms && retry_after_ms.value() > 0) { + return retry_after_ms.value(); + } + } + } + int64_t multiplier = static_cast(1) + << std::min(execution_count - 1, static_cast(6)); + int64_t delay_ms = config.retry_base_delay_ms * multiplier; + if (delay_ms > 0) { + static thread_local std::mt19937 generator( + std::random_device{}()); // NOLINT(whitespace/braces) + std::uniform_int_distribution jitter(0, delay_ms / 10); + delay_ms += jitter(generator); + } + return delay_ms; +} + +int64_t RestHttpClient::GetRetryDelayMs(int32_t execution_count, const Response* response) const { + return ComputeRetryDelayMs(config_, execution_count, response, + static_cast(std::time(nullptr))); +} + +Result RestHttpClient::Execute( + const std::string& method, const std::string& path, + const std::map& query_params, + const std::map& headers, const std::string& body) const { + if (method != "GET" && method != "POST" && method != "DELETE") { + return Status::Invalid(fmt::format("unsupported http method: {}", method)); + } + std::string url = base_uri_ + path; + if (!query_params.empty()) { + url += "?" + BuildQueryString(query_params); + } + bool idempotent = IsIdempotent(method); + std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now(); + int32_t execution_count = 0; + while (true) { + execution_count++; + bool transport_retriable = false; + Result result = ExecuteOnce(method, url, headers, body, &transport_retriable); + bool retriable; + if (result.ok()) { + retriable = IsRetriableCode(result.value().code); + } else { + retriable = idempotent && transport_retriable; + } + if (!retriable || execution_count > config_.max_retries) { + if (result.ok()) { + // Spans all attempts including the backoff sleeps. + int64_t elapsed_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count(); + // The request id ties the log line to the server side trace. + auto request_id_iter = result.value().headers.find("x-request-id"); + const std::string request_id = request_id_iter != result.value().headers.end() && + !request_id_iter->second.empty() + ? request_id_iter->second + : "unknown"; + PAIMON_LOG_DEBUG( + logger_, "%s %s finished with http status %lld in %lld ms requestId:%s", + method.c_str(), path.c_str(), + static_cast(result.value().code), // NOLINT(runtime/int) + static_cast(elapsed_ms), // NOLINT(runtime/int) + request_id.c_str()); + } + return result; + } + int64_t delay_ms = + GetRetryDelayMs(execution_count, result.ok() ? &result.value() : nullptr); + std::string reason = result.ok() ? fmt::format("http status {}", result.value().code) + : result.status().ToString(); + PAIMON_LOG_WARN(logger_, "retrying %s %s in %lld ms (retry %d/%d): %s", method.c_str(), + path.c_str(), static_cast(delay_ms), // NOLINT(runtime/int) + execution_count, config_.max_retries, reason.c_str()); + if (delay_ms > 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms)); + } + } +} + +} // namespace paimon diff --git a/src/paimon/rest/rest_http_client.h b/src/paimon/rest/rest_http_client.h new file mode 100644 index 000000000..5d21a087c --- /dev/null +++ b/src/paimon/rest/rest_http_client.h @@ -0,0 +1,125 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/logging.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +/// HTTP status codes used by the REST catalog. +struct HttpStatus { + static constexpr int64_t kOk = 200; + static constexpr int64_t kAccepted = 202; + static constexpr int64_t kNoContent = 204; + static constexpr int64_t kBadRequest = 400; + static constexpr int64_t kUnauthorized = 401; + static constexpr int64_t kForbidden = 403; + static constexpr int64_t kNotFound = 404; + static constexpr int64_t kConflict = 409; + static constexpr int64_t kTooManyRequests = 429; + static constexpr int64_t kInternalServerError = 500; + static constexpr int64_t kNotImplemented = 501; + static constexpr int64_t kServiceUnavailable = 503; +}; + +/// A blocking HTTP client for the REST catalog based on libcurl. HTTP 429/503 +/// responses are retried for all methods, transport errors only for idempotent +/// methods, with exponential backoff (honoring a `Retry-After` response header, in the +/// delta-seconds or the HTTP-date form, when it yields a positive delay). Redirects to +/// http(s) targets are followed transparently. +class RestHttpClient { + public: + struct Config { + int64_t connect_timeout_ms = 180 * 1000; + int64_t request_timeout_ms = 180 * 1000; + int32_t max_retries = 5; + int64_t retry_base_delay_ms = 1000; + }; + + struct Response { + int64_t code = 0; + std::string body; + /// Response headers with lower-cased names. + std::map headers; + + bool IsSuccessful() const { + return code == HttpStatus::kOk || code == HttpStatus::kAccepted || + code == HttpStatus::kNoContent; + } + }; + + /// Creates a client against `base_uri`. The uri is normalized: surrounding + /// whitespace and all trailing '/' are stripped and "http://" is prepended when no + /// scheme is present. + static Result> Create(const std::string& base_uri); + static Result> Create(const std::string& base_uri, + const Config& config); + + /// Executes `method` ("GET", "POST" or "DELETE") on `path` (already url-encoded, + /// starting with '/'). Query parameter keys and values are url-encoded internally. + /// Returns the final response (which may carry a non-2xx code) or an error status + /// when the request could not be transported at all. Transport errors caused by + /// unresolvable/unreachable hosts, timeouts or TLS failures are never retried. + Result Execute(const std::string& method, const std::string& path, + const std::map& query_params, + const std::map& headers, + const std::string& body) const; + + const std::string& GetBaseUri() const { + return base_uri_; + } + + static std::string NormalizeUri(const std::string& uri); + + /// Builds "k1=v1&k2=v2" with url-encoded keys and values. + static std::string BuildQueryString(const std::map& query_params); + + /// Computes the delay before the retry following the `execution_count`-th attempt, + /// in ms. A `Retry-After` header of `response` takes precedence when it yields a + /// positive delay (the delta-seconds form, or the HTTP-date form resolved against + /// `now_epoch_seconds`); otherwise the exponential backoff with up to 10% jitter + /// applies. `response` is null when the attempt failed with a transport error. + static int64_t ComputeRetryDelayMs(const Config& config, int32_t execution_count, + const Response* response, int64_t now_epoch_seconds); + + private: + RestHttpClient(const std::string& base_uri, const Config& config); + + /// On a transport failure, `transport_retriable` reports whether the failure kind + /// may be retried (see `Execute` for the non-retriable kinds). + Result ExecuteOnce(const std::string& method, const std::string& url, + const std::map& headers, + const std::string& body, bool* transport_retriable) const; + + /// Backoff before the retry following the `execution_count`-th attempt, in ms. + int64_t GetRetryDelayMs(int32_t execution_count, const Response* response) const; + + /// Keeps libcurl's global state alive for the lifetime of the client. + std::shared_ptr curl_guard_; + std::string base_uri_; + Config config_; + std::shared_ptr logger_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_http_client_test.cpp b/src/paimon/rest/rest_http_client_test.cpp new file mode 100644 index 000000000..aad0eea8d --- /dev/null +++ b/src/paimon/rest/rest_http_client_test.cpp @@ -0,0 +1,421 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_http_client.h" + +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/rest/mock_rest_server.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +namespace { +RestHttpClient::Config FastRetryConfig(int32_t max_retries) { + RestHttpClient::Config config; + config.max_retries = max_retries; + config.retry_base_delay_ms = 1; + return config; +} +} // namespace + +TEST(RestHttpClientTest, NormalizeUri) { + ASSERT_EQ("http://localhost:80", RestHttpClient::NormalizeUri("localhost:80/")); + ASSERT_EQ("http://localhost", RestHttpClient::NormalizeUri("http://localhost//")); + ASSERT_EQ("https://foo.bar", RestHttpClient::NormalizeUri(" https://foo.bar ")); +} + +TEST(RestHttpClientTest, BuildQueryString) { + ASSERT_EQ("a=1&b=x+y%2Fz", RestHttpClient::BuildQueryString({{"a", "1"}, {"b", "x y/z"}})); +} + +TEST(RestHttpClientTest, GetWithHeadersAndQuery) { + // guards last_request: the handler runs on the mock server's accept thread while + // the test thread inspects the captured request + std::mutex mutex; + MockRestServer::Request last_request; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + std::lock_guard lock(mutex); + last_request = request; + MockRestServer::Response response; + response.body = R"({"ok": true})"; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri())); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/v1/config", {{"warehouse", "wh 1"}}, + {{"Authorization", "Bearer token1"}}, "")); + ASSERT_EQ(200, response.code); + ASSERT_EQ(R"({"ok": true})", response.body); + ASSERT_EQ("application/json", response.headers.at("content-type")); + std::lock_guard lock(mutex); + ASSERT_EQ("GET", last_request.method); + ASSERT_EQ("/v1/config", last_request.path); + ASSERT_EQ("wh 1", last_request.query_params.at("warehouse")); + ASSERT_EQ("Bearer token1", last_request.headers.at("authorization")); +} + +TEST(RestHttpClientTest, PostBody) { + // guards last_request: the handler runs on the mock server's accept thread while + // the test thread inspects the captured request + std::mutex mutex; + MockRestServer::Request last_request; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + std::lock_guard lock(mutex); + last_request = request; + return MockRestServer::Response(); + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri())); + ASSERT_OK_AND_ASSIGN( + RestHttpClient::Response response, + client->Execute("POST", "/v1/databases", {}, {{"Content-Type", "application/json"}}, + R"({"name": "db1"})")); + ASSERT_EQ(200, response.code); + std::lock_guard lock(mutex); + ASSERT_EQ("POST", last_request.method); + ASSERT_EQ(R"({"name": "db1"})", last_request.body); + ASSERT_EQ("application/json", last_request.headers.at("content-type")); +} + +TEST(RestHttpClientTest, NotFoundIsNotRetried) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + request_count++; + MockRestServer::Response response; + response.code = 404; + response.body = R"({"message": "not found", "code": 404})"; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/v1/databases/db1", {}, {}, "")); + ASSERT_EQ(404, response.code); + ASSERT_FALSE(response.IsSuccessful()); + ASSERT_EQ(1, request_count.load()); +} + +TEST(RestHttpClientTest, ServiceUnavailableIsRetried) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + MockRestServer::Response response; + if (request_count++ < 2) { + response.code = 503; + } else { + response.body = R"({"ok": true})"; + } + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + // 429/503 responses are retried even for non-idempotent POST. + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("POST", "/v1/databases", {}, {}, "{}")); + ASSERT_EQ(200, response.code); + ASSERT_EQ(3, request_count.load()); +} + +TEST(RestHttpClientTest, RetryAfterHeaderIsHonored) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + MockRestServer::Response response; + if (request_count++ == 0) { + response.code = 429; + response.headers["Retry-After"] = "1"; + } + return response; + })); + // the backoff base is large so that the elapsed time below proves the Retry-After + // header took precedence: without it this test would sleep for a minute + RestHttpClient::Config config; + config.max_retries = 5; + config.retry_base_delay_ms = 60 * 1000; + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), config)); + auto start = std::chrono::steady_clock::now(); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/v1/databases", {}, {}, "")); + auto elapsed = std::chrono::steady_clock::now() - start; + ASSERT_EQ(200, response.code); + ASSERT_EQ(2, request_count.load()); + ASSERT_GE(elapsed, std::chrono::seconds(1)); + ASSERT_LT(elapsed, std::chrono::seconds(30)); +} + +TEST(RestHttpClientTest, ComputeRetryDelayMs) { + RestHttpClient::Config config; + config.retry_base_delay_ms = 1000; + // without a usable Retry-After header: exponential backoff with up to 10% jitter, + // the multiplier capped at 2^6 + int64_t delay = RestHttpClient::ComputeRetryDelayMs(config, 1, nullptr, 0); + ASSERT_GE(delay, 1000); + ASSERT_LE(delay, 1100); + delay = RestHttpClient::ComputeRetryDelayMs(config, 3, nullptr, 0); + ASSERT_GE(delay, 4000); + ASSERT_LE(delay, 4400); + delay = RestHttpClient::ComputeRetryDelayMs(config, 100, nullptr, 0); + ASSERT_GE(delay, 64000); + ASSERT_LE(delay, 70400); + + // the delta-seconds form takes precedence over the backoff, without jitter + RestHttpClient::Response response; + response.headers["retry-after"] = "7"; + ASSERT_EQ(7000, RestHttpClient::ComputeRetryDelayMs(config, 1, &response, 0)); + + // the HTTP-date form is resolved against the given "now" + constexpr int64_t kDateEpoch = 1445412480; // Wed, 21 Oct 2015 07:28:00 GMT + response.headers["retry-after"] = "Wed, 21 Oct 2015 07:28:00 GMT"; + ASSERT_EQ(5000, RestHttpClient::ComputeRetryDelayMs(config, 1, &response, kDateEpoch - 5)); + + // a date in the past falls back to the backoff + delay = RestHttpClient::ComputeRetryDelayMs(config, 1, &response, kDateEpoch + 1); + ASSERT_GE(delay, 1000); + ASSERT_LE(delay, 1100); +} + +TEST(RestHttpClientTest, InvalidRetryAfterFallsBackToBackoff) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + MockRestServer::Response response; + int32_t count = request_count++; + if (count == 0) { + response.code = 429; + // non-positive values are ignored + response.headers["Retry-After"] = "0"; + } else if (count == 2) { + response.code = 429; + // neither delta-seconds nor a valid http date + response.headers["Retry-After"] = "Sat, 32 Foo 2015 99:99:99 GMT"; + } + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/v1/databases", {}, {}, "")); + ASSERT_EQ(200, response.code); + ASSERT_EQ(2, request_count.load()); + ASSERT_OK_AND_ASSIGN(response, client->Execute("GET", "/v1/databases", {}, {}, "")); + ASSERT_EQ(200, response.code); + ASSERT_EQ(4, request_count.load()); +} + +TEST(RestHttpClientTest, RetriesExhausted) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + request_count++; + MockRestServer::Response response; + response.code = 503; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(2))); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/v1/databases", {}, {}, "")); + ASSERT_EQ(503, response.code); + // initial attempt + 2 retries + ASSERT_EQ(3, request_count.load()); +} + +TEST(RestHttpClientTest, EmptyReplyIsNotRetried) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + request_count++; + MockRestServer::Response response; + // closing without a response maps to CURLE_GOT_NOTHING, + // one of the non-retriable transport errors (like + // unresolvable hosts, timeouts and TLS failures, which + // cannot be produced deterministically in a unit test) + response.close_without_response = true; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + ASSERT_NOK(client->Execute("GET", "/v1/databases", {}, {}, "").status()); + ASSERT_EQ(1, request_count.load()); +} + +TEST(RestHttpClientTest, ConnectionRefusedIsNotRetried) { + // a closed port yields CURLE_COULDNT_CONNECT, another of the non-retriable + // transport errors; the port of a stopped mock server is known to be closed + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([](const MockRestServer::Request& request) { + return MockRestServer::Response(); + })); + std::string base_uri = server->GetBaseUri(); + server->Stop(); + // the backoff base is large so that the elapsed time below proves the failure was + // not retried: a single retry would sleep for a minute + RestHttpClient::Config config; + config.max_retries = 5; + config.retry_base_delay_ms = 60 * 1000; + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(base_uri, config)); + auto start = std::chrono::steady_clock::now(); + ASSERT_NOK(client->Execute("GET", "/v1/databases", {}, {}, "").status()); + auto elapsed = std::chrono::steady_clock::now() - start; + ASSERT_LT(elapsed, std::chrono::seconds(30)); +} + +TEST(RestHttpClientTest, DeleteWithBodyIsSent) { + // guards last_request: the handler runs on the mock server's accept thread while + // the test thread inspects the captured request + std::mutex mutex; + MockRestServer::Request last_request; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + std::lock_guard lock(mutex); + last_request = request; + return MockRestServer::Response(); + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri())); + // a body on DELETE keeps the DELETE method line instead of degrading to POST + ASSERT_OK_AND_ASSIGN( + RestHttpClient::Response response, + client->Execute("DELETE", "/v1/databases/db1", {}, {{"Content-Type", "application/json"}}, + R"({"purge": true})")); + ASSERT_EQ(200, response.code); + std::lock_guard lock(mutex); + ASSERT_EQ("DELETE", last_request.method); + ASSERT_EQ(R"({"purge": true})", last_request.body); +} + +TEST(RestHttpClientTest, TransportErrorOmitsUrlAndQuery) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([](const MockRestServer::Request& request) { + MockRestServer::Response response; + response.close_without_response = true; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri())); + Status status = + client->Execute("GET", "/v1/secret-path", {{"sig", "topsecret1"}}, {}, "").status(); + ASSERT_NOK(status); + // the error must not echo the path or query values, which may carry credentials + // (e.g. a presigned url) + ASSERT_EQ(std::string::npos, status.ToString().find("secret-path")) << status.ToString(); + ASSERT_EQ(std::string::npos, status.ToString().find("topsecret1")) << status.ToString(); +} + +TEST(RestHttpClientTest, TruncatedResponseIsRetriedForGet) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + MockRestServer::Response response; + response.body = R"({"ok": true})"; + if (request_count++ == 0) { + // cutting the response off mid-body is a retriable + // transport error, and GET is idempotent + response.missing_body_bytes = 5; + } + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/v1/databases", {}, {}, "")); + ASSERT_EQ(200, response.code); + ASSERT_EQ(R"({"ok": true})", response.body); + ASSERT_EQ(2, request_count.load()); +} + +TEST(RestHttpClientTest, TruncatedResponseIsRetriedForDelete) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + MockRestServer::Response response; + if (request_count++ == 0) { + // DELETE is the other idempotent method (see the GET + // test above) + response.missing_body_bytes = 5; + response.body = R"({"ok": true})"; + } + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("DELETE", "/v1/databases/db1", {}, {}, "")); + ASSERT_EQ(200, response.code); + ASSERT_EQ(2, request_count.load()); +} + +TEST(RestHttpClientTest, TruncatedResponseIsNotRetriedForPost) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + request_count++; + MockRestServer::Response response; + response.body = R"({"ok": true})"; + // the same retriable transport error kind as in the GET + // test above, but POST is not idempotent + response.missing_body_bytes = 5; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + ASSERT_NOK(client->Execute("POST", "/v1/databases", {}, {}, "{}").status()); + ASSERT_EQ(1, request_count.load()); +} + +TEST(RestHttpClientTest, RedirectIsFollowed) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + MockRestServer::Response response; + if (request_count++ == 0) { + response.code = 302; + response.headers["Location"] = "/v1/config"; + } else { + response.body = R"({"ok": true})"; + } + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri())); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/old", {}, {}, "")); + ASSERT_EQ(200, response.code); + ASSERT_EQ(R"({"ok": true})", response.body); + ASSERT_EQ(2, request_count.load()); + // only the final response's headers are reported, not the redirect's + ASSERT_EQ(0, response.headers.count("location")); +} + +TEST(RestHttpClientTest, UnsupportedMethod) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create("http://127.0.0.1:1")); + ASSERT_NOK_WITH_MSG(client->Execute("PATCH", "/", {}, {}, "").status(), + "unsupported http method"); +} + +} // namespace paimon::test diff --git a/src/paimon/rest/rest_messages.cpp b/src/paimon/rest/rest_messages.cpp new file mode 100644 index 000000000..99ffd056f --- /dev/null +++ b/src/paimon/rest/rest_messages.cpp @@ -0,0 +1,373 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_messages.h" + +#include + +#include "paimon/rest/rest_util.h" + +namespace paimon { + +namespace { + +constexpr const char kFieldMessage[] = "message"; +constexpr const char kFieldResourceType[] = "resourceType"; +constexpr const char kFieldResourceName[] = "resourceName"; +constexpr const char kFieldCode[] = "code"; +constexpr const char kFieldDefaults[] = "defaults"; +constexpr const char kFieldOverrides[] = "overrides"; +constexpr const char kFieldOwner[] = "owner"; +constexpr const char kFieldCreatedAt[] = "createdAt"; +constexpr const char kFieldCreatedBy[] = "createdBy"; +constexpr const char kFieldUpdatedAt[] = "updatedAt"; +constexpr const char kFieldUpdatedBy[] = "updatedBy"; +constexpr const char kFieldName[] = "name"; +constexpr const char kFieldOptions[] = "options"; +constexpr const char kFieldId[] = "id"; +constexpr const char kFieldLocation[] = "location"; +constexpr const char kFieldDatabases[] = "databases"; +constexpr const char kFieldTables[] = "tables"; +constexpr const char kFieldSnapshots[] = "snapshots"; +constexpr const char kFieldNextPageToken[] = "nextPageToken"; +constexpr const char kFieldPath[] = "path"; +constexpr const char kFieldIsExternal[] = "isExternal"; +constexpr const char kFieldSchemaId[] = "schemaId"; +constexpr const char kFieldSchema[] = "schema"; +constexpr const char kFieldIdentifier[] = "identifier"; +constexpr const char kFieldDatabase[] = "database"; +constexpr const char kFieldObject[] = "object"; +constexpr const char kFieldSource[] = "source"; +constexpr const char kFieldDestination[] = "destination"; + +void AddOptionalStringMember(rapidjson::Value* obj, const char* key, + const std::optional& value, + rapidjson::Document::AllocatorType* allocator) { + if (value) { + obj->AddMember(rapidjson::StringRef(key), + RapidJsonUtil::SerializeValue(value.value(), allocator).Move(), *allocator); + } +} + +rapidjson::Value SerializeIdentifier(const std::string& database, const std::string& table, + rapidjson::Document::AllocatorType* allocator) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldDatabase), + RapidJsonUtil::SerializeValue(database, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldObject), + RapidJsonUtil::SerializeValue(table, allocator).Move(), *allocator); + return obj; +} + +void DeserializeIdentifier(const rapidjson::Value& obj, const char* key, std::string* database, + std::string* table) { + if (!obj.IsObject() || !obj.HasMember(key) || !obj[key].IsObject()) { + throw std::invalid_argument(std::string("member '") + key + + "' must exist and be an object"); + } + const rapidjson::Value& identifier = obj[key]; + *database = RapidJsonUtil::DeserializeKeyValue(identifier, kFieldDatabase); + *table = RapidJsonUtil::DeserializeKeyValue(identifier, kFieldObject); +} + +std::string DeserializeRawJsonMember(const rapidjson::Value& obj, const char* key) { + if (!obj.IsObject() || !obj.HasMember(key) || !obj[key].IsObject()) { + throw std::invalid_argument(std::string("member '") + key + + "' must exist and be an object"); + } + return RestUtil::JsonToString(obj[key]); +} + +} // namespace + +rapidjson::Value ErrorResponse::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldMessage), + RapidJsonUtil::SerializeValue(message_, allocator).Move(), *allocator); + if (!resource_type_.empty()) { + obj.AddMember(rapidjson::StringRef(kFieldResourceType), + RapidJsonUtil::SerializeValue(resource_type_, allocator).Move(), *allocator); + } + if (!resource_name_.empty()) { + obj.AddMember(rapidjson::StringRef(kFieldResourceName), + RapidJsonUtil::SerializeValue(resource_name_, allocator).Move(), *allocator); + } + obj.AddMember(rapidjson::StringRef(kFieldCode), + RapidJsonUtil::SerializeValue(code_, allocator).Move(), *allocator); + return obj; +} + +void ErrorResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { + message_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldMessage, std::string()); + resource_type_ = + RapidJsonUtil::DeserializeKeyValue(obj, kFieldResourceType, std::string()); + resource_name_ = + RapidJsonUtil::DeserializeKeyValue(obj, kFieldResourceName, std::string()); + code_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldCode, 0); +} + +rapidjson::Value ConfigResponse::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldDefaults), + RapidJsonUtil::SerializeValue(defaults_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldOverrides), + RapidJsonUtil::SerializeValue(overrides_, allocator).Move(), *allocator); + return obj; +} + +namespace { +/// The server may send null values in `defaults`/`overrides`; merging filters them +/// out, so they are skipped here instead of failing the whole parse. +std::map DeserializeStringMapSkippingNulls(const rapidjson::Value& obj, + const char* key) { + std::map result; + if (!obj.IsObject() || !obj.HasMember(key) || obj[key].IsNull()) { + return result; + } + const rapidjson::Value& map_value = obj[key]; + if (!map_value.IsObject()) { + throw std::invalid_argument(std::string("member '") + key + "' must be an object"); + } + for (auto iter = map_value.MemberBegin(); iter != map_value.MemberEnd(); ++iter) { + if (iter->value.IsNull()) { + continue; + } + if (!iter->value.IsString()) { + throw std::invalid_argument(std::string("member '") + key + + "' must only contain string values"); + } + result[iter->name.GetString()] = iter->value.GetString(); + } + return result; +} +} // namespace + +void ConfigResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { + defaults_ = DeserializeStringMapSkippingNulls(obj, kFieldDefaults); + overrides_ = DeserializeStringMapSkippingNulls(obj, kFieldOverrides); +} + +std::map ConfigResponse::Merge( + const std::map& client_options) const { + std::map merged = defaults_; + for (const auto& [key, value] : client_options) { + merged[key] = value; + } + for (const auto& [key, value] : overrides_) { + merged[key] = value; + } + return merged; +} + +void RestAuditFields::ParseFrom(const rapidjson::Value& obj) { + owner = RapidJsonUtil::DeserializeKeyValue>(obj, kFieldOwner, + std::nullopt); + created_at = RapidJsonUtil::DeserializeKeyValue>(obj, kFieldCreatedAt, + std::nullopt); + created_by = RapidJsonUtil::DeserializeKeyValue>( + obj, kFieldCreatedBy, std::nullopt); + updated_at = RapidJsonUtil::DeserializeKeyValue>(obj, kFieldUpdatedAt, + std::nullopt); + updated_by = RapidJsonUtil::DeserializeKeyValue>( + obj, kFieldUpdatedBy, std::nullopt); +} + +void RestAuditFields::AddTo(rapidjson::Value* obj, + rapidjson::Document::AllocatorType* allocator) const { + AddOptionalStringMember(obj, kFieldOwner, owner, allocator); + if (created_at) { + obj->AddMember(rapidjson::StringRef(kFieldCreatedAt), + RapidJsonUtil::SerializeValue(created_at.value(), allocator).Move(), + *allocator); + } + AddOptionalStringMember(obj, kFieldCreatedBy, created_by, allocator); + if (updated_at) { + obj->AddMember(rapidjson::StringRef(kFieldUpdatedAt), + RapidJsonUtil::SerializeValue(updated_at.value(), allocator).Move(), + *allocator); + } + AddOptionalStringMember(obj, kFieldUpdatedBy, updated_by, allocator); +} + +void RestAuditFields::PutAuditOptionsTo(std::map* options) const { + if (owner) { + (*options)[kFieldOwner] = owner.value(); + } + if (created_at) { + (*options)[kFieldCreatedAt] = std::to_string(created_at.value()); + } + if (created_by) { + (*options)[kFieldCreatedBy] = created_by.value(); + } + if (updated_at) { + (*options)[kFieldUpdatedAt] = std::to_string(updated_at.value()); + } + if (updated_by) { + (*options)[kFieldUpdatedBy] = updated_by.value(); + } +} + +rapidjson::Value CreateDatabaseRequest::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldName), + RapidJsonUtil::SerializeValue(name_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldOptions), + RapidJsonUtil::SerializeValue(options_, allocator).Move(), *allocator); + return obj; +} + +void CreateDatabaseRequest::FromJson(const rapidjson::Value& obj) noexcept(false) { + name_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldName); + options_ = RapidJsonUtil::DeserializeKeyValue>( + obj, kFieldOptions, {}); +} + +rapidjson::Value GetDatabaseResponse::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldId), + RapidJsonUtil::SerializeValue(id_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldName), + RapidJsonUtil::SerializeValue(name_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldLocation), + RapidJsonUtil::SerializeValue(location_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldOptions), + RapidJsonUtil::SerializeValue(options_, allocator).Move(), *allocator); + audit_.AddTo(&obj, allocator); + return obj; +} + +void GetDatabaseResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { + id_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldId, std::string()); + name_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldName); + location_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldLocation, std::string()); + options_ = RapidJsonUtil::DeserializeKeyValue>( + obj, kFieldOptions, {}); + audit_.ParseFrom(obj); +} + +rapidjson::Value ListDatabasesResponse::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldDatabases), + RapidJsonUtil::SerializeValue(databases_, allocator).Move(), *allocator); + AddOptionalStringMember(&obj, kFieldNextPageToken, next_page_token_, allocator); + return obj; +} + +void ListDatabasesResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { + databases_ = + RapidJsonUtil::DeserializeKeyValue>(obj, kFieldDatabases, {}); + next_page_token_ = RapidJsonUtil::DeserializeKeyValue>( + obj, kFieldNextPageToken, std::nullopt); +} + +rapidjson::Value ListTablesResponse::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldTables), + RapidJsonUtil::SerializeValue(tables_, allocator).Move(), *allocator); + AddOptionalStringMember(&obj, kFieldNextPageToken, next_page_token_, allocator); + return obj; +} + +void ListTablesResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { + tables_ = RapidJsonUtil::DeserializeKeyValue>(obj, kFieldTables, {}); + next_page_token_ = RapidJsonUtil::DeserializeKeyValue>( + obj, kFieldNextPageToken, std::nullopt); +} + +rapidjson::Value ListSnapshotsResponse::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldSnapshots), + RapidJsonUtil::SerializeValue(snapshots_, allocator).Move(), *allocator); + AddOptionalStringMember(&obj, kFieldNextPageToken, next_page_token_, allocator); + return obj; +} + +void ListSnapshotsResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { + snapshots_ = + RapidJsonUtil::DeserializeKeyValue>(obj, kFieldSnapshots, {}); + next_page_token_ = RapidJsonUtil::DeserializeKeyValue>( + obj, kFieldNextPageToken, std::nullopt); +} + +rapidjson::Value GetTableResponse::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldId), + RapidJsonUtil::SerializeValue(id_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldName), + RapidJsonUtil::SerializeValue(name_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldPath), + RapidJsonUtil::SerializeValue(path_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldIsExternal), + RapidJsonUtil::SerializeValue(is_external_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldSchemaId), + RapidJsonUtil::SerializeValue(schema_id_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldSchema), + RestUtil::ParseToValue(schema_json_, allocator).Move(), *allocator); + audit_.AddTo(&obj, allocator); + return obj; +} + +void GetTableResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { + id_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldId, std::string()); + name_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldName, std::string()); + path_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldPath); + is_external_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldIsExternal, false); + schema_id_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldSchemaId); + schema_json_ = DeserializeRawJsonMember(obj, kFieldSchema); + audit_.ParseFrom(obj); +} + +rapidjson::Value CreateTableRequest::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldIdentifier), + SerializeIdentifier(database_, table_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldSchema), + RestUtil::ParseToValue(schema_json_, allocator).Move(), *allocator); + return obj; +} + +void CreateTableRequest::FromJson(const rapidjson::Value& obj) noexcept(false) { + DeserializeIdentifier(obj, kFieldIdentifier, &database_, &table_); + schema_json_ = DeserializeRawJsonMember(obj, kFieldSchema); +} + +rapidjson::Value RenameTableRequest::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldSource), + SerializeIdentifier(source_database_, source_table_, allocator).Move(), + *allocator); + obj.AddMember(rapidjson::StringRef(kFieldDestination), + SerializeIdentifier(destination_database_, destination_table_, allocator).Move(), + *allocator); + return obj; +} + +void RenameTableRequest::FromJson(const rapidjson::Value& obj) noexcept(false) { + DeserializeIdentifier(obj, kFieldSource, &source_database_, &source_table_); + DeserializeIdentifier(obj, kFieldDestination, &destination_database_, &destination_table_); +} + +} // namespace paimon diff --git a/src/paimon/rest/rest_messages.h b/src/paimon/rest/rest_messages.h new file mode 100644 index 000000000..d05b34c1f --- /dev/null +++ b/src/paimon/rest/rest_messages.h @@ -0,0 +1,374 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "paimon/common/utils/jsonizable.h" +#include "paimon/common/utils/rapidjson_util.h" +#include "paimon/core/snapshot.h" +#include "rapidjson/allocators.h" +#include "rapidjson/document.h" +#include "rapidjson/rapidjson.h" + +namespace paimon { + +/// Request and response objects of the REST catalog protocol. The JSON field names +/// must stay aligned with the REST catalog open api. + +class ErrorResponse : public Jsonizable { + public: + static constexpr const char* kResourceTypeDatabase = "DATABASE"; + static constexpr const char* kResourceTypeTable = "TABLE"; + + ErrorResponse(const std::string& resource_type, const std::string& resource_name, + const std::string& message, int32_t code) + : resource_type_(resource_type), + resource_name_(resource_name), + message_(message), + code_(code) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::string& GetResourceType() const { + return resource_type_; + } + const std::string& GetResourceName() const { + return resource_name_; + } + const std::string& GetMessage() const { + return message_; + } + int32_t GetCode() const { + return code_; + } + + ErrorResponse() = default; + + private: + std::string resource_type_; + std::string resource_name_; + std::string message_; + int32_t code_ = 0; +}; + +/// The response of "/v1/config". +class ConfigResponse : public Jsonizable { + public: + ConfigResponse(const std::map& defaults, + const std::map& overrides) + : defaults_(defaults), overrides_(overrides) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + /// Merges with the client options; the precedence is + /// `overrides` > `client_options` > `defaults`. + std::map Merge( + const std::map& client_options) const; + + const std::map& GetDefaults() const { + return defaults_; + } + const std::map& GetOverrides() const { + return overrides_; + } + + ConfigResponse() = default; + + private: + std::map defaults_; + std::map overrides_; +}; + +/// The audit fields shared by database/table responses. +struct RestAuditFields { + std::optional owner; + std::optional created_at; + std::optional created_by; + std::optional updated_at; + std::optional updated_by; + + void ParseFrom(const rapidjson::Value& obj); + void AddTo(rapidjson::Value* obj, rapidjson::Document::AllocatorType* allocator) const; + /// Adds the present fields to `options`. + void PutAuditOptionsTo(std::map* options) const; +}; + +class CreateDatabaseRequest : public Jsonizable { + public: + CreateDatabaseRequest(const std::string& name, + const std::map& options) + : name_(name), options_(options) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::string& GetName() const { + return name_; + } + const std::map& GetOptions() const { + return options_; + } + + CreateDatabaseRequest() = default; + + private: + std::string name_; + std::map options_; +}; + +class GetDatabaseResponse : public Jsonizable { + public: + GetDatabaseResponse(const std::string& id, const std::string& name, const std::string& location, + const std::map& options) + : id_(id), name_(name), location_(location), options_(options) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::string& GetId() const { + return id_; + } + const std::string& GetName() const { + return name_; + } + const std::string& GetLocation() const { + return location_; + } + const std::map& GetOptions() const { + return options_; + } + const RestAuditFields& GetAuditFields() const { + return audit_; + } + + GetDatabaseResponse() = default; + + private: + std::string id_; + std::string name_; + std::string location_; + std::map options_; + RestAuditFields audit_; +}; + +class ListDatabasesResponse : public Jsonizable { + public: + using ItemType = std::string; + + ListDatabasesResponse(const std::vector& databases, + const std::optional& next_page_token) + : databases_(databases), next_page_token_(next_page_token) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::vector& Data() const { + return databases_; + } + const std::optional& NextPageToken() const { + return next_page_token_; + } + + ListDatabasesResponse() = default; + + private: + std::vector databases_; + std::optional next_page_token_; +}; + +class ListTablesResponse : public Jsonizable { + public: + using ItemType = std::string; + + ListTablesResponse(const std::vector& tables, + const std::optional& next_page_token) + : tables_(tables), next_page_token_(next_page_token) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::vector& Data() const { + return tables_; + } + const std::optional& NextPageToken() const { + return next_page_token_; + } + + ListTablesResponse() = default; + + private: + std::vector tables_; + std::optional next_page_token_; +}; + +/// Each element is a snapshot in the same JSON layout as the snapshot files. +class ListSnapshotsResponse : public Jsonizable { + public: + using ItemType = Snapshot; + + ListSnapshotsResponse(const std::vector& snapshots, + const std::optional& next_page_token) + : snapshots_(snapshots), next_page_token_(next_page_token) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::vector& Data() const { + return snapshots_; + } + const std::optional& NextPageToken() const { + return next_page_token_; + } + + ListSnapshotsResponse() = default; + + private: + std::vector snapshots_; + std::optional next_page_token_; +}; + +/// The nested `schema` object (fields/partitionKeys/primaryKeys/options/comment) is +/// kept as a raw JSON string and converted to a `TableSchema` by the catalog. +class GetTableResponse : public Jsonizable { + public: + GetTableResponse(const std::string& id, const std::string& name, const std::string& path, + bool is_external, int64_t schema_id, const std::string& schema_json) + : id_(id), + name_(name), + path_(path), + is_external_(is_external), + schema_id_(schema_id), + schema_json_(schema_json) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::string& GetId() const { + return id_; + } + const std::string& GetName() const { + return name_; + } + const std::string& GetPath() const { + return path_; + } + bool IsExternal() const { + return is_external_; + } + int64_t GetSchemaId() const { + return schema_id_; + } + const std::string& GetSchemaJson() const { + return schema_json_; + } + const RestAuditFields& GetAuditFields() const { + return audit_; + } + + GetTableResponse() = default; + + private: + std::string id_; + std::string name_; + std::string path_; + bool is_external_ = false; + int64_t schema_id_ = 0; + std::string schema_json_; + RestAuditFields audit_; +}; + +/// `schema_json` uses the same schema JSON layout as `GetTableResponse`. +class CreateTableRequest : public Jsonizable { + public: + CreateTableRequest(const std::string& database, const std::string& table, + const std::string& schema_json) + : database_(database), table_(table), schema_json_(schema_json) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::string& GetDatabase() const { + return database_; + } + const std::string& GetTable() const { + return table_; + } + const std::string& GetSchemaJson() const { + return schema_json_; + } + + CreateTableRequest() = default; + + private: + std::string database_; + std::string table_; + std::string schema_json_; +}; + +class RenameTableRequest : public Jsonizable { + public: + RenameTableRequest(const std::string& source_database, const std::string& source_table, + const std::string& destination_database, + const std::string& destination_table) + : source_database_(source_database), + source_table_(source_table), + destination_database_(destination_database), + destination_table_(destination_table) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::string& GetSourceDatabase() const { + return source_database_; + } + const std::string& GetSourceTable() const { + return source_table_; + } + const std::string& GetDestinationDatabase() const { + return destination_database_; + } + const std::string& GetDestinationTable() const { + return destination_table_; + } + + RenameTableRequest() = default; + + private: + std::string source_database_; + std::string source_table_; + std::string destination_database_; + std::string destination_table_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_messages_test.cpp b/src/paimon/rest/rest_messages_test.cpp new file mode 100644 index 000000000..b75796227 --- /dev/null +++ b/src/paimon/rest/rest_messages_test.cpp @@ -0,0 +1,293 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_messages.h" + +#include +#include + +#include "gtest/gtest.h" +#include "paimon/common/utils/rapidjson_util.h" +#include "paimon/rest/resource_paths.h" +#include "paimon/rest/rest_util.h" +#include "paimon/testing/utils/testharness.h" +#include "rapidjson/document.h" + +namespace paimon::test { + +TEST(RestUtilTest, ExtractPrefixMap) { + std::map options = { + {"header.k1", "v1"}, {"header.k2", "v2"}, {"other", "v3"}, {"header.", "v4"}}; + std::map expected = {{"k1", "v1"}, {"k2", "v2"}}; + ASSERT_EQ(expected, RestUtil::ExtractPrefixMap(options, "header.")); +} + +TEST(RestUtilTest, RedactText) { + // markers are matched on the text normalized to lower-case alphanumerics, so + // separators cannot be used to smuggle a secret through + ASSERT_EQ("******", RestUtil::RedactText("invalid password=abc123")); + ASSERT_EQ("******", RestUtil::RedactText("bad ACCESS-KEY provided")); + ASSERT_EQ("******", RestUtil::RedactText("url?sig=deadbeef")); + ASSERT_EQ("table t1 not found", RestUtil::RedactText("table t1 not found")); + ASSERT_EQ("", RestUtil::RedactText("")); +} + +TEST(ResourcePathsTest, WithPrefix) { + ResourcePaths paths("my prefix"); + ASSERT_EQ("/v1/config", ResourcePaths::Config()); + ASSERT_EQ("/v1/my+prefix/databases", paths.Databases()); + ASSERT_EQ("/v1/my+prefix/databases/db%231", paths.Database("db#1")); + ASSERT_EQ("/v1/my+prefix/databases/db/tables", paths.Tables("db")); + ASSERT_EQ("/v1/my+prefix/databases/db/tables/t1", paths.Table("db", "t1")); + ASSERT_EQ("/v1/my+prefix/tables/rename", paths.RenameTable()); + ASSERT_EQ("/v1/my+prefix/databases/db/tables/t1/snapshots", paths.Snapshots("db", "t1")); +} + +TEST(ResourcePathsTest, WithoutPrefix) { + ResourcePaths paths(""); + ASSERT_EQ("/v1/databases", paths.Databases()); + ASSERT_EQ("/v1/tables/rename", paths.RenameTable()); +} + +TEST(RestMessagesTest, ErrorResponseRoundTrip) { + ErrorResponse response(ErrorResponse::kResourceTypeDatabase, "db1", "database db1 not found", + 404); + ASSERT_OK_AND_ASSIGN(std::string json, response.ToJsonString()); + ASSERT_OK_AND_ASSIGN(ErrorResponse parsed, ErrorResponse::FromJsonString(json)); + ASSERT_EQ("DATABASE", parsed.GetResourceType()); + ASSERT_EQ("db1", parsed.GetResourceName()); + ASSERT_EQ("database db1 not found", parsed.GetMessage()); + ASSERT_EQ(404, parsed.GetCode()); +} + +TEST(RestMessagesTest, ErrorResponseLenientParse) { + ASSERT_OK_AND_ASSIGN(ErrorResponse parsed, ErrorResponse::FromJsonString("{}")); + ASSERT_EQ("", parsed.GetMessage()); + ASSERT_EQ(0, parsed.GetCode()); +} + +TEST(RestMessagesTest, ConfigResponseMerge) { + // null values may be sent by the server and are filtered out + std::string json = R"({ + "defaults": {"prefix": "server-prefix", "a": "default-a", "b": "default-b", + "nullable": null}, + "overrides": {"c": "override-c", "a": "override-a"} + })"; + ASSERT_OK_AND_ASSIGN(ConfigResponse config, ConfigResponse::FromJsonString(json)); + std::map client = { + {"a", "client-a"}, {"b", "client-b"}, {"d", "client-d"}}; + std::map merged = config.Merge(client); + // overrides > client options > defaults + ASSERT_EQ("override-a", merged["a"]); + ASSERT_EQ("client-b", merged["b"]); + ASSERT_EQ("override-c", merged["c"]); + ASSERT_EQ("client-d", merged["d"]); + ASSERT_EQ("server-prefix", merged["prefix"]); + ASSERT_EQ(0, merged.count("nullable")); +} + +TEST(RestMessagesTest, ListResponsesParse) { + ASSERT_OK_AND_ASSIGN(ListDatabasesResponse databases, + ListDatabasesResponse::FromJsonString( + R"({"databases": ["db1", "db2"], "nextPageToken": "token1"})")); + ASSERT_EQ((std::vector{"db1", "db2"}), databases.Data()); + ASSERT_EQ("token1", databases.NextPageToken().value()); + + ASSERT_OK_AND_ASSIGN(ListTablesResponse tables, + ListTablesResponse::FromJsonString(R"({"tables": ["t1"]})")); + ASSERT_EQ((std::vector{"t1"}), tables.Data()); + ASSERT_FALSE(tables.NextPageToken().has_value()); + + // nextPageToken may be serialized as explicit null by the server + ASSERT_OK_AND_ASSIGN( + ListTablesResponse null_token, + ListTablesResponse::FromJsonString(R"({"tables": [], "nextPageToken": null})")); + ASSERT_FALSE(null_token.NextPageToken().has_value()); +} + +TEST(RestMessagesTest, ListSnapshotsResponseParse) { + std::string json = R"({ + "snapshots": [{ + "version": 3, + "id": 7, + "schemaId": 2, + "baseManifestList": "manifest-list-1", + "deltaManifestList": "manifest-list-2", + "commitUser": "user1", + "commitIdentifier": 9, + "commitKind": "APPEND", + "timeMillis": 1234567, + "totalRecordCount": 100, + "deltaRecordCount": 10, + "watermark": 42 + }, { + "version": 3, + "id": 8, + "schemaId": 2, + "baseManifestList": "manifest-list-3", + "deltaManifestList": "manifest-list-4", + "commitUser": "user1", + "commitIdentifier": 10, + "commitKind": "COMPACT", + "timeMillis": 1234568, + "totalRecordCount": 100, + "deltaRecordCount": 0 + }], + "nextPageToken": null + })"; + ASSERT_OK_AND_ASSIGN(ListSnapshotsResponse response, + ListSnapshotsResponse::FromJsonString(json)); + ASSERT_EQ(2, response.Data().size()); + const Snapshot& snapshot = response.Data()[0]; + ASSERT_EQ(7, snapshot.Id()); + ASSERT_EQ(2, snapshot.SchemaId()); + ASSERT_EQ("user1", snapshot.CommitUser()); + ASSERT_EQ(1234567, snapshot.TimeMillis()); + SnapshotInfo info = snapshot.ToSnapshotInfo(); + ASSERT_EQ(SnapshotInfo::CommitKind::APPEND, info.commit_kind); + ASSERT_EQ(100, info.total_record_count.value()); + ASSERT_EQ(42, info.watermark.value()); + SnapshotInfo compact_info = response.Data()[1].ToSnapshotInfo(); + ASSERT_EQ(SnapshotInfo::CommitKind::COMPACT, compact_info.commit_kind); + ASSERT_FALSE(compact_info.watermark.has_value()); +} + +TEST(RestMessagesTest, GetDatabaseResponseParse) { + std::string json = R"({ + "id": "10", + "name": "db1", + "location": "/warehouse/db1.db", + "options": {"k1": "v1"}, + "owner": "owner1", + "createdAt": 100, + "createdBy": "creator", + "updatedAt": 200, + "updatedBy": "updater" + })"; + ASSERT_OK_AND_ASSIGN(GetDatabaseResponse response, GetDatabaseResponse::FromJsonString(json)); + ASSERT_EQ("db1", response.GetName()); + ASSERT_EQ("/warehouse/db1.db", response.GetLocation()); + ASSERT_EQ("v1", response.GetOptions().at("k1")); + std::map options; + response.GetAuditFields().PutAuditOptionsTo(&options); + ASSERT_EQ("owner1", options.at("owner")); + ASSERT_EQ("100", options.at("createdAt")); + ASSERT_EQ("updater", options.at("updatedBy")); +} + +TEST(RestMessagesTest, GetTableResponseParse) { + std::string json = R"({ + "id": "42", + "name": "t1", + "path": "/warehouse/db1.db/t1", + "isExternal": false, + "schemaId": 3, + "schema": { + "fields": [ + {"id": 0, "name": "f0", "type": "INT NOT NULL"}, + {"id": 1, "name": "f1", "type": "STRING"} + ], + "partitionKeys": [], + "primaryKeys": ["f0"], + "options": {"bucket": "2"}, + "comment": "a table" + }, + "updatedAt": 300 + })"; + ASSERT_OK_AND_ASSIGN(GetTableResponse response, GetTableResponse::FromJsonString(json)); + ASSERT_EQ("42", response.GetId()); + ASSERT_EQ("t1", response.GetName()); + ASSERT_EQ("/warehouse/db1.db/t1", response.GetPath()); + ASSERT_FALSE(response.IsExternal()); + ASSERT_EQ(3, response.GetSchemaId()); + ASSERT_EQ(300, response.GetAuditFields().updated_at.value()); + rapidjson::Document schema; + schema.Parse(response.GetSchemaJson().c_str()); + ASSERT_FALSE(schema.HasParseError()); + ASSERT_EQ(2u, schema["fields"].Size()); + ASSERT_STREQ("f0", schema["fields"][0]["name"].GetString()); +} + +TEST(RestMessagesTest, UnknownFieldsAreIgnored) { + // forward compatibility with newer servers: fields the client does not know must + // be ignored + std::string json = R"({ + "id": "42", "name": "t1", "path": "p", "schemaId": 3, + "schema": {"fields": []}, + "futureField": {"nested": true}, "anotherUnknown": [1, 2] + })"; + ASSERT_OK_AND_ASSIGN(GetTableResponse response, GetTableResponse::FromJsonString(json)); + ASSERT_EQ("t1", response.GetName()); + ASSERT_EQ(3, response.GetSchemaId()); +} + +TEST(RestMessagesTest, MissingRequiredFieldsFail) { + // "path", "schemaId" and "schema" are required + ASSERT_NOK(GetTableResponse::FromJsonString(R"({"id": "42", "name": "t1"})").status()); + // a non-object identifier is rejected + CreateTableRequest bad_identifier("", "", ""); + ASSERT_NOK(RapidJsonUtil::FromJsonString(R"({"identifier": "not-an-object", "schema": {}})", + &bad_identifier)); +} + +TEST(RestMessagesTest, CreateTableRequestSerialize) { + CreateTableRequest request( + "db1", "t1", R"({"fields": [], "partitionKeys": [], "primaryKeys": [], "options": {}})"); + ASSERT_OK_AND_ASSIGN(std::string json, request.ToJsonString()); + rapidjson::Document doc; + doc.Parse(json.c_str()); + ASSERT_FALSE(doc.HasParseError()); + ASSERT_STREQ("db1", doc["identifier"]["database"].GetString()); + ASSERT_STREQ("t1", doc["identifier"]["object"].GetString()); + ASSERT_TRUE(doc["schema"].IsObject()); + ASSERT_TRUE(doc["schema"]["fields"].IsArray()); + + ASSERT_OK_AND_ASSIGN(CreateTableRequest parsed, CreateTableRequest::FromJsonString(json)); + ASSERT_EQ("db1", parsed.GetDatabase()); + ASSERT_EQ("t1", parsed.GetTable()); +} + +TEST(RestMessagesTest, InvalidSchemaJsonErrorOmitsPayload) { + CreateTableRequest request("db1", "t1", R"({"fields": [], "options": {"token": "top-secret")"); + Status status = request.ToJsonString().status(); + ASSERT_NOK_WITH_MSG(status, "invalid json"); + // the payload may carry credentials, so it must not be echoed back in the error + ASSERT_EQ(std::string::npos, status.ToString().find("top-secret")) << status.ToString(); +} + +TEST(RestMessagesTest, RenameTableRequestSerialize) { + RenameTableRequest request("db1", "t1", "db1", "t2"); + ASSERT_OK_AND_ASSIGN(std::string json, request.ToJsonString()); + rapidjson::Document doc; + doc.Parse(json.c_str()); + ASSERT_FALSE(doc.HasParseError()); + ASSERT_STREQ("t1", doc["source"]["object"].GetString()); + ASSERT_STREQ("t2", doc["destination"]["object"].GetString()); + + ASSERT_OK_AND_ASSIGN(RenameTableRequest parsed, RenameTableRequest::FromJsonString(json)); + ASSERT_EQ("db1", parsed.GetSourceDatabase()); + ASSERT_EQ("t2", parsed.GetDestinationTable()); +} + +TEST(RestMessagesTest, CreateDatabaseRequestRoundTrip) { + CreateDatabaseRequest request("db1", {{"k1", "v1"}}); + ASSERT_OK_AND_ASSIGN(std::string json, request.ToJsonString()); + ASSERT_OK_AND_ASSIGN(CreateDatabaseRequest parsed, CreateDatabaseRequest::FromJsonString(json)); + ASSERT_EQ("db1", parsed.GetName()); + ASSERT_EQ("v1", parsed.GetOptions().at("k1")); +} + +} // namespace paimon::test diff --git a/src/paimon/rest/rest_util.cpp b/src/paimon/rest/rest_util.cpp new file mode 100644 index 000000000..ffb5904a0 --- /dev/null +++ b/src/paimon/rest/rest_util.cpp @@ -0,0 +1,92 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_util.h" + +#include + +#include "fmt/format.h" +#include "paimon/common/utils/string_utils.h" +#include "rapidjson/error/en.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/writer.h" + +namespace paimon { + +std::map RestUtil::ExtractPrefixMap( + const std::map& options, const std::string& prefix) { + std::map result; + for (const auto& [key, value] : options) { + if (key.size() > prefix.size() && key.compare(0, prefix.size(), prefix) == 0) { + result[key.substr(prefix.size())] = value; + } + } + return result; +} + +std::string RestUtil::RedactText(const std::string& text) { + if (text.empty()) { + return text; + } + static constexpr const char kRedacted[] = "******"; + std::string lower = StringUtils::ToLowerCase(text); + static constexpr const char* kLiteralMarkers[] = {"sig=", "\"sig\"", "'sig'"}; + for (const char* marker : kLiteralMarkers) { + if (lower.find(marker) != std::string::npos) { + return kRedacted; + } + } + std::string normalized; + normalized.reserve(lower.size()); + for (char c : lower) { + if ((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9')) { + normalized.push_back(c); + } + } + static constexpr const char* kMarkers[] = { + "password", "secret", "token", "credential", "accesskey", "accountkey", + "encryptionkey", "authorization", "privatekey", "apikey", "signature", "sas"}; + for (const char* marker : kMarkers) { + if (normalized.find(marker) != std::string::npos) { + return kRedacted; + } + } + return text; +} + +std::string RestUtil::JsonToString(const rapidjson::Value& value) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + value.Accept(writer); + return buffer.GetString(); +} + +rapidjson::Value RestUtil::ParseToValue(const std::string& json, + rapidjson::Document::AllocatorType* allocator) { + rapidjson::Document doc; + doc.Parse(json.c_str()); + if (doc.HasParseError()) { + // The payload may carry credentials and be arbitrarily large; report only the error. + throw std::invalid_argument(fmt::format("invalid json: {} (at offset {})", + rapidjson::GetParseError_En(doc.GetParseError()), + doc.GetErrorOffset())); + } + rapidjson::Value value; + value.CopyFrom(doc, *allocator); + return value; +} + +} // namespace paimon diff --git a/src/paimon/rest/rest_util.h b/src/paimon/rest/rest_util.h new file mode 100644 index 000000000..347dc4bd8 --- /dev/null +++ b/src/paimon/rest/rest_util.h @@ -0,0 +1,54 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "rapidjson/allocators.h" +#include "rapidjson/document.h" +#include "rapidjson/rapidjson.h" + +namespace paimon { + +/// Utilities for the REST catalog; URL encoding and decoding live in `UrlUtils`. +class RestUtil { + public: + RestUtil() = delete; + ~RestUtil() = delete; + + /// Extract all options whose key starts with `prefix`, with the prefix stripped from + /// the resulting keys. A key exactly equal to the prefix is dropped instead of + /// yielding an empty key. + static std::map ExtractPrefixMap( + const std::map& options, const std::string& prefix); + + /// Redacts free-form text such as a server error message: when any marker of a + /// sensitive value ("password", "token", "sig=", ...) is present the whole text is + /// replaced by "******", since arbitrary text cannot be masked per-secret reliably. + static std::string RedactText(const std::string& text); + + /// Serialize a rapidjson value to a compact JSON string. + static std::string JsonToString(const rapidjson::Value& value); + + /// Parse a JSON string into a rapidjson value owned by `allocator`. + /// Throws `std::invalid_argument` on parse error, consistent with `Jsonizable`. + static rapidjson::Value ParseToValue(const std::string& json, + rapidjson::Document::AllocatorType* allocator); +}; + +} // namespace paimon