From beec91e40e0c4f862b3785a010ef5cae13330d42 Mon Sep 17 00:00:00 2001 From: "hotdata-automation[bot]" <267177015+hotdata-automation[bot]@users.noreply.github.com> Date: Thu, 15 Oct 2026 05:57:53 +0000 Subject: [PATCH] feat(query): add arrow ipc stream response format --- CHANGELOG.md | 1 + docs/QueryApi.md | 8 +++++--- hotdata/api/query_api.py | 9 +++++---- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d6feb0..7c027b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- feat(query): add arrow ipc stream response format - chore(query): clarify write statement handling in dialect parameter - chore(indexes): clarify vector index constraints and algorithms - chore(indexes): clarify vector precision documentation diff --git a/docs/QueryApi.md b/docs/QueryApi.md index f769189..cb351c8 100644 --- a/docs/QueryApi.md +++ b/docs/QueryApi.md @@ -16,6 +16,8 @@ Execute a SQL query scoped to a database. A database is the only window into cat Set `async: true` to execute asynchronously — returns a query run ID for polling. Optionally set `async_after_ms` to attempt synchronous execution first, falling back to async if the query exceeds the timeout. +To receive the rows as an Arrow IPC stream instead of JSON, send `Accept: application/vnd.apache.arrow.stream` (without also listing `application/json`). A query that completes within the request then returns 200 with an Arrow IPC stream body (schema, record batches, end-of-stream), and the values the JSON body would carry are sent as response headers: `X-Query-Run-Id`, `X-Result-Id` (when the result was saved), `X-Total-Row-Count`, `X-Execution-Time-Ms`, and `X-Query-Warning` (when there is a warning). Arrow keeps `±Inf` and `NaN` float values as they are. An Arrow response always holds the whole result: if the result is larger than an inline response can hold, the request returns 202 with the query run ticket and an `X-Result-Id` header instead, and the full result is read from `GET /v1/results/{id}?format=arrow` once it is ready. A query that goes async, and every error, return the same JSON bodies as without the header. + ### Example * Api Key Authentication (WorkspaceId): @@ -87,14 +89,14 @@ Name | Type | Description | Notes ### HTTP request headers - **Content-Type**: application/json - - **Accept**: application/json + - **Accept**: application/json, application/vnd.apache.arrow.stream ### HTTP response details | Status code | Description | Response headers | |-------------|-------------|------------------| -**200** | Query executed successfully | - | -**202** | Query submitted asynchronously | - | +**200** | Query executed successfully. JSON by default; an Arrow IPC stream when the request sends `Accept: application/vnd.apache.arrow.stream` (see the description for the headers that come with it). | * X-Execution-Time-Ms - Arrow responses only: query execution time in milliseconds.
* X-Query-Run-Id - Arrow responses only: the query run id.
* X-Query-Warning - Arrow responses only: present when the JSON body would carry a `warning`.
* X-Result-Id - Arrow responses only: the id to read this result again from the Results API. Absent when the result could not be saved.
* X-Total-Row-Count - Arrow responses only: the number of rows in the stream, which is the whole result.
| +**202** | Query submitted asynchronously. With an Arrow `Accept`, also returned when the result is too large to send inline; the response then carries an `X-Result-Id` header naming the result to read from the Results API. | * X-Result-Id - Arrow responses only: the id to read this result again from the Results API. Absent when the result could not be saved.
| **400** | Invalid request (no database specified, or header/body database_id conflict) | - | **404** | Database not found | - | **429** | The engine was too busy to run this query right now — too many concurrent queries, or not enough memory available (often because of other queries running at the same time). Retry after the Retry-After delay; if it persists, narrowing the query (add a filter or LIMIT) may help. | - | diff --git a/hotdata/api/query_api.py b/hotdata/api/query_api.py index b36177a..7632993 100644 --- a/hotdata/api/query_api.py +++ b/hotdata/api/query_api.py @@ -60,7 +60,7 @@ def query( ) -> QueryResponse: """Execute SQL query - Execute a SQL query scoped to a database. A database is the only window into catalogs: the query sees only that database's auto `default` catalog plus any catalogs explicitly attached to it. Select the database with EITHER the `X-Database-Id` header OR the `database_id` body field (exactly one must be given; if both are sent and disagree, that's a 400). Use standard Postgres-compatible SQL; reference the default catalog as `default..` (or just `.
` / `
`) and attached catalogs by their alias. Results are returned inline and a `result_id` is provided for later retrieval via the Results API. Set `async: true` to execute asynchronously — returns a query run ID for polling. Optionally set `async_after_ms` to attempt synchronous execution first, falling back to async if the query exceeds the timeout. + Execute a SQL query scoped to a database. A database is the only window into catalogs: the query sees only that database's auto `default` catalog plus any catalogs explicitly attached to it. Select the database with EITHER the `X-Database-Id` header OR the `database_id` body field (exactly one must be given; if both are sent and disagree, that's a 400). Use standard Postgres-compatible SQL; reference the default catalog as `default..
` (or just `.
` / `
`) and attached catalogs by their alias. Results are returned inline and a `result_id` is provided for later retrieval via the Results API. Set `async: true` to execute asynchronously — returns a query run ID for polling. Optionally set `async_after_ms` to attempt synchronous execution first, falling back to async if the query exceeds the timeout. To receive the rows as an Arrow IPC stream instead of JSON, send `Accept: application/vnd.apache.arrow.stream` (without also listing `application/json`). A query that completes within the request then returns 200 with an Arrow IPC stream body (schema, record batches, end-of-stream), and the values the JSON body would carry are sent as response headers: `X-Query-Run-Id`, `X-Result-Id` (when the result was saved), `X-Total-Row-Count`, `X-Execution-Time-Ms`, and `X-Query-Warning` (when there is a warning). Arrow keeps `±Inf` and `NaN` float values as they are. An Arrow response always holds the whole result: if the result is larger than an inline response can hold, the request returns 202 with the query run ticket and an `X-Result-Id` header instead, and the full result is read from `GET /v1/results/{id}?format=arrow` once it is ready. A query that goes async, and every error, return the same JSON bodies as without the header. :param query_request: (required) :type query_request: QueryRequest @@ -137,7 +137,7 @@ def query_with_http_info( ) -> ApiResponse[QueryResponse]: """Execute SQL query - Execute a SQL query scoped to a database. A database is the only window into catalogs: the query sees only that database's auto `default` catalog plus any catalogs explicitly attached to it. Select the database with EITHER the `X-Database-Id` header OR the `database_id` body field (exactly one must be given; if both are sent and disagree, that's a 400). Use standard Postgres-compatible SQL; reference the default catalog as `default..
` (or just `.
` / `
`) and attached catalogs by their alias. Results are returned inline and a `result_id` is provided for later retrieval via the Results API. Set `async: true` to execute asynchronously — returns a query run ID for polling. Optionally set `async_after_ms` to attempt synchronous execution first, falling back to async if the query exceeds the timeout. + Execute a SQL query scoped to a database. A database is the only window into catalogs: the query sees only that database's auto `default` catalog plus any catalogs explicitly attached to it. Select the database with EITHER the `X-Database-Id` header OR the `database_id` body field (exactly one must be given; if both are sent and disagree, that's a 400). Use standard Postgres-compatible SQL; reference the default catalog as `default..
` (or just `.
` / `
`) and attached catalogs by their alias. Results are returned inline and a `result_id` is provided for later retrieval via the Results API. Set `async: true` to execute asynchronously — returns a query run ID for polling. Optionally set `async_after_ms` to attempt synchronous execution first, falling back to async if the query exceeds the timeout. To receive the rows as an Arrow IPC stream instead of JSON, send `Accept: application/vnd.apache.arrow.stream` (without also listing `application/json`). A query that completes within the request then returns 200 with an Arrow IPC stream body (schema, record batches, end-of-stream), and the values the JSON body would carry are sent as response headers: `X-Query-Run-Id`, `X-Result-Id` (when the result was saved), `X-Total-Row-Count`, `X-Execution-Time-Ms`, and `X-Query-Warning` (when there is a warning). Arrow keeps `±Inf` and `NaN` float values as they are. An Arrow response always holds the whole result: if the result is larger than an inline response can hold, the request returns 202 with the query run ticket and an `X-Result-Id` header instead, and the full result is read from `GET /v1/results/{id}?format=arrow` once it is ready. A query that goes async, and every error, return the same JSON bodies as without the header. :param query_request: (required) :type query_request: QueryRequest @@ -214,7 +214,7 @@ def query_without_preload_content( ) -> RESTResponseType: """Execute SQL query - Execute a SQL query scoped to a database. A database is the only window into catalogs: the query sees only that database's auto `default` catalog plus any catalogs explicitly attached to it. Select the database with EITHER the `X-Database-Id` header OR the `database_id` body field (exactly one must be given; if both are sent and disagree, that's a 400). Use standard Postgres-compatible SQL; reference the default catalog as `default..
` (or just `.
` / `
`) and attached catalogs by their alias. Results are returned inline and a `result_id` is provided for later retrieval via the Results API. Set `async: true` to execute asynchronously — returns a query run ID for polling. Optionally set `async_after_ms` to attempt synchronous execution first, falling back to async if the query exceeds the timeout. + Execute a SQL query scoped to a database. A database is the only window into catalogs: the query sees only that database's auto `default` catalog plus any catalogs explicitly attached to it. Select the database with EITHER the `X-Database-Id` header OR the `database_id` body field (exactly one must be given; if both are sent and disagree, that's a 400). Use standard Postgres-compatible SQL; reference the default catalog as `default..
` (or just `.
` / `
`) and attached catalogs by their alias. Results are returned inline and a `result_id` is provided for later retrieval via the Results API. Set `async: true` to execute asynchronously — returns a query run ID for polling. Optionally set `async_after_ms` to attempt synchronous execution first, falling back to async if the query exceeds the timeout. To receive the rows as an Arrow IPC stream instead of JSON, send `Accept: application/vnd.apache.arrow.stream` (without also listing `application/json`). A query that completes within the request then returns 200 with an Arrow IPC stream body (schema, record batches, end-of-stream), and the values the JSON body would carry are sent as response headers: `X-Query-Run-Id`, `X-Result-Id` (when the result was saved), `X-Total-Row-Count`, `X-Execution-Time-Ms`, and `X-Query-Warning` (when there is a warning). Arrow keeps `±Inf` and `NaN` float values as they are. An Arrow response always holds the whole result: if the result is larger than an inline response can hold, the request returns 202 with the query run ticket and an `X-Result-Id` header instead, and the full result is read from `GET /v1/results/{id}?format=arrow` once it is ready. A query that goes async, and every error, return the same JSON bodies as without the header. :param query_request: (required) :type query_request: QueryRequest @@ -306,7 +306,8 @@ def _query_serialize( if 'Accept' not in _header_params: _header_params['Accept'] = self.api_client.select_header_accept( [ - 'application/json' + 'application/json', + 'application/vnd.apache.arrow.stream' ] )