From a75f8fc6b143e3c82d2881e1a19e0477ec73f249 Mon Sep 17 00:00:00 2001 From: Markus Cozowicz Date: Sun, 11 Oct 2026 19:04:45 +0200 Subject: [PATCH 1/4] feat: propose HTTP Arrow execution profile and TypeScript client --- .github/workflows/execution-client-ci.yml | 71 ++ .gitignore | 1 + README.md | 1 + docs/index.md | 9 + docs/proposals/execute-query-http.md | 594 ++++++++++++++++ .../execution-http/execution.schema.json | 431 ++++++++++++ docs/proposals/execution-http/openapi.json | 232 ++++++ typescript/execution/LICENSE | 201 ++++++ typescript/execution/NOTICE | 5 + typescript/execution/README.md | 402 +++++++++++ typescript/execution/package-lock.json | 665 ++++++++++++++++++ typescript/execution/package.json | 36 + typescript/execution/src/arrow.ts | 443 ++++++++++++ typescript/execution/src/client.ts | 480 +++++++++++++ typescript/execution/src/errors.ts | 64 ++ typescript/execution/src/index.ts | 23 + typescript/execution/src/transport.ts | 174 +++++ typescript/execution/src/types.ts | 184 +++++ typescript/execution/src/validation.ts | 363 ++++++++++ .../execution/test/arrow-metadata.test.mjs | 148 ++++ typescript/execution/test/arrow.test.mjs | 421 +++++++++++ typescript/execution/test/browser.test.mjs | 40 ++ .../execution/test/browser/smoke.test.mjs | 300 ++++++++ typescript/execution/test/fixtures.mjs | 234 ++++++ typescript/execution/test/lifecycle.test.mjs | 285 ++++++++ typescript/execution/test/origins.test.mjs | 134 ++++ typescript/execution/test/pagination.test.mjs | 320 +++++++++ typescript/execution/test/protocol.test.mjs | 221 ++++++ typescript/execution/test/review.test.mjs | 186 +++++ typescript/execution/test/schema.test.mjs | 165 +++++ typescript/execution/tsconfig.json | 19 + 31 files changed, 6852 insertions(+) create mode 100644 .github/workflows/execution-client-ci.yml create mode 100644 docs/proposals/execute-query-http.md create mode 100644 docs/proposals/execution-http/execution.schema.json create mode 100644 docs/proposals/execution-http/openapi.json create mode 100644 typescript/execution/LICENSE create mode 100644 typescript/execution/NOTICE create mode 100644 typescript/execution/README.md create mode 100644 typescript/execution/package-lock.json create mode 100644 typescript/execution/package.json create mode 100644 typescript/execution/src/arrow.ts create mode 100644 typescript/execution/src/client.ts create mode 100644 typescript/execution/src/errors.ts create mode 100644 typescript/execution/src/index.ts create mode 100644 typescript/execution/src/transport.ts create mode 100644 typescript/execution/src/types.ts create mode 100644 typescript/execution/src/validation.ts create mode 100644 typescript/execution/test/arrow-metadata.test.mjs create mode 100644 typescript/execution/test/arrow.test.mjs create mode 100644 typescript/execution/test/browser.test.mjs create mode 100644 typescript/execution/test/browser/smoke.test.mjs create mode 100644 typescript/execution/test/fixtures.mjs create mode 100644 typescript/execution/test/lifecycle.test.mjs create mode 100644 typescript/execution/test/origins.test.mjs create mode 100644 typescript/execution/test/pagination.test.mjs create mode 100644 typescript/execution/test/protocol.test.mjs create mode 100644 typescript/execution/test/review.test.mjs create mode 100644 typescript/execution/test/schema.test.mjs create mode 100644 typescript/execution/tsconfig.json diff --git a/.github/workflows/execution-client-ci.yml b/.github/workflows/execution-client-ci.yml new file mode 100644 index 000000000..e4b28c295 --- /dev/null +++ b/.github/workflows/execution-client-ci.yml @@ -0,0 +1,71 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Execution Client CI + +on: + push: + branches: ["main"] + paths: + - "typescript/execution/**" + - "docs/proposals/execute-query-http.md" + - "docs/proposals/execution-http/**" + - ".github/workflows/execution-client-ci.yml" + pull_request: + branches: ["main"] + paths: + - "typescript/execution/**" + - "docs/proposals/execute-query-http.md" + - "docs/proposals/execution-http/**" + - ".github/workflows/execution-client-ci.yml" + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + node-version: ["22", "24"] + defaults: + run: + working-directory: typescript/execution + steps: + - name: Checkout project + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Set up Node.js + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + with: + node-version: ${{ matrix.node-version }} + cache: npm + cache-dependency-path: typescript/execution/package-lock.json + - name: Install locked dependencies + run: npm ci + - name: Check types, contract and client + run: npm test + - name: Exercise a real browser + if: matrix.node-version == '24' + run: | + export OSSIE_BROWSER_BINARY="$(command -v google-chrome)" + test -n "$OSSIE_BROWSER_BINARY" + npm run test:browser diff --git a/.gitignore b/.gitignore index f6a84d9ca..e61a8874b 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ **/coverage.xml **/htmlcov/ **/dist/ +**/node_modules/ **/target/ **/.tom/ *.py[cod] diff --git a/README.md b/README.md index 94ed3f38a..a4b619880 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ Apache Ossie provides a single JSON- and YAML-based specification that any tool - [`examples/`](examples/) — Example semantic models, including a complete TPC-DS model. - [`validation/`](validation/) — Tooling for validating semantic models against the Ossie schema. - [`docs/`](docs/) — Project documentation and overview. +- [`typescript/execution/`](typescript/execution/) — Experimental browser-first HTTP/Arrow execution client for the [proposed Layer 3 execution profile](docs/proposals/execute-query-http.md), not an adopted standard or query engine. ## Get involved diff --git a/docs/index.md b/docs/index.md index fa71bb83c..ee189b164 100644 --- a/docs/index.md +++ b/docs/index.md @@ -63,6 +63,15 @@ The specification supports multiple SQL dialects (`ANSI_SQL`, `SNOWFLAKE`, `DATA For the full specification, see [core-spec/spec.md](../core-spec/spec.md). For validation tooling, see [validation/validate.py](../validation/validate.py). For a complete example, see the [TPC-DS semantic model](../examples/tpcds_semantic_model.yaml). +### Proposed Query Execution Profile + +The optional [HTTP/Arrow execution proposal](proposals/execute-query-http.md) +defines a browser-first binding for model-resolved Layer 3 queries, execution +status and typed results. Its [experimental TypeScript client](../typescript/execution/README.md) +implements the proposed wire contract; it is not a query engine, a native vendor +adapter, or proof of equivalent answers across engines. The profile and its +Foundation/MCP dependencies remain proposals and do not change core model conformance. + ### Participating Organizations Ossie is supported by a broad coalition of 50+ organizations across the data ecosystem, including: diff --git a/docs/proposals/execute-query-http.md b/docs/proposals/execute-query-http.md new file mode 100644 index 000000000..f1b236072 --- /dev/null +++ b/docs/proposals/execute-query-http.md @@ -0,0 +1,594 @@ + + +# Ossie HTTP execution — DRAFT 0.1 + +**Status:** proposal `0.1-draft`, not an adopted Apache Ossie standard. +The associated TypeScript package is a client and local protocol-validation +prototype, not a server, semantic engine, or vendor adapter. + +This proposal binds structured Ossie query submission and status to HTTP/JSON, +and completed tabular results to Apache Arrow IPC streams. It targets browser +applications without requiring gRPC. It does not define a new query language. + +## 1. Normative basis and scope + +The words **MUST**, **MUST NOT**, **SHOULD**, **SHOULD NOT**, and **MAY** have the +meaning given in [BCP 14](https://www.rfc-editor.org/rfc/rfc8174). + +The following proposal revisions are pinned; both upstream pull requests remain +open, and their inclusion here does not imply adoption: + +| Dependency | Pinned revision | +| --- | --- | +| [MCP proposal #529](https://github.com/apache/ossie/pull/529), query profile `0.4-draft` | `b5418ee3aa19f117a1c464733de21f5a0dc0326b` | +| [Foundation proposal #246](https://github.com/apache/ossie/pull/246) | `cc0d07099a3ef31e85af1fd93458a52186fae2c6` | + +[execution.schema.json](execution-http/execution.schema.json) is the normative +JSON Schema (draft 2020-12). +[openapi.json](execution-http/openapi.json) describes the HTTP operations using +references to those definitions, not a second set of object schemas. These +documents specify structure; the rules below additionally specify behavior. + +The lowercase `expression`, `projection`, `predicate`, and `query` definitions +are copied unchanged from the [pinned MCP query schema][query-schema]. +The public `Query` definition adds only a portable integer bound to `limit`. +All wire property names are case-sensitive lower snake case; schema definition +names such as `Execution` are not wire keys. + +In scope: one read-only query, immutable model binding, bounded synchronous wait, +polling, cancellation, errors, and one materialized result delivered as a single +stream or immutable pages. Out of scope: +model/catalog CRUD, native-query escape, query compilation, semantic evaluation, +sessions, transactions, mutations, SSE/progress subscriptions, multi-result or +live-result delivery, cursor APIs, result-link renewal, and automatic repair or +re-execution. +Flight is a possible future mapping, not implemented or negotiated by this draft. + +## 2. JSON objects and discovery + +JSON messages MUST conform to their named schema definitions. Unknown core +properties MUST be rejected. Optional `extensions` values are namespaced JSON +objects; clients MAY ignore unfamiliar namespaces, which MUST NOT change core +meaning or relax required validation. Unknown feature identifiers confer no +capability. Core text fields and extension namespace keys MUST be nonblank when +present; arbitrary text inside opaque extensions is not subject to that rule. +Extension keys SHOULD use reverse-DNS naming to avoid collisions; domain-style +syntax is a naming convention, not an additional schema/client validity check. + +Identifiers are opaque, nonblank strings. Clients MUST encode identifiers as a +single URL path segment, preserving an embedded slash as an encoded component. +Clients MUST reject `.` and `..` where URL normalization would change the path. +Model identity consists of `id` and `revision`; +`revision` is an opaque immutable revision, not necessarily a content hash. +JSON counters (`limit`, `page_index`, `row_offset`, `row_count`, `byte_count`) MUST be integers in +`0..9007199254740991`, inclusive. JavaScript clients MUST reject values outside +that range rather than round them. This binding restriction is intentionally +stricter than the pinned MCP schema's unbounded nonnegative `limit`. + +`GET /capabilities` returns `Capabilities` for the current caller: + +```json +{ + "contract_version": "0.1-draft", + "foundation_revision": "cc0d07099a3ef31e85af1fd93458a52186fae2c6", + "query_schema_version": "mcp-0.4-draft", + "result_formats": ["arrow-stream"], + "query_shapes": ["aggregation", "scalar"], + "features": [], + "max_wait_seconds": 10 +} +``` + +The three version values MUST match this draft exactly. `query_shapes` is a +nonempty unique list; `features` is a possibly empty unique list. These advertise +support, not semantic-conformance certificates. `max_wait_seconds` MUST be an +integer from 0 through 60. + +`GET /bindings/{id}` returns a `Binding`; `{id}` is its `data_source_id`: + +```json +{ + "data_source_id": "sales-v7", + "model": { "id": "sales", "revision": "7" }, + "query_shapes": ["aggregation", "scalar"], + "features": [] +} +``` + +A binding id MUST identify exactly one model revision throughout its lifetime +and MUST NOT be reassigned. Publishing a new revision requires a new binding +id. Executions and results MUST retain that identity. Binding support MUST NOT +exceed service capabilities; servers MUST reject shapes unsupported by either +the service or binding. Clients SHOULD discover both before selecting a query +shape. The binding is not the model document: models are obtained separately. + +## 3. HTTP operations + +Paths are relative to a deployment-selected base URL. Control requests and +successful responses use `application/json`. Servers MUST NOT redirect these +operations. A non-2xx response MUST use `application/problem+json` with +`ProblemDetails`; this draft uses 4xx/5xx for failures, not redirects. + +| Operation | Successful response | +| --- | --- | +| `GET /capabilities` | `200 Capabilities` | +| `GET /bindings/{id}` | `200 Binding` | +| `POST /queries` with `Request` | `200 Execution` if terminal; `202 Execution` if pending | +| `GET /queries/{id}` | `200 Execution` in any state | +| `POST /queries/{id}/cancel`, no body | `200 Execution` if terminal; `202 Execution` if pending | +| `GET` the successful manifest's `url` or a `pages[i].url` | `200 application/vnd.apache.arrow.stream` | + +### Submission + +`Request` contains only `data_source_id` and `query`. An aggregation query has +at least one nonempty `dimensions` or `measures` projection and no `fields`. +A scalar query has nonempty `fields` and no `dimensions`, `measures`, or +`having`. Both support `where`, `order_by`, and `limit`; only aggregation +supports `having`. Predicates are a nonblank expression or a nonempty list of +nonblank expressions. Order direction is `ASC`/`DESC`; null ordering is +`FIRST`/`LAST`. Interpretation and omitted-clause defaults belong to the pinned +query proposals, not to HTTP or this client's shape validator. + +```http +POST /queries HTTP/1.1 +Content-Type: application/json +Accept: application/json +Prefer: wait=5 + +{"data_source_id":"sales-v7","query":{"dimensions":["customers.region"],"measures":["revenue"],"order_by":[{"field":"revenue","direction":"DESC","nulls":"LAST"}],"limit":100}} +``` + +Scalar request: + +```json +{ + "data_source_id": "sales-v7", + "query": { + "fields": ["orders.order_id", "orders.amount"], + "where": ["orders.amount > 0"], + "limit": 100 + } +} +``` + +The optional preference is `Prefer: wait=N`, where `N` is an integer from 0 +through 60, no greater than `max_wait_seconds`. An omitted preference means +`wait=0`. Unsupported or malformed values MUST be rejected, not silently +treated as a longer wait. The server MAY return earlier. It MUST NOT hold the +POST beyond `N` seconds waiting for execution completion. It returns `200` if +terminal or `202` if pending, without cancelling the execution at wait expiry. +Clients MAY discover/cache the advertised bound or validate only the protocol +range locally and rely on the server to enforce its advertised maximum. +The TypeScript prototype does not automatically preflight capabilities or +bindings and does not cache discovery results for submission-time enforcement. +Its explicit getters return validated data; applications can use that data to +choose requests, while the server enforces its advertised restrictions. +`200` does not mean the query succeeded: inspect `state`. +No submission idempotency key or automatic retry is defined. An interrupted +submission can leave an execution running even if the client did not receive +its id; blindly resubmitting can create a second execution. + +### Bounded wait versus synchronous and asynchronous execution + +There is always **one execution job**, not separate synchronous and asynchronous +query modes. `Prefer: wait=0` returns immediately with `202` unless the execution +is already terminal, in which case it returns `200`. `Prefer: wait=N` holds the +POST for at most `N` seconds: fast completion returns terminal `200`; otherwise +the server returns pending `202` and the client polls **the same execution id**. +Terminal `200` can mean `succeeded`, `failed`, or `cancelled`. + +Unbounded synchronous waiting risks long-held connections and intermediary +timeouts; pure asynchronous submission typically needs extra polling round trips. +Bounded wait optimizes the fast path without requiring a long-lived POST. +The wait preference is neither an execution deadline nor a result-download +timeout, and its expiry does not cancel the job. A JavaScript `execute()` promise +is asynchronous independently of this HTTP choice: it can hide polling until +that same job finishes, then download and validate its result. + +### Lifecycle and cancellation + +`Execution` always carries `contract_version`, `execution_id`, +`data_source_id`, `model`, `state`, and `warnings` (an array, possibly empty). +Its identity MUST remain unchanged across observations. + +| Current state | Permitted next states | +| --- | --- | +| `queued` | `running`, `cancelling`, `succeeded`, `failed`, `cancelled` | +| `running` | `cancelling`, `succeeded`, `failed`, `cancelled` | +| `cancelling` | `succeeded`, `failed`, `cancelled` | +| `succeeded`, `failed`, `cancelled` | No different state | + +Repeated observations of the same state are allowed. Terminal representations, +including result manifests, MUST be immutable. Only pending states MAY carry +`progress`, with optional `fraction` in `[0,1]` and optional `phase`. +Progress is advisory, not evidence of result completeness. + +```json +{ + "contract_version": "0.1-draft", + "execution_id": "q-42", + "data_source_id": "sales-v7", + "model": { "id": "sales", "revision": "7" }, + "state": "running", + "warnings": [], + "progress": { "fraction": 0.25, "phase": "executing" } +} +``` + +Only `succeeded` carries `result`; only `failed` carries `error`. Neither field +is allowed while pending or cancelled. Terminal responses MUST NOT carry +`progress`. Clients MUST poll with `GET /queries/{id}` to observe completion; +they SHOULD space requests and impose an application-selected deadline. + +Cancellation is best-effort and idempotent for a given execution. +`POST /queries/q-42/cancel` MAY return a terminal result if completion wins +the race, or `202` with `cancelling` or another observed pending state. +A client MUST NOT interpret acceptance as confirmed cancellation. +Aborting a request, stopping an iterator, or reaching a client timeout MUST +NOT be represented as a server-side cancellation; cancellation requires the +explicit endpoint. + +### Errors + +`QueryError` contains `code`, `message`, `retryable`, and optional +`replacement_query` and `extensions`. A [Foundation error code][foundation-semantics] MUST appear +directly in `error.code`, without a transport-specific prefix or replacement +taxonomy. Unknown codes MUST remain inspectable. This draft does not pretend +the Foundation enumerates transport, authorization, or local client errors. + +```http +HTTP/1.1 422 Unprocessable Content +Content-Type: application/problem+json + +{"type":"about:blank","title":"Query cannot be resolved","status":422,"error":{"code":"E_AMBIGUOUS_PATH","message":"More than one model relationship path applies.","retryable":false}} +``` + +`ProblemDetails.status` MUST equal the HTTP status. A failed execution returned +in a successful HTTP response carries the same `QueryError` shape in `error`. +`retryable` and `replacement_query` are advisory: clients MUST NOT automatically +repair, retry, or re-execute queries. They MUST NOT fall back to SQL, DAX, or +another native query. The package's low-level methods return failed/cancelled +executions for inspection; `wait` and `execute` throw for those states. + +## 4. Results and Arrow contract + +A successful result is one immutable materialization, available only after +execution succeeds. Its `ResultManifest` MUST provide **exactly one** delivery +form: `url` for one self-contained Arrow IPC stream, or a nonempty `pages` array +of `ResultPage` descriptors, each pointing to a self-contained Arrow IPC stream. +Downloading batches or pages is not streaming live rows from a running query. +No Arrow file container, base64 JSON, range retrieval, multi-stream concatenation, +or external dictionary dependency is permitted. + +Illustrative successful `Execution` (the counts describe a hypothetical +payload, not an executable semantic fixture): + +```json +{ + "contract_version": "0.1-draft", + "execution_id": "q-42", + "data_source_id": "sales-v7", + "model": { "id": "sales", "revision": "7" }, + "state": "succeeded", + "warnings": [], + "result": { + "result_id": "r-42", + "format": "arrow-stream", + "url": "https://api.example.test/results/r-42", + "row_count": 2, + "byte_count": 1536, + "completeness": "complete", + "expires_at": "2026-10-12T00:00:00Z", + "columns": [ + {"column_id":"c-1","name":"region","logical_type":"String","nullable":true,"role":"dimension","model_ref":"customers.region"}, + {"column_id":"c-2","name":"revenue","logical_type":"Decimal","nullable":true,"role":"measure","model_ref":"revenue","label":"Revenue","unit":"currency","currency":"USD"} + ] + } +} +``` + +`row_count` and `byte_count` are required exact counts of the delivered +materialization, including when truncated. `byte_count` includes all IPC +framing, metadata, padding, and record batches **after HTTP decompression**. +For paged delivery these are whole-result totals: bytes are summed across pages, +including every repeated schema and each page's framing. +HTTP `Content-Length` may describe compressed transfer bytes and is not a +substitute. Counts detect length/count mismatches; they are not checksums or +cryptographic integrity guarantees. + +`completeness` MUST be `complete` or `truncated`. A truncated result MUST include +a nonblank `truncation_reason`; a complete result MUST omit it. A server-imposed +row/byte/time cap MUST NOT be disguised as a complete result. An explicit query +`limit` defines the requested result, so satisfying it is not by itself +truncation. Partial rows MUST NOT be emitted. `expires_at` is RFC 3339 and applies +to the whole result; result URLs, including every page URL, MUST NOT be used at +or after expiry. This draft offers no renewal or automatic re-execution. Expiry or +authorization failure does not change terminal state. + +### Immutable result pagination + +The server chooses page sizes and a fixed partition of the completed result. +There is no new query clause, requested page size, or query offset: `limit` +remains a semantic cap on the requested result, not a page size. Pagination +does not re-execute a query with arbitrary SQL `OFFSET`, define a cursor API, +or supply a server-side query engine. + +The entire immutable page manifest MUST be returned with the successful +execution. A client can select a descriptor by index or traverse the array in +order; fetching pages MUST NOT create new executions or change that partition. +The complete manifest remains subject to the client's 1 MiB control-JSON limit; +pagination does not provide a cursor-based fallback for an oversized manifest. + +Each `ResultPage` is a closed object with exactly `page_index`, `row_offset`, +`row_count`, `byte_count`, and `url`. All counters are safe nonnegative integers. +For a manifest with `N` pages: + +- `page_index` MUST equal the descriptor's array position, from `0` to `N-1`. +- The first `row_offset` MUST be `0`; every later offset MUST equal the preceding + offset plus its row count. Offsets describe this materialization, not a query + clause or re-execution instruction. +- Descriptor row and decoded-byte counts MUST sum exactly to the root + `row_count` and `byte_count`, respectively. +- A zero-row page is forbidden except as the sole page of a zero-row result. + Every page MUST have a positive `byte_count`, including that empty result's + self-contained schema. + +These contiguity, sum, and empty-page rules are normative runtime invariants, +not expressible by the descriptor's structural JSON Schema alone. + +Illustrative paged `Execution` (hypothetical payload counts): + +```json +{ + "contract_version": "0.1-draft", + "execution_id": "q-43", + "data_source_id": "sales-v7", + "model": { "id": "sales", "revision": "7" }, + "state": "succeeded", + "warnings": [], + "result": { + "result_id": "r-43", + "format": "arrow-stream", + "pages": [ + {"page_index":0,"row_offset":0,"row_count":1,"byte_count":1024,"url":"https://api.example.test/results/r-43/pages/0"}, + {"page_index":1,"row_offset":1,"row_count":1,"byte_count":1024,"url":"https://api.example.test/results/r-43/pages/1"} + ], + "row_count": 2, + "byte_count": 2048, + "completeness": "complete", + "expires_at": "2026-10-12T00:00:00Z", + "columns": [ + {"column_id":"c-1","name":"region","logical_type":"String","nullable":true,"role":"dimension","model_ref":"customers.region"}, + {"column_id":"c-2","name":"revenue","logical_type":"Decimal","nullable":true,"role":"measure","model_ref":"revenue","label":"Revenue","unit":"currency","currency":"USD"} + ] + } +} +``` + +`complete` or `truncated` describes the entire materialization, not an individual +page or the subset a user has browsed. Browsing selected pages is not verified +complete-result consumption, even when the manifest says `complete`. +Applications MAY explicitly read an immutable page again before expiry; this +is not an automatic network retry. A failed download MUST NOT automatically +resubmit the query, renew links, or retry the network operation. + +The existing single-stream `url` form remains supported unchanged alongside +pages in this unreleased `0.1-draft` profile. Clients implementing this draft +MUST support both forms without an additional capability feature token. +Servers can retain the `url` form for interoperability with older +single-stream-only clients; those older clients must not be assumed to support +a paged manifest. + +### Identity and semantic descriptors + +The Arrow schema MUST contain UTF-8 JSON metadata at +`org.apache.ossie.execution`, conforming to `ExecutionMetadata`: + +```json +{ + "contract_version": "0.1-draft", + "execution_id": "q-42", + "result_id": "r-42", + "data_source_id": "sales-v7", + "model": { "id": "sales", "revision": "7" } +} +``` + +For paged delivery, `ExecutionMetadata` MUST additionally contain `page_index` +and `row_offset` together, matching the selected descriptor. Both fields MUST +be absent for single-stream `url` delivery. For example, page 1 above carries: + +```json +{ + "contract_version": "0.1-draft", + "execution_id": "q-43", + "result_id": "r-43", + "data_source_id": "sales-v7", + "model": { "id": "sales", "revision": "7" }, + "page_index": 1, + "row_offset": 1 +} +``` + +The paired fields bind an Arrow stream to its page and reject a response that +otherwise identifies the right result but contains the wrong page. + +Each field MUST contain UTF-8 JSON metadata at `org.apache.ossie.column`, +conforming to `ColumnMetadata`, for example `{"column_id":"c-2"}`. +These identities MUST equal the execution and manifest identities. +`column_id` values MUST be unique within the result. Display/physical names +MAY repeat, so clients MUST NOT key results only by name. + +The manifest is authoritative for semantic descriptors: `logical_type`, +optional `role`, `model_ref`, `label`, `description`, `unit`, `currency`, and +`extensions`. Its nonempty columns array MUST match Arrow field count, order, names, +nullability, ids, and the physical/logical mapping below. Labels or duplicate +names MUST NOT change field identity. Clients MUST reject mismatches rather +than infer a replacement schema. + +Every page MUST repeat the same self-contained schema: identical field names, +order, ids, nullability, actual physical types, and field metadata. Schema +metadata MUST also remain identical except for the two page coordinates in +`org.apache.ossie.execution`. Choosing another permitted physical type for the +same logical type on a later page is forbidden: integer widths, decimal +precision/scale, and timestamp unit/timezone, for example, MUST remain fixed. +Consumers traversing pages MUST reject physical-schema drift. + +### Baseline physical types + +| Manifest `logical_type` | Permitted Arrow physical types | +| --- | --- | +| `String` | `Utf8` | +| `Integer` | Signed or unsigned `Int8`, `Int16`, `Int32`, `Int64` | +| `Float` | `Float32`, `Float64` | +| `Boolean` | `Bool` | +| `Decimal` | `Decimal128`, `Decimal256`, retaining declared precision and scale | +| `Date` | `Date32`, day unit | +| `Time` | `Time32` second/millisecond; `Time64` microsecond/nanosecond | +| `DateTime` | Timestamp without a timezone: civil date-time, not an inferred UTC instant | +| `DateTimeTz` | Timestamp with a nonempty timezone: instant, preserving timezone metadata | + +Timestamp units MAY be second, millisecond, microsecond, or nanosecond. +The baseline is flat, nondictionary, uncompressed little-endian IPC with V4 or +V5 metadata. A stream MAY terminate at a valid message boundary by EOF or use +the Arrow end-of-stream marker. Nested, opaque/extension, +dictionary-encoded, compressed-buffer, or other unlisted physical types MUST be +refused rather than guessed or converted. Ordinary HTTP content compression +MAY be used independently of the prohibited Arrow IPC buffer compression. + +Consumers MUST preserve Arrow values and physical type information, including +64-bit integers, decimal precision/scale, timestamp units, and nulls. +The TypeScript package exposes `RecordBatch`/`Table`; it does not convert to +JavaScript row objects, `Number`, or `Date`. This guarantee concerns physical +buffers and type metadata, not every Arrow convenience accessor. In particular, +timestamp vector `get()` may convert nanoseconds to a millisecond JavaScript +number. Exact consumers must inspect typed `.data` values buffers with their +declared units; later convenience conversions are outside this guarantee. + +### Completion and resource limits + +Consumers MUST validate framing, required metadata, field mapping, and final +row/decoded-byte counts for every downloaded stream. For paged delivery, +per-page counts and page identity MUST match the selected descriptor; full +consumption also requires all pages, a consistent physical schema, and exact +aggregate totals. Batches yielded before complete-result validation are +**provisional**. A later decode, count, truncation, or network error means the +consumer has not obtained a verified complete result; applications MUST discard +or explicitly label previously consumed partial data. + +The package's `readBatches` exposes that provisional iteration boundary and +traverses all pages in manifest order. `execute` materializes and returns only +after validating the entire result. Both enforce aggregate limits across all +pages, defaulting to 64 MiB decoded IPC bytes and 1,000,000 rows. +`readPage(execution: SuccessfulExecution, pageIndex, options)` returns a +`Promise` for just one fully validated page, with `allowTruncated` and +`signal` options. It applies the configured row/byte limits to that page, so it +can browse a page even when the whole result exceeds those limits; it does not +assert that unrequested pages were downloaded or validated. For unpaged +delivery, index `0` reads the single stream and every other index is rejected. +Limits are configurable; they are client admission limits, not a request to +silently truncate. The package +rejects declared truncation unless the caller explicitly opts in; opting in +does not waive framing, identity, count, or resource-limit validation. +Arrow decoding and application allocations can exceed the IPC byte size; the +byte limit is not a heap-size guarantee. +The prototype additionally caps a control-plane JSON response and each IPC +metadata message at 1 MiB; these are client resource limits, not semantic rules. +The client checks FlatBuffer bounds and flat-schema vector counts before +decoding, and limits expanded metadata per message to `maxResultBytes`, +including repeated references. This is an explicit admission policy, not a +claim that decoded Arrow memory usage equals wire size. +Cross-call lifecycle history is bounded to the 64 most-recently observed +executions and may be released locally with `forgetExecution`. Active waits +retain their own identity/transition baseline across history eviction. +Overlapping requests may return stale pending observations; those cannot +overwrite newer observations, while identity and terminal immutability checks +remain mandatory. Forgetting history does not cancel execution or delete +server results, and the client cannot compare history it no longer retains. + +## 5. Authorization and browser deployment + +Every operation MUST authorize the same caller against the same immutable +model binding. Server-side policy, including row-level security and isolation +of execution handles and result objects, remains a server obligation. +The client and schema do not prove authorization or RLS correctness. + +All URLs MUST use HTTPS except explicit local testing. The manifest's `url` +and every page's `url` MUST be absolute; relative references are not supported +in this draft. Each MUST contain no userinfo or fragment and MUST NOT redirect. +Every page is subject to the same caller authorization, origin checks, and +whole-result expiry protections as single-stream delivery. Same-origin +result requests MAY use the API bearer token. Cross-origin results require an +explicit client origin allowlist and MUST omit account authorization and +cookies. Origins MUST be compared structurally, not by hostname suffix or URL +prefix. Signed result URLs are bearer secrets: diagnostics, logs, and error +messages MUST NOT expose them. Query/error text and extension metadata are +untrusted data, not executable instructions. +The TypeScript client uses Fetch `credentials: "omit"` for every request, +including same-origin API requests; an explicitly attached same-origin bearer +header is separate from ambient browser credentials. + +A browser deployment MUST configure both API and result hosts for its approved +application origins. Preflight MUST permit `GET`, `POST`, and `OPTIONS` and the +used `Authorization`, `Content-Type`, `Accept`, and `Prefer` headers. +Responses SHOULD expose `Content-Type`, `Content-Length`, and +`Content-Encoding` for diagnostics; no custom response header is required to +discover an execution or result. CORS is not authorization. + +The package accepts an access-token callback; it does not acquire identities +or implement login. A deployment MAY provide a trusted token broker or a +same-origin backend-for-frontend that authenticates the user and performs the +upstream calls. Long-lived service credentials MUST NOT be embedded in browser +bundles. A broker/proxy must preserve the wire and caller-isolation contract; +none is implemented here. + +## 6. Relationship to existing work + +The existing [Ossie REST discussion][rest-thread] concerns model/catalog +interchange rather than this execution binding; [Polaris #4961][polaris] +concerns model CRUD integration, not an Ossie query execution endpoint. +The proposed binding reuses the pinned query shape and Foundation identities +and errors; it does not amend either upstream proposal. + +HTTP handles, bounded wait, polling, cancellation, result manifests, and Arrow +downloads have prior art, notably [Databricks Statement Execution][databricks]. +[dbt's GraphQL Semantic Layer API][dbt] also combines query handles and Arrow +results. These patterns are not claimed as inventions or as proof of +cross-vendor semantic equivalence. A focused, endpoint-specific comparison is +in the [package README](../../typescript/execution/README.md#feature-comparison). + +[Arrow Flight][flight] is an Arrow-oriented RPC transport; Flight SQL adds a +SQL service protocol. [ADBC][adbc] is a database driver API, not a replacement +HTTP wire format. A later Flight mapping would need its own authentication, +capability negotiation, identity, and completion rules. This draft and package +implement neither Flight nor ADBC. + +Local fixtures and JSON Schema validation establish only exercised transport +behavior. They do not establish expression-language conformance, correct +joins/aggregation, production authorization, or native vendor interoperability. + +[query-schema]: https://raw.githubusercontent.com/apache/ossie/b5418ee3aa19f117a1c464733de21f5a0dc0326b/validation/mcp/execute-query-query.schema.json +[foundation-semantics]: https://github.com/apache/ossie/blob/cc0d07099a3ef31e85af1fd93458a52186fae2c6/core-spec/foundational_semantics.md +[rest-thread]: https://lists.apache.org/thread/xvpomkdvmgv1m5krprq590l8mzccks5n +[polaris]: https://github.com/apache/polaris/pull/4961 +[databricks]: https://docs.databricks.com/api/statement-execution/v1/execute-statement +[dbt]: https://docs.getdbt.com/docs/dbt-apis/sl-graphql +[flight]: https://arrow.apache.org/docs/format/Flight.html +[adbc]: https://arrow.apache.org/adbc/current/format/specification.html diff --git a/docs/proposals/execution-http/execution.schema.json b/docs/proposals/execution-http/execution.schema.json new file mode 100644 index 000000000..922da6455 --- /dev/null +++ b/docs/proposals/execution-http/execution.schema.json @@ -0,0 +1,431 @@ +{ + "$comment": "Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See NOTICE for copyright ownership. Licensed under the Apache License, Version 2.0; http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, distributed on an AS IS BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND.", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:apache:ossie:execution:http:0.1-draft", + "title": "Ossie HTTP execution proposal 0.1-draft", + "description": "Not an adopted standard. Select a definition by fragment; Request is the default. Structural validation does not establish expression validity, authorization, or semantic conformance.", + "$ref": "#/$defs/Request", + "$defs": { + "Request": { + "type": "object", + "additionalProperties": false, + "required": ["data_source_id", "query"], + "properties": { + "data_source_id": { "$ref": "#/$defs/Identifier" }, + "query": { "$ref": "#/$defs/Query" } + } + }, + "Query": { + "description": "Pinned MCP profile 0.4 query shape, with the HTTP binding's additional portable safe-integer limit constraint.", + "allOf": [ + { "$ref": "#/$defs/query" }, + { + "type": "object", + "properties": { + "limit": { "$ref": "#/$defs/Counter" } + } + } + ] + }, + "Capabilities": { + "type": "object", + "additionalProperties": false, + "required": ["contract_version", "foundation_revision", "query_schema_version", "result_formats", "query_shapes", "features", "max_wait_seconds"], + "properties": { + "contract_version": { "$ref": "#/$defs/ContractVersion" }, + "foundation_revision": { "const": "cc0d07099a3ef31e85af1fd93458a52186fae2c6" }, + "query_schema_version": { "const": "mcp-0.4-draft" }, + "result_formats": { + "type": "array", + "prefixItems": [{ "const": "arrow-stream" }], + "minItems": 1, + "maxItems": 1, + "items": false + }, + "query_shapes": { "$ref": "#/$defs/QueryShapes" }, + "features": { "$ref": "#/$defs/Features" }, + "max_wait_seconds": { "type": "integer", "minimum": 0, "maximum": 60 }, + "extensions": { "$ref": "#/$defs/Extensions" } + } + }, + "Binding": { + "type": "object", + "additionalProperties": false, + "required": ["data_source_id", "model", "query_shapes", "features"], + "properties": { + "data_source_id": { "$ref": "#/$defs/Identifier" }, + "model": { "$ref": "#/$defs/ModelIdentity" }, + "query_shapes": { "$ref": "#/$defs/QueryShapes" }, + "features": { "$ref": "#/$defs/Features" }, + "extensions": { "$ref": "#/$defs/Extensions" } + } + }, + "Execution": { + "type": "object", + "additionalProperties": false, + "required": ["contract_version", "execution_id", "data_source_id", "model", "state", "warnings"], + "properties": { + "contract_version": { "$ref": "#/$defs/ContractVersion" }, + "execution_id": { "$ref": "#/$defs/Identifier" }, + "data_source_id": { "$ref": "#/$defs/Identifier" }, + "model": { "$ref": "#/$defs/ModelIdentity" }, + "state": { "$ref": "#/$defs/State" }, + "warnings": { "type": "array", "items": { "$ref": "#/$defs/Warning" } }, + "extensions": { "$ref": "#/$defs/Extensions" }, + "progress": { "$ref": "#/$defs/Progress" }, + "result": { "$ref": "#/$defs/ResultManifest" }, + "error": { "$ref": "#/$defs/QueryError" } + }, + "oneOf": [ + { + "properties": { "state": { "enum": ["queued", "running", "cancelling"] } }, + "not": { "anyOf": [{ "required": ["result"] }, { "required": ["error"] }] } + }, + { + "properties": { "state": { "const": "succeeded" } }, + "required": ["result"], + "not": { "anyOf": [{ "required": ["progress"] }, { "required": ["error"] }] } + }, + { + "properties": { "state": { "const": "failed" } }, + "required": ["error"], + "not": { "anyOf": [{ "required": ["progress"] }, { "required": ["result"] }] } + }, + { + "properties": { "state": { "const": "cancelled" } }, + "not": { "anyOf": [{ "required": ["progress"] }, { "required": ["result"] }, { "required": ["error"] }] } + } + ] + }, + "ResultManifest": { + "description": "One immutable materialized result, delivered by exactly one of url or pages. Root row_count and byte_count are totals, including when truncated. Page indices and row offsets must be contiguous from zero; descriptor row and decoded-byte sums must equal the root totals. These cross-page invariants require runtime validation.", + "type": "object", + "additionalProperties": false, + "required": ["result_id", "format", "row_count", "byte_count", "completeness", "expires_at", "columns"], + "properties": { + "result_id": { "$ref": "#/$defs/Identifier" }, + "format": { "const": "arrow-stream" }, + "url": { "$ref": "#/$defs/ResultUrl" }, + "pages": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/ResultPage" }, + "description": "Entire immutable server-selected partition, available only after success; not a cursor API. No zero-row page except the sole page of a zero-row result. Every page is a self-contained IPC stream with the same physical schema and field metadata." + }, + "row_count": { "$ref": "#/$defs/Counter" }, + "byte_count": { + "$ref": "#/$defs/Counter", + "description": "Total IPC byte length after HTTP content decoding, summed across all pages including their repeated schemas when paged; not Content-Length under compression and not a cryptographic integrity guarantee." + }, + "completeness": { "enum": ["complete", "truncated"] }, + "truncation_reason": { "$ref": "#/$defs/NonEmptyString" }, + "expires_at": { + "type": "string", + "format": "date-time", + "description": "Whole-result expiry applying to the single stream or every page; no link renewal." + }, + "columns": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/Column" }, + "description": "In Arrow field order. column_id must be unique across this array; names may repeat. Uniqueness by column_id is a runtime invariant." + }, + "extensions": { "$ref": "#/$defs/Extensions" } + }, + "oneOf": [{ "required": ["url"] }, { "required": ["pages"] }], + "if": { "properties": { "completeness": { "const": "truncated" } }, "required": ["completeness"] }, + "then": { "required": ["truncation_reason"] }, + "else": { "not": { "required": ["truncation_reason"] } } + }, + "ResultPage": { + "description": "One immutable self-contained IPC stream. page_index must equal its zero-based array index; row_offset must equal the sum of preceding page row counts. Descriptor counts are exact after HTTP decoding. Only a zero-row result may have a zero-row page, which must be its sole page.", + "type": "object", + "additionalProperties": false, + "required": ["page_index", "row_offset", "row_count", "byte_count", "url"], + "properties": { + "page_index": { "$ref": "#/$defs/Counter" }, + "row_offset": { "$ref": "#/$defs/Counter" }, + "row_count": { "$ref": "#/$defs/Counter" }, + "byte_count": { + "$ref": "#/$defs/Counter", + "minimum": 1, + "description": "Positive decoded IPC byte length including this page's schema, even for a zero-row result." + }, + "url": { "$ref": "#/$defs/ResultUrl" } + } + }, + "ResultUrl": { + "type": "string", + "format": "uri", + "pattern": "^https?://", + "description": "Absolute HTTPS URL; explicit same-origin localhost testing is the only HTTP exception. No userinfo, fragment, or redirects. Cross-origin requires an explicit client allowlist. Identical authorization, origin, and expiry protections apply to every page." + }, + "Column": { + "type": "object", + "additionalProperties": false, + "required": ["column_id", "name", "logical_type", "nullable"], + "properties": { + "column_id": { "$ref": "#/$defs/Identifier" }, + "name": { "$ref": "#/$defs/NonEmptyString" }, + "logical_type": { "$ref": "#/$defs/LogicalType" }, + "nullable": { "type": "boolean" }, + "role": { "enum": ["dimension", "measure", "field"] }, + "model_ref": { "$ref": "#/$defs/NonEmptyString" }, + "label": { "$ref": "#/$defs/NonEmptyString" }, + "description": { "$ref": "#/$defs/NonEmptyString" }, + "unit": { "$ref": "#/$defs/NonEmptyString" }, + "currency": { "$ref": "#/$defs/NonEmptyString" }, + "extensions": { "$ref": "#/$defs/Extensions" } + } + }, + "QueryError": { + "type": "object", + "additionalProperties": false, + "required": ["code", "message", "retryable"], + "properties": { + "code": { + "$ref": "#/$defs/NonEmptyString", + "description": "Foundation error code directly, without a transport prefix or wrapper taxonomy. A string preserves codes this client does not recognize." + }, + "message": { "$ref": "#/$defs/NonEmptyString" }, + "retryable": { "type": "boolean" }, + "replacement_query": { "$ref": "#/$defs/Query" }, + "extensions": { "$ref": "#/$defs/Extensions" } + } + }, + "ProblemDetails": { + "type": "object", + "additionalProperties": false, + "required": ["type", "title", "status", "error"], + "properties": { + "type": { "type": "string", "minLength": 1, "pattern": "\\S", "format": "uri-reference" }, + "title": { "$ref": "#/$defs/NonEmptyString" }, + "status": { "type": "integer", "minimum": 400, "maximum": 599 }, + "detail": { "$ref": "#/$defs/NonEmptyString" }, + "error": { "$ref": "#/$defs/QueryError" } + } + }, + "Progress": { + "type": "object", + "additionalProperties": false, + "properties": { + "fraction": { "type": "number", "minimum": 0, "maximum": 1 }, + "phase": { "$ref": "#/$defs/NonEmptyString" } + } + }, + "Warning": { + "type": "object", + "additionalProperties": false, + "required": ["code", "message"], + "properties": { + "code": { "$ref": "#/$defs/NonEmptyString" }, + "message": { "$ref": "#/$defs/NonEmptyString" } + } + }, + "ExecutionMetadata": { + "description": "JSON value stored under Arrow schema metadata key org.apache.ossie.execution. page_index and row_offset must appear together, must match the selected ResultPage when paged, and are forbidden for single-stream delivery. Matching delivery mode and descriptor identity requires runtime validation.", + "type": "object", + "additionalProperties": false, + "required": ["contract_version", "execution_id", "result_id", "data_source_id", "model"], + "properties": { + "contract_version": { "$ref": "#/$defs/ContractVersion" }, + "execution_id": { "$ref": "#/$defs/Identifier" }, + "result_id": { "$ref": "#/$defs/Identifier" }, + "data_source_id": { "$ref": "#/$defs/Identifier" }, + "model": { "$ref": "#/$defs/ModelIdentity" }, + "page_index": { "$ref": "#/$defs/Counter" }, + "row_offset": { "$ref": "#/$defs/Counter" } + }, + "dependentRequired": { + "page_index": ["row_offset"], + "row_offset": ["page_index"] + } + }, + "ColumnMetadata": { + "description": "JSON value stored under each Arrow field metadata key org.apache.ossie.column.", + "type": "object", + "additionalProperties": false, + "required": ["column_id"], + "properties": { "column_id": { "$ref": "#/$defs/Identifier" } } + }, + "ModelIdentity": { + "type": "object", + "additionalProperties": false, + "required": ["id", "revision"], + "properties": { + "id": { "$ref": "#/$defs/Identifier" }, + "revision": { "$ref": "#/$defs/Identifier" } + } + }, + "ContractVersion": { "const": "0.1-draft" }, + "State": { "enum": ["queued", "running", "cancelling", "succeeded", "failed", "cancelled"] }, + "LogicalType": { "enum": ["String", "Integer", "Decimal", "Float", "Boolean", "Date", "Time", "DateTime", "DateTimeTz"] }, + "QueryShape": { "enum": ["aggregation", "scalar"] }, + "QueryShapes": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "$ref": "#/$defs/QueryShape" } + }, + "Features": { + "type": "array", + "uniqueItems": true, + "items": { "$ref": "#/$defs/NonEmptyString" } + }, + "Identifier": { "$ref": "#/$defs/NonEmptyString" }, + "NonEmptyString": { "type": "string", "minLength": 1, "pattern": "\\S" }, + "Counter": { "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, + "PreferWait": { "type": "string", "pattern": "^wait=([0-9]|[1-5][0-9]|60)$" }, + "Extensions": { + "type": "object", + "description": "Opaque JSON objects with nonblank namespace keys. Reverse-DNS naming is recommended, not syntactically required. Extensions must not redefine core fields or relax required behavior.", + "propertyNames": { "$ref": "#/$defs/NonEmptyString" }, + "additionalProperties": { "type": "object" } + }, + "expression": { + "type": "string", + "minLength": 1, + "pattern": "\\S" + }, + "projection": { + "type": "array", + "items": { + "$ref": "#/$defs/expression" + } + }, + "predicate": { + "oneOf": [ + { + "$ref": "#/$defs/expression" + }, + { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/expression" + } + } + ] + }, + "query": { + "type": "object", + "additionalProperties": false, + "properties": { + "dimensions": { + "$ref": "#/$defs/projection" + }, + "measures": { + "$ref": "#/$defs/projection" + }, + "fields": { + "allOf": [ + { + "$ref": "#/$defs/projection" + } + ], + "minItems": 1 + }, + "where": { + "$ref": "#/$defs/predicate" + }, + "having": { + "$ref": "#/$defs/predicate" + }, + "order_by": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "field" + ], + "properties": { + "field": { + "$ref": "#/$defs/expression" + }, + "direction": { + "enum": [ + "ASC", + "DESC" + ] + }, + "nulls": { + "enum": [ + "FIRST", + "LAST" + ] + } + } + } + }, + "limit": { + "type": "integer", + "minimum": 0 + } + }, + "oneOf": [ + { + "not": { + "required": [ + "fields" + ] + }, + "anyOf": [ + { + "required": [ + "dimensions" + ], + "properties": { + "dimensions": { + "minItems": 1 + } + } + }, + { + "required": [ + "measures" + ], + "properties": { + "measures": { + "minItems": 1 + } + } + } + ] + }, + { + "required": [ + "fields" + ], + "not": { + "anyOf": [ + { + "required": [ + "dimensions" + ] + }, + { + "required": [ + "measures" + ] + }, + { + "required": [ + "having" + ] + } + ] + } + } + ] + } + }, + "x-query-provenance": { + "source": "https://raw.githubusercontent.com/apache/ossie/b5418ee3aa19f117a1c464733de21f5a0dc0326b/validation/mcp/execute-query-query.schema.json", + "source_id": "urn:apache:ossie:execute-query:query:0.4-draft", + "source_revision": "b5418ee3aa19f117a1c464733de21f5a0dc0326b", + "copied_definitions": ["expression", "projection", "predicate", "query"], + "note": "The four lowercase definitions are copied without changes. Query adds only the HTTP safe-integer limit bound; no expression parser or semantic validator is implied." + } +} diff --git a/docs/proposals/execution-http/openapi.json b/docs/proposals/execution-http/openapi.json new file mode 100644 index 000000000..810503c30 --- /dev/null +++ b/docs/proposals/execution-http/openapi.json @@ -0,0 +1,232 @@ +{ + "openapi": "3.1.0", + "x-license": { + "$comment": "Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See NOTICE for copyright ownership. Licensed under the Apache License, Version 2.0; http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, distributed on an AS IS BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND." + }, + "info": { + "title": "Ossie HTTP execution proposal", + "version": "0.1-draft", + "description": "DRAFT, not an adopted standard or a deployed service. Browser-oriented HTTP/JSON control plane and one materialized result delivered as a single Arrow IPC stream or immutable self-contained pages. All schemas are defined in execution.schema.json. Authentication is deployment supplied; this document does not implement an identity system.", + "license": { + "name": "Apache-2.0", + "identifier": "Apache-2.0" + } + }, + "jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema", + "paths": { + "/capabilities": { + "get": { + "operationId": "getCapabilities", + "summary": "Discover the caller-visible contract and query capabilities", + "responses": { + "200": { + "description": "Capabilities for this authorized caller; declarations do not prove semantic conformance.", + "content": { + "application/json": { + "schema": { "$ref": "execution.schema.json#/$defs/Capabilities" } + } + } + }, + "default": { "$ref": "#/components/responses/Problem" } + } + } + }, + "/bindings/{id}": { + "parameters": [{ "$ref": "#/components/parameters/Id" }], + "get": { + "operationId": "getBinding", + "summary": "Resolve an immutable data source binding", + "responses": { + "200": { + "description": "The id is data_source_id. This id MUST NOT be rebound to another model revision.", + "content": { + "application/json": { + "schema": { "$ref": "execution.schema.json#/$defs/Binding" } + } + } + }, + "default": { "$ref": "#/components/responses/Problem" } + } + } + }, + "/queries": { + "post": { + "operationId": "submit", + "summary": "Submit one read-only structured query", + "description": "Submit one execution job, not separate sync/async jobs. Omitting Prefer means wait=0: return immediately with 202 unless already terminal (200). wait=N holds POST at most N seconds, returning terminal 200 or pending 202 for the same execution_id; the server may return earlier. Bounded wait optimizes fast completion without unbounded synchronous connections or mandatory polling round trips. It is neither an execution deadline nor a download timeout and never cancels the query. No automatic retries, query repair, native SQL escape, or idempotency guarantee. Pagination partitions the completed result, not the query: no page-size/offset clause; limit remains a semantic cap.", + "parameters": [ + { + "name": "Prefer", + "in": "header", + "required": false, + "description": "One wait=N preference; integer N from 0 through 60 and no greater than advertised max_wait_seconds. Bounds POST waiting only; expiry returns the same job without cancellation. Invalid or unsupported waits are rejected.", + "schema": { "$ref": "execution.schema.json#/$defs/PreferWait" } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { "$ref": "execution.schema.json#/$defs/Request" }, + "example": { + "data_source_id": "sales-v7", + "query": { "dimensions": ["region"], "measures": ["revenue"], "limit": 100 } + } + } + } + }, + "responses": { + "200": { + "description": "Terminal Execution: succeeded, failed, or cancelled. HTTP success alone does not imply query success. Success includes the entire immutable result manifest: exactly one of url or pages, subject to the client's 1 MiB control-JSON limit.", + "content": { + "application/json": { + "schema": { "$ref": "execution.schema.json#/$defs/Execution" }, + "examples": { + "single-stream": { + "summary": "Existing single-stream delivery (illustrative IPC counts)", + "value": { + "contract_version": "0.1-draft", + "execution_id": "q-42", + "data_source_id": "sales-v7", + "model": { "id": "sales", "revision": "7" }, + "state": "succeeded", + "warnings": [], + "result": { + "result_id": "r-42", + "format": "arrow-stream", + "url": "https://api.example.test/results/r-42", + "row_count": 2, + "byte_count": 1536, + "completeness": "complete", + "expires_at": "2026-10-12T00:00:00Z", + "columns": [ + { "column_id": "c-1", "name": "region", "logical_type": "String", "nullable": true } + ] + } + } + }, + "paged": { + "summary": "Server-selected immutable pages (illustrative IPC counts)", + "value": { + "contract_version": "0.1-draft", + "execution_id": "q-43", + "data_source_id": "sales-v7", + "model": { "id": "sales", "revision": "7" }, + "state": "succeeded", + "warnings": [], + "result": { + "result_id": "r-43", + "format": "arrow-stream", + "pages": [ + { "page_index": 0, "row_offset": 0, "row_count": 1, "byte_count": 1024, "url": "https://api.example.test/results/r-43/pages/0" }, + { "page_index": 1, "row_offset": 1, "row_count": 1, "byte_count": 1024, "url": "https://api.example.test/results/r-43/pages/1" } + ], + "row_count": 2, + "byte_count": 2048, + "completeness": "complete", + "expires_at": "2026-10-12T00:00:00Z", + "columns": [ + { "column_id": "c-1", "name": "region", "logical_type": "String", "nullable": true } + ] + } + } + } + } + } + } + }, + "202": { + "description": "Pending Execution: queued, running, or cancelling. Poll using execution_id.", + "content": { + "application/json": { + "schema": { "$ref": "execution.schema.json#/$defs/Execution" } + } + } + }, + "default": { "$ref": "#/components/responses/Problem" } + } + } + }, + "/queries/{id}": { + "parameters": [{ "$ref": "#/components/parameters/Id" }], + "get": { + "operationId": "getExecution", + "summary": "Read the current execution state", + "responses": { + "200": { + "description": "An Execution in any state; terminal states and manifests are immutable.", + "content": { + "application/json": { + "schema": { "$ref": "execution.schema.json#/$defs/Execution" } + } + } + }, + "default": { "$ref": "#/components/responses/Problem" } + } + } + }, + "/queries/{id}/cancel": { + "parameters": [{ "$ref": "#/components/parameters/Id" }], + "post": { + "operationId": "cancel", + "summary": "Request best-effort cancellation without a request body", + "description": "Idempotent for an execution. Completion may win the race. Cancelling is pending, not proof of cancellation. Aborting a local HTTP request does not invoke this operation.", + "responses": { + "200": { + "description": "Terminal Execution: cancelled, succeeded, or failed.", + "content": { + "application/json": { + "schema": { "$ref": "execution.schema.json#/$defs/Execution" } + } + } + }, + "202": { + "description": "Pending Execution: cancelling or another currently observed pending state.", + "content": { + "application/json": { + "schema": { "$ref": "execution.schema.json#/$defs/Execution" } + } + } + }, + "default": { "$ref": "#/components/responses/Problem" } + } + } + } + }, + "components": { + "parameters": { + "Id": { + "name": "id", + "in": "path", + "required": true, + "description": "Opaque identifier, encoded as a single path segment. A binding id is data_source_id; a query id is execution_id.", + "schema": { "$ref": "execution.schema.json#/$defs/Identifier" } + } + }, + "responses": { + "Problem": { + "description": "Non-2xx failure using RFC 9457 ProblemDetails. status matches the HTTP status; error.code is the Foundation code directly. Redirects are not allowed.", + "content": { + "application/problem+json": { + "schema": { "$ref": "execution.schema.json#/$defs/ProblemDetails" } + } + } + } + } + }, + "x-result-download": { + "method": "GET", + "url": "Execution.result.url OR Execution.result.pages[i].url, only when state is succeeded", + "manifest-schema": { "$ref": "execution.schema.json#/$defs/ResultManifest" }, + "page-schema": { "$ref": "execution.schema.json#/$defs/ResultPage" }, + "success-status": 200, + "media-type": "application/vnd.apache.arrow.stream", + "description": "Dynamic absolute same-origin or explicitly allowlisted HTTPS URL, not a fixed API path or cursor API. Every download is one immutable self-contained IPC stream with required Ossie schema/field metadata; no concatenation, redirects, or Range requests. Server-selected page_index and row_offset are contiguous from zero; page row/decoded-byte sums equal root totals, including repeated schemas. No empty page except the sole page of a zero-row result; page bytes are positive. Every page has identical fields and actual physical types (including decimal scale and timestamp unit/timezone) and metadata except ExecutionMetadata page_index/row_offset, which must appear together and match its descriptor; both are forbidden unpaged. Cross-origin fetch omits account authorization and cookies. The same authorization, origin, and whole-result expires_at protections apply to each page. HTTP content compression is allowed; Arrow IPC buffer compression is not. Decode bytes before comparing counts. Selected-page browsing does not establish complete-result consumption; completeness describes the whole result. Explicit repeated reads before expiry are allowed, but no automatic retries, resubmission, link renewal, live execution rows, or Flight are defined." + }, + "x-browser-cors": { + "allow-methods": ["GET", "POST", "OPTIONS"], + "allow-headers": ["Authorization", "Content-Type", "Accept", "Prefer"], + "expose-headers": ["Content-Type", "Content-Length", "Content-Encoding"], + "description": "Servers and result hosts must explicitly allow approved application origins and satisfy preflight requests. A token broker or backend-for-frontend is deployment infrastructure, not supplied by this package." + } +} diff --git a/typescript/execution/LICENSE b/typescript/execution/LICENSE new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/typescript/execution/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/typescript/execution/NOTICE b/typescript/execution/NOTICE new file mode 100644 index 000000000..e4bc78e2b --- /dev/null +++ b/typescript/execution/NOTICE @@ -0,0 +1,5 @@ +Apache Ossie (incubating) +Copyright 2026 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/typescript/execution/README.md b/typescript/execution/README.md new file mode 100644 index 000000000..6bd7b7245 --- /dev/null +++ b/typescript/execution/README.md @@ -0,0 +1,402 @@ + + +# Ossie execution client — DRAFT + +A browser-oriented TypeScript client prototype for +[Ossie HTTP execution `0.1-draft`](../../docs/proposals/execute-query-http.md): +HTTP/JSON submission, status and cancellation, followed by materialized +Apache Arrow IPC results, either in one stream or immutable pages. + +**This is a proposal, not an adopted standard or a published-service promise.** +The package does not implement an execution server, semantic evaluator, +authentication system, Flight transport, or vendor adapters. Local tests +exercise the protocol against fixtures; they do not prove production vendor +interoperability, Foundation expression semantics, or row-level security. + +## Contract + +- [Normative draft](../../docs/proposals/execute-query-http.md). +- [JSON Schema, draft 2020-12](../../docs/proposals/execution-http/execution.schema.json), + including `Request`, `Query`, `Capabilities`, `Binding`, `Execution`, + `ResultManifest`, `ResultPage`, `Column`, `QueryError`, and `ProblemDetails`. +- [OpenAPI 3.1](../../docs/proposals/execution-http/openapi.json), referencing + those same definitions. +- [Public TypeScript types](src/types.ts). + +The draft pins the still-open [MCP #529][mcp] at +`b5418ee3aa19f117a1c464733de21f5a0dc0326b` (query profile `0.4-draft`) and +[Foundation #246][foundation] at +`cc0d07099a3ef31e85af1fd93458a52186fae2c6`. +The MCP query definitions are copied unchanged into the HTTP schema; its +`Query` wrapper additionally restricts `limit` to a nonnegative safe integer. +This is shape reuse, not a second expression language or a semantic validator. + +## Client surface + +`OssieClient` accepts a base URL, an optional `fetch` implementation, and an +optional access-token callback. The package is intended for a browser bundler +or Node.js 22+ with Fetch, Web Streams, and `AbortController`. +It is a repository-local package; the examples do not imply npm publication. + +| Method (optional options argument omitted below) | Purpose / return type | +| --- | --- | +| `getCapabilities()` | `Promise`; validate versions, result format, wait bound, shapes, and features. | +| `getBinding(dataSourceId)` | `Promise`; resolve the model id/revision and supported shapes. | +| `submit(request)` | `Promise`; submit once and return any execution state. Accepts `SubmitOptions`. | +| `getExecution(executionId)` | `Promise`; read state, including failed or cancelled. | +| `wait(execution)` | `Promise`; poll to success, otherwise throw. Accepts `WaitOptions`. | +| `cancel(executionId)` | `Promise`; request best-effort cancellation and return the observed state. | +| `forgetExecution(executionId)` | Release local lifecycle history; no network request, cancellation, or server-side deletion. | +| `readBatches(successfulExecution)` | `AsyncGenerator`; provisional batches until successful exhaustion. Accepts `RequestOptions` plus `allowTruncated`. | +| `readPage(successfulExecution, pageIndex)` | `Promise
` for one zero-based page; accepts `RequestOptions` plus `allowTruncated`. Only that page is fetched and validated. | +| `execute(request)` | `Promise<{ execution: SuccessfulExecution; table: Table }>`; submit, wait, and return verified materialization. Accepts `ExecuteOptions`. | + +Unless noted otherwise, methods accept `RequestOptions` with an optional +`signal`. Types, `OssieClient`, default-limit constants, and error classes are +exported from the package entry point. + +### Example + +Once built and linked as a local dependency, the package imports as +`@apache-ossie/execution-client`. The host application supplies a short-lived +token; the example does not implement or prescribe a token broker. + +```ts +import { OssieClient } from "@apache-ossie/execution-client"; + +export async function loadRevenue( + getAccessToken: () => Promise, + signal?: AbortSignal, +) { + const client = new OssieClient({ + baseUrl: "https://api.example.test/ossie", + getAccessToken, + maxResultBytes: 64 * 1024 * 1024, + maxResultRows: 1_000_000, + }); + const { execution, table } = await client.execute( + { + data_source_id: "sales-v7", + query: { + dimensions: ["customers.region"], + measures: ["revenue"], + limit: 100, + }, + }, + signal ? { signal } : {}, + ); + // Keep Arrow values; do not turn exact integers/decimals into Number. + return { table, columns: execution.result.columns }; +} +``` + +The URL and model are hypothetical. No included server can execute this +example. Low-level consumers may instead call `submit`, `wait` with the returned +execution object, and `readBatches` with the successful execution. + +### Bounded wait is a synchronous fast path for one asynchronous job + +`waitSeconds` controls how long the **initial submission response** may wait, +not how long the query may run. The server may return earlier. + +| Pattern | Submission response | +| --- | --- | +| Immediate asynchronous (`waitSeconds: 0`, also the default) | Return the handle promptly; `202` while pending, or `200` if already terminal. | +| Bounded wait (`waitSeconds: 5`) | Wait up to five seconds. Return `200` if terminal, otherwise `202` with the **same running execution**. | +| Unbounded synchronous | Hold the connection until completion regardless of duration. This draft does not offer that submission mode. | + +For a 200 ms query with `waitSeconds: 5`, the submission can return success +without polling. For a 20 s query, the submission returns pending at or before +five seconds and the client polls the same id. It does not cancel, restart or +submit another query. Results are still downloaded separately after success. +HTTP `200` means terminal, which may also be failed or cancelled. +JavaScript methods always return promises; this is separate from the server's +sync/async response policy. `execute()` hides polling and resolves only after +the result is fully read. `timeoutMs` instead bounds the whole client operation, +including polling/download, and stopping it does not cancel server execution. + +### Pagination + +The server partitions one immutable materialized result and returns **either** +the existing `result.url` **or** a `result.pages` manifest. Each descriptor has +`page_index`, `row_offset`, `row_count`, `byte_count`, and `url`; the root counts +describe all pages. Pages are zero-based, ordered and contiguous. Their sizes +are server-selected, not extra query clauses. The same query `limit` can span +many pages: a page boundary is neither query truncation nor a new execution. + +```ts +const submitted = await client.submit( + { data_source_id: "sales-v7", query: { measures: ["revenue"], limit: 1000 } }, + { waitSeconds: 5 }, +); +const completed = await client.wait(submitted); +const firstPage = await client.readPage(completed, 0); +const pageCount = completed.result.pages?.length ?? 1; +// Read another page only when the application requests it. +if (pageCount > 1) { + const secondPage = await client.readPage(completed, 1); +} +``` + +`readPage` checks its selected page against configured row/byte limits, even +when the full result exceeds them. It returns only after validating that +page's Arrow schema, identity and counts. A single-stream result is page zero. +Explicitly rereading a page before expiry is allowed; automatic retries, +offset-based query reruns, cursor APIs and link renewal are not implemented. +The complete page manifest must fit the existing 1 MiB control-response limit. + +`readBatches` traverses every page in order without prefetching, while `execute` +returns one combined table. Both enforce aggregate limits and physical-schema +consistency across pages. Reading only selected pages proves nothing about +unread pages or cross-page schema consistency. Each page's Arrow metadata +includes `page_index` and `row_offset`; individual batches preserve those +coordinates. The combined table's schema metadata describes the full result +and omits page coordinates. Original batch buffers are not converted. + +### Configuration + +| `ClientOptions` | Meaning | +| --- | --- | +| `baseUrl` | HTTP API base URL; HTTPS required except explicit localhost testing. | +| `fetch` | Optional Fetch-compatible transport. It must preserve redirect refusal, cancellation, decoded response bytes, and header/credential behavior. | +| `getAccessToken` | Optional `() => string \| Promise`; supplies a bearer token, not a login flow. | +| `allowedResultOrigins` | Explicit cross-origin result allowlist. No account token or cookies are sent to those origins. | +| `allowInsecureLocalhost` | Explicit local-test exception; not a production TLS bypass. | +| `pollIntervalMs`, `timeoutMs` | Defaults: 250 ms poll spacing and 60,000 ms operation deadline. `execute` shares one deadline across submission, polling, and download. A local deadline is not server cancellation. | +| `maxResultBytes` | Decoded IPC admission limit, default 64 MiB; whole-result operations use totals, `readPage` uses its selected page. Not a JavaScript heap-size guarantee. | +| `maxResultRows` | Row limit, default 1,000,000; whole result or selected page as above. Exceeding it is an error, not silent truncation. | + +Per-request `signal` supports local abort. `SubmitOptions.waitSeconds` controls +the submission's `Prefer: wait=N`. The client checks the protocol range +`0..60`; the server enforces its advertised lower `max_wait_seconds` and +binding support. `getCapabilities` and `getBinding` are explicit validated +reads, not a cached preflight policy. Submission performs no automatic discovery +or cached capability/binding enforcement. Applications can use explicit +discovery results to choose a supported shape and wait preference. +`WaitOptions.onProgress` receives execution observations. +`ExecuteOptions.allowTruncated` explicitly accepts a server-declared truncated +materialization without waiving count, identity, framing, or resource checks. +Control-plane JSON responses and individual IPC metadata messages have separate +fixed 1 MiB admission caps; increasing `maxResultBytes` does not raise those caps. +Before Arrow decoding, the client checks FlatBuffer table, vector and string +bounds, flat-schema node/buffer counts, and unsupported nested encodings. +Repeated metadata references also count against the expanded-metadata budget +of `maxResultBytes` per message; valid but unusually expanded metadata can +therefore be rejected with `RESULT_LIMIT_EXCEEDED`. +The client retains at most 64 most-recently observed executions +(`MAX_TRACKED_EXECUTIONS`) for cross-call lifecycle validation. Older history is +evicted; `forgetExecution` can release it sooner. An active `wait` keeps its own +previous observation even after eviction. Server immutability is still required, +but the client cannot compare forgotten history. Overlapping polls cannot +replace a newer cancellation observation with an older pending state; identity +checks and immutable terminal-response checks still apply. + +The wire `Query` is either: + +```ts +// Aggregation shape: at least one nonempty dimensions/measures projection. +const aggregate = { + data_source_id: "sales-v7", + query: { + dimensions: ["customers.region"], + measures: ["revenue"], + limit: 100, + }, +}; + +// Scalar shape: fields, without dimensions, measures, or having. +const scalar = { + data_source_id: "sales-v7", + query: { + fields: ["orders.order_id", "orders.amount"], + where: ["orders.amount > 0"], + limit: 100, + }, +}; +``` + +These are structural examples against a hypothetical binding, not executable +semantic fixtures. The server must authorize and evaluate the query correctly. +Both shapes support `order_by` entries with `field`, optional `ASC`/`DESC` +`direction`, and optional `FIRST`/`LAST` `nulls`. Structurally invalid queries +are refused locally; the server enforces advertised shape support. +The client does not repair the query or substitute a native query. + +## What result validation means + +- Result URLs exist only on `succeeded`. Each identifies one immutable, + self-contained Arrow IPC **stream**, not live rows produced during execution. + A result uses one URL or an ordered page manifest, never both. +- The stream's `org.apache.ossie.execution` JSON metadata binds contract version, + execution, result, data source, and model revision. Each field's + `org.apache.ossie.column` JSON metadata binds its `column_id`. + Paged streams also bind the page index and row offset to prevent unnoticed + page substitution; unpaged streams omit both. +- The manifest supplies authoritative semantic descriptors. Field ids are + unique; names may duplicate. Field order, names, nullability, ids, and physical + versus logical types must agree. +- Supported physical types are flat, nondictionary `Utf8`; signed/unsigned + integers 8–64 bit; `Float32/64`; `Bool`; `Decimal128/256`; `Date32` days; + `Time32` seconds/milliseconds; `Time64` microseconds/nanoseconds; and timestamps + with or without timezone. No-timezone timestamps are civil date-times, not + inferred UTC instants. Unsupported, nested, opaque, dictionary, or + Arrow-buffer-compressed payloads are refused. Streams use little-endian V4/V5 + metadata and terminate at a valid EOF boundary or Arrow end-of-stream marker. +- Arrow `RecordBatch`/`Table` objects preserve exact physical buffers and type + metadata. The package performs no row-object, `Number`, or `Date` conversion. + This is not a guarantee about every Arrow convenience accessor: + a timestamp vector's `get()` may convert nanoseconds to a millisecond + JavaScript number and lose precision. Use the typed `.data` values buffers + with their declared units when exact temporal values are required. + Convenience conversions to `Number`/`Date` are the caller's choice and can + also lose 64-bit integer or decimal precision. +- `row_count` and `byte_count` are exact counts of the delivered materialization. + Bytes are counted **after HTTP decompression**, including IPC framing. + HTTP compression is distinct from unsupported Arrow buffer compression. + Counts are not cryptographic checksums. +- `readBatches` can yield before a late framing/count/network failure. Only + successful exhaustion of all pages validates the whole result. Early + termination leaves the result unverified. Do not present a consumed prefix + or selected page as the complete result. +- `execute` buffers within configured limits and returns only after validation. + A complete result differs from a declared truncated result; the latter must + include `truncation_reason`, even when explicitly accepted by the caller. + +## Errors and operational boundaries + +HTTP success is not query success. `submit`, `getExecution`, and `cancel` expose +terminal failures and cancellation for inspection. `wait` and `execute` turn +them into errors. Foundation codes are retained directly in `error.code`; +non-2xx responses carry RFC 9457 `ProblemDetails`. +Server error details and replacement queries are untrusted advisory data. +There is no automatic repair, retry, resubmission, or native-query fallback. +`QueryExecutionError.error` retains the server query error; +`HttpProblemError.problem` retains the problem document. Other exported classes +include `ExecutionCancelledError`, `ProtocolError`, `TimeoutError`, and their +base `OssieError`. Local abort uses `OssieError` code `ABORTED`; result-bound +failures use `RESULT_LIMIT_EXCEEDED` or `TRUNCATED_RESULT`. +Other local codes include `RESULT_EXPIRED`, `AUTHENTICATION_ERROR`, and +`TRANSPORT_ERROR`. +Structured error properties may contain sensitive server +data even though the client-generated messages avoid result URLs. + +Cancellation can race completion: `202` means pending, not confirmed cancelled. +Request abort and timeout stop local work; use `cancel` to request server-side +cancellation. An aborted submission may have executed without returning its id. +Do not resubmit blindly. + +Result URLs must be absolute; relative references are not supported. +The target is subject to the same HTTPS and origin policy. +All page URLs share RFC 3339 `expires_at`; there is no link renewal, byte-range +resume, or re-execution fallback. Explicit page rereads are allowed until +expiry. Redirects are refused. Signed URLs +are credentials: do not log complete executions, manifests, raw error bodies, +or access tokens. Cross-origin result access requires an explicit origin +allowlist and omits API account credentials. +All client Fetch calls use `credentials: "omit"`, including same-origin API +calls. The optional bearer `Authorization` header is attached explicitly to +same-origin requests; it is not an ambient cookie credential. + +For browser use, configure API and result-host CORS for approved app origins: +allow the used `GET`/`POST`/`OPTIONS` methods and +`Authorization`/`Content-Type`/`Accept`/`Prefer` request headers. Expose +`Content-Type`, `Content-Length`, and `Content-Encoding` when needed for +diagnostics. CORS is not authorization. A separately deployed token broker or +backend-for-frontend can keep long-lived credentials off the browser; neither +is supplied here. All operations and result delivery remain subject to the +same caller's authorized model and server-enforced RLS. + +## Local development + +From the repository root, using PowerShell: + +```powershell +Set-Location .\typescript\execution +npm ci +npm run build +npm test +``` + +`npm run check` runs the same build-and-test path as `npm test`. The package is +private and uses Apache Arrow JavaScript `21.2.0`. `npm test` includes the +esbuild browser-bundle check but does not require a browser executable. +Tests and a successful bundle validate only exercised client/protocol behavior, +not a real service integration or query engine. + +`npm run test:browser` runs the separate real-browser smoke, which bundles the public entry point with esbuild +and exercises synthetic loopback HTTP in an installed Chromium, Chrome, or +Edge browser. From the package directory, after building: + +```powershell +npm run test:browser +``` + +The harness uses Node.js 22+'s built-in Fetch/WebSocket support, discovers common +browser installations, and fails rather than skips if none is found. +Set `OSSIE_BROWSER_BINARY` to an installed executable when discovery is +insufficient; an invalid explicit value also fails. For example: + +```powershell +$env:OSSIE_BROWSER_BINARY = "C:\Program Files\Google\Chrome\Application\chrome.exe" +npm run test:browser +``` + +This test installs no browser or browser framework. Its fixture server is +test infrastructure, not a delivered execution server or vendor adapter. + +## Feature comparison + +This table compares **named interfaces**, not entire products. Documentation +was checked on 2026-10-11. “Arrow” does not imply that a product's REST endpoint +uses Arrow, or that two systems implement the same query semantics. +**This package contains no adapters for any endpoint below.** + +| Interface | Submission/lifecycle | Result representation | Difference from this package | +| --- | --- | --- | --- | +| **This draft/client prototype** | Structured Ossie JSON; bounded wait, handle/poll/cancel | Single stream or indexed immutable Arrow pages; per-page identity/count checks, full-result schema/total checks | Local protocol implementation only; no query server or vendor semantic proof. | +| [Databricks Statement Execution][databricks] | SQL over HTTP; bounded synchronous wait, handles, polling, cancellation, manifests | `ARROW_STREAM` **only with `EXTERNAL_LINKS`**; inline supports `JSON_ARRAY`, not Arrow | Closest lifecycle/download prior art; additional formats and fresh link resolution remain beyond this client. External downloads must omit account authorization. | +| [dbt Semantic Layer GraphQL][dbt-graphql] | `createQuery`, then poll `query` with the id | Paginated base64-encoded Arrow in `arrowResult`; JSON is also available | Semantic query API, but different request schema and delivery encoding. [Its JDBC interface][dbt-jdbc] separately uses Arrow Flight SQL. | +| [Snowflake SQL API][snowflake-sql] | SQL HTTP submission, status handles, cancellation | Partitioned JSON `ResultSet`/`jsonv2` | Not an Arrow REST result API in the cited contract. [Python connector Arrow fetch APIs][snowflake-python] are a separate connector surface. | +| [Power BI REST Execute Queries][powerbi-rest] | DAX request against a semantic model | JSON; limits include 100,000 rows or 1,000,000 values, 15 MB; some limited-data/error responses are HTTP `200` | No Arrow result in this REST contract. Applications must inspect the body, not just status. | +| [Power BI/Analysis Services XMLA with ADOMD][xmla] | A separate Analysis Services client/protocol surface | XMLA text or binary XML, with transport compression options | Binary XML is not evidence of Arrow. Arrow on this path has not been verified. | +| [Cube REST API][cube] | Cube JSON semantic query, including `/v1/load` | JSON data plus annotations such as title, short title, and type | Prior art for semantic result descriptors; its JSON “columnar” layout is not Arrow IPC. | +| [Arrow Flight / Flight SQL][flight] | Arrow-oriented RPC; Flight SQL adds a SQL service protocol | Arrow record batches with tickets/endpoints | A distinct transport, not implemented by this browser HTTP client. A future mapping requires separate design. | +| [ADBC][adbc] | Database driver API with statements and driver-dependent capabilities | Arrow-oriented result consumption | An API abstraction, not an HTTP wire protocol or implemented adapter here. | + +The [existing Ossie REST discussion][rest-thread] addresses model/catalog +interchange; [Polaris #4961][polaris] implements semantic-model CRUD. +Neither establishes this execution endpoint. The draft narrows familiar +patterns into a proposed browser contract; it does not claim novel transport +mechanisms or parity with a native vendor client. + +[mcp]: https://github.com/apache/ossie/pull/529 +[foundation]: https://github.com/apache/ossie/pull/246 +[databricks]: https://docs.databricks.com/api/statement-execution/v1/execute-statement +[dbt-graphql]: https://docs.getdbt.com/docs/dbt-apis/sl-graphql +[dbt-jdbc]: https://docs.getdbt.com/docs/dbt-apis/sl-jdbc +[snowflake-sql]: https://docs.snowflake.com/en/developer-guide/sql-api/handling-responses +[snowflake-python]: https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-api +[powerbi-rest]: https://learn.microsoft.com/en-us/rest/api/power-bi/datasets/execute-queries +[xmla]: https://learn.microsoft.com/en-us/analysis-services/instances/connection-string-properties-analysis-services +[cube]: https://docs.cube.dev/reference/core-data-apis/rest-api/reference +[flight]: https://arrow.apache.org/docs/format/Flight.html +[adbc]: https://arrow.apache.org/adbc/current/format/specification.html +[rest-thread]: https://lists.apache.org/thread/xvpomkdvmgv1m5krprq590l8mzccks5n +[polaris]: https://github.com/apache/polaris/pull/4961 diff --git a/typescript/execution/package-lock.json b/typescript/execution/package-lock.json new file mode 100644 index 000000000..5e26c3ac3 --- /dev/null +++ b/typescript/execution/package-lock.json @@ -0,0 +1,665 @@ +{ + "name": "@apache-ossie/execution-client", + "version": "0.1.0-draft.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@apache-ossie/execution-client", + "version": "0.1.0-draft.0", + "license": "Apache-2.0", + "dependencies": { + "apache-arrow": "21.2.0" + }, + "devDependencies": { + "@types/node": "^22.0.0", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "esbuild": "^0.25.0", + "typescript": "^5.9.3" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.25.12", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", + "integrity": "sha1-gPy+NhMOWLdnBRHoiLjoiiWe12w=", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.25.12", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@esbuild/android-arm/-/android-arm-0.25.12.tgz", + "integrity": "sha1-MAcSEB9/UPHSYnoWLm4JsQm2dno=", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.25.12", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", + "integrity": "sha1-iqSWX40KeYLcIXNL9mATI6Ztp1I=", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.25.12", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@esbuild/android-x64/-/android-x64-0.25.12.tgz", + "integrity": "sha1-h9+ycWEgK9yVjvSLthsJx1j67hY=", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.25.12", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", + "integrity": "sha1-eRl4mOwf90XSHAceHHzDyALwwf0=", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.25.12", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", + "integrity": "sha1-FGQAqFYhM/RcTS6tzzfd0JcYB54=", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.12", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", + "integrity": "sha1-HF+bpyBuFY/SskxZ+i0si7R8oP4=", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.25.12", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", + "integrity": "sha1-6mMfSja+qsS5J5+g/MbKKerusrM=", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.25.12", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", + "integrity": "sha1-RSzWayCTLQi9xTqLYcDjC69DSLk=", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.25.12", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", + "integrity": "sha1-4QZrzlg5TxsRQd7shVel8KIvWXc=", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.25.12", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", + "integrity": "sha1-sk+KzEW89UGSx/LzvhtT5lUer+A=", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.25.12", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", + "integrity": "sha1-+c//p/yDIlcfvEyLMmjK8VvYGtA=", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.25.12", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", + "integrity": "sha1-V1oUvXRkT/q4ka3H1+YNJ1KW8s0=", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.25.12", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", + "integrity": "sha1-dbmccKlfvV93OddpK+/mBgFZGGk=", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.25.12", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", + "integrity": "sha1-LjJZRAMhpE553fdTXDJQV9qHXNY=", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.25.12", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", + "integrity": "sha1-F2dsq7/lko2lsqDW311YzQjbJmM=", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.25.12", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", + "integrity": "sha1-BYN3VoXKggZtBMNQfwlSTTzXowY=", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.12", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", + "integrity": "sha1-8ExAScsuJS/paxb+2Q9wdGsT9KQ=", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.25.12", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", + "integrity": "sha1-d9oNCg2CbXySHuo9QCklSLJYoHY=", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.12", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", + "integrity": "sha1-Ypb1hnrt7yioGyKrIAnHhqlS3M0=", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.25.12", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", + "integrity": "sha1-+NIzAzYOJ7Fs8GWyO7/0PBQUJnk=", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.25.12", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", + "integrity": "sha1-SeC3aHRKOSS+DX/ZfdbOmykj2I0=", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.25.12", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", + "integrity": "sha1-pu19Z3jWflKMgfsWWyP0kRubE9Y=", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.25.12", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", + "integrity": "sha1-msFMN44bZTrxfQjn0840yu9YcyM=", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.25.12", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", + "integrity": "sha1-kYlC3LuzXMFPyjmvuRteaj0Scmc=", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.25.12", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", + "integrity": "sha1-m9rYF2vngRrRSNH4dyNZBB9GxsU=", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@types/node": { + "version": "22.20.5", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@types/node/-/node-22.20.5.tgz", + "integrity": "sha512-U2+DNr+wSjpsTS/wZGYHq7GcwfuSmKiKvoPvK22zwTlRhU91yOniN4qRR5KhIjvif7ysw/dz/hKmfDH0Ris4aA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha1-MEs2Nq3Yi6fZNnYN1Q7OAG3qlfk=", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha1-PV3HYryhdnnDwup+kK1rdTIwlXg=", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/apache-arrow": { + "version": "21.2.0", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/apache-arrow/-/apache-arrow-21.2.0.tgz", + "integrity": "sha1-OtwW64FiAmh3ggZYNs2mCS/321E=", + "license": "Apache-2.0", + "dependencies": { + "@types/node": "^25.2.0", + "flatbuffers": "^25.1.24", + "json-with-bigint": "^3.5.3", + "tslib": "^2.6.2" + }, + "bin": { + "arrow2csv": "bin/arrow2csv.js" + } + }, + "node_modules/apache-arrow/node_modules/@types/node": { + "version": "25.9.9", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/@types/node/-/node-25.9.9.tgz", + "integrity": "sha512-b4e2xxj/yMeT2hNlD6x7zNFiQK2Dlmkb9CQ6v3/1G2RezKN4YS2lcEv8lzGm1mhs4f2lM4yNAMI/zUDBZBBt7w==", + "license": "MIT", + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, + "node_modules/apache-arrow/node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha1-YSdbSF1/1OnSacfPBOwoc8nMD5E=", + "license": "MIT" + }, + "node_modules/esbuild": { + "version": "0.25.12", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/esbuild/-/esbuild-0.25.12.tgz", + "integrity": "sha1-l6HQQfSrAML84vg40rmWmi0ql6U=", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.12", + "@esbuild/android-arm": "0.25.12", + "@esbuild/android-arm64": "0.25.12", + "@esbuild/android-x64": "0.25.12", + "@esbuild/darwin-arm64": "0.25.12", + "@esbuild/darwin-x64": "0.25.12", + "@esbuild/freebsd-arm64": "0.25.12", + "@esbuild/freebsd-x64": "0.25.12", + "@esbuild/linux-arm": "0.25.12", + "@esbuild/linux-arm64": "0.25.12", + "@esbuild/linux-ia32": "0.25.12", + "@esbuild/linux-loong64": "0.25.12", + "@esbuild/linux-mips64el": "0.25.12", + "@esbuild/linux-ppc64": "0.25.12", + "@esbuild/linux-riscv64": "0.25.12", + "@esbuild/linux-s390x": "0.25.12", + "@esbuild/linux-x64": "0.25.12", + "@esbuild/netbsd-arm64": "0.25.12", + "@esbuild/netbsd-x64": "0.25.12", + "@esbuild/openbsd-arm64": "0.25.12", + "@esbuild/openbsd-x64": "0.25.12", + "@esbuild/openharmony-arm64": "0.25.12", + "@esbuild/sunos-x64": "0.25.12", + "@esbuild/win32-arm64": "0.25.12", + "@esbuild/win32-ia32": "0.25.12", + "@esbuild/win32-x64": "0.25.12" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha1-On1WtVnWy8PrUSMlJE5hmmXGxSU=", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.8", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/fast-uri/-/fast-uri-3.1.8.tgz", + "integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/flatbuffers": { + "version": "25.9.23", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/flatbuffers/-/flatbuffers-25.9.23.tgz", + "integrity": "sha1-NGgRVX/pMSq1ZHU155PHYenIHrE=", + "license": "Apache-2.0" + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha1-rnvLNlard6c7pcSb9lTzjmtoYOI=", + "dev": true, + "license": "MIT" + }, + "node_modules/json-with-bigint": { + "version": "3.5.12", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/json-with-bigint/-/json-with-bigint-3.5.12.tgz", + "integrity": "sha1-n2v3IqmEftcvqbIXsrZhgSJ8RUw=", + "license": "MIT" + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha1-iaf92TgmEmcxjq/hT5wy5ZjDaQk=", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha1-YS7+TtI11Wfoq6Xypfq3AoCt6D8=", + "license": "0BSD" + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha1-W09Z4VMQqxeiFvXWz1PuR27eZw8=", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha1-aR0ArzkJvpOn+qE75hs6W1DvEss=", + "dev": true, + "license": "MIT" + } + } +} diff --git a/typescript/execution/package.json b/typescript/execution/package.json new file mode 100644 index 000000000..e3ced339d --- /dev/null +++ b/typescript/execution/package.json @@ -0,0 +1,36 @@ +{ + "name": "@apache-ossie/execution-client", + "version": "0.1.0-draft.0", + "private": true, + "description": "Experimental, unpublished browser-first implementation of the Ossie HTTP execution proposal.", + "license": "Apache-2.0", + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "files": ["dist", "README.md", "LICENSE", "NOTICE"], + "engines": { + "node": ">=22" + }, + "scripts": { + "build": "tsc -p tsconfig.json", + "test": "npm run build && node --test test/*.test.mjs", + "test:browser": "npm run build && node --test test/browser/*.test.mjs", + "check": "npm run test" + }, + "dependencies": { + "apache-arrow": "21.2.0" + }, + "devDependencies": { + "@types/node": "^22.0.0", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "esbuild": "^0.25.0", + "typescript": "^5.9.3" + } +} diff --git a/typescript/execution/src/arrow.ts b/typescript/execution/src/arrow.ts new file mode 100644 index 000000000..cccafdf4a --- /dev/null +++ b/typescript/execution/src/arrow.ts @@ -0,0 +1,443 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { DataType, DateUnit, Message, MessageHeader, MetadataVersion, Precision, RecordBatchReader, TimeUnit, Type } from "apache-arrow"; +import type { RecordBatch, Schema } from "apache-arrow"; +import { OssieError, ProtocolError } from "./errors.js"; +import { BodySource, Operation } from "./transport.js"; +import { COLUMN_METADATA_KEY, EXECUTION_METADATA_KEY } from "./types.js"; +import type { Column, ResultPage, SuccessfulExecution } from "./types.js"; +import { assert, count, equivalent, object, validateMetadata } from "./validation.js"; + +const timeUnits = [TimeUnit.SECOND, TimeUnit.MILLISECOND, TimeUnit.MICROSECOND, TimeUnit.NANOSECOND]; + +function supported(type: DataType, column: Column): void { + let matches = false; + switch (column.logical_type) { + case "String": matches = DataType.isUtf8(type); break; + case "Integer": + matches = DataType.isInt(type) && [8, 16, 32, 64].includes(type.bitWidth) + && typeof type.isSigned === "boolean"; + break; + case "Float": + matches = DataType.isFloat(type) && [Precision.SINGLE, Precision.DOUBLE].includes(type.precision); + break; + case "Boolean": matches = DataType.isBool(type); break; + case "Decimal": + matches = DataType.isDecimal(type) && [128, 256].includes(type.bitWidth) + && Number.isInteger(type.precision) && type.precision >= 1 + && type.precision <= (type.bitWidth === 128 ? 38 : 76) && Number.isInteger(type.scale); + break; + case "Date": matches = DataType.isDate(type) && type.unit === DateUnit.DAY; break; + case "Time": + matches = DataType.isTime(type) && ((type.bitWidth === 32 + && [TimeUnit.SECOND, TimeUnit.MILLISECOND].includes(type.unit)) + || (type.bitWidth === 64 && [TimeUnit.MICROSECOND, TimeUnit.NANOSECOND].includes(type.unit))); + break; + case "DateTime": + matches = DataType.isTimestamp(type) && timeUnits.includes(type.unit) + && (type.timezone === undefined || type.timezone === null || type.timezone === ""); + break; + case "DateTimeTz": + matches = DataType.isTimestamp(type) && timeUnits.includes(type.unit) + && typeof type.timezone === "string" && type.timezone.trim().length > 0; + break; + } + assert(matches && (!type.children || type.children.length === 0), + "An Arrow physical type is unsupported or does not match its logical type."); +} + +function metadataJson(value: string | undefined): unknown { + assert(typeof value === "string", "Required Ossie Arrow metadata is missing."); + try { return JSON.parse(value) as unknown; } + catch { throw new ProtocolError("Ossie Arrow metadata is not valid JSON."); } +} + +function schemaMatches(schema: Schema, execution: SuccessfulExecution, page?: ResultPage): void { + const metadata = validateMetadata(metadataJson(schema.metadata.get(EXECUTION_METADATA_KEY))); + assert(equivalent(metadata, { + contract_version: execution.contract_version, + execution_id: execution.execution_id, + result_id: execution.result.result_id, + data_source_id: execution.data_source_id, + model: execution.model, + ...(page === undefined ? {} : { page_index: page.page_index, row_offset: page.row_offset }), + }), "Arrow execution metadata does not match the execution."); + assert(schema.fields.length === execution.result.columns.length, "Arrow column count mismatch."); + schema.fields.forEach((field, index) => { + const column = execution.result.columns[index]!; + assert(field.name === column.name && field.nullable === column.nullable, + "Arrow column name, order or nullability mismatch."); + const metadata = object(metadataJson(field.metadata.get(COLUMN_METADATA_KEY)), ["column_id"], ["column_id"]); + assert(metadata.column_id === column.column_id, "Arrow column identity mismatch."); + assert(![...field.metadata.keys()].some(key => key.startsWith("ARROW:extension:")), + "Arrow extension types, including Opaque, are unsupported."); + supported(field.type, column); + }); +} + +type FlatTable = readonly (number | undefined)[]; +interface FlatVector { start: number; length: number } + +/** Arrow's generated FlatBuffer accessors do not check bounds. Verify every + * reachable flat-profile table, vector and string before either decoder sees it. + */ +class MetadataVerifier { + private readonly view: DataView; + private remaining: number; + private visits = 0; + + constructor(private readonly bytes: Uint8Array, private readonly operation: Operation, maxBytes: number) { + this.view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + this.remaining = maxBytes; + } + + private range(offset: number, length: number, alignment = 1): void { + assert(Number.isSafeInteger(offset) && Number.isSafeInteger(length) && offset >= 0 && length >= 0 + && offset % alignment === 0 && offset <= this.bytes.length - length, + "Arrow metadata reference is unaligned or out of bounds."); + } + + private consume(length: number): void { + // Count repeated references too: individually valid shared vectors/strings + // must not expand into more metadata than the configured result byte budget. + this.remaining -= length; + if (this.remaining < 0) { + throw new OssieError("RESULT_LIMIT_EXCEEDED", "Expanded Arrow metadata exceeds the configured byte limit."); + } + if ((this.visits++ & 255) === 0) this.operation.check(); + } + + private reference(location: number): number { + this.range(location, 4, 4); + const distance = this.view.getUint32(location, true); + assert(distance > 0, "Arrow metadata contains a null or cyclic reference."); + const target = location + distance; + this.range(target, 4, 4); + return target; + } + + private table(position: number, widths: readonly number[]): FlatTable { + this.range(position, 4, 4); + const distance = this.view.getInt32(position, true); + assert(distance !== 0, "Arrow metadata has an invalid vtable reference."); + const vtable = position - distance; + this.range(vtable, 4, 2); + const vtableSize = this.view.getUint16(vtable, true); + const objectSize = this.view.getUint16(vtable + 2, true); + assert(vtableSize >= 4 && vtableSize % 2 === 0 && objectSize >= 4, + "Arrow metadata has an invalid table size."); + this.range(vtable, vtableSize, 2); + this.range(position, objectSize, 4); + this.consume(vtableSize + objectSize); + const fields: (number | undefined)[] = new Array(widths.length); + for (let index = 0; index < (vtableSize - 4) / 2; index++) { + const offset = this.view.getUint16(vtable + 4 + index * 2, true); + if (offset === 0) continue; + const width = widths[index]; + assert(width !== undefined && offset >= 4 && offset <= objectSize - width, + "Arrow metadata field is outside its table or unsupported."); + this.range(position + offset, width, width); + fields[index] = position + offset; + } + return fields; + } + + private vector(location: number | undefined, stride: number, alignment: number, + expected?: number): FlatVector { + if (location === undefined) { + assert(expected === undefined || expected === 0, "Required Arrow metadata vector is missing."); + return { start: 0, length: 0 }; + } + const position = this.reference(location); + const length = this.view.getUint32(position, true); + const start = position + 4; + assert(length <= Math.floor((this.bytes.length - start) / stride), + "Arrow metadata vector exceeds its actual buffer."); + assert(expected === undefined || length === expected, "Arrow metadata vector does not match the flat schema."); + this.range(start, length * stride, alignment); + this.consume(4 + length * stride); + return { start, length }; + } + + private string(location: number | undefined, required = false): void { + if (location === undefined) { + assert(!required, "Required Arrow metadata string is missing."); + return; + } + const position = this.reference(location); + const length = this.view.getUint32(position, true); + this.range(position + 4, length + 1); + assert(this.bytes[position + 4 + length] === 0, "Arrow metadata string is not terminated."); + this.consume(5 + length); + } + + private customMetadata(location: number | undefined): void { + const vector = this.vector(location, 4, 4); + for (let index = 0; index < vector.length; index++) { + const entry = this.table(this.reference(vector.start + index * 4), [4, 4]); + this.string(entry[0]); + this.string(entry[1]); + } + } + + private signed16(location: number | undefined, fallback = 0): number { + return location === undefined ? fallback : this.view.getInt16(location, true); + } + + private unsigned8(location: number | undefined, fallback = 0): number { + return location === undefined ? fallback : this.view.getUint8(location); + } + + private counter(location: number | undefined): number { + if (location === undefined) return 0; + const value = this.view.getBigInt64(location, true); + assert(value >= 0n && value <= BigInt(Number.MAX_SAFE_INTEGER), "Invalid Arrow metadata counter."); + return Number(value); + } + + private field(position: number): void { + const field = this.table(position, [4, 1, 1, 4, 4, 4, 4]); + this.string(field[0], true); + assert(this.unsigned8(field[1]) <= 1, "Invalid Arrow nullable flag."); + assert(field[4] === undefined, "Dictionary-encoded Arrow fields are unsupported."); + // Children are forbidden by this flat profile, so no recursive decoder + // traversal is allowed, including children hidden on primitive fields. + this.vector(field[5], 4, 4, 0); + this.customMetadata(field[6]); + const typeId = this.unsigned8(field[2]); + const layouts: Partial> = { + [Type.Int]: [4, 1], [Type.Float]: [2], [Type.Utf8]: [], [Type.Bool]: [], + [Type.Decimal]: [4, 4, 4], [Type.Date]: [2], [Type.Time]: [2, 4], [Type.Timestamp]: [2, 4], + }; + const widths = layouts[typeId as Type]; + assert(widths !== undefined && field[3] !== undefined, "Unsupported or missing Arrow physical type."); + const physical = this.table(this.reference(field[3]), widths); + if (typeId === Type.Int) assert(this.unsigned8(physical[1]) <= 1, "Invalid Arrow signedness flag."); + if (typeId === Type.Timestamp) this.string(physical[1]); + } + + verify(schema: Schema | undefined, columnCount: number): void { + const message = this.table(this.reference(0), [2, 1, 4, 8, 4]); + assert([MetadataVersion.V4, MetadataVersion.V5].includes(this.signed16(message[0])), + "Unsupported Arrow metadata version."); + const type = this.unsigned8(message[1]); + assert(message[2] !== undefined, "Required Arrow message header is missing."); + this.customMetadata(message[4]); + const bodyLength = this.counter(message[3]); + if (type === MessageHeader.Schema) { + assert(schema === undefined && bodyLength === 0, "The IPC stream must have exactly one schema."); + const header = this.table(this.reference(message[2]), [2, 4, 4, 4]); + // Arrow JS discards endianness when decoding. + assert(this.signed16(header[0]) === 0, "Big-endian Arrow streams are unsupported."); + const fields = this.vector(header[1], 4, 4, columnCount); + this.customMetadata(header[2]); + this.vector(header[3], 8, 8); + for (let index = 0; index < fields.length; index++) this.field(this.reference(fields.start + index * 4)); + } else { + assert(type === MessageHeader.RecordBatch && schema !== undefined, + "Only uncompressed record batches may follow the schema."); + const header = this.table(this.reference(message[2]), [8, 4, 4, 4, 4]); + this.counter(header[0]); + this.vector(header[1], 16, 8, schema.fields.length); + const buffers = schema.fields.reduce((sum, field) => sum + (DataType.isUtf8(field.type) ? 3 : 2), 0); + this.vector(header[2], 16, 8, buffers); + assert(header[3] === undefined, "Compressed Arrow IPC is unsupported."); + this.vector(header[4], 8, 8, 0); + } + this.operation.check(); + } +} + +function valueBytes(type: DataType, rows: number): number { + if (DataType.isBool(type)) return Math.ceil(rows / 8); + if (DataType.isInt(type) || DataType.isTime(type) || DataType.isDecimal(type)) return rows * type.bitWidth / 8; + if (DataType.isFloat(type)) return rows * (type.precision === Precision.SINGLE ? 4 : 8); + if (DataType.isDate(type)) return rows * 4; + if (DataType.isTimestamp(type)) return rows * 8; + throw new ProtocolError("Unsupported Arrow value buffer."); +} + +type BatchHeader = ReturnType["header"]>; + +function validateBatchHeader(header: BatchHeader, schema: Schema, bodyLength: number): void { + count(header.length); + assert(header.compression === null, "Compressed Arrow IPC is unsupported."); + assert(header.variadicBufferCounts.length === 0, "Variadic Arrow buffers are unsupported."); + assert(header.nodes.length === schema.fields.length, "Arrow field-node count mismatch."); + const bufferCount = schema.fields.reduce((sum, field) => sum + (DataType.isUtf8(field.type) ? 3 : 2), 0); + assert(header.buffers.length === bufferCount, "Arrow buffer count mismatch."); + let end = 0; + for (const buffer of header.buffers) { + count(buffer.offset); + count(buffer.length); + assert(buffer.offset % 8 === 0 && buffer.offset >= end && buffer.offset + buffer.length <= bodyLength, + "Arrow buffer is unaligned, overlapping or out of bounds."); + end = buffer.offset + buffer.length; + } + for (let i = 0; i < header.nodes.length; i++) { + const node = header.nodes[i]!; + count(node.length); + count(node.nullCount); + assert(node.length === header.length && node.nullCount <= node.length, "Arrow field length mismatch."); + assert(schema.fields[i]!.nullable || node.nullCount === 0, "Nonnullable Arrow field contains nulls."); + } +} + +function validateBatchBody(header: BatchHeader, schema: Schema, body: Uint8Array): void { + let bufferIndex = 0; + const next = (): Uint8Array => { + const region = header.buffers[bufferIndex++]!; + return body.subarray(region.offset, region.offset + region.length); + }; + for (let i = 0; i < schema.fields.length; i++) { + const field = schema.fields[i]!, node = header.nodes[i]!; + const validity = next(); + assert(validity.byteLength === 0 || validity.byteLength >= Math.ceil(node.length / 8), + "Arrow validity buffer is too short."); + assert(node.nullCount === 0 || validity.byteLength > 0, "Arrow null bitmap is missing."); + let nulls = 0; + if (validity.byteLength > 0) { + for (let row = 0; row < node.length; row++) { + if ((validity[row >> 3]! & (1 << (row & 7))) === 0) nulls++; + } + } + assert(nulls === node.nullCount, "Arrow null count does not match the validity bitmap."); + if (DataType.isUtf8(field.type)) { + const offsets = next(), values = next(); + assert((node.length === 0 && offsets.byteLength === 0) + || offsets.byteLength >= (node.length + 1) * 4, "Arrow string offsets are too short."); + if (offsets.byteLength === 0) continue; + const view = new DataView(offsets.buffer, offsets.byteOffset, offsets.byteLength); + let previous = view.getInt32(0, true); + assert(previous >= 0 && previous <= values.byteLength, "Invalid Arrow string offset."); + const decoder = new TextDecoder("utf-8", { fatal: true }); + for (let row = 0; row < node.length; row++) { + const offset = view.getInt32((row + 1) * 4, true); + assert(offset >= previous && offset <= values.byteLength, "Invalid Arrow string offsets."); + if (validity.byteLength === 0 || (validity[row >> 3]! & (1 << (row & 7))) !== 0) { + decoder.decode(values.subarray(previous, offset)); + } + previous = offset; + } + } else { + assert(next().byteLength >= valueBytes(field.type, node.length), "Arrow value buffer is too short."); + } + } +} + +/** Validates IPC framing before Arrow allocates or silently accepts schema replacement. */ +async function* frames(source: BodySource, operation: Operation, execution: SuccessfulExecution, + maxRows: number, maxBytes: number, page?: ResultPage): AsyncGenerator { + let schema: Schema | undefined; + let rows = 0; + const counts = page ?? execution.result; + const finish = (): void => { + assert(schema !== undefined, "The IPC stream has no schema."); + assert(source.bytes === counts.byte_count, "Decoded IPC byte count mismatch."); + assert(rows === counts.row_count, "Arrow row count mismatch."); + }; + while (true) { + operation.check(); + const first = await source.exact(4, true); + if (first === undefined) { finish(); return; } + let prefix = first; + let length = new DataView(first.buffer).getInt32(0, true); + if (length === -1) { + const second = (await source.exact(4))!; + prefix = new Uint8Array(8); + prefix.set(first); + prefix.set(second, 4); + length = new DataView(second.buffer).getInt32(0, true); + } + if (length === 0) { + await source.end(); + finish(); + yield prefix; + return; + } + assert(length > 0 && length <= Math.min(maxBytes, 1024 * 1024), "Invalid Arrow metadata length."); + assert((length + prefix.byteLength) % 8 === 0, "Unaligned Arrow metadata."); + const bytes = (await source.exact(length))!; + new MetadataVerifier(bytes, operation, maxBytes).verify(schema, execution.result.columns.length); + const message = Message.decode(bytes); + assert([MetadataVersion.V4, MetadataVersion.V5].includes(message.version), "Unsupported Arrow metadata version."); + count(message.bodyLength); + assert(message.bodyLength <= maxBytes && message.bodyLength % 8 === 0, "Invalid Arrow body length."); + let header: BatchHeader | undefined; + if (message.isSchema()) { + assert(schema === undefined && message.bodyLength === 0, "The IPC stream must have exactly one schema."); + schema = message.header(); + schemaMatches(schema, execution, page); + } else { + assert(schema !== undefined && message.isRecordBatch(), "Only uncompressed record batches may follow the schema."); + header = message.header(); + validateBatchHeader(header, schema, message.bodyLength); + rows += header.length; + if (!Number.isSafeInteger(rows) || rows > maxRows) { + throw new OssieError("RESULT_LIMIT_EXCEEDED", "The result exceeded the configured row limit."); + } + assert(rows <= counts.row_count, "Arrow row count exceeds the manifest."); + } + // Retain at most one complete IPC message, not the whole result. + const body = (await source.exact(message.bodyLength))!; + if (header) validateBatchBody(header, schema!, body); + operation.check(); + yield prefix; + yield bytes; + if (body.byteLength) yield body; + } +} + +export async function* arrowBatches(response: Response, execution: SuccessfulExecution, + operation: Operation, maxBytes: number, maxRows: number, page?: ResultPage, + remainingBytes = maxBytes): AsyncGenerator { + const source = new BodySource(response, operation, Math.min(maxBytes, remainingBytes)); + const input = frames(source, operation, execution, maxRows, maxBytes, page); + let reader: RecordBatchReader | undefined; + try { + reader = await operation.race(() => RecordBatchReader.from(input)); + await operation.race(() => reader!.open({ autoDestroy: false })); + assert(reader.isStream() && reader.schema, "Expected an Arrow IPC stream with a schema."); + schemaMatches(reader.schema, execution, page); + while (true) { + const next = await operation.race(() => reader!.next()); + if (next.done) break; + const batch = next.value as RecordBatch; + if (new Set(reader.schema.fields.map(field => field.name)).size !== reader.schema.fields.length) { + // Arrow 21's RecordBatch constructor merges field metadata by display + // name. Restore the already-validated positional IPC schema; keep the + // exact decoded batch and all its column buffers untouched. + Object.defineProperty(batch, "schema", { value: reader.schema }); + } + schemaMatches(batch.schema, execution, page); + operation.check(); + yield batch; + } + } catch (error) { + operation.check(); + if (error instanceof OssieError) throw error; + throw new ProtocolError("The result is not a valid supported Arrow IPC stream."); + } finally { + source.close(); + // Closing the source first also unblocks an outstanding Arrow read on abort. + if (reader) void Promise.resolve(reader.cancel()).catch(() => {}); + void input.return(undefined).catch(() => {}); + } +} diff --git a/typescript/execution/src/client.ts b/typescript/execution/src/client.ts new file mode 100644 index 000000000..f4eb4c67d --- /dev/null +++ b/typescript/execution/src/client.ts @@ -0,0 +1,480 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Schema, Table } from "apache-arrow"; +import type { RecordBatch } from "apache-arrow"; +import { arrowBatches } from "./arrow.js"; +import { ExecutionCancelledError, HttpProblemError, OssieError, ProtocolError, QueryExecutionError } from "./errors.js"; +import { ARROW_MEDIA_TYPE, EXECUTION_METADATA_KEY } from "./types.js"; +import type { Binding, Capabilities, ClientOptions, ExecuteOptions, ExecuteRequest, Execution, + RequestOptions, ResultPage, SubmitOptions, SuccessfulExecution, WaitOptions } from "./types.js"; +import { Operation, cancelBody, mediaType, readJson } from "./transport.js"; +import { assert, count, equivalent, terminal, text, transition, validateBinding, validateCapabilities, + validateExecution, validateProblem, validateRequest } from "./validation.js"; + +export const DEFAULT_TIMEOUT_MS = 60_000; +export const DEFAULT_POLL_INTERVAL_MS = 250; +export const DEFAULT_MAX_RESULT_BYTES = 64 * 1024 * 1024; +export const DEFAULT_MAX_RESULT_ROWS = 1_000_000; +export const MAX_TRACKED_EXECUTIONS = 64; + +interface Observation { + previous: Execution | undefined; + fallback?: Execution; +} + +function integerOption(value: unknown, fallback: number, minimum: number, maximum = Number.MAX_SAFE_INTEGER): number { + if (value === undefined) return fallback; + count(value); + assert(value >= minimum && value <= maximum, "Invalid numeric client option."); + return value; +} + +function url(value: string): URL { + text(value); + assert(value === value.trim() && !/[\u0000-\u0020\u007f\\]/.test(value), "Invalid URL."); + const authority = /^(?:[a-z][a-z0-9+.-]*:)?\/{2,}([^/?#]*)/i.exec(value)?.[1]; + assert(!authority?.includes("@"), "URL user information is forbidden."); + try { + const parsed = new URL(value); + assert(!parsed.username && !parsed.password && !parsed.href.includes("#"), "URL credentials and fragments are forbidden."); + return parsed; + } catch { + throw new ProtocolError("Invalid or unsafe URL."); + } +} + +function local(url: URL): boolean { + return ["localhost", "127.0.0.1", "[::1]"].includes(url.hostname); +} + +function idPath(id: string): string { + text(id); + assert(id !== "." && id !== "..", "Dot-segment identifiers are unsupported."); + try { return encodeURIComponent(id); } + catch { throw new ProtocolError("The identifier is not valid Unicode."); } +} + +function waitPreference(options: SubmitOptions): string | undefined { + if (options.waitSeconds === undefined) return undefined; + count(options.waitSeconds); + assert(options.waitSeconds <= 60, "waitSeconds must be an integer between zero and sixty."); + return `wait=${options.waitSeconds}`; +} + +function successful(execution: Execution): SuccessfulExecution { + if (execution.state === "failed") throw new QueryExecutionError(execution); + if (execution.state === "cancelled") throw new ExecutionCancelledError(execution); + assert(execution.state === "succeeded", "The execution has not succeeded."); + return execution; +} + +function schemaIdentity(schema: Schema): unknown { + const metadata: Record = Object.fromEntries(schema.metadata); + const identity = JSON.parse(schema.metadata.get(EXECUTION_METADATA_KEY)!) as Record; + delete identity.page_index; + delete identity.row_offset; + metadata[EXECUTION_METADATA_KEY] = identity; + return { + metadata, + fields: schema.fields.map(field => ({ + name: field.name, nullable: field.nullable, type: field.type, + metadata: Object.fromEntries(field.metadata), + })), + }; +} + +/** Experimental browser-first client. It never retries or repairs a submitted query. + * Aborting a local operation does not cancel the backend execution; use cancel(). + */ +export class OssieClient { + private readonly base: URL; + private readonly fetcher: typeof globalThis.fetch; + private readonly getAccessToken: ClientOptions["getAccessToken"]; + private readonly origins = new Set(); + private readonly timeoutMs: number; + private readonly pollIntervalMs: number; + private readonly maxResultBytes: number; + private readonly maxResultRows: number; + private readonly executions = new Map(); + + constructor(options: ClientOptions) { + assert(options !== null && typeof options === "object", "Client options are required."); + if (options.allowInsecureLocalhost !== undefined) { + assert(typeof options.allowInsecureLocalhost === "boolean", "allowInsecureLocalhost must be a boolean."); + } + this.base = url(options.baseUrl); + assert(!this.base.href.includes("?"), "The base URL cannot contain a query."); + assert(this.base.protocol === "https:" || (options.allowInsecureLocalhost === true + && this.base.protocol === "http:" && local(this.base)), "The base URL must use HTTPS."); + if (!this.base.pathname.endsWith("/")) this.base.pathname += "/"; + this.fetcher = options.fetch ?? globalThis.fetch?.bind(globalThis); + assert(typeof this.fetcher === "function", "A Fetch implementation is required."); + this.getAccessToken = options.getAccessToken; + assert(this.getAccessToken === undefined || typeof this.getAccessToken === "function", + "getAccessToken must be a function."); + this.timeoutMs = integerOption(options.timeoutMs, DEFAULT_TIMEOUT_MS, 1, 2_147_483_647); + this.pollIntervalMs = integerOption(options.pollIntervalMs, DEFAULT_POLL_INTERVAL_MS, 1, 2_147_483_647); + this.maxResultBytes = integerOption(options.maxResultBytes, DEFAULT_MAX_RESULT_BYTES, 1); + this.maxResultRows = integerOption(options.maxResultRows, DEFAULT_MAX_RESULT_ROWS, 0); + if (options.allowedResultOrigins !== undefined) { + assert(Array.isArray(options.allowedResultOrigins), "allowedResultOrigins must be an array."); + for (const origin of options.allowedResultOrigins) { + const parsed = url(origin); + assert(parsed.protocol === "https:" && parsed.pathname === "/" && !parsed.href.includes("?"), + "Allowed result origins must be HTTPS origins, without paths or queries."); + this.origins.add(parsed.origin); + } + } + } + + /** Reads only the configured endpoint; does not enumerate vendors or data sources. */ + async getCapabilities(options: RequestOptions = {}): Promise { + return this.run(options, async operation => + validateCapabilities(await this.control("capabilities", "GET", operation, [200]))); + } + + async getBinding(id: string, options: RequestOptions = {}): Promise { + const path = `bindings/${idPath(id)}`; + return this.run(options, async operation => { + const binding = validateBinding(await this.control(path, "GET", operation, [200])); + assert(binding.data_source_id === id, "Binding identity mismatch."); + return binding; + }); + } + + /** Sends exactly one POST. Shape validation is not an expression compiler. */ + async submit(request: ExecuteRequest, options: SubmitOptions = {}): Promise { + const body = this.requestBody(request); + const prefer = waitPreference(options); + return this.run(options, operation => this.submitInternal(body, prefer, operation)); + } + + /** Returns failed/cancelled states as data, while enforcing identity and lifecycle. */ + async getExecution(id: string, options: RequestOptions = {}): Promise { + idPath(id); + return this.run(options, operation => this.getInternal(id, operation)); + } + + /** Polls a known execution. Progress fractions may change in either direction. */ + async wait(execution: Execution, options: WaitOptions = {}): Promise { + validateExecution(execution); + const known = this.executions.get(execution.execution_id); + if (known) this.sameIdentity(known, execution); + const snapshot = this.remember(known && !terminal(execution) ? known : execution); + this.validateProgressCallback(options); + return this.run(options, operation => this.waitInternal(snapshot, options, operation)); + } + + /** Explicit backend cancellation; races can still return a succeeded execution. */ + async cancel(executionId: string, options: RequestOptions = {}): Promise { + const path = `queries/${idPath(executionId)}/cancel`; + const observation: Observation = { previous: this.executions.get(executionId) }; + return this.run(options, async operation => this.remember(validateExecution( + await this.control(path, "POST", operation, [200, 202], undefined, undefined, true)), + executionId, undefined, observation)); + } + + /** Drops local lifecycle history only; does not cancel or delete server data. */ + forgetExecution(executionId: string): void { + idPath(executionId); + this.executions.delete(executionId); + } + + /** Streams batches from one completed immutable result. Full counts are only + * verified after exhaustion; early return cancels/releases the response reader. + */ + async *readBatches(execution: SuccessfulExecution, + options: RequestOptions & { allowTruncated?: boolean } = {}): AsyncGenerator { + const snapshot = successful(this.remember(validateExecution(execution))); + this.validateTruncated(options); + const operation = new Operation(options.signal, this.timeoutMs); + try { + yield* this.readInternal(snapshot, options.allowTruncated === true, operation); + } finally { + operation.close(); + } + } + + /** Reads one immutable page without fetching others. Index zero also works + * for an unpaged result. Verifying a page does not verify the whole result. + */ + async readPage(execution: SuccessfulExecution, pageIndex: number, + options: RequestOptions & { allowTruncated?: boolean } = {}): Promise
{ + const snapshot = successful(this.remember(validateExecution(execution))); + this.validateTruncated(options); + count(pageIndex); + const pages = snapshot.result.pages; + assert(pageIndex < (pages?.length ?? 1), "Result page index is out of range."); + const page = pages?.[pageIndex]; + this.admitResult(snapshot, options.allowTruncated === true, page); + return this.run(options, async operation => { + const batches: RecordBatch[] = []; + for await (const batch of this.readStream(snapshot, operation, page)) batches.push(batch); + operation.check(); + assert(batches.length > 0, "The Arrow reader did not preserve the result schema."); + return new Table(batches[0]!.schema, batches); + }); + } + + /** Bounded materialization preserving Arrow's exact types and metadata. */ + async execute(request: ExecuteRequest, options: ExecuteOptions = {}): + Promise<{ execution: SuccessfulExecution; table: Table }> { + const body = this.requestBody(request); + const prefer = waitPreference(options); + this.validateProgressCallback(options); + this.validateTruncated(options); + return this.run(options, async operation => { + const submitted = await this.submitInternal(body, prefer, operation); + const execution = await this.waitInternal(submitted, options, operation); + const batches: RecordBatch[] = []; + for await (const batch of this.readInternal(execution, options.allowTruncated === true, operation)) batches.push(batch); + operation.check(); + assert(batches.length > 0, "The Arrow reader did not preserve the result schema."); + let schema = batches[0]!.schema; + if (execution.result.pages) { + const metadata = new Map(schema.metadata); + metadata.set(EXECUTION_METADATA_KEY, JSON.stringify({ + contract_version: execution.contract_version, execution_id: execution.execution_id, + result_id: execution.result.result_id, data_source_id: execution.data_source_id, model: execution.model, + })); + schema = new Schema(schema.fields, metadata); + } + return { execution, table: new Table(schema, batches) }; + }); + } + + private requestBody(request: ExecuteRequest): string { + validateRequest(request); + try { + const body = JSON.stringify(request); + // The serialized shape is the actual contract (including unusual JS inputs). + validateRequest(JSON.parse(body) as unknown); + return body; + } catch (error) { + if (error instanceof OssieError) throw error; + throw new ProtocolError("The query request is not serializable."); + } + } + + private validateProgressCallback(options: WaitOptions): void { + assert(options.onProgress === undefined || typeof options.onProgress === "function", + "onProgress must be a function."); + } + + private validateTruncated(options: { allowTruncated?: boolean }): void { + assert(options.allowTruncated === undefined || typeof options.allowTruncated === "boolean", + "allowTruncated must be a boolean."); + } + + private async run(options: RequestOptions, callback: (operation: Operation) => Promise): Promise { + const operation = new Operation(options.signal, this.timeoutMs); + try { operation.check(); return await callback(operation); } + finally { operation.close(); } + } + + private sameIdentity(previous: Execution, next: Execution): void { + assert(previous.execution_id === next.execution_id && previous.data_source_id === next.data_source_id + && equivalent(previous.model, next.model), "Execution identity changed."); + } + + private remember(execution: Execution, id?: string, source?: string, observation?: Observation): Execution { + if (id !== undefined) assert(execution.execution_id === id, "Execution identifier mismatch."); + if (source !== undefined) assert(execution.data_source_id === source, "Execution data-source mismatch."); + const previous = this.executions.get(execution.execution_id); + if (observation?.previous) transition(observation.previous, execution); + if (observation?.fallback) transition(observation.fallback, execution); + if (previous) this.sameIdentity(previous, execution); + // Overlapping requests can deliver an older pending observation after a + // cancellation response. Never let it overwrite a newer observed state. + if (observation && previous && previous !== observation.previous && !terminal(execution)) { + this.executions.delete(previous.execution_id); + this.executions.set(previous.execution_id, previous); + return structuredClone(previous); + } + if (previous) transition(previous, execution); + // A caller mutating a returned object cannot rewrite the client's history. + const snapshot = structuredClone(execution); + this.executions.delete(execution.execution_id); + this.executions.set(execution.execution_id, structuredClone(snapshot)); + if (this.executions.size > MAX_TRACKED_EXECUTIONS) { + this.executions.delete(this.executions.keys().next().value!); + } + return snapshot; + } + + private async submitInternal(body: string, prefer: string | undefined, operation: Operation): Promise { + const request = JSON.parse(body) as ExecuteRequest; + const result = await this.control("queries", "POST", operation, [200, 202], body, prefer, true); + const execution = validateExecution(result); + return this.remember(execution, undefined, request.data_source_id); + } + + private async getInternal(id: string, operation: Operation, fallback?: Execution): Promise { + const observation: Observation = { + previous: this.executions.get(id), ...(fallback === undefined ? {} : { fallback }), + }; + return this.remember(validateExecution( + await this.control(`queries/${idPath(id)}`, "GET", operation, [200])), id, undefined, observation); + } + + private async waitInternal(initial: Execution, options: WaitOptions, operation: Operation): Promise { + let execution = initial; + while (true) { + operation.check(); + if (options.onProgress) { + await operation.race(() => options.onProgress!(structuredClone(execution))); + } + if (terminal(execution)) return successful(execution); + await operation.sleep(this.pollIntervalMs); + execution = await this.getInternal(execution.execution_id, operation, execution); + } + } + + private resultUrl(value: string): URL { + const result = url(value); + if (result.origin === this.base.origin) { + assert(result.protocol === this.base.protocol, "Unsafe result URL."); + } else { + assert(result.protocol === "https:" && this.origins.has(result.origin), + "The result origin is not explicitly allowed."); + } + return result; + } + + private async *readInternal(execution: SuccessfulExecution, allowTruncated: boolean, + operation: Operation): AsyncGenerator { + operation.check(); + this.admitResult(execution, allowTruncated); + let schema: unknown; + let remainingBytes = this.maxResultBytes; + for (const page of execution.result.pages ?? [undefined]) { + for await (const batch of this.readStream(execution, operation, page, remainingBytes)) { + const current = schemaIdentity(batch.schema); + assert(schema === undefined || equivalent(schema, current), "Arrow schema or metadata changed between result pages."); + schema ??= structuredClone(current); + yield batch; + } + remainingBytes -= (page ?? execution.result).byte_count; + } + } + + private admitResult(execution: SuccessfulExecution, allowTruncated: boolean, page?: ResultPage): void { + const manifest = execution.result; + if (manifest.completeness === "truncated" && !allowTruncated) { + throw new OssieError("TRUNCATED_RESULT", "The result is truncated; explicit acceptance is required."); + } + const counts = page ?? manifest; + if (counts.byte_count > this.maxResultBytes || counts.row_count > this.maxResultRows) { + throw new OssieError("RESULT_LIMIT_EXCEEDED", "The result exceeds configured materialization limits."); + } + if (Date.parse(manifest.expires_at) <= Date.now()) { + throw new OssieError("RESULT_EXPIRED", "The materialized result has expired."); + } + } + + private async *readStream(execution: SuccessfulExecution, operation: Operation, + page?: ResultPage, remainingBytes = this.maxResultBytes): AsyncGenerator { + operation.check(); + const target = page?.url ?? execution.result.url; + assert(target !== undefined, "The result has no stream URL."); + const response = await this.fetchResponse(this.resultUrl(target), "GET", operation, ARROW_MEDIA_TYPE, + undefined, undefined, Date.parse(execution.result.expires_at)); + try { + if (response.status >= 400) await this.problem(response, operation); + assert(response.status === 200 && mediaType(response) === ARROW_MEDIA_TYPE, + "Expected a successful Arrow IPC stream response."); + yield* arrowBatches(response, execution, operation, this.maxResultBytes, this.maxResultRows, page, remainingBytes); + } finally { + cancelBody(response); + } + } + + private async control(path: string, method: string, operation: Operation, statuses: readonly number[], + body?: string, prefer?: string, lifecyclePost = false): Promise { + const response = await this.fetchResponse(new URL(path, this.base), method, operation, "application/json", body, prefer); + try { + if (response.status >= 400) await this.problem(response, operation); + assert(statuses.includes(response.status), "Unexpected HTTP success status."); + assert(mediaType(response) === "application/json", "Expected an application/json response."); + const result = await readJson(response, operation); + if (lifecyclePost) { + const execution = validateExecution(result); + assert((response.status === 200) === terminal(execution), "HTTP status does not match execution state."); + } + return result; + } finally { + cancelBody(response); + } + } + + private async problem(response: Response, operation: Operation): Promise { + assert(mediaType(response) === "application/problem+json", "Expected an application/problem+json error."); + throw new HttpProblemError(validateProblem(await readJson(response, operation), response.status)); + } + + private async fetchResponse(target: URL, method: string, operation: Operation, accept: string, + body?: string, prefer?: string, expiresAt?: number): Promise { + const headers = new Headers({ Accept: accept }); + if (body !== undefined) headers.set("Content-Type", "application/json"); + if (prefer !== undefined) headers.set("Prefer", prefer); + if (target.origin === this.base.origin && this.getAccessToken) { + let token: string; + try { + token = await operation.race(() => this.getAccessToken!()); + assert(typeof token === "string" && token.trim().length > 0 && !/[\r\n]/.test(token), + "The access-token callback returned an invalid token."); + headers.set("Authorization", `Bearer ${token}`); + } catch { + operation.check(); + throw new OssieError("AUTHENTICATION_ERROR", "An access token could not be obtained."); + } + } + const expired = new OssieError("RESULT_EXPIRED", "The materialized result has expired."); + let response: Response; + try { + response = await operation.race(async () => { + if (expiresAt !== undefined && expiresAt <= Date.now()) throw expired; + const received = await this.fetcher(target.href, { + method, headers, ...(body === undefined ? {} : { body }), signal: operation.signal, + redirect: "error", credentials: "omit", referrerPolicy: "no-referrer", cache: "no-store", + }); + // A custom Fetch implementation may ignore AbortSignal and resolve late. + if (operation.signal.aborted) { + cancelBody(received); + operation.check(); + } + return received; + }); + } catch (error) { + if (error === expired) throw expired; + operation.check(); + throw new OssieError("TRANSPORT_ERROR", "The HTTP request could not be completed."); + } + if (operation.signal.aborted) { + cancelBody(response); + operation.check(); + } + if (response.redirected) { + cancelBody(response); + throw new ProtocolError("HTTP redirects are forbidden."); + } + return response; + } +} diff --git a/typescript/execution/src/errors.ts b/typescript/execution/src/errors.ts new file mode 100644 index 000000000..120c4fa41 --- /dev/null +++ b/typescript/execution/src/errors.ts @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import type { Execution, ProblemDetails, QueryError } from "./types.js"; + +/** Base error. Messages deliberately exclude URLs, credentials and server bodies. + * Structured server details are untrusted data and must not be logged blindly. + */ +export class OssieError extends Error { + constructor(public readonly code: string, message: string) { + super(message); + this.name = new.target.name; + } +} + +export class ProtocolError extends OssieError { + constructor(message = "The server returned an invalid execution protocol response.") { + super("PROTOCOL_ERROR", message); + } +} + +export class QueryExecutionError extends OssieError { + readonly error: QueryError; + constructor(public readonly execution: Extract) { + super(execution.error.code, "The query execution failed."); + this.error = execution.error; + } +} + +export class ExecutionCancelledError extends OssieError { + constructor(public readonly execution: Extract) { + super("EXECUTION_CANCELLED", "The query execution was cancelled."); + } +} + +export class HttpProblemError extends OssieError { + readonly status: number; + constructor(public readonly problem: ProblemDetails) { + super(problem.error.code, "The server rejected the HTTP request."); + this.status = problem.status; + } +} + +export class TimeoutError extends OssieError { + constructor() { + super("TIMEOUT", "The client operation exceeded its deadline."); + } +} diff --git a/typescript/execution/src/index.ts b/typescript/execution/src/index.ts new file mode 100644 index 000000000..7ec48c90f --- /dev/null +++ b/typescript/execution/src/index.ts @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export * from "./types.js"; +export * from "./errors.js"; +export { OssieClient, DEFAULT_TIMEOUT_MS, DEFAULT_POLL_INTERVAL_MS, + DEFAULT_MAX_RESULT_BYTES, DEFAULT_MAX_RESULT_ROWS, MAX_TRACKED_EXECUTIONS } from "./client.js"; diff --git a/typescript/execution/src/transport.ts b/typescript/execution/src/transport.ts new file mode 100644 index 000000000..6273b7397 --- /dev/null +++ b/typescript/execution/src/transport.ts @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { OssieError, ProtocolError, TimeoutError } from "./errors.js"; +import { assert } from "./validation.js"; + +/** One deadline spans every phase of a public operation, including user callbacks. */ +export class Operation { + readonly controller = new AbortController(); + readonly signal = this.controller.signal; + private readonly timer: ReturnType; + private readonly deadline: number; + private readonly parentAbort = (): void => { + this.controller.abort(new OssieError("ABORTED", "The client operation was aborted.")); + }; + + constructor(private readonly parent: AbortSignal | undefined, timeoutMs: number) { + this.deadline = performance.now() + timeoutMs; + this.timer = setTimeout(() => this.controller.abort(new TimeoutError()), timeoutMs); + if (parent?.aborted) this.parentAbort(); + else parent?.addEventListener("abort", this.parentAbort, { once: true }); + } + + check(): void { + if (!this.signal.aborted && performance.now() >= this.deadline) { + this.controller.abort(new TimeoutError()); + } + if (this.signal.aborted) throw this.signal.reason as OssieError; + } + + async race(operation: () => T | PromiseLike): Promise { + this.check(); + return new Promise((resolve, reject) => { + const abort = (): void => reject(this.signal.reason); + this.signal.addEventListener("abort", abort, { once: true }); + Promise.resolve().then(() => { + this.check(); + return operation(); + }).then(resolve, reject).finally(() => this.signal.removeEventListener("abort", abort)); + }); + } + + async sleep(milliseconds: number): Promise { + let timer: ReturnType | undefined; + try { + await this.race(() => new Promise(resolve => { timer = setTimeout(resolve, milliseconds); })); + } finally { + clearTimeout(timer); + } + } + + close(): void { + clearTimeout(this.timer); + this.parent?.removeEventListener("abort", this.parentAbort); + } +} + +export function cancelBody(response: Response): void { + if (response.body && !response.body.locked) void response.body.cancel().catch(() => {}); +} + +/** Reads decoded fetch bytes, never Content-Length (which can describe compressed HTTP data). */ +export class BodySource { + private readonly reader: ReadableStreamDefaultReader; + private readonly abort = (): void => { void this.reader.cancel().catch(() => {}); }; + private closed = false; + private pending: Uint8Array = new Uint8Array(0); + private offset = 0; + bytes = 0; + + constructor(response: Response, private readonly operation: Operation, private readonly maxBytes: number) { + assert(response.body, "The HTTP response has no readable body."); + this.reader = response.body.getReader(); + operation.signal.addEventListener("abort", this.abort, { once: true }); + if (operation.signal.aborted) this.abort(); + } + + async chunk(): Promise { + let result: ReadableStreamReadResult; + try { + result = await this.operation.race(() => this.reader.read()); + } catch { + this.operation.check(); + throw new OssieError("TRANSPORT_ERROR", "The HTTP response body could not be read."); + } + this.operation.check(); + if (result.done) return undefined; + assert(result.value instanceof Uint8Array, "The transport did not provide bytes."); + this.bytes += result.value.byteLength; + if (!Number.isSafeInteger(this.bytes) || this.bytes > this.maxBytes) { + throw new OssieError("RESULT_LIMIT_EXCEEDED", "The response exceeded the configured byte limit."); + } + return result.value; + } + + async exact(length: number, allowEof = false): Promise { + assert(Number.isSafeInteger(length) && length >= 0 && length <= this.maxBytes, + "Invalid IPC buffer length."); + const output = new Uint8Array(length); + let filled = 0; + while (filled < length) { + if (this.offset === this.pending.byteLength) { + const chunk = await this.chunk(); + if (chunk === undefined) { + if (allowEof && filled === 0) return undefined; + throw new ProtocolError("The IPC stream ended inside a message."); + } + this.pending = chunk; + this.offset = 0; + if (chunk.byteLength === 0) continue; + } + const amount = Math.min(length - filled, this.pending.byteLength - this.offset); + output.set(this.pending.subarray(this.offset, this.offset + amount), filled); + this.offset += amount; + filled += amount; + } + return output; + } + + async end(): Promise { + assert(this.offset === this.pending.byteLength, "Unexpected bytes after the IPC stream."); + let chunk: Uint8Array | undefined; + while ((chunk = await this.chunk()) !== undefined) { + assert(chunk.byteLength === 0, "Unexpected bytes after the IPC stream."); + } + } + + close(): void { + if (this.closed) return; + this.closed = true; + this.operation.signal.removeEventListener("abort", this.abort); + void this.reader.cancel().catch(() => {}); + this.reader.releaseLock(); + } +} + +export function mediaType(response: Response): string { + return response.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() ?? ""; +} + +export async function readJson(response: Response, operation: Operation): Promise { + const source = new BodySource(response, operation, 1024 * 1024); + try { + const chunks: Uint8Array[] = []; + let chunk: Uint8Array | undefined; + while ((chunk = await source.chunk()) !== undefined) if (chunk.byteLength) chunks.push(chunk); + const bytes = new Uint8Array(source.bytes); + let offset = 0; + for (const part of chunks) { bytes.set(part, offset); offset += part.byteLength; } + try { + return JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)) as unknown; + } catch { + throw new ProtocolError("The server returned invalid JSON."); + } + } finally { + source.close(); + } +} diff --git a/typescript/execution/src/types.ts b/typescript/execution/src/types.ts new file mode 100644 index 000000000..b6bb8e254 --- /dev/null +++ b/typescript/execution/src/types.ts @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export const CONTRACT_VERSION = "0.1-draft" as const; +export const FOUNDATION_REVISION = "cc0d07099a3ef31e85af1fd93458a52186fae2c6" as const; +export const QUERY_SCHEMA_VERSION = "mcp-0.4-draft" as const; +export const ARROW_MEDIA_TYPE = "application/vnd.apache.arrow.stream" as const; +export const EXECUTION_METADATA_KEY = "org.apache.ossie.execution" as const; +export const COLUMN_METADATA_KEY = "org.apache.ossie.column" as const; + +export type JsonValue = null | boolean | number | string | JsonValue[] | { [key: string]: JsonValue }; +export type Extensions = Record; +export type Predicate = string | readonly string[]; +export interface OrderBy { + field: string; + direction?: "ASC" | "DESC"; + nulls?: "FIRST" | "LAST"; +} +interface QueryClauses { + where?: Predicate; + order_by?: readonly OrderBy[]; + limit?: number; +} +export interface AggregationQuery extends QueryClauses { + dimensions?: readonly string[]; + measures?: readonly string[]; + having?: Predicate; + fields?: never; +} +export interface ScalarQuery extends QueryClauses { + fields: readonly string[]; + dimensions?: never; + measures?: never; + having?: never; +} +export type Query = AggregationQuery | ScalarQuery; +export type QueryShape = "aggregation" | "scalar"; +export interface ExecuteRequest { + data_source_id: string; + query: Query; +} +export interface ModelIdentity { + id: string; + revision: string; +} +export interface Capabilities { + contract_version: typeof CONTRACT_VERSION; + foundation_revision: typeof FOUNDATION_REVISION; + query_schema_version: typeof QUERY_SCHEMA_VERSION; + result_formats: readonly ["arrow-stream"]; + query_shapes: readonly QueryShape[]; + features: readonly string[]; + max_wait_seconds: number; + extensions?: Extensions; +} +export interface Binding { + data_source_id: string; + model: ModelIdentity; + query_shapes: readonly QueryShape[]; + features: readonly string[]; + extensions?: Extensions; +} +export interface QueryError { + code: string; + message: string; + retryable: boolean; + replacement_query?: Query; + extensions?: Extensions; +} +export interface ProblemDetails { + type: string; + title: string; + status: number; + detail?: string; + error: QueryError; +} +export interface Progress { + fraction?: number; + phase?: string; +} +export interface Warning { + code: string; + message: string; +} +export type LogicalType = "String" | "Integer" | "Decimal" | "Float" | "Boolean" + | "Date" | "Time" | "DateTime" | "DateTimeTz"; +export interface Column { + column_id: string; + name: string; + logical_type: LogicalType; + nullable: boolean; + role?: "dimension" | "measure" | "field"; + model_ref?: string; + label?: string; + description?: string; + unit?: string; + currency?: string; + extensions?: Extensions; +} +export interface ResultPage { + page_index: number; + row_offset: number; + row_count: number; + byte_count: number; + url: string; +} +interface ResultManifestBase { + result_id: string; + format: "arrow-stream"; + row_count: number; + byte_count: number; + completeness: "complete" | "truncated"; + truncation_reason?: string; + expires_at: string; + columns: readonly Column[]; + extensions?: Extensions; +} +export type ResultManifest = ResultManifestBase & ( + | { url: string; pages?: never } + | { url?: never; pages: readonly ResultPage[] } +); +interface ExecutionBase { + contract_version: typeof CONTRACT_VERSION; + execution_id: string; + data_source_id: string; + model: ModelIdentity; + warnings: readonly Warning[]; + extensions?: Extensions; +} +export type Execution = ExecutionBase & ( + | { state: "queued" | "running" | "cancelling"; progress?: Progress; result?: never; error?: never } + | { state: "succeeded"; result: ResultManifest; progress?: never; error?: never } + | { state: "failed"; error: QueryError; progress?: never; result?: never } + | { state: "cancelled"; progress?: never; result?: never; error?: never } +); +export type SuccessfulExecution = Extract; +export interface ExecutionMetadata { + contract_version: typeof CONTRACT_VERSION; + execution_id: string; + result_id: string; + data_source_id: string; + model: ModelIdentity; + page_index?: number; + row_offset?: number; +} +export interface ClientOptions { + baseUrl: string; + fetch?: typeof globalThis.fetch; + getAccessToken?: () => string | Promise; + allowedResultOrigins?: readonly string[]; + allowInsecureLocalhost?: boolean; + pollIntervalMs?: number; + timeoutMs?: number; + maxResultBytes?: number; + maxResultRows?: number; +} +export interface RequestOptions { + signal?: AbortSignal; +} +export interface SubmitOptions extends RequestOptions { + waitSeconds?: number; +} +export interface WaitOptions extends RequestOptions { + onProgress?: (execution: Execution) => void; +} +export interface ExecuteOptions extends SubmitOptions, WaitOptions { + allowTruncated?: boolean; +} diff --git a/typescript/execution/src/validation.ts b/typescript/execution/src/validation.ts new file mode 100644 index 000000000..f2f6fbf67 --- /dev/null +++ b/typescript/execution/src/validation.ts @@ -0,0 +1,363 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ProtocolError } from "./errors.js"; +import { CONTRACT_VERSION, FOUNDATION_REVISION, QUERY_SCHEMA_VERSION } from "./types.js"; +import type { Binding, Capabilities, ExecuteRequest, Execution, ExecutionMetadata, + ProblemDetails, Query, QueryError, ResultManifest } from "./types.js"; + +type ObjectValue = Record; +const own = (value: ObjectValue, key: string): boolean => Object.hasOwn(value, key); + +export function assert(condition: unknown, message: string): asserts condition { + if (!condition) throw new ProtocolError(message); +} + +export function object(value: unknown, allowed: readonly string[], required: readonly string[]): ObjectValue { + assert(value !== null && typeof value === "object" && !Array.isArray(value), + "Expected a protocol object."); + const record = value as ObjectValue; + assert(Object.getPrototypeOf(record) === Object.prototype || Object.getPrototypeOf(record) === null, + "Expected a plain protocol object."); + assert(Object.keys(record).every(key => allowed.includes(key)), "Unknown core protocol field."); + assert(required.every(key => own(record, key)), "Missing required protocol field."); + return record; +} + +export function text(value: unknown): asserts value is string { + assert(typeof value === "string" && value.trim().length > 0, "Expected a nonempty string."); +} + +export function count(value: unknown): asserts value is number { + assert(typeof value === "number" && Number.isSafeInteger(value) && value >= 0, + "Expected a nonnegative safe integer."); +} + +function strings(value: unknown, nonempty = false): asserts value is string[] { + assert(Array.isArray(value) && (!nonempty || value.length > 0), "Expected a string array."); + for (const item of value) text(item); +} + +function enumeration(value: unknown, choices: readonly unknown[]): void { + assert(choices.includes(value), "Unsupported protocol value."); +} + +function boolean(value: unknown): void { + assert(typeof value === "boolean", "Expected a boolean."); +} + +function json(value: unknown, depth = 0, seen = new Set()): void { + assert(depth <= 32, "Extension nesting exceeds the supported limit."); + if (value === null || typeof value === "string" || typeof value === "boolean") return; + if (typeof value === "number") { + assert(Number.isFinite(value), "Extension numbers must be finite."); + return; + } + assert(typeof value === "object" && !seen.has(value), "Invalid JSON extension value."); + seen.add(value); + for (const item of Object.values(value)) json(item, depth + 1, seen); + seen.delete(value); +} + +function extensions(value: ObjectValue): void { + if (!own(value, "extensions")) return; + assert(value.extensions !== null && typeof value.extensions === "object" && !Array.isArray(value.extensions), + "Extensions must be a namespaced object."); + for (const [key, extension] of Object.entries(value.extensions)) { + text(key); + assert(extension !== null && typeof extension === "object" && !Array.isArray(extension), + "Each extension must be an object."); + json(extension); + } +} + +function predicate(value: unknown): void { + if (Array.isArray(value)) strings(value, true); + else text(value); +} + +export function validateQuery(value: unknown): Query { + const query = object(value, ["fields", "dimensions", "measures", "where", "having", "order_by", "limit"], []); + if (own(query, "fields")) { + assert(!own(query, "dimensions") && !own(query, "measures") && !own(query, "having"), + "Scalar fields cannot be combined with aggregation clauses."); + strings(query.fields, true); + } else { + if (own(query, "dimensions")) strings(query.dimensions); + if (own(query, "measures")) strings(query.measures); + assert((Array.isArray(query.dimensions) && query.dimensions.length > 0) + || (Array.isArray(query.measures) && query.measures.length > 0), + "An aggregation needs at least one dimension or measure."); + if (own(query, "having")) predicate(query.having); + } + if (own(query, "where")) predicate(query.where); + if (own(query, "limit")) count(query.limit); + if (own(query, "order_by")) { + assert(Array.isArray(query.order_by) && query.order_by.length > 0, "order_by must be a nonempty array."); + for (const item of query.order_by) { + const order = object(item, ["field", "direction", "nulls"], ["field"]); + text(order.field); + if (own(order, "direction")) enumeration(order.direction, ["ASC", "DESC"]); + if (own(order, "nulls")) enumeration(order.nulls, ["FIRST", "LAST"]); + } + } + return value as Query; +} + +export function validateRequest(value: unknown): ExecuteRequest { + const request = object(value, ["data_source_id", "query"], ["data_source_id", "query"]); + text(request.data_source_id); + validateQuery(request.query); + return value as ExecuteRequest; +} + +function model(value: unknown): void { + const identity = object(value, ["id", "revision"], ["id", "revision"]); + text(identity.id); + text(identity.revision); +} + +function shapes(value: unknown): void { + strings(value, true); + value.forEach(item => enumeration(item, ["scalar", "aggregation"])); + assert(new Set(value).size === value.length, "Duplicate query shapes."); +} + +export function validateCapabilities(value: unknown): Capabilities { + const result = object(value, ["contract_version", "foundation_revision", "query_schema_version", + "result_formats", "query_shapes", "features", "max_wait_seconds", "extensions"], + ["contract_version", "foundation_revision", "query_schema_version", "result_formats", + "query_shapes", "features", "max_wait_seconds"]); + enumeration(result.contract_version, [CONTRACT_VERSION]); + enumeration(result.foundation_revision, [FOUNDATION_REVISION]); + enumeration(result.query_schema_version, [QUERY_SCHEMA_VERSION]); + assert(Array.isArray(result.result_formats) && result.result_formats.length === 1 + && result.result_formats[0] === "arrow-stream", "Unsupported result formats."); + shapes(result.query_shapes); + strings(result.features); + assert(new Set(result.features).size === result.features.length, "Duplicate features."); + count(result.max_wait_seconds); + assert(result.max_wait_seconds <= 60, "Unsupported maximum wait."); + extensions(result); + return value as Capabilities; +} + +export function validateBinding(value: unknown): Binding { + const result = object(value, ["data_source_id", "model", "query_shapes", "features", "extensions"], + ["data_source_id", "model", "query_shapes", "features"]); + text(result.data_source_id); + model(result.model); + shapes(result.query_shapes); + strings(result.features); + assert(new Set(result.features).size === result.features.length, "Duplicate features."); + extensions(result); + return value as Binding; +} + +export function validateQueryError(value: unknown): QueryError { + const error = object(value, ["code", "message", "retryable", "replacement_query", "extensions"], + ["code", "message", "retryable"]); + text(error.code); + text(error.message); + boolean(error.retryable); + if (own(error, "replacement_query")) validateQuery(error.replacement_query); + extensions(error); + return value as QueryError; +} + +export function validateProblem(value: unknown, status: number): ProblemDetails { + const problem = object(value, ["type", "title", "status", "detail", "error"], + ["type", "title", "status", "error"]); + text(problem.type); + text(problem.title); + count(problem.status); + assert(problem.status === status && status >= 400 && status <= 599, "Problem status mismatch."); + if (own(problem, "detail")) text(problem.detail); + validateQueryError(problem.error); + return value as ProblemDetails; +} + +function timestamp(value: unknown): void { + text(value); + const match = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d+)?(?:Z|([+-])(\d{2}):(\d{2}))$/i.exec(value); + assert(match, "Expected an RFC3339 timestamp."); + const [, y, m, d, h, minute, second, , oh = "0", om = "0"] = match; + const year = Number(y), month = Number(m), day = Number(d); + const leap = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0); + const days = [31, leap ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; + assert(month >= 1 && month <= 12 && day >= 1 && day <= days[month - 1]! + && Number(h) <= 23 && Number(minute) <= 59 && Number(second) <= 59 + && Number(oh) <= 23 && Number(om) <= 59 && Number.isFinite(Date.parse(value)), + "Invalid RFC3339 timestamp."); +} + +function resultUrl(value: unknown): void { + text(value); + assert(/^https?:\/\//i.test(value) && !/[\u0000-\u0020\u007f\\]/.test(value), + "Result URLs must be absolute HTTP(S) URLs."); + try { + const parsed = new URL(value); + const authority = /^https?:\/\/([^/?#]*)/i.exec(value)?.[1]; + assert(!parsed.username && !parsed.password && !authority?.includes("@") + && !parsed.href.includes("#"), "Result URL credentials and fragments are forbidden."); + } catch { + throw new ProtocolError("Invalid or unsafe result URL."); + } +} + +export function validateManifest(value: unknown): ResultManifest { + const manifest = object(value, ["result_id", "format", "url", "pages", "row_count", "byte_count", + "completeness", "truncation_reason", "expires_at", "columns", "extensions"], + ["result_id", "format", "row_count", "byte_count", "completeness", "expires_at", "columns"]); + text(manifest.result_id); + enumeration(manifest.format, ["arrow-stream"]); + count(manifest.row_count); + count(manifest.byte_count); + assert(own(manifest, "url") !== own(manifest, "pages"), "Expected exactly one result delivery mode."); + if (own(manifest, "url")) resultUrl(manifest.url); + else { + assert(Array.isArray(manifest.pages) && manifest.pages.length > 0, "Expected nonempty result pages."); + let rows = 0, bytes = 0; + for (let index = 0; index < manifest.pages.length; index++) { + const page = object(manifest.pages[index], ["page_index", "row_offset", "row_count", "byte_count", "url"], + ["page_index", "row_offset", "row_count", "byte_count", "url"]); + count(page.page_index); + count(page.row_offset); + count(page.row_count); + count(page.byte_count); + resultUrl(page.url); + assert(page.page_index === index && page.row_offset === rows, "Result pages are not contiguous and ordered."); + assert(page.byte_count > 0, "Every page must include a self-contained Arrow schema."); + assert(page.row_count > 0 || manifest.pages.length === 1, "Only a single empty-result page may have zero rows."); + rows += page.row_count; + bytes += page.byte_count; + count(rows); + count(bytes); + } + assert(rows === manifest.row_count && bytes === manifest.byte_count, "Result page totals do not match the manifest."); + } + enumeration(manifest.completeness, ["complete", "truncated"]); + if (manifest.completeness === "truncated") { + assert(own(manifest, "truncation_reason"), "A truncated result must give a truncation reason."); + } + if (own(manifest, "truncation_reason")) { + text(manifest.truncation_reason); + assert(manifest.completeness === "truncated", "A complete result cannot have a truncation reason."); + } + timestamp(manifest.expires_at); + assert(Array.isArray(manifest.columns) && manifest.columns.length > 0, "Expected result columns."); + const ids = new Set(); + for (const item of manifest.columns) { + const column = object(item, ["column_id", "name", "logical_type", "nullable", "role", "model_ref", + "label", "description", "unit", "currency", "extensions"], ["column_id", "name", "logical_type", "nullable"]); + text(column.column_id); + assert(!ids.has(column.column_id), "Duplicate result column identifiers."); + ids.add(column.column_id); + text(column.name); + enumeration(column.logical_type, ["String", "Integer", "Decimal", "Float", "Boolean", + "Date", "Time", "DateTime", "DateTimeTz"]); + boolean(column.nullable); + if (own(column, "role")) enumeration(column.role, ["dimension", "measure", "field"]); + for (const key of ["model_ref", "label", "description", "unit", "currency"]) { + if (own(column, key)) text(column[key]); + } + extensions(column); + } + extensions(manifest); + return value as ResultManifest; +} + +export function validateExecution(value: unknown): Execution { + const execution = object(value, ["contract_version", "execution_id", "data_source_id", "model", + "warnings", "extensions", "state", "progress", "result", "error"], + ["contract_version", "execution_id", "data_source_id", "model", "warnings", "state"]); + enumeration(execution.contract_version, [CONTRACT_VERSION]); + text(execution.execution_id); + text(execution.data_source_id); + model(execution.model); + assert(Array.isArray(execution.warnings), "Expected execution warnings."); + for (const item of execution.warnings) { + const warning = object(item, ["code", "message"], ["code", "message"]); + text(warning.code); + text(warning.message); + } + enumeration(execution.state, ["queued", "running", "cancelling", "succeeded", "failed", "cancelled"]); + if (execution.state === "succeeded") validateManifest(execution.result); + else assert(!own(execution, "result"), "Only succeeded executions can have a result."); + if (execution.state === "failed") validateQueryError(execution.error); + else assert(!own(execution, "error"), "Only failed executions can have an error."); + if (["queued", "running", "cancelling"].includes(execution.state as string)) { + if (own(execution, "progress")) { + const progress = object(execution.progress, ["fraction", "phase"], []); + if (own(progress, "fraction")) { + assert(typeof progress.fraction === "number" && Number.isFinite(progress.fraction) + && progress.fraction >= 0 && progress.fraction <= 1, "Invalid progress fraction."); + } + if (own(progress, "phase")) text(progress.phase); + } + } else assert(!own(execution, "progress"), "Terminal executions cannot have progress."); + extensions(execution); + return value as Execution; +} + +export function validateMetadata(value: unknown): ExecutionMetadata { + const metadata = object(value, ["contract_version", "execution_id", "result_id", "data_source_id", "model", + "page_index", "row_offset"], + ["contract_version", "execution_id", "result_id", "data_source_id", "model"]); + enumeration(metadata.contract_version, [CONTRACT_VERSION]); + text(metadata.execution_id); + text(metadata.result_id); + text(metadata.data_source_id); + model(metadata.model); + assert(own(metadata, "page_index") === own(metadata, "row_offset"), "Page metadata requires both index and offset."); + if (own(metadata, "page_index")) { + count(metadata.page_index); + count(metadata.row_offset); + } + return value as ExecutionMetadata; +} + +export function terminal(execution: Execution): boolean { + return ["succeeded", "failed", "cancelled"].includes(execution.state); +} + +/** Canonical comparison is independent of JSON object key order. */ +export function equivalent(left: unknown, right: unknown): boolean { + if (left === right) return true; + if (left === null || right === null || typeof left !== "object" || typeof right !== "object") return false; + if (Array.isArray(left) !== Array.isArray(right)) return false; + const a = left as ObjectValue, b = right as ObjectValue; + const keys = Object.keys(a); + return keys.length === Object.keys(b).length && keys.every(key => own(b, key) && equivalent(a[key], b[key])); +} + +export function transition(previous: Execution, next: Execution): void { + assert(previous.execution_id === next.execution_id && previous.data_source_id === next.data_source_id + && equivalent(previous.model, next.model), "Execution identity changed."); + const allowed: Record = { + queued: ["queued", "running", "cancelling", "succeeded", "failed", "cancelled"], + running: ["running", "cancelling", "succeeded", "failed", "cancelled"], + cancelling: ["cancelling", "succeeded", "failed", "cancelled"], + succeeded: ["succeeded"], failed: ["failed"], cancelled: ["cancelled"], + }; + assert(allowed[previous.state].includes(next.state), "Execution lifecycle regressed."); + if (terminal(previous)) { + assert(equivalent(previous, next), "An immutable terminal execution changed."); + } +} diff --git a/typescript/execution/test/arrow-metadata.test.mjs b/typescript/execution/test/arrow-metadata.test.mjs new file mode 100644 index 000000000..9af574043 --- /dev/null +++ b/typescript/execution/test/arrow-metadata.test.mjs @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from "node:assert/strict"; +import test from "node:test"; +import { Message, MessageHeader, Table, tableToIPC } from "apache-arrow"; +import { OssieClient, ProtocolError } from "../dist/index.js"; +import { ORIGIN, arrowFixture, arrowResponse, collect } from "./fixtures.mjs"; + +function metadataFrame(fixture, type) { + let offset = 0; + while (offset + 8 <= fixture.bytes.length) { + const prefix = new DataView(fixture.bytes.buffer, fixture.bytes.byteOffset + offset); + const prefixSize = prefix.getInt32(0, true) === -1 ? 8 : 4; + const size = prefix.getInt32(prefixSize - 4, true); + if (size === 0) break; + const bytes = fixture.bytes.subarray(offset + prefixSize, offset + prefixSize + size); + const message = Message.decode(bytes); + if ((type === MessageHeader.Schema && message.isSchema()) + || (type === MessageHeader.RecordBatch && message.isRecordBatch())) return bytes; + offset += prefixSize + size + message.bodyLength; + } + throw new Error("Fixture has no requested metadata frame"); +} + +function flatAccess(bytes) { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const reference = location => location + view.getUint32(location, true); + const field = (table, index) => { + const vtable = table - view.getInt32(table, true); + assert.ok(4 + index * 2 < view.getUint16(vtable, true)); + const offset = view.getUint16(vtable + 4 + index * 2, true); + assert.ok(offset > 0, `Fixture field ${index} must be present`); + return table + offset; + }; + const root = view.getUint32(0, true); + const header = reference(field(root, 2)); + return { view, reference, field, root, header }; +} + +const mutations = [ + ["record-batch node vector allocation", MessageHeader.RecordBatch, a => + a.view.setUint32(a.reference(a.field(a.header, 1)), 200_000, true)], + ["record-batch buffer vector allocation", MessageHeader.RecordBatch, a => + a.view.setUint32(a.reference(a.field(a.header, 2)), 200_000, true)], + ["in-bounds node count inconsistent with the flat schema", MessageHeader.RecordBatch, a => + a.view.setUint32(a.reference(a.field(a.header, 1)), 1, true)], + ["schema fields vector allocation", MessageHeader.Schema, a => + a.view.setUint32(a.reference(a.field(a.header, 1)), 200_000, true)], + ["custom metadata vector allocation", MessageHeader.Schema, a => + a.view.setUint32(a.reference(a.field(a.header, 2)), 200_000, true)], + ["metadata string allocation", MessageHeader.Schema, a => { + const vector = a.reference(a.field(a.header, 2)); + const entry = a.reference(vector + 4); + a.view.setUint32(a.reference(a.field(entry, 1)), 200_000, true); + }], + ["unterminated metadata string", MessageHeader.Schema, a => { + const vector = a.reference(a.field(a.header, 2)); + const entry = a.reference(vector + 4); + const value = a.reference(a.field(entry, 1)); + a.view.setUint8(value + 4 + a.view.getUint32(value, true), 1); + }], + ["schema field reference outside metadata", MessageHeader.Schema, a => { + const vector = a.reference(a.field(a.header, 1)); + a.view.setUint32(vector + 4, 0xffff_fffc, true); + }], + ["null message header reference", MessageHeader.Schema, a => + a.view.setUint32(a.field(a.root, 2), 0, true)], + ["vtable outside metadata", MessageHeader.Schema, a => + a.view.setInt32(a.root, 0x7fff_fffc, true)], +]; + +for (const [name, type, mutate] of mutations) { + test(`predecode bounds reject ${name} before Arrow allocation`, async context => { + const fixture = arrowFixture(); + const metadata = metadataFrame(fixture, type); + mutate(flatAccess(metadata)); + const originalDecode = Message.decode; + let unsafeDecode = false; + context.mock.method(Message, "decode", bytes => { + if (bytes.length === metadata.length && bytes.every((value, index) => value === metadata[index])) { + unsafeDecode = true; + throw new Error("Unsafe metadata reached Arrow decoder"); + } + return originalDecode.call(Message, bytes); + }); + const client = new OssieClient({ + baseUrl: ORIGIN, fetch: async () => arrowResponse(fixture.bytes), + }); + await assert.rejects(collect(client.readBatches(fixture.succeeded)), ProtocolError); + assert.equal(unsafeDecode, false, "Reject malformed lengths before allocating decoded vectors"); + }); +} + +test("shared metadata strings cannot amplify decoding beyond the configured budget", async context => { + const fixture = arrowFixture(); + fixture.schema.metadata.set("example.test.large", "x".repeat(4096)); + for (let index = 0; index < 32; index++) fixture.schema.metadata.set(`example.test.${index}`, "small"); + fixture.bytes = tableToIPC(new Table(fixture.schema, fixture.batches), "stream"); + fixture.succeeded.result.byte_count = fixture.bytes.length; + const metadata = metadataFrame(fixture, MessageHeader.Schema); + const a = flatAccess(metadata); + const vector = a.reference(a.field(a.header, 2)); + const length = a.view.getUint32(vector, true); + const entries = Array.from({ length }, (_, index) => a.reference(vector + 4 + index * 4)); + const values = entries.map(entry => a.reference(a.field(entry, 1))); + const largest = values.reduce((best, value) => + a.view.getUint32(value, true) > a.view.getUint32(best, true) ? value : best); + let aliases = 0; + for (const entry of entries) { + const pointer = a.field(entry, 1); + if (pointer < largest && a.reference(pointer) !== largest) { + a.view.setUint32(pointer, largest - pointer, true); + aliases++; + } + } + assert.ok(aliases >= 8, "Fixture must exercise repeated valid forward string references"); + const originalDecode = Message.decode; + let unsafeDecode = false; + context.mock.method(Message, "decode", bytes => { + if (bytes.length === metadata.length && bytes.every((value, index) => value === metadata[index])) { + unsafeDecode = true; + throw new Error("Expanded metadata reached Arrow decoder"); + } + return originalDecode.call(Message, bytes); + }); + const client = new OssieClient({ + baseUrl: ORIGIN, fetch: async () => arrowResponse(fixture.bytes), maxResultBytes: fixture.bytes.length * 2, + }); + await assert.rejects(collect(client.readBatches(fixture.succeeded)), error => error.code === "RESULT_LIMIT_EXCEEDED"); + assert.equal(unsafeDecode, false); +}); diff --git a/typescript/execution/test/arrow.test.mjs b/typescript/execution/test/arrow.test.mjs new file mode 100644 index 000000000..4afe9eca9 --- /dev/null +++ b/typescript/execution/test/arrow.test.mjs @@ -0,0 +1,421 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from "node:assert/strict"; +import test from "node:test"; +import { + Bool, DateDay, Decimal, Dictionary, Field, Float64, Int32, Int64, List, + TimeMicrosecond, TimestampMicrosecond, TimestampNanosecond, Utf8, + Table, makeData, tableToIPC, +} from "apache-arrow"; +import { + ARROW_MEDIA_TYPE, COLUMN_METADATA_KEY, OssieClient, ProtocolError, +} from "../dist/index.js"; +import { + MODEL, ORIGIN, arrowFixture, arrowResponse, assertSafeError, + chunksResponse, collect, mockFetch, rewriteFirstBatchHeader, +} from "./fixtures.mjs"; + +const client = (fetch, options = {}) => new OssieClient({ + baseUrl: ORIGIN, fetch, timeoutMs: 1000, ...options, +}); +const read = (fixture, options = {}, response) => { + const mock = mockFetch(() => response ?? arrowResponse(fixture.bytes)); + return collect(client(mock.fetch, options).readBatches(fixture.succeeded)); +}; + +test("real Arrow IPC survives arbitrary network chunk boundaries and retains schema metadata", async () => { + const fixture = arrowFixture(); + const batches = await read(fixture, {}, chunksResponse(fixture.bytes, 1)); + assert.equal(batches.reduce((n, batch) => n + batch.numRows, 0), 2); + assert.equal(batches[0].getChildAt(1).get(0), 9007199254740993n); + assert.equal(batches[0].getChildAt(1).get(1), null); + assert.deepEqual(batches[0].schema.metadata, fixture.schema.metadata); +}); + +test("zero-row result is a valid typed table, not absent data", async () => { + const fixture = arrowFixture({ rows: [] }); + const batches = await read(fixture); + assert.equal(batches.reduce((count, batch) => count + batch.numRows, 0), 0); +}); + +test("schema-only zero-row stream preserves its typed columns during execute", async () => { + const fixture = arrowFixture({ rows: [] }); + fixture.bytes = tableToIPC(new Table(fixture.schema, []), "stream"); + fixture.succeeded.result.byte_count = fixture.bytes.length; + const mock = mockFetch(call => call.method === "POST" + ? new Response(JSON.stringify(fixture.succeeded), { headers: { "Content-Type": "application/json" } }) + : arrowResponse(fixture.bytes)); + const result = await client(mock.fetch).execute({ data_source_id: "sales", query: { fields: ["region"], limit: 0 } }); + assert.equal(result.table.numRows, 0); + assert.equal(result.table.numCols, 2); + assert.deepEqual(result.table.schema.metadata, fixture.schema.metadata); +}); + +test("duplicate display names and labels are allowed when column ids are stable and unique", async () => { + const fixture = arrowFixture({ columns: [ + { column_id: "first", name: "value", label: "Value", logical_type: "String", nullable: true }, + { column_id: "second", name: "value", label: "Value", logical_type: "Integer", nullable: true }, + ] }); + const batches = await read(fixture); + assert.equal(batches[0].schema.fields[0].name, "value"); + assert.equal(batches[0].schema.fields[1].name, "value"); + assert.equal(batches[0].getChildAt(1).get(0), 9007199254740993n); +}); + +test("precision and literal nulls remain Arrow data, not guessed JSON/JavaScript scalars", async () => { + const columns = [ + { column_id: "integer", name: "integer", logical_type: "Integer", nullable: true }, + { column_id: "decimal", name: "decimal", logical_type: "Decimal", nullable: true }, + { column_id: "date", name: "date", logical_type: "Date", nullable: true }, + { column_id: "timestamp", name: "timestamp", logical_type: "DateTime", nullable: true }, + { column_id: "literal", name: "literal", logical_type: "String", nullable: true }, + ]; + const types = [new Int64(), new Decimal(2, 38, 128), new DateDay(), new TimestampMicrosecond(), new Utf8()]; + const fields = columns.map((column, index) => new Field(column.name, types[index], true, + new Map([[COLUMN_METADATA_KEY, JSON.stringify({ column_id: column.column_id })]]))); + const decimal = new Uint32Array([0xffffffff, 0x00200000, 0, 0]); + const fixture = arrowFixture({ + columns, fields, + rows: [ + [9007199254740993n, decimal, new Date("2024-02-29T00:00:00Z"), 1709164800123.456, "null"], + [null, null, null, null, null], + ], + }); + + const [batch] = await read(fixture); + assert.equal(batch.getChildAt(0).get(0), 9007199254740993n); + assert.deepEqual(Array.from(batch.getChildAt(1).get(0)), Array.from(decimal)); + assert.equal(batch.schema.fields[1].type.scale, 2); + assert.equal(batch.schema.fields[1].type.precision, 38); + for (const index of [2, 3]) { + assert.equal(batch.schema.fields[index].type.toString(), fixture.schema.fields[index].type.toString()); + const actual = batch.getChildAt(index).data[0].values; + const original = fixture.batches[0].getChildAt(index).data[0].values; + assert.deepEqual(actual, original.subarray(0, actual.length)); + } + assert.equal(batch.getChildAt(4).get(0), "null"); + for (let index = 0; index < columns.length; index++) assert.equal(batch.getChildAt(index).get(1), null); +}); + +test("Boolean, Float, Time and timezone-bearing timestamps keep their physical Arrow types", async () => { + const definitions = [ + ["Boolean", new Bool(), true], + ["Float", new Float64(), 1.125], + ["Time", new TimeMicrosecond(), 43200123456n], + ["DateTimeTz", new TimestampMicrosecond("UTC"), 1709164800123.456], + ]; + const columns = definitions.map(([logical_type], index) => ({ + column_id: `typed-${index}`, name: `typed-${index}`, logical_type, nullable: true, + })); + const fields = definitions.map(([, type], index) => new Field(columns[index].name, type, true, + new Map([[COLUMN_METADATA_KEY, JSON.stringify({ column_id: columns[index].column_id })]]))); + const fixture = arrowFixture({ + columns, fields, + rows: [definitions.map(([, , value]) => value), definitions.map(() => null)], + }); + const [batch] = await read(fixture); + assert.equal(batch.getChildAt(0).get(0), true); + assert.equal(batch.getChildAt(1).get(0), 1.125); + for (let index = 0; index < columns.length; index++) { + assert.equal(batch.schema.fields[index].type.toString(), fields[index].type.toString()); + assert.equal(batch.getChildAt(index).get(1), null); + } + assert.equal(batch.getChildAt(2).data[0].values[0], 43200123456n); + assert.equal(batch.schema.fields[3].type.timezone, "UTC"); +}); + +test("nanosecond timestamps preserve exact Arrow buffers without promising lossless convenience accessors", async () => { + const nanoseconds = 1709164800123456789n; + const integer = 9007199254740993n; + const types = [new TimestampNanosecond("UTC"), new Int64()]; + const columns = [ + { column_id: "instant", name: "instant", logical_type: "DateTimeTz", nullable: true }, + { column_id: "integer", name: "integer", logical_type: "Integer", nullable: true }, + ]; + const fields = columns.map((column, index) => new Field(column.name, types[index], true, + new Map([[COLUMN_METADATA_KEY, JSON.stringify({ column_id: column.column_id })]]))); + const columnData = [nanoseconds, integer].map((value, index) => makeData({ + type: types[index], length: 2, data: BigInt64Array.of(value, 0n), + nullBitmap: Uint8Array.of(1), nullCount: 1, + })); + const fixture = arrowFixture({ + fields, columns, columnData, rows: [[null, null], [null, null]], + }); + const [batch] = await read(fixture); + // Timestamp get() may convert to a millisecond Number; inspect the exact native buffer. + assert.equal(batch.getChildAt(0).data[0].values[0], nanoseconds); + assert.equal(batch.getChildAt(0).type.unit, types[0].unit); + assert.equal(batch.getChildAt(0).get(1), null); + assert.equal(batch.getChildAt(1).get(0), integer); + assert.equal(batch.getChildAt(1).get(1), null); +}); + +for (const [name, metadata] of [ + ["execution id", { execution_id: "another-execution" }], + ["result id", { result_id: "another-result" }], + ["data source", { data_source_id: "another-source" }], + ["model revision", { model: { ...MODEL, revision: "another-revision" } }], + ["contract version", { contract_version: "unknown-version" }], + ["unknown metadata field", { extra: true }], +]) { + test(`Arrow schema metadata must match envelope: ${name}`, async () => { + const good = arrowFixture(); + const value = JSON.parse(good.schema.metadata.values().next().value); + await assert.rejects(read(arrowFixture({ metadata: { ...value, ...metadata } })), ProtocolError); + }); +} + +test("Arrow schema metadata must be valid JSON", async () => { + await assert.rejects(read(arrowFixture({ metadata: "{bad" })), ProtocolError); +}); + +test("Arrow column metadata id must match its manifest column", async () => { + await assert.rejects(read(arrowFixture({ columnMetadata: [{ column_id: "wrong-column" }] })), ProtocolError); +}); + +test("Arrow column metadata rejects unknown core properties", async () => { + await assert.rejects(read(arrowFixture({ + columnMetadata: [{ column_id: "col-region", inferred: true }], + })), ProtocolError); +}); + +test("missing required schema or field metadata is rejected", async () => { + for (const missing of ["schema", "field"]) { + const fixture = arrowFixture(); + if (missing === "schema") fixture.schema.metadata.clear(); + else fixture.schema.fields[0].metadata.clear(); + fixture.bytes = tableToIPC(fixture.table, "stream"); + fixture.succeeded.result.byte_count = fixture.bytes.length; + await assert.rejects(read(fixture), ProtocolError); + } +}); + +test("dictionary and nested Arrow types are outside the flat result profile", async () => { + for (const type of [ + new Dictionary(new Utf8(), new Int32(), 0, false), + new List(new Field("item", new Utf8(), true)), + ]) { + const normal = arrowFixture(); + const first = new Field("region", type, true, normal.schema.fields[0].metadata); + const value = type instanceof List ? ["West"] : "West"; + const fixture = arrowFixture({ + fields: [first, normal.schema.fields[1]], rows: [[value, 1n]], + }); + await assert.rejects(read(fixture), ProtocolError); + } +}); + +test("Arrow extension types cannot masquerade as supported primitive data", async () => { + const normal = arrowFixture(); + normal.schema.fields[0].metadata.set("ARROW:extension:name", "example.opaque"); + const fixture = arrowFixture({ fields: normal.schema.fields }); + await assert.rejects(read(fixture), ProtocolError); +}); + +for (const [name, mutate] of [ + ["name", columns => { columns[0].name = "different"; }], + ["logical type", columns => { columns[0].logical_type = "Integer"; }], + ["nullability", columns => { columns[0].nullable = false; }], + ["column count", columns => { columns.pop(); }], +]) { + test(`Arrow field ${name} must match result columns`, async () => { + const fixture = arrowFixture(); + mutate(fixture.succeeded.result.columns); + await assert.rejects(read(fixture), ProtocolError); + }); +} + +for (const [name, headers] of [ + ["JSON MIME", { "Content-Type": "application/json" }], + ["file IPC MIME", { "Content-Type": "application/vnd.apache.arrow.file" }], +]) { + test(`result refuses ${name}`, async () => { + const fixture = arrowFixture(); + await assert.rejects(read(fixture, {}, arrowResponse(fixture.bytes, headers)), ProtocolError); + }); +} + +test("HTTP content decoding precedes byte_count checks; HTTP compression is not IPC compression", async () => { + const fixture = arrowFixture(); + const batches = await read(fixture, {}, arrowResponse(fixture.bytes, { + "Content-Encoding": "gzip", "Content-Length": "100", + })); + assert.equal(batches[0].numRows, 2); +}); + +for (const [name, change] of [ + ["too few rows", result => { result.row_count--; }], + ["too many rows", result => { result.row_count++; }], + ["too few bytes", result => { result.byte_count--; }], + ["too many bytes", result => { result.byte_count++; }], +]) { + test(`exact result counts reject ${name}`, async () => { + const fixture = arrowFixture(); + change(fixture.succeeded.result); + await assert.rejects(read(fixture), ProtocolError); + }); +} + +test("malformed Arrow bytes are rejected even when byte count matches", async () => { + const fixture = arrowFixture(); + fixture.bytes = new TextEncoder().encode("not an Arrow stream"); + fixture.succeeded.result.byte_count = fixture.bytes.length; + await assert.rejects(read(fixture), ProtocolError); +}); + +for (const [name, mutate] of [ + ["IPC compression declaration", header => { header._compression = { type: 0, method: 0 }; }], + ["field length mismatch", header => { header.nodes[0].length++; }], + ["null count mismatch", header => { header.nodes[0].nullCount = 0; }], + ["missing validity bytes", header => { header.buffers[0].length = 0; }], + ["out-of-bounds buffer", header => { header.buffers[1].length = 1_000_000; }], + ["unaligned buffer offset", header => { header.buffers[1].offset++; }], +]) { + test(`malformed or unsupported Arrow batch rejects ${name}`, async () => { + const fixture = rewriteFirstBatchHeader(arrowFixture(), mutate); + await assert.rejects(read(fixture), ProtocolError); + }); +} + +test("Arrow file encoding is rejected even with stream MIME", async () => { + const fixture = arrowFixture(); + fixture.bytes = tableToIPC(fixture.table, "file"); + fixture.succeeded.result.byte_count = fixture.bytes.length; + await assert.rejects(read(fixture), ProtocolError); +}); + +test("a second schema cannot replace metadata mid-result", async () => { + const fixture = arrowFixture(); + const bytes = new Uint8Array(fixture.bytes.length * 2 - 8); + bytes.set(fixture.bytes.subarray(0, -8)); + bytes.set(fixture.bytes, fixture.bytes.length - 8); + fixture.bytes = bytes; + fixture.succeeded.result.byte_count = bytes.length; + fixture.succeeded.result.row_count *= 2; + await assert.rejects(read(fixture), ProtocolError); +}); + +test("trailing bytes after the IPC end marker are rejected", async () => { + const fixture = arrowFixture(); + const bytes = new Uint8Array(fixture.bytes.length + 8); + bytes.set(fixture.bytes); + bytes.fill(42, fixture.bytes.length); + fixture.bytes = bytes; + fixture.succeeded.result.byte_count = bytes.length; + await assert.rejects(read(fixture), ProtocolError); +}); + +for (const removeBytes of [12, 40]) { + test(`truncated IPC missing ${removeBytes} final record bytes is not accepted as a complete result`, async () => { + const fixture = arrowFixture(); + fixture.bytes = fixture.bytes.slice(0, -removeBytes); + fixture.succeeded.result.byte_count = fixture.bytes.length; + await assert.rejects(read(fixture), ProtocolError); + }); +} + +test("truncated query result requires explicit opt-in independent of well-formed IPC", async () => { + const fixture = arrowFixture({ manifest: { completeness: "truncated", truncation_reason: "server-row-limit" } }); + const mock = mockFetch(() => arrowResponse(fixture.bytes)); + const api = client(mock.fetch); + await assert.rejects(collect(api.readBatches(fixture.succeeded)), assertSafeError); + assert.equal(mock.calls.length, 0); + const batches = await collect(api.readBatches(fixture.succeeded, { allowTruncated: true })); + assert.equal(batches[0].numRows, 2); +}); + +test("configured result size limits reject manifests before fetching", async () => { + const fixture = arrowFixture(); + for (const limits of [{ maxResultBytes: fixture.bytes.length - 1 }, { maxResultRows: 1 }]) { + const mock = mockFetch(() => arrowResponse(fixture.bytes)); + await assert.rejects(collect(client(mock.fetch, limits).readBatches(fixture.succeeded)), assertSafeError); + assert.equal(mock.calls.length, 0); + } +}); + +test("default byte/row limits reject oversized manifests before fetching", async () => { + for (const oversized of [{ byte_count: 64 * 1024 * 1024 + 1 }, { row_count: 1_000_001 }]) { + const fixture = arrowFixture({ manifest: oversized }); + const mock = mockFetch(() => arrowResponse(fixture.bytes)); + await assert.rejects(collect(client(mock.fetch).readBatches(fixture.succeeded)), assertSafeError); + assert.equal(mock.calls.length, 0); + } +}); + +test("actual streamed bytes cannot evade limits through an underreported manifest", async () => { + const fixture = arrowFixture(); + const limit = fixture.bytes.length - 16; + fixture.succeeded.result.byte_count = limit; + await assert.rejects(read(fixture, { maxResultBytes: limit }, chunksResponse(fixture.bytes)), assertSafeError); +}); + +test("readBatches yields the first batch before consuming the complete result; early break cancels body", async () => { + const fixture = arrowFixture({ + batchRows: Array.from({ length: 20 }, (_, index) => + Array.from({ length: 128 }, () => [`batch-${index}-${"x".repeat(64)}`, BigInt(index)])), + }); + let delivered = 0; + let cancelled = false; + const mock = mockFetch(() => chunksResponse(fixture.bytes, 97, { + onPull(offset) { delivered = Math.min(offset + 97, fixture.bytes.length); }, + onCancel() { cancelled = true; }, + })); + for await (const batch of client(mock.fetch).readBatches(fixture.succeeded)) { + assert.equal(batch.numRows, 128); + assert.ok(delivered < fixture.bytes.length / 2, "Result bytes are not fully accumulated before first yield"); + break; + } + assert.ok(cancelled, "Early iterator return releases/cancels the underlying response body"); +}); + +test("result timeout covers a stalled Arrow response body", { timeout: 2000 }, async () => { + const fixture = arrowFixture(); + let cancelled = false; + const mock = mockFetch(() => new Response(new ReadableStream({ + start(controller) { controller.enqueue(fixture.bytes.slice(0, 12)); }, + cancel() { cancelled = true; }, + }), { headers: { "Content-Type": ARROW_MEDIA_TYPE } })); + await assert.rejects(collect(client(mock.fetch, { timeoutMs: 30 }).readBatches(fixture.succeeded)), assertSafeError); + assert.ok(cancelled); +}); + +test("Arrow body transport failures are typed and do not expose signed result details", async () => { + const fixture = arrowFixture(); + const secret = "ARROW_STREAM_SYNTHETIC_SECRET"; + const response = new Response(new ReadableStream({ + start(controller) { controller.error(new Error(`socket failed https://host.invalid/?token=${secret}`)); }, + }), { headers: { "Content-Type": ARROW_MEDIA_TYPE } }); + await assert.rejects(read(fixture, {}, response), error => assertSafeError(error, [secret])); +}); + +test("result abort covers a stalled Arrow response body", { timeout: 2000 }, async () => { + const fixture = arrowFixture(); + const controller = new AbortController(); + let cancelled = false; + const mock = mockFetch(() => new Response(new ReadableStream({ + start(stream) { + stream.enqueue(fixture.bytes.slice(0, 12)); + setTimeout(() => controller.abort(), 20); + }, + cancel() { cancelled = true; }, + }), { headers: { "Content-Type": ARROW_MEDIA_TYPE } })); + await assert.rejects(collect(client(mock.fetch).readBatches(fixture.succeeded, { signal: controller.signal })), assertSafeError); + assert.ok(cancelled); +}); diff --git a/typescript/execution/test/browser.test.mjs b/typescript/execution/test/browser.test.mjs new file mode 100644 index 000000000..3f219a7c0 --- /dev/null +++ b/typescript/execution/test/browser.test.mjs @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from "node:assert/strict"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; +import { build } from "esbuild"; + +test("public package bundles for a browser without Node builtins, externals or polyfills", async () => { + const packageDirectory = fileURLToPath(new URL("../", import.meta.url)); + const result = await build({ + absWorkingDir: packageDirectory, + stdin: { + contents: "import * as Ossie from './dist/index.js'; globalThis.Ossie = Ossie;", + resolveDir: packageDirectory, sourcefile: "browser-validation.mjs", loader: "js", + }, + bundle: true, platform: "browser", format: "iife", target: "es2022", + write: false, metafile: true, logLevel: "silent", + }); + assert.equal(result.outputFiles.length, 1); + for (const output of Object.values(result.metafile.outputs)) assert.deepEqual(output.imports, []); + assert.ok(result.outputFiles[0].text.includes("OssieClient")); + assert.ok(!result.warnings.length); +}); diff --git a/typescript/execution/test/browser/smoke.test.mjs b/typescript/execution/test/browser/smoke.test.mjs new file mode 100644 index 000000000..8eca9365a --- /dev/null +++ b/typescript/execution/test/browser/smoke.test.mjs @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import { existsSync } from "node:fs"; +import { mkdtemp, rm } from "node:fs/promises"; +import { createServer } from "node:http"; +import { delimiter, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; +import { build } from "esbuild"; +import { ARROW_MEDIA_TYPE } from "../../dist/index.js"; +import { BINDING, CAPABILITIES, ERROR, REQUEST, arrowFixture, execution, pagedFixture } from "../fixtures.mjs"; + +const packageDirectory = fileURLToPath(new URL("../../", import.meta.url)); +const browserCandidates = process.env.OSSIE_BROWSER_BINARY ? [process.env.OSSIE_BROWSER_BINARY] : [ + process.env.PROGRAMFILES && join(process.env.PROGRAMFILES, "Microsoft", "Edge", "Application", "msedge.exe"), + process.env["PROGRAMFILES(X86)"] && join(process.env["PROGRAMFILES(X86)"], "Microsoft", "Edge", "Application", "msedge.exe"), + process.env.PROGRAMFILES && join(process.env.PROGRAMFILES, "Google", "Chrome", "Application", "chrome.exe"), + "google-chrome", "chromium", "chromium-browser", +].filter(Boolean); +function resolveBrowser(candidate) { + if (existsSync(candidate)) return candidate; + if (/[\\/]/.test(candidate)) return undefined; + for (const directory of (process.env.PATH ?? "").split(delimiter)) { + for (const suffix of process.platform === "win32" ? ["", ".exe"] : [""]) { + const path = join(directory, candidate + suffix); + if (existsSync(path)) return path; + } + } + return undefined; +} +const browser = browserCandidates.map(resolveBrowser).find(Boolean); + +const delay = milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds)); + +async function devtools(endpoint) { + const socket = new WebSocket(endpoint); + await once(socket, "open"); + let nextId = 0; + const pending = new Map(); + socket.addEventListener("message", event => { + const message = JSON.parse(String(event.data)); + const request = pending.get(message.id); + if (!request) return; + pending.delete(message.id); + clearTimeout(request.timer); + if (message.error) request.reject(new Error(JSON.stringify(message.error))); + else request.resolve(message.result); + }); + return { + close: () => socket.close(), + send(method, params = {}, sessionId) { + return new Promise((resolve, reject) => { + const id = ++nextId; + const timer = setTimeout(() => { + pending.delete(id); + reject(new Error(`DevTools command timed out: ${method}`)); + }, 5000); + pending.set(id, { resolve, reject, timer }); + socket.send(JSON.stringify({ id, method, params, sessionId })); + }); + }, + }; +} + +async function bundle(contents) { + return build({ + absWorkingDir: packageDirectory, + stdin: { contents, resolveDir: packageDirectory, sourcefile: "browser-validation.mjs", loader: "js" }, + bundle: true, platform: "browser", format: "iife", target: "es2022", + write: false, metafile: true, logLevel: "silent", + }); +} + +test("real browser validates synthetic HTTP Arrow execution, polling, CORS and late failure", { + timeout: 40_000, +}, async () => { + assert.ok(browser, "Set OSSIE_BROWSER_BINARY to an installed Chromium/Chrome/Edge executable"); + const fixture = arrowFixture(); + const paginated = pagedFixture(); + const requests = []; + const polls = { asynchronous: 0, "late-failure": 0 }; + let pageOrigin; + let bundleText = ""; + const pageServer = createServer((request, response) => { + if (request.url === "/") { + response.writeHead(200, { "Content-Type": "text/html" }); + response.end('Running'); + } else if (request.url === "/client.js") { + response.writeHead(200, { "Content-Type": "text/javascript" }); + response.end(bundleText); + } else { + response.writeHead(204); + response.end(); + } + }); + const apiServer = createServer(async (request, response) => { + const observation = { + path: request.url, method: request.method, origin: request.headers.origin, + authorization: request.headers.authorization, + requestedHeaders: request.headers["access-control-request-headers"], + }; + requests.push(observation); + const denied = request.url.startsWith("/cors-denied/"); + if (!denied) { + response.setHeader("Access-Control-Allow-Origin", pageOrigin); + response.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS"); + response.setHeader("Access-Control-Allow-Headers", "Authorization, Content-Type, Prefer"); + } + if (request.method === "OPTIONS") { + response.writeHead(204); + response.end(); + return; + } + if (request.url === "/results/result-1") { + response.writeHead(200, { "Content-Type": ARROW_MEDIA_TYPE }); + response.end(fixture.bytes); + return; + } + const page = paginated.streams.find(stream => + new URL(stream.descriptor.url).pathname === request.url); + if (page) { + response.writeHead(200, { "Content-Type": ARROW_MEDIA_TYPE }); + response.end(page.bytes); + return; + } + const bodyParts = []; + for await (const chunk of request) bodyParts.push(chunk); + observation.body = Buffer.concat(bodyParts).toString("utf8"); + const mode = request.url.split("/")[1]; + const pendingSubmission = request.method === "POST" && !["immediate", "paginated"].includes(mode); + response.writeHead(pendingSubmission ? 202 : 200, { "Content-Type": "application/json" }); + if (request.url.endsWith("/capabilities")) response.end(JSON.stringify(CAPABILITIES)); + else if (request.url.endsWith("/bindings/sales")) response.end(JSON.stringify(BINDING)); + else if (mode === "paginated") response.end(JSON.stringify(paginated.succeeded)); + else if (mode === "immediate") response.end(JSON.stringify(fixture.succeeded)); + else if (request.method === "POST") response.end(JSON.stringify(execution("queued"))); + else { + const finished = ++polls[mode] >= 2; + response.end(JSON.stringify(!finished ? execution("running", { progress: { fraction: 0.5 } }) + : mode === "late-failure" ? execution("failed", { error: ERROR }) : fixture.succeeded)); + } + }); + for (const server of [pageServer, apiServer]) { + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + } + pageOrigin = `http://127.0.0.1:${pageServer.address().port}`; + const apiOrigin = `http://127.0.0.1:${apiServer.address().port}`; + fixture.succeeded.result.url = `${apiOrigin}/results/result-1`; + for (const stream of paginated.streams) { + stream.descriptor.url = `${apiOrigin}/results/result-1/pages/${stream.descriptor.page_index}`; + } + const profile = await mkdtemp(join(packageDirectory, "test", "browser", ".browser-profile-")); + try { + const result = await bundle(` + import { OssieClient, QueryExecutionError } from './dist/index.js'; + const check = (condition, label) => { if (!condition) throw new Error(label); }; + (async () => { + const createClient = mode => new OssieClient({ + baseUrl: ${JSON.stringify(apiOrigin)} + '/' + mode, + allowInsecureLocalhost: true, pollIntervalMs: 1, timeoutMs: 10000, + getAccessToken: () => 'SYNTHETIC_BROWSER_TOKEN' + }); + for (const mode of ['immediate', 'asynchronous']) { + const api = createClient(mode); + check((await api.getCapabilities()).result_formats[0] === 'arrow-stream', 'capabilities'); + check((await api.getBinding('sales')).model.id === 'sales-model', 'binding'); + const progress = []; + const { table, execution } = await api.execute(${JSON.stringify(REQUEST)}, { + waitSeconds: 0, onProgress: value => progress.push(value.state) + }); + check(execution.state === 'succeeded', 'success state'); + check(table.numRows === 2, 'row count'); + check(table.getChildAt(1).get(0) === 9007199254740993n, 'int64 precision'); + check(table.getChildAt(0).get(1) === null, 'null'); + if (mode === 'asynchronous') { + check(progress.includes('queued') && progress.includes('running'), 'queued/running progress'); + } + } + const pagedApi = createClient('paginated'); + const pagedExecution = await pagedApi.submit(${JSON.stringify(REQUEST)}, { waitSeconds: 5 }); + const selectedPage = await pagedApi.readPage(pagedExecution, 1); + check(selectedPage.numRows === 1 && selectedPage.getChildAt(0).get(0) === null, 'selected page'); + const allRows = []; + for await (const batch of pagedApi.readBatches(pagedExecution)) { + for (let row = 0; row < batch.numRows; row++) allRows.push(batch.getChildAt(1).get(row)); + } + check(allRows.length === 2 && allRows[0] === 9007199254740993n && allRows[1] === null, 'ordered page traversal'); + let failure; + try { await createClient('late-failure').execute(${JSON.stringify(REQUEST)}); } + catch (error) { failure = error; } + check(failure instanceof QueryExecutionError, 'late failure classification'); + check(failure.code === 'INVALID_QUERY', 'late failure error code'); + check(failure.error.replacement_query.measures[0] === 'revenue', 'replacement query retained'); + const denied = new OssieClient({ + baseUrl: ${JSON.stringify(apiOrigin)} + '/cors-denied', + allowInsecureLocalhost: true, timeoutMs: 10000 + }); + let corsFailure; + try { await denied.getCapabilities(); } catch (error) { corsFailure = error; } + check(corsFailure?.code === 'TRANSPORT_ERROR', 'browser enforces missing CORS permission'); + document.body.dataset.result = 'pass'; + document.body.textContent = 'OSSIE_BROWSER_PASS'; + })().catch(error => { + document.body.dataset.result = 'fail'; + document.body.textContent = 'OSSIE_BROWSER_FAIL ' + error.name + ': ' + error.message; + }); + `); + bundleText = result.outputFiles[0].text; + const child = spawn(browser, [ + "--headless=new", "--disable-gpu", "--no-first-run", "--no-default-browser-check", + "--disable-background-networking", "--disable-extensions", "--disable-component-update", + "--disable-dev-shm-usage", "--no-sandbox", + `--user-data-dir=${profile}`, "--remote-debugging-port=0", + "--remote-debugging-address=127.0.0.1", "about:blank", + ], { stdio: ["ignore", "pipe", "pipe"], windowsHide: true }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", chunk => { stdout += chunk; }); + child.stderr.on("data", chunk => { stderr += chunk; }); + let cdp; + try { + const startDeadline = Date.now() + 10_000; + let endpoint; + while (Date.now() < startDeadline && child.exitCode === null) { + endpoint = /DevTools listening on (ws:\/\/[^\s]+)/.exec(stderr)?.[1]; + if (endpoint) break; + await delay(50); + } + assert.ok(endpoint, `Browser did not expose DevTools: ${stderr}`); + cdp = await devtools(endpoint); + const { targetId } = await cdp.send("Target.createTarget", { url: pageOrigin }); + const { sessionId } = await cdp.send("Target.attachToTarget", { targetId, flatten: true }); + const deadline = Date.now() + 15_000; + let state; + while (Date.now() < deadline) { + const result = await cdp.send("Runtime.evaluate", { + expression: "JSON.stringify({status:document.body?.dataset.result,text:document.body?.textContent})", + returnByValue: true, + }, sessionId); + state = JSON.parse(result.result.value); + if (state.status === "pass" || state.status === "fail") break; + await delay(50); + } + assert.equal(state?.status, "pass", JSON.stringify({ state, requests, stdout, stderr })); + assert.match(state.text, /OSSIE_BROWSER_PASS/); + assert.deepEqual(polls, { asynchronous: 2, "late-failure": 2 }); + assert.ok(requests.some(request => request.method === "OPTIONS" + && request.requestedHeaders.includes("authorization")), "Real authorization preflight is exercised"); + assert.equal(requests.filter(request => request.method === "GET" && request.path === "/results/result-1").length, 2); + assert.equal(requests.filter(request => request.method === "GET" && request.path === "/results/result-1/pages/0").length, 1); + assert.equal(requests.filter(request => request.method === "GET" && request.path === "/results/result-1/pages/1").length, 2); + for (const mode of ["immediate", "asynchronous", "paginated", "late-failure"]) { + const submissions = requests.filter(request => request.method === "POST" && request.path === `/${mode}/queries`); + assert.equal(submissions.length, 1, "Neither polling nor replacement queries create additional submissions"); + assert.deepEqual(JSON.parse(submissions[0].body), REQUEST); + } + for (const request of requests.filter(value => value.method !== "OPTIONS" && !value.path.startsWith("/cors-denied/"))) { + assert.equal(request.origin, pageOrigin); + assert.equal(request.authorization, "Bearer SYNTHETIC_BROWSER_TOKEN"); + } + assert.equal(requests.filter(request => request.path === "/cors-denied/capabilities").length, 1); + } finally { + if (cdp) { + await cdp.send("Browser.close").catch(() => {}); + cdp.close(); + } + if (child.exitCode === null) { + await Promise.race([once(child, "close"), delay(1000)]); + if (child.exitCode === null) child.kill(); + } + } + } finally { + for (const server of [pageServer, apiServer]) { + server.closeAllConnections(); + await new Promise(resolve => server.close(resolve)); + } + await rm(profile, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); + } +}); diff --git a/typescript/execution/test/fixtures.mjs b/typescript/execution/test/fixtures.mjs new file mode 100644 index 000000000..a164ffe8c --- /dev/null +++ b/typescript/execution/test/fixtures.mjs @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from "node:assert/strict"; +import { + Field, Int64, Message, RecordBatch, Schema, Struct, Table, Utf8, + makeData, tableToIPC, vectorFromArray, +} from "apache-arrow"; +import { + ARROW_MEDIA_TYPE, COLUMN_METADATA_KEY, CONTRACT_VERSION, + EXECUTION_METADATA_KEY, FOUNDATION_REVISION, QUERY_SCHEMA_VERSION, +} from "../dist/index.js"; + +export const ORIGIN = "https://ossie.example"; +export const MODEL = { id: "sales-model", revision: "revision-1" }; +export const REQUEST = { + data_source_id: "sales", + query: { dimensions: ["region"], measures: ["revenue"], limit: 0 }, +}; +export const CAPABILITIES = { + contract_version: CONTRACT_VERSION, + foundation_revision: FOUNDATION_REVISION, + query_schema_version: QUERY_SCHEMA_VERSION, + result_formats: ["arrow-stream"], + query_shapes: ["aggregation", "scalar"], + features: [], + max_wait_seconds: 30, +}; +export const BINDING = { + data_source_id: "sales", model: MODEL, + query_shapes: ["aggregation", "scalar"], features: [], +}; +export const ERROR = { + code: "INVALID_QUERY", message: "Choose a known measure", retryable: false, + replacement_query: { measures: ["revenue"] }, +}; + +export function execution(state = "running", extra = {}) { + return { + contract_version: CONTRACT_VERSION, + execution_id: "exec-1", data_source_id: "sales", model: { ...MODEL }, + warnings: [], state, ...extra, + }; +} + +export function json(value, status = 200, headers = {}) { + return new Response(JSON.stringify(value), { + status, headers: { "Content-Type": "application/json", ...headers }, + }); +} + +export function arrowResponse(bytes, headers = {}) { + return new Response(bytes, { + headers: { "Content-Type": ARROW_MEDIA_TYPE, ...headers }, + }); +} + +export function mockFetch(handler) { + const calls = []; + const fetch = async (input, init = {}) => { + const url = new URL(input instanceof Request ? input.url : input); + const call = { + url, method: init.method ?? "GET", + headers: new Headers(init.headers), body: init.body, signal: init.signal, + credentials: init.credentials, redirect: init.redirect, + }; + calls.push(call); + return handler(call, calls.length - 1); + }; + return { fetch, calls }; +} + +export async function collect(iterable) { + const batches = []; + for await (const batch of iterable) batches.push(batch); + return batches; +} + +export function assertSafeError(error, secrets = []) { + assert.ok(error instanceof Error); + assert.ok(typeof error.code === "string" && error.code.length > 0); + const publicText = `${error.name} ${error.message} ${error.stack} ${JSON.stringify(error)}`; + for (const secret of secrets) assert.ok(!publicText.includes(secret), `Error exposed ${secret}`); + return true; +} + +export function arrowFixture({ + rows = [["West", 9007199254740993n], [null, null]], + batchRows, + columns, + fields, + metadata, + columnMetadata, + columnData, + manifest = {}, +} = {}) { + const defaultColumns = [ + { column_id: "col-region", name: "region", logical_type: "String", nullable: true, role: "dimension" }, + { column_id: "col-count", name: "count", logical_type: "Integer", nullable: true, role: "measure" }, + ]; + const resultColumns = columns ?? defaultColumns; + const arrowFields = fields ?? [ + new Field(resultColumns[0].name, new Utf8(), true, + new Map([[COLUMN_METADATA_KEY, JSON.stringify(columnMetadata?.[0] ?? { column_id: resultColumns[0].column_id })]])), + new Field(resultColumns[1].name, new Int64(), true, + new Map([[COLUMN_METADATA_KEY, JSON.stringify(columnMetadata?.[1] ?? { column_id: resultColumns[1].column_id })]])), + ]; + const metadataValue = metadata ?? { + contract_version: CONTRACT_VERSION, execution_id: "exec-1", + result_id: "result-1", data_source_id: "sales", model: MODEL, + }; + const schema = new Schema(arrowFields, new Map([ + [EXECUTION_METADATA_KEY, typeof metadataValue === "string" ? metadataValue : JSON.stringify(metadataValue)], + ])); + const chunks = batchRows ?? [rows]; + const batches = chunks.map(chunk => { + const children = arrowFields.map((field, index) => columnData?.[index] + ?? vectorFromArray(chunk.map(row => row[index]), field.type).data[0]); + const batch = new RecordBatch(schema, makeData({ + type: new Struct(arrowFields), length: chunk.length, + children, + })); + // Arrow JS Schema.assign merges by name inside RecordBatch construction. + // Restore the position-preserving schema so this fixture can exercise duplicate names. + batch.schema = schema; + return batch; + }); + const table = new Table(schema, batches); + const bytes = tableToIPC(table, "stream"); + const succeeded = execution("succeeded", { + result: { + result_id: "result-1", format: "arrow-stream", + url: `${ORIGIN}/results/result-1`, + row_count: chunks.reduce((count, chunk) => count + chunk.length, 0), + byte_count: bytes.byteLength, + completeness: "complete", expires_at: "2099-01-01T00:00:00Z", + columns: resultColumns, ...manifest, + }, + }); + return { bytes, table, schema, batches, succeeded }; +} + +export function chunksResponse(bytes, chunkSize = 17, { onCancel, onPull } = {}) { + let offset = 0; + return new Response(new ReadableStream({ + pull(controller) { + onPull?.(offset); + if (offset === bytes.length) return controller.close(); + const end = Math.min(offset + chunkSize, bytes.length); + controller.enqueue(bytes.slice(offset, end)); + offset = end; + }, + cancel(reason) { onCancel?.(reason); }, + }, { highWaterMark: 0 }), { headers: { "Content-Type": ARROW_MEDIA_TYPE } }); +} + +export function pagedFixture({ + pageRows = [[["West", 9007199254740993n]], [[null, null]]], + pageOptions = [], + manifest = {}, +} = {}) { + let offset = 0; + const streams = pageRows.map((rows, index) => { + const fixture = arrowFixture({ + rows, + metadata: { + contract_version: CONTRACT_VERSION, execution_id: "exec-1", + result_id: "result-1", data_source_id: "sales", model: MODEL, + page_index: index, row_offset: offset, + }, + ...pageOptions[index], + }); + const descriptor = { + page_index: index, row_offset: offset, row_count: rows.length, + byte_count: fixture.bytes.length, url: `${ORIGIN}/results/result-1/pages/${index}`, + }; + offset += rows.length; + return { ...fixture, descriptor }; + }); + const succeeded = structuredClone(streams[0].succeeded); + delete succeeded.result.url; + Object.assign(succeeded.result, { + pages: streams.map(stream => stream.descriptor), row_count: offset, + byte_count: streams.reduce((total, stream) => total + stream.bytes.length, 0), + ...manifest, + }); + return { streams, succeeded }; +} + +export function rewriteFirstBatchHeader(fixture, mutate) { + let offset = 0; + while (offset < fixture.bytes.length) { + const prefix = new DataView(fixture.bytes.buffer, fixture.bytes.byteOffset + offset); + const prefixSize = prefix.getInt32(0, true) === -1 ? 8 : 4; + const metadataSize = prefix.getInt32(prefixSize - 4, true); + if (metadataSize === 0) break; + const bodyStart = offset + prefixSize + metadataSize; + const message = Message.decode(fixture.bytes.subarray(offset + prefixSize, bodyStart)); + if (message.isRecordBatch()) { + const header = message.header(); + mutate(header); + const metadata = Message.encode(Message.from(header, message.bodyLength)); + const paddedLength = Math.ceil(metadata.length / 8) * 8; + const bytes = new Uint8Array(fixture.bytes.length - prefixSize - metadataSize + 8 + paddedLength); + bytes.set(fixture.bytes.subarray(0, offset)); + const view = new DataView(bytes.buffer, offset); + view.setInt32(0, -1, true); + view.setInt32(4, paddedLength, true); + bytes.set(metadata, offset + 8); + bytes.set(fixture.bytes.subarray(bodyStart), offset + 8 + paddedLength); + fixture.bytes = bytes; + fixture.succeeded.result.byte_count = bytes.length; + return fixture; + } + offset = bodyStart + message.bodyLength; + } + throw new Error("Fixture has no record batch to rewrite"); +} diff --git a/typescript/execution/test/lifecycle.test.mjs b/typescript/execution/test/lifecycle.test.mjs new file mode 100644 index 000000000..4be0a28b1 --- /dev/null +++ b/typescript/execution/test/lifecycle.test.mjs @@ -0,0 +1,285 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from "node:assert/strict"; +import { once } from "node:events"; +import { createServer } from "node:http"; +import test from "node:test"; +import { + ARROW_MEDIA_TYPE, HttpProblemError, OssieClient, ProtocolError, QueryExecutionError, +} from "../dist/index.js"; +import { + BINDING, CAPABILITIES, ERROR, ORIGIN, REQUEST, + arrowFixture, arrowResponse, assertSafeError, execution, json, mockFetch, +} from "./fixtures.mjs"; + +const client = fetch => new OssieClient({ baseUrl: ORIGIN, fetch, pollIntervalMs: 1, timeoutMs: 1000 }); + +test("capabilities and binding are explicit discovery operations", async () => { + const mock = mockFetch(call => { + if (call.url.pathname === "/capabilities") return json(CAPABILITIES); + assert.equal(call.url.pathname, "/bindings/sales"); + return json(BINDING); + }); + const api = client(mock.fetch); + assert.deepEqual(await api.getCapabilities(), CAPABILITIES); + assert.deepEqual(await api.getBinding("sales"), BINDING); + assert.equal(mock.calls.length, 2); +}); + +test("submit preserves Layer 3 structure and limit zero; it never sends SQL", async () => { + const mock = mockFetch(() => json(execution("queued"), 202)); + const api = client(mock.fetch); + assert.equal((await api.submit(REQUEST, { waitSeconds: 2 })).state, "queued"); + assert.equal(mock.calls.length, 1); + const [call] = mock.calls; + assert.equal(call.url.pathname, "/queries"); + assert.equal(call.method, "POST"); + assert.deepEqual(JSON.parse(call.body), REQUEST); + assert.equal(call.headers.get("Content-Type"), "application/json"); + assert.match(call.headers.get("Prefer") ?? "", /wait=2/); +}); + +test("scalar query is preserved without aggregation clauses", async () => { + const request = { data_source_id: "sales", query: { fields: ["region"], limit: 0 } }; + const mock = mockFetch(() => json(execution("queued"), 202)); + await client(mock.fetch).submit(request); + assert.deepEqual(JSON.parse(mock.calls[0].body), request); +}); + +test("immediate success executes exactly one submit and one materialized result fetch", async () => { + const fixture = arrowFixture(); + const mock = mockFetch(call => call.method === "POST" ? json(fixture.succeeded) : arrowResponse(fixture.bytes)); + const result = await client(mock.fetch).execute(REQUEST); + assert.equal(result.execution.execution_id, "exec-1"); + assert.equal(result.table.numRows, 2); + assert.equal(result.table.getChildAt(1).get(0), 9007199254740993n); + assert.equal(result.table.getChildAt(0).get(1), null); + assert.deepEqual(mock.calls.map(call => call.method), ["POST", "GET"]); +}); + +test("queued/running/progress/succeeded poll loop preserves execution identity", async () => { + const fixture = arrowFixture(); + const states = [ + execution("queued"), + execution("running", { progress: { fraction: 0.25, phase: "scan" } }), + fixture.succeeded, + ]; + const progress = []; + const mock = mockFetch(call => json(states.shift(), call.method === "POST" ? 202 : 200)); + const api = client(mock.fetch); + const initial = await api.submit(REQUEST); + const completed = await api.wait(initial, { onProgress: value => progress.push(value) }); + assert.equal(completed.state, "succeeded"); + assert.ok(progress.some(value => value.state === "running" && value.progress?.fraction === 0.25)); + assert.equal(mock.calls.filter(call => call.method === "POST").length, 1); + for (const call of mock.calls.slice(1)) assert.equal(call.url.pathname, "/queries/exec-1"); +}); + +test("HTTP 200 failed execution preserves replacement query without automatic execution", async () => { + const failed = execution("failed", { error: ERROR }); + const mock = mockFetch(() => json(failed)); + const api = client(mock.fetch); + assert.deepEqual(await api.submit(REQUEST), failed); + await assert.rejects(api.execute(REQUEST), error => { + assert.ok(error instanceof QueryExecutionError); + const serialized = JSON.stringify(error); + assert.ok(serialized.includes("replacement_query"), "Query error retains structured replacement query"); + assert.ok(serialized.includes("revenue")); + return true; + }); + assert.equal(mock.calls.length, 2, "One submit per explicit request, no replacement execution"); +}); + +test("poll failure is a query failure, not an HTTP transport failure", async () => { + const mock = mockFetch(() => json(execution("failed", { error: ERROR }))); + await assert.rejects(client(mock.fetch).wait(execution()), QueryExecutionError); + assert.equal(mock.calls.length, 1); +}); + +test("cancellation acknowledges cancelling then observes cancelled", async () => { + const mock = mockFetch(call => json(execution(call.method === "POST" ? "cancelling" : "cancelled"), + call.method === "POST" ? 202 : 200)); + const api = client(mock.fetch); + const cancelled = await api.cancel("exec-1"); + assert.equal(cancelled.state, "cancelling"); + assert.equal(mock.calls[0].url.pathname, "/queries/exec-1/cancel"); + await assert.rejects(api.wait(cancelled), assertSafeError); + assert.equal(mock.calls.length, 2); +}); + +test("cancellation racing completion preserves succeeded state and result", async () => { + const fixture = arrowFixture(); + const mock = mockFetch(() => json(fixture.succeeded)); + assert.deepEqual(await client(mock.fetch).cancel("exec-1"), fixture.succeeded); + assert.equal(mock.calls.length, 1); +}); + +for (const status of [429, 500, 502, 503]) { + test(`submit is not retried on transient HTTP ${status}`, async () => { + const mock = mockFetch(() => json({ + type: "about:blank", title: "Unavailable", status, + error: { code: "UNAVAILABLE", message: "Please retry", retryable: true }, + }, status, { "Retry-After": "0", "Content-Type": "application/problem+json" })); + await assert.rejects(client(mock.fetch).submit(REQUEST), error => { + assert.ok(error instanceof HttpProblemError); + assert.equal(error.code, "UNAVAILABLE"); + assert.equal(error.status, status); + return true; + }); + assert.equal(mock.calls.length, 1); + }); +} + +test("HTTP problem details retain query replacement without running it", async () => { + const problem = { type: "about:blank", title: "Invalid query", status: 400, error: ERROR }; + const mock = mockFetch(() => json(problem, 400, { "Content-Type": "application/problem+json" })); + await assert.rejects(client(mock.fetch).execute(REQUEST), error => { + assert.ok(error instanceof HttpProblemError); + assert.deepEqual(error.problem, problem); + return true; + }); + assert.equal(mock.calls.length, 1); +}); + +test("a timeout also bounds an unresponsive credential callback, before fetch", { timeout: 2000 }, async () => { + const mock = mockFetch(() => json(CAPABILITIES)); + const api = new OssieClient({ + baseUrl: ORIGIN, fetch: mock.fetch, timeoutMs: 20, + getAccessToken: () => new Promise(() => {}), + }); + await assert.rejects(api.getCapabilities(), assertSafeError); + assert.equal(mock.calls.length, 0); +}); + +test("wait uses an overall deadline rather than resetting it on every successful poll", { timeout: 2000 }, async () => { + const mock = mockFetch(() => json(execution("running"))); + const api = new OssieClient({ baseUrl: ORIGIN, fetch: mock.fetch, timeoutMs: 200, pollIntervalMs: 5 }); + await assert.rejects(api.wait(execution("queued")), error => { + assert.equal(error.code, "TIMEOUT"); + return true; + }); + assert.ok(mock.calls.length >= 1); + assert.ok(mock.calls.every(call => call.method === "GET"), "Timeout is local; it does not issue backend cancellation"); +}); + +test("caller progress callback cannot mutate the execution used for polling", async () => { + const fixture = arrowFixture(); + const mock = mockFetch(() => json(fixture.succeeded)); + const completed = await client(mock.fetch).wait(execution("queued"), { + onProgress(value) { value.execution_id = "malicious-change"; value.model.revision = "different"; }, + }); + assert.equal(completed.execution_id, "exec-1"); + assert.equal(completed.model.revision, "revision-1"); + assert.equal(mock.calls[0].url.pathname, "/queries/exec-1"); +}); + +test("submit transport failure is sanitized and never retried", async () => { + const secret = "transport-bearer-secret"; + const mock = mockFetch(() => { throw new TypeError(`fetch failed https://secret.invalid/?token=${secret}`); }); + await assert.rejects(client(mock.fetch).submit(REQUEST), error => assertSafeError(error, [secret])); + assert.equal(mock.calls.length, 1); +}); + +test("JSON body transport failures are typed, sanitized and never cause a submit retry", async () => { + const secret = "BODY_STREAM_SYNTHETIC_SECRET"; + const mock = mockFetch(() => new Response(new ReadableStream({ + start(controller) { controller.error(new Error(`socket failed https://host.invalid/?token=${secret}`)); }, + }), { headers: { "Content-Type": "application/json" } })); + await assert.rejects(client(mock.fetch).submit(REQUEST), error => assertSafeError(error, [secret])); + assert.equal(mock.calls.length, 1); +}); + +test("AbortSignal stops a request and an already-aborted call makes no network request", async () => { + const controller = new AbortController(); + controller.abort(); + const mock = mockFetch(() => json(execution())); + await assert.rejects(client(mock.fetch).submit(REQUEST, { signal: controller.signal }), assertSafeError); + assert.equal(mock.calls.length, 0); +}); + +test("request timeout includes a stalled JSON response body", { timeout: 2000 }, async () => { + let cancelled = false; + const mock = mockFetch(() => new Response(new ReadableStream({ + start(controller) { controller.enqueue(new TextEncoder().encode("{")); }, + cancel() { cancelled = true; }, + }), { headers: { "Content-Type": "application/json" } })); + const api = new OssieClient({ baseUrl: ORIGIN, fetch: mock.fetch, timeoutMs: 30 }); + await assert.rejects(api.submit(REQUEST), assertSafeError); + assert.ok(cancelled, "Timed out JSON stream is cancelled"); +}); + +test("abort includes a stalled JSON response body", { timeout: 2000 }, async () => { + const controller = new AbortController(); + const mock = mockFetch(() => new Response(new ReadableStream({ + start(stream) { + stream.enqueue(new TextEncoder().encode("{")); + setTimeout(() => controller.abort(), 20); + }, + }), { headers: { "Content-Type": "application/json" } })); + await assert.rejects(client(mock.fetch).submit(REQUEST, { signal: controller.signal }), assertSafeError); +}); + +test("two synthetic HTTP behaviours work over real loopback fetch, not vendor engines", async t => { + for (const behaviour of ["immediate", "asynchronous"]) { + await t.test(behaviour, { timeout: 5000 }, async () => { + const fixture = arrowFixture({ rows: behaviour === "immediate" ? [["Now", 3n]] : [["Later", 8n], ["Later", 9n]] }); + const requests = []; + let polls = 0; + const server = createServer(async (request, response) => { + requests.push({ method: request.method, url: request.url }); + if (request.url === "/results/result-1") { + response.writeHead(200, { "Content-Type": ARROW_MEDIA_TYPE }); + response.end(fixture.bytes); + return; + } + const body = []; + for await (const chunk of request) body.push(chunk); + if (request.method === "POST") assert.deepEqual(JSON.parse(Buffer.concat(body).toString()), REQUEST); + response.writeHead(behaviour === "asynchronous" && request.method === "POST" ? 202 : 200, + { "Content-Type": "application/json" }); + if (behaviour === "immediate" || (request.method === "GET" && ++polls >= 2)) { + response.end(JSON.stringify(fixture.succeeded)); + } else { + response.end(JSON.stringify(execution(request.method === "POST" ? "queued" : "running"))); + } + }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const baseUrl = `http://127.0.0.1:${server.address().port}`; + fixture.succeeded.result.url = `${baseUrl}/results/result-1`; + try { + const api = new OssieClient({ baseUrl, allowInsecureLocalhost: true, pollIntervalMs: 1, timeoutMs: 2000 }); + const result = await api.execute(REQUEST); + assert.equal(result.table.numRows, behaviour === "immediate" ? 1 : 2); + assert.equal(result.table.getChildAt(0).get(0), behaviour === "immediate" ? "Now" : "Later"); + assert.equal(requests.filter(request => request.method === "POST").length, 1); + assert.equal(polls, behaviour === "immediate" ? 0 : 2); + } finally { + server.closeAllConnections(); + await new Promise(resolve => server.close(resolve)); + } + }); + } +}); + +test("malformed JSON is a protocol error, not a query failure", async () => { + const mock = mockFetch(() => new Response("{broken", { headers: { "Content-Type": "application/json" } })); + await assert.rejects(client(mock.fetch).submit(REQUEST), ProtocolError); +}); diff --git a/typescript/execution/test/origins.test.mjs b/typescript/execution/test/origins.test.mjs new file mode 100644 index 000000000..61c857de4 --- /dev/null +++ b/typescript/execution/test/origins.test.mjs @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from "node:assert/strict"; +import test from "node:test"; +import { OssieClient } from "../dist/index.js"; +import { + CAPABILITIES, ORIGIN, arrowFixture, arrowResponse, assertSafeError, + collect, json, mockFetch, +} from "./fixtures.mjs"; + +test("same-origin requests use authentication but disable redirect following", async () => { + const fixture = arrowFixture(); + const mock = mockFetch(call => call.url.pathname === "/capabilities" ? json(CAPABILITIES) : arrowResponse(fixture.bytes)); + const api = new OssieClient({ + baseUrl: ORIGIN, fetch: mock.fetch, getAccessToken: async () => "test-token-not-real", + }); + await api.getCapabilities(); + await collect(api.readBatches(fixture.succeeded)); + assert.equal(mock.calls.length, 2); + for (const call of mock.calls) { + assert.equal(call.headers.get("Authorization"), "Bearer test-token-not-real"); + assert.ok(["same-origin", "omit"].includes(call.credentials)); + assert.ok(["manual", "error"].includes(call.redirect), "Redirects cannot forward authentication to another origin"); + } +}); + +test("explicit allowlisted cross-origin results omit Authorization and browser credentials", async () => { + const fixture = arrowFixture({ manifest: { url: "https://results.example/result?sig=not-a-real-signature" } }); + let tokenRequests = 0; + const mock = mockFetch(() => arrowResponse(fixture.bytes)); + const api = new OssieClient({ + baseUrl: ORIGIN, fetch: mock.fetch, allowedResultOrigins: ["https://results.example"], + getAccessToken() { tokenRequests++; return "control-plane-token"; }, + }); + await collect(api.readBatches(fixture.succeeded)); + assert.equal(mock.calls[0].headers.get("Authorization"), null); + assert.equal(mock.calls[0].credentials, "omit"); + assert.equal(tokenRequests, 0); +}); + +for (const url of [ + "https://untrusted.example/result", + "https://results.example.attacker.example/result", + "https://user:password@results.example/result", + "https://results.example/result#fragment", + "http://results.example/result", + "file:///C:/private/file.arrow", + "data:application/vnd.apache.arrow.stream;base64,AAAA", + "/results/result-1", +]) { + test(`unsafe result destination is rejected before fetch: ${new URL(url, ORIGIN).protocol} ${new URL(url, ORIGIN).hostname}`, async () => { + const fixture = arrowFixture({ manifest: { url } }); + const mock = mockFetch(() => arrowResponse(fixture.bytes)); + const api = new OssieClient({ + baseUrl: ORIGIN, fetch: mock.fetch, allowedResultOrigins: ["https://results.example"], + allowInsecureLocalhost: true, + }); + await assert.rejects(collect(api.readBatches(fixture.succeeded)), assertSafeError); + assert.equal(mock.calls.length, 0); + }); +} + +for (const baseUrl of [ + "http://ossie.example", "http://127.0.0.1:8000", + "https://user:password@ossie.example", "https://ossie.example/#private", + "https://ossie.example/?secret=token", +]) { + test(`unsafe control plane URL is rejected: ${baseUrl}`, () => { + assert.throws(() => new OssieClient({ baseUrl }), assertSafeError); + }); +} + +test("explicit local-development exception permits loopback HTTP, never arbitrary cleartext hosts", () => { + for (const baseUrl of ["http://127.0.0.1:8000", "http://localhost:8000", "http://[::1]:8000"]) { + assert.doesNotThrow(() => new OssieClient({ baseUrl, allowInsecureLocalhost: true })); + } + for (const baseUrl of ["http://ossie.example", "http://localhost.attacker.example", "http://10.0.0.1"]) { + assert.throws(() => new OssieClient({ baseUrl, allowInsecureLocalhost: true }), assertSafeError); + } +}); + +test("result redirects are not followed even to another allowlisted origin", async () => { + const fixture = arrowFixture(); + const mock = mockFetch(() => new Response(null, { status: 302, headers: { Location: "https://results.example/next" } })); + const api = new OssieClient({ + baseUrl: ORIGIN, fetch: mock.fetch, allowedResultOrigins: ["https://results.example"], + getAccessToken: () => "test-token-not-real", + }); + await assert.rejects(collect(api.readBatches(fixture.succeeded)), assertSafeError); + assert.equal(mock.calls.length, 1); + assert.ok(["manual", "error"].includes(mock.calls[0].redirect)); +}); + +test("control-plane redirects are not followed", async () => { + const mock = mockFetch(() => new Response(null, { status: 307, headers: { Location: "https://ossie.example/new" } })); + const api = new OssieClient({ baseUrl: ORIGIN, fetch: mock.fetch }); + await assert.rejects(api.getCapabilities(), assertSafeError); + assert.equal(mock.calls.length, 1); + assert.ok(["manual", "error"].includes(mock.calls[0].redirect)); +}); + +test("signed result URL and token do not appear in result transport errors", async () => { + const signature = "SYNTHETIC_SECRET_SIGNATURE"; + const token = "SYNTHETIC_SECRET_BEARER"; + const fixture = arrowFixture({ manifest: { url: `${ORIGIN}/result?sig=${signature}` } }); + const mock = mockFetch(call => { throw new Error(`fetch ${call.url.href} Authorization: Bearer ${token}`); }); + const api = new OssieClient({ baseUrl: ORIGIN, fetch: mock.fetch, getAccessToken: () => token }); + await assert.rejects(collect(api.readBatches(fixture.succeeded)), error => + assertSafeError(error, [signature, token, fixture.succeeded.result.url])); +}); + +test("expired result is rejected without fetching its signed URL", async () => { + const fixture = arrowFixture({ manifest: { expires_at: "2000-01-01T00:00:00Z" } }); + const mock = mockFetch(() => arrowResponse(fixture.bytes)); + await assert.rejects(collect(new OssieClient({ baseUrl: ORIGIN, fetch: mock.fetch }).readBatches(fixture.succeeded)), assertSafeError); + assert.equal(mock.calls.length, 0); +}); diff --git a/typescript/execution/test/pagination.test.mjs b/typescript/execution/test/pagination.test.mjs new file mode 100644 index 000000000..776ffc53f --- /dev/null +++ b/typescript/execution/test/pagination.test.mjs @@ -0,0 +1,320 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from "node:assert/strict"; +import test from "node:test"; +import { Table, tableToIPC, vectorFromArray, Int32 } from "apache-arrow"; +import { OssieClient, ProtocolError, EXECUTION_METADATA_KEY } from "../dist/index.js"; +import { + ORIGIN, REQUEST, arrowFixture, arrowResponse, chunksResponse, collect, json, mockFetch, pagedFixture, +} from "./fixtures.mjs"; + +function clientFor(fixture, options = {}, handler) { + const mock = mockFetch(call => { + if (handler) return handler(call); + if (call.method === "POST") return json(fixture.succeeded); + const stream = fixture.streams.find(item => item.descriptor.url === call.url.href); + assert.ok(stream, "Fetch only one of the advertised pages"); + return arrowResponse(stream.bytes); + }); + return { api: new OssieClient({ baseUrl: ORIGIN, fetch: mock.fetch, ...options }), mock }; +} + +function changePage(fixture, index, transform) { + const previous = fixture.streams[index]; + transform(previous); + previous.bytes = tableToIPC(new Table(previous.schema, previous.batches), "stream"); + previous.descriptor.byte_count = previous.bytes.length; + fixture.succeeded.result.byte_count = fixture.streams.reduce((sum, page) => sum + page.bytes.length, 0); +} + +test("execute traverses ordered result pages once without resubmitting or modifying limit", async () => { + const fixture = pagedFixture(); + const { api, mock } = clientFor(fixture); + const { table, execution } = await api.execute(REQUEST, { waitSeconds: 5 }); + assert.equal(table.numRows, 2); + assert.equal(table.getChildAt(1).get(0), 9007199254740993n); + assert.equal(table.getChildAt(0).get(1), null); + assert.deepEqual(execution, fixture.succeeded); + assert.equal(Object.hasOwn(JSON.parse(table.schema.metadata.get(EXECUTION_METADATA_KEY)), "page_index"), false); + assert.equal(JSON.parse(table.batches[1].schema.metadata.get(EXECUTION_METADATA_KEY)).page_index, 1); + assert.deepEqual(mock.calls.map(call => call.method), ["POST", "GET", "GET"]); + assert.deepEqual(JSON.parse(mock.calls[0].body), REQUEST); + assert.equal(mock.calls[0].headers.get("Prefer"), "wait=5"); + assert.deepEqual(mock.calls.slice(1).map(call => call.url.href), fixture.succeeded.result.pages.map(page => page.url)); +}); + +test("readPage allows indexed access and explicit re-reading of an immutable page", async () => { + const fixture = pagedFixture(); + const { api, mock } = clientFor(fixture); + const second = await api.readPage(fixture.succeeded, 1); + assert.equal(second.numRows, 1); + assert.equal(second.getChildAt(0).get(0), null); + await api.readPage(fixture.succeeded, 1); + assert.deepEqual(mock.calls.map(call => call.url.href), [fixture.streams[1].descriptor.url, fixture.streams[1].descriptor.url]); +}); + +test("readPage zero also works for legacy single-stream results", async () => { + const fixture = arrowFixture(); + const mock = mockFetch(() => arrowResponse(fixture.bytes)); + const api = new OssieClient({ baseUrl: ORIGIN, fetch: mock.fetch }); + assert.equal((await api.readPage(fixture.succeeded, 0)).numRows, 2); + await assert.rejects(api.readPage(fixture.succeeded, 1), ProtocolError); + assert.equal(mock.calls.length, 1); +}); + +test("page reads enforce per-page limits, while full traversal enforces aggregate limits", async () => { + const fixture = pagedFixture(); + const pageBytes = Math.max(...fixture.streams.map(page => page.bytes.length)); + const { api, mock } = clientFor(fixture, { maxResultRows: 1, maxResultBytes: pageBytes + 100 }); + assert.equal((await api.readPage(fixture.succeeded, 1)).numRows, 1); + await assert.rejects(collect(api.readBatches(fixture.succeeded)), error => error.code === "RESULT_LIMIT_EXCEEDED"); + assert.equal(mock.calls.length, 1, "Full-result admission rejects before fetching pages"); +}); + +test("oversized selected page rejects before download", async () => { + const fixture = pagedFixture(); + const { api, mock } = clientFor(fixture, { maxResultBytes: 10 }); + await assert.rejects(api.readPage(fixture.succeeded, 0), error => error.code === "RESULT_LIMIT_EXCEEDED"); + assert.equal(mock.calls.length, 0); +}); + +for (const index of [-1, 0.5, Number.MAX_SAFE_INTEGER + 1, NaN, Infinity, 2]) { + test(`invalid page index ${index} is rejected without a download`, async () => { + const fixture = pagedFixture(); + const { api, mock } = clientFor(fixture); + await assert.rejects(api.readPage(fixture.succeeded, index), ProtocolError); + assert.equal(mock.calls.length, 0); + }); +} + +for (const [name, mutate] of [ + ["both delivery modes", m => { m.url = `${ORIGIN}/other`; }], + ["neither delivery mode", m => { delete m.pages; }], + ["empty page list", m => { m.pages = []; }], + ["unknown page field", m => { m.pages[0].unexpected = true; }], + ["index gap", m => { m.pages[1].page_index = 2; }], + ["offset gap", m => { m.pages[1].row_offset = 2; }], + ["offset overlap", m => { m.pages[1].row_offset = 0; }], + ["missing total rows", m => { m.row_count++; }], + ["missing total bytes", m => { m.byte_count++; }], + ["unsafe page count", m => { m.pages[1].row_count = Number.MAX_SAFE_INTEGER + 1; }], + ["byte sum overflow", m => { m.pages[1].byte_count = Number.MAX_SAFE_INTEGER; }], + ["zero-byte page", m => { m.pages[1].byte_count = 0; }], + ["empty page amongst others", m => { m.pages[1].row_count = 0; }], + ["relative page URL", m => { m.pages[1].url = "/page"; }], +]) { + test(`invalid pagination manifest rejects ${name}`, async () => { + const fixture = pagedFixture(); + mutate(fixture.succeeded.result); + const mock = mockFetch(() => json(fixture.succeeded)); + const api = new OssieClient({ baseUrl: ORIGIN, fetch: mock.fetch }); + await assert.rejects(api.getExecution("exec-1"), ProtocolError); + }); +} + +for (const [name, mutate] of [ + ["swapped page index", m => { m.page_index = 1; }], + ["wrong row offset", m => { m.row_offset = 5; }], + ["missing coordinates", m => { delete m.page_index; delete m.row_offset; }], + ["half coordinates", m => { delete m.row_offset; }], + ["wrong result", m => { m.result_id = "another-result"; }], +]) { + test(`Arrow page identity rejects ${name}`, async () => { + const fixture = pagedFixture(); + changePage(fixture, 0, page => { + const metadata = JSON.parse(page.schema.metadata.get(EXECUTION_METADATA_KEY)); + mutate(metadata); + page.schema.metadata.set(EXECUTION_METADATA_KEY, JSON.stringify(metadata)); + }); + const { api } = clientFor(fixture); + await assert.rejects(api.readPage(fixture.succeeded, 0), ProtocolError); + }); +} + +test("identical page payloads cannot be swapped unnoticed", async () => { + const fixture = pagedFixture({ pageRows: [[["same", 1n]], [["same", 1n]]] }); + const { api } = clientFor(fixture, {}, () => arrowResponse(fixture.streams[1].bytes)); + await assert.rejects(api.readPage(fixture.succeeded, 0), ProtocolError); +}); + +test("unpaged results reject paged Arrow metadata", async () => { + const fixture = pagedFixture(); + const unpaged = fixture.streams[0].succeeded; + const api = new OssieClient({ baseUrl: ORIGIN, fetch: async () => arrowResponse(fixture.streams[0].bytes) }); + await assert.rejects(api.readPage(unpaged, 0), ProtocolError); +}); + +test("physical integer widths cannot change between otherwise valid logical pages", async () => { + const fixture = pagedFixture(); + changePage(fixture, 1, page => { + page.schema.fields[1].type = new Int32(); + page.batches[0].data.children[1] = vectorFromArray([null], new Int32()).data[0]; + }); + const { api } = clientFor(fixture); + await assert.rejects(collect(api.readBatches(fixture.succeeded)), ProtocolError); +}); + +test("caller mutation of a yielded batch cannot change the cross-page schema baseline", async () => { + const fixture = pagedFixture(); + const { api } = clientFor(fixture); + const reader = api.readBatches(fixture.succeeded); + const first = await reader.next(); + first.value.schema.fields[1].type.bitWidth = 32; + assert.equal((await reader.next()).value.numRows, 1); + assert.equal((await reader.next()).done, true); +}); + +for (const level of ["field", "schema"]) { + test(`additional ${level} metadata cannot drift between pages`, async () => { + const fixture = pagedFixture(); + changePage(fixture, 1, page => { + const metadata = level === "field" ? page.schema.fields[0].metadata : page.schema.metadata; + metadata.set("example.test.label", "different"); + }); + const { api } = clientFor(fixture); + await assert.rejects(collect(api.readBatches(fixture.succeeded)), ProtocolError); + }); +} + +test("schema metadata compares by keys and execution identity rather than JSON serialization order", async () => { + const fixture = pagedFixture(); + changePage(fixture, 1, page => { + const identity = JSON.parse(page.schema.metadata.get(EXECUTION_METADATA_KEY)); + page.schema.metadata.set(EXECUTION_METADATA_KEY, JSON.stringify(Object.fromEntries(Object.entries(identity).reverse()), null, 2)); + }); + const { api } = clientFor(fixture); + assert.equal((await api.execute(REQUEST)).table.numRows, 2); +}); + +test("underreported later pages cannot reset the aggregate decoded-byte budget", async () => { + const fixture = pagedFixture({ + pageRows: [[["large-".repeat(1024), 1n]], [["large-".repeat(1024), 2n]]], + }); + const firstBytes = fixture.streams[0].bytes.length; + const budget = firstBytes + 1024; + fixture.succeeded.result.pages[1].byte_count = 1024; + fixture.succeeded.result.byte_count = budget; + const { api } = clientFor(fixture, { maxResultBytes: budget }); + let rows = 0; + await assert.rejects(async () => { + for await (const batch of api.readBatches(fixture.succeeded)) rows += batch.numRows; + }, error => error.code === "RESULT_LIMIT_EXCEEDED"); + assert.equal(rows, 1, "Do not yield data from a page that exceeds the remaining aggregate budget"); +}); + +test("one schema-only page represents a zero-row result", async () => { + const fixture = pagedFixture({ pageRows: [[]] }); + const page = fixture.streams[0]; + page.bytes = tableToIPC(new Table(page.schema, []), "stream"); + page.descriptor.byte_count = page.bytes.length; + fixture.succeeded.result.byte_count = page.bytes.length; + const { api } = clientFor(fixture); + assert.equal((await api.readPage(fixture.succeeded, 0)).numCols, 2); + assert.equal((await api.execute(REQUEST)).table.numRows, 0); +}); + +test("breaking batch traversal does not prefetch another page", async () => { + const batchRows = Array.from({ length: 10 }, (_, index) => + Array.from({ length: 128 }, () => [`batch-${index}`, BigInt(index)])); + const fixture = pagedFixture({ + pageRows: [batchRows.flat(), [["last", 1n]]], pageOptions: [{ batchRows }], + }); + let cancelled = false; + const { api, mock } = clientFor(fixture, {}, () => + chunksResponse(fixture.streams[0].bytes, 97, { onCancel: () => { cancelled = true; } })); + for await (const batch of api.readBatches(fixture.succeeded)) { + assert.equal(batch.numRows, 128); + break; + } + assert.equal(mock.calls.length, 1); + assert.equal(cancelled, true); +}); + +test("a late page error rejects full materialization without rerunning the query", async () => { + const fixture = pagedFixture(); + const { api, mock } = clientFor(fixture, {}, call => { + if (call.method === "POST") return json(fixture.succeeded); + if (call.url.href === fixture.streams[0].descriptor.url) return arrowResponse(fixture.streams[0].bytes); + throw new Error("synthetic download failure"); + }); + await assert.rejects(api.execute(REQUEST), error => error.code === "TRANSPORT_ERROR"); + assert.equal(mock.calls.filter(call => call.method === "POST").length, 1); + assert.equal(mock.calls.length, 3); +}); + +test("page byte and row counts are checked independently of the manifest totals", async () => { + const fixture = pagedFixture(); + fixture.succeeded.result.pages[0].byte_count += 8; + fixture.succeeded.result.pages[1].byte_count -= 8; + const { api } = clientFor(fixture); + await assert.rejects(api.readPage(fixture.succeeded, 0), ProtocolError); +}); + +test("page-level origin policy never forwards account credentials to an external host", async () => { + const fixture = pagedFixture(); + fixture.streams[1].descriptor.url = "https://download.example/page1?signature=secret"; + let tokens = 0; + const { api, mock } = clientFor(fixture, { + allowedResultOrigins: ["https://download.example"], + getAccessToken: () => { tokens++; return "ACCOUNT_TOKEN"; }, + }); + await api.readPage(fixture.succeeded, 1); + assert.equal(mock.calls[0].headers.has("Authorization"), false); + assert.equal(mock.calls[0].credentials, "omit"); + assert.equal(mock.calls[0].redirect, "error"); + assert.equal(tokens, 0); +}); + +test("unallowlisted page origin is refused before download", async () => { + const fixture = pagedFixture(); + fixture.streams[1].descriptor.url = "https://untrusted.example/page"; + const { api, mock } = clientFor(fixture); + await assert.rejects(api.readPage(fixture.succeeded, 1), ProtocolError); + assert.equal(mock.calls.length, 0); +}); + +test("page expiry is rechecked after token refresh", async context => { + let now = Date.now(); + context.mock.method(Date, "now", () => now); + const fixture = pagedFixture({ manifest: { expires_at: new Date(now + 1000).toISOString() } }); + const { api, mock } = clientFor(fixture, { getAccessToken: async () => { now += 2000; return "TOKEN"; } }); + await assert.rejects(api.readPage(fixture.succeeded, 1), error => error.code === "RESULT_EXPIRED"); + assert.equal(mock.calls.length, 0); +}); + +test("truncation opt-in does not confuse an individual page with a complete result", async () => { + const fixture = pagedFixture({ manifest: { completeness: "truncated", truncation_reason: "server limit" } }); + const { api, mock } = clientFor(fixture); + await assert.rejects(api.readPage(fixture.succeeded, 0), error => error.code === "TRUNCATED_RESULT"); + assert.equal(mock.calls.length, 0); + assert.equal((await api.readPage(fixture.succeeded, 0, { allowTruncated: true })).numRows, 1); + assert.equal(fixture.succeeded.result.completeness, "truncated"); +}); + +test("terminal page descriptors remain immutable across observations", async () => { + const fixture = pagedFixture(); + const changed = structuredClone(fixture.succeeded); + changed.result.pages[1].url = `${ORIGIN}/different`; + const responses = [fixture.succeeded, changed]; + const mock = mockFetch(() => json(responses.shift())); + const api = new OssieClient({ baseUrl: ORIGIN, fetch: mock.fetch }); + await api.getExecution("exec-1"); + await assert.rejects(api.getExecution("exec-1"), ProtocolError); +}); diff --git a/typescript/execution/test/protocol.test.mjs b/typescript/execution/test/protocol.test.mjs new file mode 100644 index 000000000..fe31f35cb --- /dev/null +++ b/typescript/execution/test/protocol.test.mjs @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from "node:assert/strict"; +import test from "node:test"; +import { OssieClient, ProtocolError } from "../dist/index.js"; +import { + BINDING, CAPABILITIES, ERROR, ORIGIN, REQUEST, + arrowFixture, execution, json, mockFetch, +} from "./fixtures.mjs"; + +const client = fetch => new OssieClient({ baseUrl: ORIGIN, fetch, pollIntervalMs: 1, timeoutMs: 1000 }); + +const badQueries = [ + ["mixed scalar and aggregation", { fields: ["region"], measures: ["revenue"] }], + ["scalar having", { fields: ["region"], having: "revenue > 0" }], + ["empty scalar", { fields: [] }], + ["empty aggregation", {}], + ["empty dimensions and measures", { dimensions: [], measures: [] }], + ["negative limit", { measures: ["revenue"], limit: -1 }], + ["fractional limit", { measures: ["revenue"], limit: 0.5 }], + ["unsafe integer limit", { measures: ["revenue"], limit: Number.MAX_SAFE_INTEGER + 1 }], + ["nonfinite limit", { measures: ["revenue"], limit: Infinity }], + ["unknown clause", { measures: ["revenue"], sql: "select * from sales" }], + ["empty measure", { measures: [""] }], + ["invalid ordering", { measures: ["revenue"], order_by: [{ field: "revenue", direction: "SIDEWAYS" }] }], + ["empty ordering", { measures: ["revenue"], order_by: [] }], +]; + +for (const [name, query] of badQueries) { + test(`rejects ${name} before submission`, async () => { + const mock = mockFetch(() => json(execution())); + await assert.rejects(async () => client(mock.fetch).submit({ ...REQUEST, query }), ProtocolError); + assert.equal(mock.calls.length, 0); + }); +} + +for (const [name, extra] of [ + ["unknown contract version", { contract_version: "future-1" }], + ["unknown state", { state: "done" }], + ["unknown property", { surprising: true }], + ["running execution with error", { error: ERROR }], + ["running execution with result", { result: arrowFixture().succeeded.result }], + ["invalid progress", { progress: { fraction: 1.1 } }], + ["invalid warning", { warnings: [{ message: "No code" }] }], + ["nonobject extension", { extensions: { "com.example.custom": true } }], +]) { + test(`rejects ${name} in execution envelope`, async () => { + const mock = mockFetch(() => json(execution("running", extra))); + await assert.rejects(client(mock.fetch).getExecution("exec-1"), ProtocolError); + }); +} + +for (const state of ["failed", "succeeded"]) { + test(`rejects ${state} missing its mandatory payload`, async () => { + const mock = mockFetch(() => json(execution(state))); + await assert.rejects(client(mock.fetch).getExecution("exec-1"), ProtocolError); + }); +} + +for (const [name, mutation] of [ + ["execution id", value => { value.execution_id = "exec-2"; }], + ["data source", value => { value.data_source_id = "other-source"; }], + ["model id", value => { value.model.id = "other-model"; }], + ["model revision", value => { value.model.revision = "revision-2"; }], +]) { + test(`poll rejects mutable ${name}`, async () => { + const changed = execution(); + mutation(changed); + const mock = mockFetch(() => json(changed)); + await assert.rejects(client(mock.fetch).wait(execution()), ProtocolError); + assert.equal(mock.calls.length, 1); + }); +} + +test("getExecution rejects a response for another execution id", async () => { + const mock = mockFetch(() => json(execution("running", { execution_id: "exec-other" }))); + await assert.rejects(client(mock.fetch).getExecution("exec-1"), ProtocolError); +}); + +test("terminal execution state and immutable result cannot change on later reads", async () => { + for (const mutation of [ + value => { value.state = "running"; delete value.result; }, + value => { value.result.result_id = "replacement"; }, + value => { value.result.url = `${ORIGIN}/replacement`; }, + value => { value.result.row_count++; }, + ]) { + const { succeeded } = arrowFixture(); + const altered = structuredClone(succeeded); + mutation(altered); + const responses = [succeeded, altered]; + const mock = mockFetch(() => json(responses.shift())); + const api = client(mock.fetch); + await api.getExecution("exec-1"); + await assert.rejects(api.getExecution("exec-1"), ProtocolError); + } +}); + +test("client snapshots execution history instead of trusting caller-mutated responses", async () => { + const mock = mockFetch(() => json(execution())); + const api = client(mock.fetch); + const returned = await api.getExecution("exec-1"); + returned.model.revision = "mutated"; + assert.equal((await api.getExecution("exec-1")).model.revision, "revision-1"); +}); + +test("submit rejects a response for another data source", async () => { + const mock = mockFetch(() => json(execution("queued", { data_source_id: "other-source" }))); + await assert.rejects(client(mock.fetch).submit(REQUEST), ProtocolError); +}); + +for (const operation of ["submit", "cancel"]) { + test(`${operation} enforces HTTP 200 terminal and HTTP 202 pending state consistency`, async () => { + const { succeeded } = arrowFixture(); + for (const [value, status] of [[execution("queued"), 200], [succeeded, 202]]) { + const mock = mockFetch(() => json(value, status)); + const api = client(mock.fetch); + await assert.rejects(operation === "submit" ? api.submit(REQUEST) : api.cancel("exec-1"), ProtocolError); + assert.equal(mock.calls.length, 1); + } + }); +} + +for (const [name, change] of [ + ["unsafe row count", value => { value.row_count = Number.MAX_SAFE_INTEGER + 1; }], + ["unsafe byte count", value => { value.byte_count = Number.MAX_SAFE_INTEGER + 1; }], + ["negative count", value => { value.row_count = -1; }], + ["fractional count", value => { value.byte_count = 0.5; }], + ["duplicate column id", value => { value.columns[1].column_id = value.columns[0].column_id; }], + ["unknown logical type", value => { value.columns[0].logical_type = "GuessFromJSON"; }], + ["unknown column attribute", value => { value.columns[0].guess = true; }], + ["missing truncation reason", value => { value.completeness = "truncated"; }], + ["complete with truncation reason", value => { value.truncation_reason = "unneeded"; }], + ["invalid expiry", value => { value.expires_at = "never"; }], + ["relative result URL", value => { value.url = "/results/result-1"; }], +]) { + test(`rejects malformed result manifest: ${name}`, async () => { + const { succeeded } = arrowFixture(); + change(succeeded.result); + const mock = mockFetch(() => json(succeeded)); + await assert.rejects(client(mock.fetch).getExecution("exec-1"), ProtocolError); + }); +} + +test("capabilities revision mismatches and unknown fields fail closed", async () => { + for (const mutation of [ + { contract_version: "0.2" }, { foundation_revision: "old" }, + { query_schema_version: "invented" }, { result_formats: ["json"] }, + { silent_feature: true }, { max_wait_seconds: Number.MAX_SAFE_INTEGER + 1 }, + ]) { + const mock = mockFetch(() => json({ ...CAPABILITIES, ...mutation })); + await assert.rejects(client(mock.fetch).getCapabilities(), ProtocolError); + } +}); + +test("binding identity is checked and extensions remain namespaced", async () => { + const mock = mockFetch(() => json({ ...BINDING, data_source_id: "wrong" })); + await assert.rejects(client(mock.fetch).getBinding("sales"), ProtocolError); + const extension = { "com.example.metrics": { region: "west", enabled: true } }; + const valid = mockFetch(() => json(execution("running", { extensions: extension }))); + assert.deepEqual((await client(valid.fetch).getExecution("exec-1")).extensions, extension); +}); + +test("binding path values cannot escape the configured control-plane base path", async () => { + const id = "sales/../../private?token=x#fragment"; + const mock = mockFetch(() => json({ ...BINDING, data_source_id: id })); + const api = new OssieClient({ baseUrl: `${ORIGIN}/api/v1`, fetch: mock.fetch }); + await api.getBinding(id); + assert.equal(mock.calls.length, 1); + assert.ok(mock.calls[0].url.pathname.startsWith("/api/v1/bindings/")); + assert.equal(mock.calls[0].url.search, ""); + assert.equal(mock.calls[0].url.hash, ""); +}); + +test("literal dot-segment opaque IDs are rejected before any network request", async () => { + for (const method of ["getBinding", "getExecution", "cancel"]) { + for (const id of [".", ".."]) { + assert.equal(encodeURIComponent(id), id, "URI encoding alone does not protect dot segments"); + const mock = mockFetch(() => json(execution())); + const api = new OssieClient({ baseUrl: `${ORIGIN}/api/v1`, fetch: mock.fetch }); + await assert.rejects(async () => api[method](id), ProtocolError); + assert.equal(mock.calls.length, 0, `${method} must reject ${id} before URL normalization or fetch`); + } + } +}); + +test("encoded dots and slashes remain opaque IDs instead of becoming traversal segments", async () => { + for (const id of ["%2e", "%2e%2e", "../admin", "a/../admin"]) { + const mock = mockFetch(() => json(execution("running", { execution_id: id }))); + const api = new OssieClient({ baseUrl: `${ORIGIN}/api/v1`, fetch: mock.fetch }); + assert.equal((await api.getExecution(id)).execution_id, id); + assert.equal(mock.calls.length, 1); + assert.equal(mock.calls[0].url.pathname, `/api/v1/queries/${encodeURIComponent(id)}`); + assert.equal(mock.calls[0].url.search, ""); + assert.equal(mock.calls[0].url.hash, ""); + } +}); + +test("waitSeconds is a bounded nonnegative safe integer", async () => { + for (const waitSeconds of [-1, 0.5, Infinity, Number.MAX_SAFE_INTEGER + 1]) { + const mock = mockFetch(() => json(execution())); + await assert.rejects(async () => client(mock.fetch).submit(REQUEST, { waitSeconds }), ProtocolError); + assert.equal(mock.calls.length, 0); + } +}); diff --git a/typescript/execution/test/review.test.mjs b/typescript/execution/test/review.test.mjs new file mode 100644 index 000000000..7ebe68c98 --- /dev/null +++ b/typescript/execution/test/review.test.mjs @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from "node:assert/strict"; +import test from "node:test"; +import { ExecutionCancelledError, MAX_TRACKED_EXECUTIONS, OssieClient, ProtocolError } from "../dist/index.js"; +import { ERROR, ORIGIN, arrowFixture, collect, execution, json, mockFetch } from "./fixtures.mjs"; + +function deferred() { + let resolve; + const promise = new Promise(done => { resolve = done; }); + return { promise, resolve }; +} + +test("a pending poll arriving after confirmed cancellation preserves cancellation", async () => { + const started = deferred(), held = deferred(); + const mock = mockFetch(call => { + if (call.method === "POST") return json(execution("cancelled")); + started.resolve(); + return held.promise; + }); + const api = new OssieClient({ baseUrl: ORIGIN, fetch: mock.fetch, pollIntervalMs: 1 }); + const waiting = assert.rejects(api.wait(execution("queued")), ExecutionCancelledError); + await started.promise; + assert.equal((await api.cancel("exec-1")).state, "cancelled"); + held.resolve(json(execution("running"))); + await waiting; + assert.equal(mock.calls.length, 2); +}); + +test("a stale concurrent poll still cannot change execution identity", async () => { + const started = deferred(), held = deferred(); + const mock = mockFetch(call => { + if (call.method === "POST") return json(execution("cancelled")); + started.resolve(); + return held.promise; + }); + const api = new OssieClient({ baseUrl: ORIGIN, fetch: mock.fetch, pollIntervalMs: 1 }); + const waiting = assert.rejects(api.wait(execution("queued")), ProtocolError); + await started.promise; + await api.cancel("exec-1"); + held.resolve(json(execution("running", { data_source_id: "different-source" }))); + await waiting; +}); + +test("waiting with an older submitted object observes an already confirmed cancellation", async () => { + const mock = mockFetch(() => json(execution("cancelled"))); + const api = new OssieClient({ baseUrl: ORIGIN, fetch: mock.fetch }); + await api.cancel("exec-1"); + await assert.rejects(api.wait(execution("queued")), ExecutionCancelledError); + assert.equal(mock.calls.length, 1); +}); + +test("sequential lifecycle regressions remain protocol errors", async () => { + let calls = 0; + const mock = mockFetch(() => json(execution(calls++ === 0 ? "running" : "queued"))); + const api = new OssieClient({ baseUrl: ORIGIN, fetch: mock.fetch }); + await api.getExecution("exec-1"); + await assert.rejects(api.getExecution("exec-1"), ProtocolError); +}); + +for (const [name, mutate] of [ + ["error code", state => { state.error.code = "CHANGED"; }], + ["replacement query", state => { state.error.replacement_query = { fields: ["other"] }; }], + ["terminal warnings", state => { state.warnings = [{ code: "CHANGED", message: "changed" }]; }], + ["terminal extensions", state => { state.extensions = { "example.test": { changed: true } }; }], +]) { + test(`terminal immutability includes ${name}`, async () => { + const original = execution("failed", { error: structuredClone(ERROR) }); + const changed = structuredClone(original); + mutate(changed); + const responses = [original, changed]; + const mock = mockFetch(() => json(responses.shift())); + const api = new OssieClient({ baseUrl: ORIGIN, fetch: mock.fetch }); + await api.getExecution("exec-1"); + await assert.rejects(api.getExecution("exec-1"), ProtocolError); + }); +} + +test("terminal comparison ignores JSON object property order", async () => { + const original = execution("failed", { error: structuredClone(ERROR) }); + const reordered = Object.fromEntries(Object.entries(original).reverse()); + const responses = [original, reordered]; + const mock = mockFetch(() => json(responses.shift())); + const api = new OssieClient({ baseUrl: ORIGIN, fetch: mock.fetch }); + await api.getExecution("exec-1"); + assert.deepEqual(await api.getExecution("exec-1"), original); +}); + +test("execution history is bounded and can be explicitly released without network calls", async () => { + const mock = mockFetch(call => json(execution("cancelled", { + execution_id: decodeURIComponent(call.url.pathname.split("/").at(-1)), + extensions: { "example.test": { padding: "x".repeat(128 * 1024) } }, + }))); + const api = new OssieClient({ baseUrl: ORIGIN, fetch: mock.fetch }); + for (let index = 0; index < MAX_TRACKED_EXECUTIONS * 2; index++) { + await api.getExecution(`exec-${index}`); + } + assert.equal(api.executions.size, MAX_TRACKED_EXECUTIONS); + assert.equal(api.executions.has("exec-0"), false); + const count = mock.calls.length; + api.forgetExecution(`exec-${MAX_TRACKED_EXECUTIONS * 2 - 1}`); + assert.equal(api.executions.size, MAX_TRACKED_EXECUTIONS - 1); + api.forgetExecution("never-seen"); + assert.equal(mock.calls.length, count); +}); + +test("an active wait validates its previous state even after history eviction", async () => { + const started = deferred(), held = deferred(); + const mock = mockFetch(call => { + const id = decodeURIComponent(call.url.pathname.split("/").at(-1)); + if (id !== "exec-1") return json(execution("cancelled", { execution_id: id })); + started.resolve(); + return held.promise; + }); + const api = new OssieClient({ baseUrl: ORIGIN, fetch: mock.fetch, pollIntervalMs: 1 }); + const waiting = assert.rejects(api.wait(execution("running")), ProtocolError); + await started.promise; + for (let index = 0; index < MAX_TRACKED_EXECUTIONS; index++) { + await api.getExecution(`other-${index}`); + } + assert.equal(api.executions.has("exec-1"), false); + held.resolve(json(execution("queued"))); + await waiting; +}); + +test("result expiry is rechecked after asynchronous token refresh before issuing GET", async context => { + let now = Date.now(); + context.mock.method(Date, "now", () => now); + const expires = now + 1000; + const fixture = arrowFixture({ manifest: { expires_at: new Date(expires).toISOString() } }); + let fetches = 0; + const api = new OssieClient({ + baseUrl: ORIGIN, + getAccessToken: async () => { + now = expires + 1; + return "SYNTHETIC_TOKEN"; + }, + fetch: async () => { + fetches++; + throw new Error("Expired URL must never be fetched"); + }, + }); + await assert.rejects(collect(api.readBatches(fixture.succeeded)), error => error.code === "RESULT_EXPIRED"); + assert.equal(fetches, 0); +}); + +test("repopulated history cannot replace an active wait's own model identity", async () => { + const started = deferred(), resume = deferred(); + const fixture = arrowFixture(); + fixture.succeeded.model = { id: "sales-model", revision: "different-revision" }; + const mock = mockFetch(call => { + const id = decodeURIComponent(call.url.pathname.split("/").at(-1)); + return json(id === "exec-1" ? fixture.succeeded : execution("cancelled", { execution_id: id })); + }); + const api = new OssieClient({ baseUrl: ORIGIN, fetch: mock.fetch, pollIntervalMs: 1 }); + const waiting = assert.rejects(api.wait(execution("running"), { + onProgress: async () => { + started.resolve(); + await resume.promise; + }, + }), ProtocolError); + await started.promise; + for (let index = 0; index < MAX_TRACKED_EXECUTIONS; index++) { + await api.getExecution(`other-${index}`); + } + await api.getExecution("exec-1"); + resume.resolve(); + await waiting; +}); diff --git a/typescript/execution/test/schema.test.mjs b/typescript/execution/test/schema.test.mjs new file mode 100644 index 000000000..813279247 --- /dev/null +++ b/typescript/execution/test/schema.test.mjs @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; +import Ajv2020 from "ajv/dist/2020.js"; +import addFormats from "ajv-formats"; +import { + COLUMN_METADATA_KEY, EXECUTION_METADATA_KEY, +} from "../dist/index.js"; +import { + BINDING, CAPABILITIES, ERROR, REQUEST, arrowFixture, execution, pagedFixture, +} from "./fixtures.mjs"; + +const schema = JSON.parse(await readFile(new URL("../../../docs/proposals/execution-http/execution.schema.json", import.meta.url), "utf8")); +const openapi = JSON.parse(await readFile(new URL("../../../docs/proposals/execution-http/openapi.json", import.meta.url), "utf8")); +const ajv = new Ajv2020({ allErrors: true, strict: false }); +addFormats(ajv); +ajv.addSchema(schema); +const validate = name => ajv.getSchema(`${schema.$id}#/$defs/${name}`); + +test("independent JSON Schema validators accept every fixture envelope and Arrow metadata document", () => { + const fixture = arrowFixture(); + const paged = pagedFixture(); + const examples = [ + ["Request", REQUEST], + ["Request", { data_source_id: "sales", query: { fields: ["region"], limit: 0 } }], + ["Capabilities", CAPABILITIES], ["Binding", BINDING], + ...["queued", "running", "cancelling", "cancelled"].map(state => ["Execution", execution(state)]), + ["Execution", fixture.succeeded], ["Execution", execution("failed", { error: ERROR })], + ["Execution", paged.succeeded], + ...paged.streams.flatMap(page => [ + ["ResultPage", page.descriptor], + ["ExecutionMetadata", JSON.parse(page.schema.metadata.get(EXECUTION_METADATA_KEY))], + ]), + ["QueryError", ERROR], + ["ExecutionMetadata", JSON.parse(fixture.schema.metadata.get(EXECUTION_METADATA_KEY))], + ...fixture.schema.fields.map(field => ["ColumnMetadata", JSON.parse(field.metadata.get(COLUMN_METADATA_KEY))]), + ]; + for (const [name, value] of examples) { + const validator = validate(name); + assert.ok(validator, `Definition ${name} exists`); + assert.ok(validator(value), `${name}: ${ajv.errorsText(validator.errors)}`); + } +}); + +test("JSON Schema rejects malformed query shapes, unsafe counters and contradictory states", () => { + const fixture = arrowFixture(); + const paged = pagedFixture(); + const examples = [ + ["Request", { ...REQUEST, query: { fields: ["region"], measures: ["revenue"] } }], + ["Request", { ...REQUEST, query: { fields: ["region"], having: "revenue > 1" } }], + ["Request", { ...REQUEST, query: {} }], + ["Request", { ...REQUEST, query: { fields: [] } }], + ["Request", { ...REQUEST, query: { measures: ["revenue"], limit: Number.MAX_SAFE_INTEGER + 1 } }], + ["Request", { ...REQUEST, query: { measures: ["revenue"], order_by: [] } }], + ["Request", { ...REQUEST, query: { measures: ["revenue"], sql: "SELECT 1" } }], + ["Capabilities", { ...CAPABILITIES, contract_version: "2.0" }], + ["Execution", execution("running", { result: fixture.succeeded.result })], + ["Execution", execution("running", { error: ERROR })], + ["Execution", execution("failed")], + ["Execution", execution("succeeded")], + ["Execution", execution("cancelled", { progress: { fraction: 1 } })], + ["Execution", { ...fixture.succeeded, result: { ...fixture.succeeded.result, byte_count: Number.MAX_SAFE_INTEGER + 1 } }], + ["Execution", { ...fixture.succeeded, result: { ...fixture.succeeded.result, completeness: "truncated" } }], + ["Execution", { ...fixture.succeeded, result: { ...fixture.succeeded.result, truncation_reason: "contradiction" } }], + ["Execution", { ...fixture.succeeded, unexpected: true }], + ["ExecutionMetadata", { contract_version: "0.1-draft", execution_id: "exec-1" }], + ["ColumnMetadata", { column_id: "id", unexpected: true }], + ["ResultManifest", { ...paged.succeeded.result, url: "https://example.test/result" }], + ["ResultManifest", { ...paged.succeeded.result, pages: [] }], + ["ResultManifest", { ...fixture.succeeded.result, url: undefined }], + ["ResultPage", { ...paged.streams[0].descriptor, row_offset: -1 }], + ["ResultPage", { ...paged.streams[0].descriptor, byte_count: 0 }], + ["ResultPage", { ...paged.streams[0].descriptor, page_index: Number.MAX_SAFE_INTEGER + 1 }], + ["ResultPage", { ...paged.streams[0].descriptor, unexpected: true }], + ["ExecutionMetadata", { ...JSON.parse(paged.streams[0].schema.metadata.get(EXECUTION_METADATA_KEY)), row_offset: undefined }], + ]; + for (const [name, value] of examples) { + assert.equal(validate(name)(value), false, `${name} unexpectedly accepted ${JSON.stringify(value)}`); + } +}); + +test("OpenAPI defines lifecycle and references only available execution schema definitions", () => { + assert.match(openapi.openapi, /^3\.1\./); + for (const path of ["/capabilities", "/bindings/{id}", "/queries", "/queries/{id}", "/queries/{id}/cancel"]) { + assert.ok(openapi.paths[path], `Missing lifecycle path ${path}`); + } + let references = 0; + function walk(value) { + if (!value || typeof value !== "object") return; + if (typeof value.$ref === "string" && value.$ref.includes("execution.schema.json#/$defs/")) { + const name = value.$ref.split("#/$defs/")[1]; + assert.ok(schema.$defs[name], `Unresolved schema reference ${name}`); + references++; + } + for (const item of Object.values(value)) walk(item); + } + walk(openapi); + assert.ok(references > 0, "OpenAPI must use the shared schema, not a divergent inline model"); +}); + +test("OpenAPI media-type examples validate against their referenced contract definitions", () => { + let examples = 0; + function walk(value) { + if (!value || typeof value !== "object") return; + const reference = value.schema?.$ref; + if (typeof reference === "string" && reference.includes("execution.schema.json#/$defs/")) { + const validator = validate(reference.split("#/$defs/")[1]); + const values = [ + ...(Object.hasOwn(value, "example") ? [value.example] : []), + ...Object.values(value.examples ?? {}).filter(example => Object.hasOwn(example, "value")).map(example => example.value), + ]; + for (const example of values) { + assert.ok(validator(example), ajv.errorsText(validator.errors)); + examples++; + } + } + for (const child of Object.values(value)) walk(child); + } + walk(openapi); + assert.ok(examples > 0, "At least one executable request example must be present"); +}); + +test("normative draft JSON and HTTP-body examples conform structurally to the shared schema", async () => { + const document = await readFile(new URL("../../../docs/proposals/execute-query-http.md", import.meta.url), "utf8"); + const definitions = new Set(); + for (const [, language, contents] of document.matchAll(/```(json|http)\r?\n([\s\S]*?)\r?\n```/g)) { + const body = language === "http" ? contents.replace(/^[\s\S]*?\r?\n\r?\n/, "") : contents; + const value = JSON.parse(body); + const name = "foundation_revision" in value ? "Capabilities" + : "state" in value ? "Execution" + : "result_id" in value && "execution_id" in value ? "ExecutionMetadata" + : "query" in value ? "Request" + : "status" in value && "error" in value ? "ProblemDetails" + : "model" in value && "query_shapes" in value ? "Binding" + : undefined; + assert.ok(name, "Every documented JSON object must identify a known contract definition"); + const validator = validate(name); + assert.ok(validator(value), `${name}: ${ajv.errorsText(validator.errors)}`); + if (language === "http" && name === "ProblemDetails") { + assert.equal(value.status, Number(/^HTTP\/1\.1 (\d+)/.exec(contents)?.[1])); + } + definitions.add(name); + } + assert.deepEqual([...definitions].sort(), + ["Binding", "Capabilities", "Execution", "ExecutionMetadata", "ProblemDetails", "Request"]); +}); diff --git a/typescript/execution/tsconfig.json b/typescript/execution/tsconfig.json new file mode 100644 index 000000000..5cf2bb43a --- /dev/null +++ b/typescript/execution/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "types": [], + "rootDir": "src", + "outDir": "dist", + "strict": true, + "noUncheckedIndexedAccess": true, + "exactOptionalPropertyTypes": true, + "verbatimModuleSyntax": true, + "declaration": true, + "sourceMap": true, + "skipLibCheck": true + }, + "include": ["src/**/*.ts"] +} From 9fe8deff5522139b49cf2ff6e5f49e60f09fa5df Mon Sep 17 00:00:00 2001 From: Markus Cozowicz Date: Mon, 12 Oct 2026 15:41:47 +0200 Subject: [PATCH 2/4] docs: clarify HTTP version negotiation and transport scope --- docs/proposals/execute-query-http.md | 24 ++++++++++++++++++++++ docs/proposals/execution-http/openapi.json | 2 +- typescript/execution/README.md | 15 ++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/docs/proposals/execute-query-http.md b/docs/proposals/execute-query-http.md index f1b236072..395cb41c3 100644 --- a/docs/proposals/execute-query-http.md +++ b/docs/proposals/execute-query-http.md @@ -61,6 +61,30 @@ live-result delivery, cursor APIs, result-link renewal, and automatic repair or re-execution. Flight is a possible future mapping, not implemented or negotiated by this draft. +### HTTP versions and transport behavior + +This is a binding to [HTTP semantics](https://www.rfc-editor.org/rfc/rfc9110), +not an HTTP/1.1-only protocol. The same methods, status codes, headers, JSON +objects, and Arrow IPC payloads can be carried over HTTP/1.1, HTTP/2, or +HTTP/3 where the client, server, and intermediaries support them. The profile +does not require every deployment to support every HTTP version. +The `HTTP/1.1` request/response examples below use readable textual notation; +they do not require that wire framing for HTTP/2 or HTTP/3. + +Transport-version negotiation belongs to the HTTP stack and deployment, not +to the Ossie query or capability schema. The browser Fetch client does not +select or force a version. HTTPS and the authorization, origin, redirect, +expiry, and completeness rules apply regardless of the negotiated version. + +HTTP/2 and HTTP/3 support multiplexed streams, but multiplexing alone does not +supply Flight's endpoint/ticket conventions, distributed-result coordination, +or application-level flow control. The prototype traverses result pages +sequentially without prefetching. Pull-based consumption through Web Streams +does not guarantee end-to-end backpressure or bounded buffering in browsers, +proxies, servers, or query engines. No Flight performance-parity claim is made. +Current integration fixtures use loopback HTTP/1.1; they do not establish +HTTP/2 or HTTP/3 deployment interoperability. + ## 2. JSON objects and discovery JSON messages MUST conform to their named schema definitions. Unknown core diff --git a/docs/proposals/execution-http/openapi.json b/docs/proposals/execution-http/openapi.json index 810503c30..84dccc7cd 100644 --- a/docs/proposals/execution-http/openapi.json +++ b/docs/proposals/execution-http/openapi.json @@ -6,7 +6,7 @@ "info": { "title": "Ossie HTTP execution proposal", "version": "0.1-draft", - "description": "DRAFT, not an adopted standard or a deployed service. Browser-oriented HTTP/JSON control plane and one materialized result delivered as a single Arrow IPC stream or immutable self-contained pages. All schemas are defined in execution.schema.json. Authentication is deployment supplied; this document does not implement an identity system.", + "description": "DRAFT, not an adopted standard or a deployed service. Browser-oriented HTTP/JSON control plane and one materialized result delivered as a single Arrow IPC stream or immutable self-contained pages. This binding uses HTTP semantics, not HTTP/1.1-specific framing: HTTP/1.1, HTTP/2, or HTTP/3 may carry it where supported, with version negotiation left to the HTTP stack and deployment. Supporting every version is not required. Multiplexing does not imply parallel page fetching or Flight-equivalent flow control; the prototype traverses pages sequentially. Current loopback integration fixtures cover HTTP/1.1, not HTTP/2/3 interoperability. All schemas are defined in execution.schema.json. Authentication is deployment supplied; this document does not implement an identity system.", "license": { "name": "Apache-2.0", "identifier": "Apache-2.0" diff --git a/typescript/execution/README.md b/typescript/execution/README.md index 6bd7b7245..b2888d40c 100644 --- a/typescript/execution/README.md +++ b/typescript/execution/README.md @@ -30,6 +30,21 @@ authentication system, Flight transport, or vendor adapters. Local tests exercise the protocol against fixtures; they do not prove production vendor interoperability, Foundation expression semantics, or row-level security. +## HTTP versions + +The binding is **not HTTP/1.1-only**: its HTTP semantics and payloads can run +over HTTP/1.1, HTTP/2, or HTTP/3 where supported. Fetch and the deployment +negotiate the transport; the client exposes no version-selection option. +Textual `HTTP/1.1` examples in the spec are illustrative, not a restriction. +No deployment is required to support all three versions. + +HTTP/2 or HTTP/3 multiplexing does not turn this into Flight or introduce +parallel page downloads: full-result traversal remains sequential. +Web Streams provide pull-based consumption, not a guarantee of end-to-end +backpressure or bounded intermediary buffering. The existing integration +fixtures use loopback HTTP/1.1; HTTP/2/3 interoperability and performance parity +with Flight have not been demonstrated. + ## Contract - [Normative draft](../../docs/proposals/execute-query-http.md). From 5b106d5c04b5ec1611825cf9cc7f74cd93fd8c01 Mon Sep 17 00:00:00 2001 From: Markus Cozowicz Date: Mon, 12 Oct 2026 16:25:13 +0200 Subject: [PATCH 3/4] feat: map execution backends and demonstrate ADBC materialization --- .github/workflows/execution-client-ci.yml | 19 + docs/proposals/execute-query-http.md | 98 +++-- .../execution-http/backend-mapping.md | 103 +++++ examples/execution-adbc/README.md | 148 +++++++ examples/execution-adbc/demo.py | 80 ++++ examples/execution-adbc/materialize.py | 286 ++++++++++++ examples/execution-adbc/pyproject.toml | 35 ++ .../execution-adbc/tests/test_materialize.py | 331 ++++++++++++++ examples/execution-adbc/uv.lock | 414 ++++++++++++++++++ examples/execution-adbc/verify-client.mjs | 89 ++++ typescript/execution/README.md | 13 +- 11 files changed, 1581 insertions(+), 35 deletions(-) create mode 100644 docs/proposals/execution-http/backend-mapping.md create mode 100644 examples/execution-adbc/README.md create mode 100644 examples/execution-adbc/demo.py create mode 100644 examples/execution-adbc/materialize.py create mode 100644 examples/execution-adbc/pyproject.toml create mode 100644 examples/execution-adbc/tests/test_materialize.py create mode 100644 examples/execution-adbc/uv.lock create mode 100644 examples/execution-adbc/verify-client.mjs diff --git a/.github/workflows/execution-client-ci.yml b/.github/workflows/execution-client-ci.yml index e4b28c295..ba854843c 100644 --- a/.github/workflows/execution-client-ci.yml +++ b/.github/workflows/execution-client-ci.yml @@ -24,6 +24,7 @@ on: branches: ["main"] paths: - "typescript/execution/**" + - "examples/execution-adbc/**" - "docs/proposals/execute-query-http.md" - "docs/proposals/execution-http/**" - ".github/workflows/execution-client-ci.yml" @@ -31,6 +32,7 @@ on: branches: ["main"] paths: - "typescript/execution/**" + - "examples/execution-adbc/**" - "docs/proposals/execute-query-http.md" - "docs/proposals/execution-http/**" - ".github/workflows/execution-client-ci.yml" @@ -69,3 +71,20 @@ jobs: export OSSIE_BROWSER_BINARY="$(command -v google-chrome)" test -n "$OSSIE_BROWSER_BINARY" npm run test:browser + - name: Set up Python for ADBC example + if: matrix.node-version == '24' + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.13" + - name: Install uv + if: matrix.node-version == '24' + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + version: "0.12.19" + - name: Exercise ADBC-to-TypeScript materialization + if: matrix.node-version == '24' + working-directory: . + run: | + uv run --frozen --project examples/execution-adbc pytest examples/execution-adbc + uv run --frozen --project examples/execution-adbc python examples/execution-adbc/demo.py --output "$RUNNER_TEMP/ossie-adbc-capture" + node examples/execution-adbc/verify-client.mjs "$RUNNER_TEMP/ossie-adbc-capture" diff --git a/docs/proposals/execute-query-http.md b/docs/proposals/execute-query-http.md index 395cb41c3..6381d2834 100644 --- a/docs/proposals/execute-query-http.md +++ b/docs/proposals/execute-query-http.md @@ -26,6 +26,9 @@ prototype, not a server, semantic engine, or vendor adapter. This proposal binds structured Ossie query submission and status to HTTP/JSON, and completed tabular results to Apache Arrow IPC streams. It targets browser applications without requiring gRPC. It does not define a new query language. +It is a complementary access binding, not a replacement for Flight SQL or +ADBC. Engines may retain those interfaces behind an HTTP adapter; Layer 3 +interpretation remains a separate semantic responsibility. ## 1. Normative basis and scope @@ -517,35 +520,13 @@ aggregate totals. Batches yielded before complete-result validation are consumer has not obtained a verified complete result; applications MUST discard or explicitly label previously consumed partial data. -The package's `readBatches` exposes that provisional iteration boundary and -traverses all pages in manifest order. `execute` materializes and returns only -after validating the entire result. Both enforce aggregate limits across all -pages, defaulting to 64 MiB decoded IPC bytes and 1,000,000 rows. -`readPage(execution: SuccessfulExecution, pageIndex, options)` returns a -`Promise
` for just one fully validated page, with `allowTruncated` and -`signal` options. It applies the configured row/byte limits to that page, so it -can browse a page even when the whole result exceeds those limits; it does not -assert that unrequested pages were downloaded or validated. For unpaged -delivery, index `0` reads the single stream and every other index is rejected. -Limits are configurable; they are client admission limits, not a request to -silently truncate. The package -rejects declared truncation unless the caller explicitly opts in; opting in -does not waive framing, identity, count, or resource-limit validation. -Arrow decoding and application allocations can exceed the IPC byte size; the -byte limit is not a heap-size guarantee. -The prototype additionally caps a control-plane JSON response and each IPC -metadata message at 1 MiB; these are client resource limits, not semantic rules. -The client checks FlatBuffer bounds and flat-schema vector counts before -decoding, and limits expanded metadata per message to `maxResultBytes`, -including repeated references. This is an explicit admission policy, not a -claim that decoded Arrow memory usage equals wire size. -Cross-call lifecycle history is bounded to the 64 most-recently observed -executions and may be released locally with `forgetExecution`. Active waits -retain their own identity/transition baseline across history eviction. -Overlapping requests may return stale pending observations; those cannot -overwrite newer observations, while identity and terminal immutability checks -remain mandatory. Forgetting history does not cancel execution or delete -server results, and the client cannot compare history it no longer retains. +Clients MAY impose local admission limits and MUST fail explicitly rather than +silently truncate when those limits are exceeded. A client accepting a declared +truncated result MUST still enforce identity, schema, count, and framing rules. +Selected-page consumption MUST NOT be represented as whole-result validation. +Prototype-specific defaults, memory policies, and public library methods are +documented in the non-normative implementation notes below; they are not +requirements to implement this HTTP binding. ## 5. Authorization and browser deployment @@ -599,15 +580,66 @@ cross-vendor semantic equivalence. A focused, endpoint-specific comparison is in the [package README](../../typescript/execution/README.md#feature-comparison). [Arrow Flight][flight] is an Arrow-oriented RPC transport; Flight SQL adds a -SQL service protocol. [ADBC][adbc] is a database driver API, not a replacement -HTTP wire format. A later Flight mapping would need its own authentication, -capability negotiation, identity, and completion rules. This draft and package -implement neither Flight nor ADBC. +database service protocol. [ADBC][adbc] is a database driver API, not an HTTP +wire format. The [backend mapping](execution-http/backend-mapping.md) describes +corresponding operations and obligations that cannot be delegated automatically. +In particular, existing Flight SQL and ADBC endpoints do not gain Ossie query +semantics merely by receiving a new request envelope. + +A separate [ADBC materialization example](../../examples/execution-adbc/README.md) +demonstrates reusing a real ADBC SQLite result reader behind the existing +materialized-result contract. It is not an HTTP service, general Layer 3 +compiler, Flight implementation, or adapter certification for other drivers. +The TypeScript client itself implements neither Flight nor an ADBC driver. Local fixtures and JSON Schema validation establish only exercised transport behavior. They do not establish expression-language conformance, correct joins/aggregation, production authorization, or native vendor interoperability. +## 7. Experimental status and implementation evidence (non-normative) + +The proposed progression criterion is two independently implemented backends +demonstrating the same declared Layer 3 subset through a common client. Evidence +should cover expected answers for grain-sensitive queries, unsupported-query +refusals, caller/model isolation, cancellation races, paging, and partial or +failed result delivery. The scope and capability differences must be explicit; +passing a finite corpus is not proof of general semantic equivalence. + +The local TypeScript fixtures and the single ADBC SQLite example do not satisfy +that criterion. There are no confirmed independent engine commitments recorded +by this proposal. Keeping the PR experimental avoids treating a prototype as +community adoption; adoption remains subject to the project's normal process. + +A future streaming-partition mode, different metadata attachment rules, or +renewable access references should be motivated by adapter evidence and reviewed +as protocol changes. They are not implicit relaxations of this draft's exact +counts, immutable manifests, page identity, or expiry requirements. + +## 8. Prototype implementation notes (non-normative) + +The TypeScript package's `readBatches` yields provisional batches in page order; +`execute` returns a fully validated table; `readPage` validates only a selected +page (index zero for unpaged delivery). Whole-result operations use aggregate +limits, while `readPage` uses selected-page limits. Defaults are 64 MiB decoded +IPC bytes and 1,000,000 rows. Declared truncation requires explicit caller +acceptance. These are configurable admission policies, not protocol defaults. +Arrow decoding and application allocations may exceed IPC byte size. + +The client limits control JSON and each IPC metadata message to 1 MiB. Its +predecode FlatBuffer guard checks bounds and schema-vector counts, with repeated +metadata references charged against `maxResultBytes`. That is prototype +hardening, not a new Arrow format or a prescribed verifier implementation. +Replacing generic binary verification with maintained upstream/library support +is preferable if equivalent pre-allocation safety and resource limits can be +preserved; Ossie identity/count/type checks would still be needed. + +Cross-call history retains 64 recently observed executions and can be released +with `forgetExecution`, without changing server state. Active waits preserve +their own identity baseline. Reordered concurrent pending responses cannot +replace newer observations; terminal and identity checks still apply. +The client cannot compare forgotten history. Full implementation details and +security limitations are in the [client README](../../typescript/execution/README.md). + [query-schema]: https://raw.githubusercontent.com/apache/ossie/b5418ee3aa19f117a1c464733de21f5a0dc0326b/validation/mcp/execute-query-query.schema.json [foundation-semantics]: https://github.com/apache/ossie/blob/cc0d07099a3ef31e85af1fd93458a52186fae2c6/core-spec/foundational_semantics.md [rest-thread]: https://lists.apache.org/thread/xvpomkdvmgv1m5krprq590l8mzccks5n diff --git a/docs/proposals/execution-http/backend-mapping.md b/docs/proposals/execution-http/backend-mapping.md new file mode 100644 index 000000000..bc5c6bb27 --- /dev/null +++ b/docs/proposals/execution-http/backend-mapping.md @@ -0,0 +1,103 @@ + + +# Backend mapping: materialized HTTP, Flight SQL, and ADBC + +**Design note for [HTTP execution 0.1-draft](../execute-query-http.md), not an +additional wire contract or adopted standard.** Flight/ADBC and a browser HTTP +binding are complementary, not replacements for each other. Reusing established +backend execution/data APIs can reduce gateway work without pretending that +they already provide Ossie semantic planning or the HTTP draft's guarantees. + +## Concrete boundaries + +```text +structured Ossie query + immutable binding + -> authorization / semantic planning / backend compilation (external) + -> Flight SQL or ADBC execution (backend-specific) + -> validate, annotate, materialize immutable Arrow IPC (adapter) + -> HTTP execution job + manifest + result delivery (gateway) +``` + +Native backend SQL exists only behind the planner boundary. It is not a new +HTTP `query` variant or a fallback after semantic validation fails. A fixed SQL +fixture is not a general Layer 3/Foundation compiler. + +| Concern | Established backend API | Work still required by this HTTP draft | +| --- | --- | --- | +| Submission | [Flight SQL][flight-sql] packs `CommandStatementQuery` in Protobuf `Any` in a CMD `FlightDescriptor`, obtains `FlightInfo` through `GetFlightInfo`, then reads tickets with `DoGet`. [ADBC][adbc] executes a statement; Python DBAPI `execute` followed by `fetch_record_batch` exposes an Arrow reader. | Compile the authorized structured query for that backend. Assign and retain Ossie execution, result, binding, and model identities. Neither interface supplies those semantics automatically. | +| Pending execution | [Flight][flight] `PollFlightInfo`, when the server supports the applicable command, can yield evolving `FlightInfo` and a retry descriptor. An ADBC driver may expose incremental partition execution. | A durable job/id mapping and worker retain state across HTTP requests. The gateway bounds `Prefer: wait=N`, polls **the same** job, and publishes the whole immutable manifest only after materialization. An ordinary synchronous DBAPI call is not a durable asynchronous job service. | +| Data | `DoGet` streams Arrow batches; endpoint locations are alternatives for the same partition, not additional partitions. ADBC readers expose Arrow batches. Partition execution/read support is driver-specific; the [SQLite driver][sqlite] does not support partitioned result sets. | Fetch every required partition once, validate flat supported types and fixed schema, attach Ossie schema/column identities, and serialize self-contained IPC streams. ADBC/C data transfer or Flight transport is not itself the required HTTP IPC resource. | +| Counts | [FlightInfo][flight-proto] `total_records`/`total_bytes` may be unknown (`-1`, **not zero**). Backend counts need not describe the final reserialized HTTP IPC byte length. | Exact decoded IPC counts, including framing, rewritten metadata, and repeated page schemas, normally require spooling before success. Do not translate unknown into empty or equate compressed `Content-Length` with decoded bytes. | +| Ordering | `FlightInfo.ordered=true` defines concatenation in endpoint order. With `ordered=false`, inter-endpoint ordering is not promised; each endpoint retains its own row order. ADBC exposes the backend reader's order. | Preserve the planner's `ORDER BY` semantics and reader/partition order. Arbitrarily sorting endpoint descriptors cannot manufacture the missing global row order. Require a backend ordering guarantee or a planner-owned merge/sort, otherwise refuse that ordered execution. | +| Replay and expiry | A Flight endpoint with `expiration_time` permits repeated `DoGet` before expiry; absent expiry, retryability is application-defined. Optional `RenewFlightEndpoint` extends access. ADBC partition tokens have driver-specific lifetime/reuse semantics. | Pages must refer to one retained immutable materialization, with contiguous indices/offsets and one whole-result expiry. Snapshot/provider identity must survive mapping; a refetch must not silently re-execute against changed data. Renewal of a data reference is not a change to result data. This HTTP draft currently has **no renewal**. | +| Cancellation | Flight `DoAction(CancelFlightInfo)` reports cancelled, cancelling, not-cancellable, or unspecified/unknown outcomes. ADBC 1.1 introduced cancellation; Python exposes `adbc_cancel`, with actual support depending on the driver/backend. | Keep best-effort cancellation and completion races distinct. Accepted cancellation is not confirmed cancellation, and closing a local reader is not a durable backend cancel operation. The gateway owns the state machine. | +| Errors | Flight status codes and ADBC portable status categories can carry provider-specific details such as vendor codes/SQLSTATE. | Preserve genuine planner/Foundation semantic codes directly; do not invent an `E_*` semantic diagnosis from a generic backend status. Sanitize provider details, keep private diagnostics separate, and leave unknown codes inspectable. Retryability is a reviewed policy decision, not automatic re-execution. | + +Raw Flight permits application-defined commands. A server accepting raw SQL +bytes in a custom Flight descriptor is **not automatically Flight SQL +compatible**: Flight SQL defines specific messages and their RPC behavior. +Likewise, merely returning Arrow does not establish either Ossie or Flight SQL +semantic conformance. + +The gateway additionally owns authorization at submission/status/result access, +RLS-aware backend credentials and session selection, origin/redirect/CORS policy, +and isolation of provider credentials/tickets from browser tokens. None of the +adapter mechanics authorize a query. Durable queues, job retention, bounded +wait, identity mapping, and terminal publication remain gateway responsibilities. + +## Can upstream HTTP locations be reused? + +[Flight locations][flight-proto] can advertise HTTP URLs, including formats +other than IPC. Reuse is possible **only** when the gateway can establish all +of this draft's requirements: supported self-contained IPC/media type, Ossie +schema and column metadata, immutable binding/result/page identity, physical +schema/type compatibility, exact decoded byte/row counts, required ordering, +retention/replay and whole-result expiry, and applicable authorization/origin/ +redirect/token-isolation protections. + +An Arrow-looking content type or a presigned URL alone is insufficient. If any +guarantee is missing, consume and validate the upstream result, rewrite metadata, +and reframe/spool it into owned resources rather than passing the location +through. Even when some Arrow buffers can be shared in-process, metadata +rewriting and IPC serialization still incur work: no zero-copy, live-streaming, +or Flight performance-parity claim follows. + +## Evidence first, then possible extensions + +The [ADBC SQLite example](../../../examples/execution-adbc/README.md) implements +one narrow path today: trusted SQL or an existing ADBC reader into immutable, +single-stream/paged IPC using native PyArrow, with resource ceilings and exact +final counts. Its capture is consumed by the unchanged TypeScript client's +validators through synthetic Fetch. It does not run a Flight SQL service or a +live HTTP engine, implement a planner/job gateway, or prove multiple vendors +adopt the profile. + +This makes the existing materialized contract testable before weakening it. +A future streaming-partition mode could expose evolving endpoints/unknown +totals; a future optional-metadata mode could reduce annotation requirements. +Both would change observable guarantees and need explicit negotiation, +completeness/ordering/expiry rules, and evidence from additional integrations. +They are follow-up questions, **not changes made by this mapping or example**. + +[flight]: https://arrow.apache.org/docs/format/Flight.html +[flight-sql]: https://arrow.apache.org/docs/format/FlightSql.html +[flight-proto]: https://github.com/apache/arrow/blob/main/format/Flight.proto +[adbc]: https://arrow.apache.org/adbc/current/format/specification.html +[sqlite]: https://arrow.apache.org/adbc/current/driver/sqlite.html diff --git a/examples/execution-adbc/README.md b/examples/execution-adbc/README.md new file mode 100644 index 000000000..b6babe355 --- /dev/null +++ b/examples/execution-adbc/README.md @@ -0,0 +1,148 @@ + + +# ADBC SQLite materialization example + +One backend-side reference adapter for the existing +[HTTP execution 0.1-draft](../../docs/proposals/execute-query-http.md) result +contract. This is **not a server, Foundation compiler, conformance certificate, +or evidence of vendor adoption**. The [backend mapping](../../docs/proposals/execution-http/backend-mapping.md) +explains the complementary roles of Flight/Flight SQL, ADBC, and the HTTP gateway. + +The actual data path is: + +```text +trusted backend SQL -> ADBC SQLite execute/fetch_record_batch + -> PyArrow IPC materialization -> existing TypeScript client +``` + +The CLI seeds a synthetic SQLite fixture, then opens it read-only through ADBC. +It queries deliberately unsorted input with `ORDER BY sequence`, preserving +`9007199254740993`, a null, and `-7` exactly. There is no public SQL submission +endpoint. The fixed SQL in `demo.py` is a fixture, **not** a compiler for the +structured HTTP query shape. + +## Run from the repository root + +Python 3.13, uv, and the existing Node/TypeScript client build are sufficient. +Dependencies are isolated in this example's ignored `.venv`; `uv.lock` pins +the packages and hashes. Nothing is installed globally. + +```powershell +uv sync --project examples\execution-adbc --python 3.13 --locked +uv run --project examples\execution-adbc --python 3.13 --locked python -m pytest examples\execution-adbc\tests --basetemp=examples\execution-adbc\.test-output -q -W error +uv run --project examples\execution-adbc --python 3.13 --locked python examples\execution-adbc\demo.py --output examples\execution-adbc\.capture-paged +npm --prefix typescript\execution run build +node examples\execution-adbc\verify-client.mjs examples\execution-adbc\.capture-paged +``` + +Use a **new explicit directory** for each demo run; existing directories are +refused. Absolute output paths are also accepted. Add `--single-stream` to the +Python demo and use a different output directory to exercise the `url` form. +The default demo emits two pages (two rows, then one). Delete the generated +capture and test directories when finished; do not commit their contents. + +Each capture has `execution` (the full succeeded execution) and `resources` +(objects with `url` and a generated relative `file` basename). The only durable +demo outputs are `capture.json` and `.arrow` streams. The fixture database is +closed and removed. URLs use `https://example.test` and are **not live links**. +The 2099 expiry is only a test value, not a retention commitment. + +`verify-client.mjs` reads the Python-written bytes via synthetic Fetch, retaining +the existing TypeScript client's framing, FlatBuffer, metadata, count, expiry, +and physical-type validators. It verifies selected pages, full `readBatches` +exhaustion, and `execute`, including bigint/null/order. Malformed captures or +unsupported IPC fail nonzero. No Node Arrow writer creates the payload. +The synthetic request uses the captured binding id and column model references; +the client checks captured execution/model/page identities against native IPC metadata. +This proves native ADBC-to-IPC-to-client interoperability for this fixture, +**not live engine HTTP, browser deployment, or Flight SQL interoperability**. + +## Integration boundary and ownership + +`materialize_sqlite(database, trusted_sql, **options)` owns its read-only +connection, cursor, and result reader. It opens an existing database with +`mode=ro` and enables SQLite `query_only`. These backend settings are tested; +they are not a SQL sandbox or a substitute for authorization. There is no +regex SQL filtering. + +Alternatively, an integration can call +`materialize(cursor.fetch_record_batch(), **options)` with an already-executed +ADBC cursor. `materialize` takes ownership of and closes the reader, including +on failure. The caller owns/closes that cursor and its connection. Both paths +use the same materializer, not two backend implementations. + +The caller supplies: + +- Trusted, already-authorized/compiled SQL **or** an existing Arrow reader. + Native SQL never becomes an HTTP-profile `query`. +- `execution_id`, `result_id`, `data_source_id`, and `model={id, revision}`. + A gateway/planner must ensure binding immutability and unique execution/result + identity; this example has no registry enforcing lifetime uniqueness. +- `columns` in physical field order: `column_id`, `name`, `logical_type`, + `nullable`, optionally `role`, `model_ref`, `label`, `description`, `unit`, + and `currency`. Descriptor extensions are intentionally not supported here. + Semantic descriptors come from the planner, not guesses from SQL aliases. +- An existing private `output` directory chosen by trusted application code, + `expires_at`, and optional `page_rows`, `max_rows`, `max_bytes`. + The row/decoded-IPC-byte defaults are 100,000/64 MiB; `page_rows=None` chooses + a single stream. Pagination never re-executes SQL or adds query offsets. + +## Guarantees and deliberate limits + +- Arrow primitive physical types and values are retained without coercion. + Unsupported types, schema drift, extension metadata, and descriptor + mismatches fail closed. SQLite's dynamically inferred Arrow types are + driver-specific; an empty query may not infer the type a planner expects. + The adapter rejects mismatches rather than manufacturing a replacement type. +- Native PyArrow writes uncompressed little-endian V5 IPC. Schema and field + metadata are rewritten with the required Ossie identities. Provider metadata + is deliberately not forwarded. This requires IPC serialization and schema + rewriting; there is **no zero-copy claim**. +- Pages preserve reader order, repeat the same field schema, and carry + `page_index`/`row_offset`. Empty results still get one schema-bearing stream. + Exact row and decoded-byte totals include repeated schemas and IPC framing. +- No success descriptor is returned until exhaustion, writer finalization, + reader closure, and an expiry recheck. A bounded sink limits actual IPC bytes + written, including metadata/framing. Limits fail the operation, never return + success-shaped truncation. The final control JSON is capped at 1 MiB. + Driver batches, Arrow allocations, metadata, and query-engine work are not + bounded by the on-disk byte ceiling; no heap/deadline guarantee is claimed. +- Generated UUID basenames are opened exclusively; wire IDs never choose paths. + A failed call removes only its own files, preserving unrelated caller files. + Error-path SQLite statement preparation clears a retained failed-step status + before close; it does not execute or retry the query. +- Local `MaterializationError.public_detail()` exposes a fixed category/message, + never native SQL, backend diagnostics, credentials, or file paths. These + categories are **not Foundation semantic error codes** or ready-made + `QueryError` objects. A production gateway owns reviewed error translation + and private diagnostics. + +The caller must keep the spool private until success and preserve the published +bytes thereafter. There is no crash recovery, atomic durable publication, +job store, HTTP service, authorization/RLS, cancellation, renewal, expiry +deletion scheduler, provider-token forwarding, or general semantic planning. +Existing-query cancellation and durable job lifecycle belong to the gateway, +not to closing this synchronous reader. + +Tests validate emitted objects against the existing JSON Schema, plus runtime +identity/count/order invariants and actual SQLite/ADBC values. Primitive-type +unit fixtures test the materializer independently; they do not claim SQLite +supports all those types. Further backend work should follow measured +integration evidence rather than relaxing the current contract in advance. diff --git a/examples/execution-adbc/demo.py b/examples/execution-adbc/demo.py new file mode 100644 index 000000000..8f1cac40a --- /dev/null +++ b/examples/execution-adbc/demo.py @@ -0,0 +1,80 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Fixed trusted backend SQL fixture, not an Ossie/Foundation compiler.""" + +import argparse +import json +import sqlite3 +from contextlib import closing +from pathlib import Path +from uuid import uuid4 + +from materialize import MaterializationError, materialize_sqlite + +SQL = "SELECT sequence, amount FROM sample ORDER BY sequence" +COLUMNS = [ + dict(column_id="sequence-id", name="sequence", logical_type="Integer", + nullable=True, role="field", model_ref="sample.sequence"), + dict(column_id="amount-id", name="amount", logical_type="Integer", + nullable=True, role="field", model_ref="sample.amount", + label="Exact amount", description="Synthetic integer fixture", unit="units"), +] +IDENTITY = dict(execution_id="adbc-demo-query", result_id="adbc-demo-result", + data_source_id="sqlite-demo-v1", model={"id": "sqlite-demo", "revision": "1"}, + columns=COLUMNS, expires_at="2099-01-01T00:00:00Z") + + +def seed_database(path): + # Fixture setup alone writes SQLite. The adapter subsequently opens mode=ro. + with closing(sqlite3.connect(path)) as connection, connection: + connection.execute("CREATE TABLE sample (sequence INTEGER, amount INTEGER)") + connection.executemany("INSERT INTO sample VALUES (?, ?)", + [(3, -7), (1, 9007199254740993), (2, None)]) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True, + help="New private output directory (must not already exist)") + parser.add_argument("--single-stream", action="store_true") + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=False) + database = args.output / f"fixture-{uuid4().hex}.sqlite" + try: + seed_database(database) + capture = materialize_sqlite(database, SQL, output=args.output, + page_rows=None if args.single_stream else 2, **IDENTITY) + with (args.output / "capture.json").open("x", encoding="utf-8") as stream: + json.dump(capture, stream, indent=2) + stream.write("\n") + except Exception: + # This CLI created the directory exclusively; no pre-existing caller data. + for path in args.output.iterdir(): + path.unlink() + args.output.rmdir() + raise + finally: + database.unlink(missing_ok=True) + print(f"Wrote {len(capture['resources'])} immutable IPC resource(s) to {args.output}") + + +if __name__ == "__main__": + try: + main() + except MaterializationError as error: + raise SystemExit(json.dumps(error.public_detail())) from None diff --git a/examples/execution-adbc/materialize.py b/examples/execution-adbc/materialize.py new file mode 100644 index 000000000..cc895c2b7 --- /dev/null +++ b/examples/execution-adbc/materialize.py @@ -0,0 +1,286 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Backend-only materialization; no HTTP query parser or semantic compiler.""" + +import copy +import io +import json +import sys +from contextlib import ExitStack, suppress +from datetime import datetime, timezone +from pathlib import Path +from uuid import uuid4 + +import pyarrow as pa + +EXECUTION_KEY = b"org.apache.ossie.execution" +COLUMN_KEY = b"org.apache.ossie.column" +MAX_COUNTER = 9007199254740991 +MESSAGES = { + "invalid_input": "Invalid materialization configuration.", + "unsupported_type": "Backend schema does not match the supported result profile.", + "resource_limit": "Materialization exceeded its configured resource limit.", + "materialization_failed": "Backend result could not be materialized.", +} + + +class MaterializationError(Exception): + """Local adapter category, deliberately not a Foundation QueryError code.""" + + def __init__(self, category): + self.category = category + super().__init__(MESSAGES[category]) + + def public_detail(self): + return {"category": self.category, "message": str(self)} + + +def _require(condition, category="invalid_input"): + if not condition: + raise MaterializationError(category) + + +def _text(value): + return isinstance(value, str) and bool(value.strip()) + + +def _counter(value, minimum=0): + _require(type(value) is int and minimum <= value <= MAX_COUNTER) + + +def _json(value): + return json.dumps(value, ensure_ascii=True, separators=(",", ":"), allow_nan=False).encode() + + +def _unexpired(value): + _require(isinstance(value, str)) + # Require an explicit RFC 3339 offset; emit canonical UTC rather than a local time. + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + raise MaterializationError("invalid_input") from None + _require("T" in value and parsed.tzinfo is not None) + _require(parsed > datetime.now(timezone.utc)) + return parsed.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _discard(paths): + failed = False + for path in paths: + try: + path.unlink(missing_ok=True) + except OSError: + failed = True + if failed: + raise MaterializationError("materialization_failed") from None + + +def _logical_type(t): + if pa.types.is_string(t): + return "String" + if pa.types.is_integer(t): + return "Integer" + if t in (pa.float32(), pa.float64()): + return "Float" + if pa.types.is_boolean(t): + return "Boolean" + if pa.types.is_decimal128(t) or pa.types.is_decimal256(t): + return "Decimal" + if pa.types.is_date32(t): + return "Date" + if pa.types.is_time32(t) or pa.types.is_time64(t): + return "Time" + if pa.types.is_timestamp(t): + _require(t.tz is None or _text(t.tz), "unsupported_type") + return "DateTimeTz" if t.tz else "DateTime" + raise MaterializationError("unsupported_type") + + +def _fields(schema, columns): + required = {"column_id", "name", "logical_type", "nullable"} + optional = {"role", "model_ref", "label", "description", "unit", "currency"} + _require(isinstance(columns, list) and len(columns) == len(schema) > 0) + ids, fields = set(), [] + for field, column in zip(schema, columns): + _require(isinstance(column, dict) and required <= column.keys()) + _require(column.keys() <= required | optional) + _require(all(_text(v) for k, v in column.items() if k != "nullable")) + _require(type(column["nullable"]) is bool and column["column_id"] not in ids) + _require("role" not in column or column["role"] in ("field", "dimension", "measure")) + ids.add(column["column_id"]) + _require(not any(k.startswith(b"ARROW:extension:") for k in (field.metadata or {})), + "unsupported_type") + _require(field.name == column["name"] and field.nullable == column["nullable"] + and _logical_type(field.type) == column["logical_type"], "unsupported_type") + fields.append(field.with_metadata({COLUMN_KEY: _json({"column_id": column["column_id"]})})) + return fields + + +class _BoundedSink(io.RawIOBase): + def __init__(self, raw, remaining): + super().__init__() + self.raw, self.remaining = raw, remaining + + def writable(self): + return True + + def tell(self): + return self.raw.tell() + + def write(self, data): + _require(len(data) <= self.remaining[0], "resource_limit") + count = self.raw.write(data) + self.remaining[0] -= count + return count + + +def materialize(reader, *, output, execution_id, result_id, data_source_id, model, + columns, expires_at, page_rows=1000, max_rows=100000, max_bytes=64 * 1024 * 1024): + """Consume and close an owned Arrow reader, returning a complete capture. + + Inputs are supplied by an already-authorized planner/backend integration. + `output` must be an existing private directory, not derived from a wire id. + `page_rows=None` selects single-stream delivery. No partial success is returned. + Only generated files owned by this call are deleted on failure. + """ + created, complete = [], False + try: + with ExitStack() as stack: + stack.callback(reader.close) + _require(sys.byteorder == "little", "unsupported_type") + for value in (execution_id, result_id, data_source_id): + _require(_text(value) and value not in (".", "..")) + _require(isinstance(model, dict) and model.keys() == {"id", "revision"}) + _require(all(_text(v) and v not in (".", "..") for v in model.values())) + _counter(max_rows) + _counter(max_bytes, 1) + if page_rows is not None: + _counter(page_rows, 1) + expiry = _unexpired(expires_at) + columns, model = copy.deepcopy(columns), copy.deepcopy(model) + source_schema = reader.schema + fields = _fields(source_schema, columns) + output = Path(output) + _require(output.is_dir()) + identity = dict(contract_version="0.1-draft", execution_id=execution_id, + result_id=result_id, data_source_id=data_source_id, model=model) + rows, pages, resources, remaining = 0, [], [], [max_bytes] + + def batches(): + received = 0 + for batch in reader: + _require(batch.schema.equals(source_schema, check_metadata=True), "unsupported_type") + batch.validate(full=True) + _require(all(f.nullable or a.null_count == 0 + for f, a in zip(source_schema, batch.columns)), "unsupported_type") + received += batch.num_rows + _require(received <= max_rows, "resource_limit") + if batch.num_rows: + yield batch + + iterator = batches() + pending = next(iterator, None) + while pending is not None or not pages: + index, offset, page_count = len(pages), rows, 0 + metadata = dict(identity) + if page_rows is not None: + metadata.update(page_index=index, row_offset=offset) + schema = pa.schema(fields, metadata={EXECUTION_KEY: _json(metadata)}) + name = f"page-{uuid4().hex}.arrow" + path = output / name + with path.open("xb") as raw: + created.append(path) + with _BoundedSink(raw, remaining) as sink: + options = pa.ipc.IpcWriteOptions(compression=None, metadata_version=pa.ipc.MetadataVersion.V5) + with pa.ipc.new_stream(sink, schema, options=options) as writer: + while pending is not None: + take = pending.num_rows if page_rows is None else min( + pending.num_rows, page_rows - page_count) + part = pending.slice(0, take) + writer.write_batch(pa.RecordBatch.from_arrays(part.columns, schema=schema)) + rows += take + page_count += take + pending = (pending.slice(take) if take < pending.num_rows + else next(iterator, None)) + if page_rows is not None and page_count == page_rows: + break + byte_count = raw.tell() + url = f"https://example.test/results/{name}" + pages.append(dict(page_index=index, row_offset=offset, row_count=page_count, + byte_count=byte_count, url=url)) + resources.append(dict(url=url, file=name)) + + result = dict(result_id=result_id, format="arrow-stream", row_count=rows, + byte_count=max_bytes - remaining[0], completeness="complete", + expires_at=expiry, columns=columns) + if page_rows is None: + result["url"] = pages[0]["url"] + else: + result["pages"] = pages + execution = {k: v for k, v in identity.items() if k != "result_id"} + execution.update(state="succeeded", warnings=[], result=result) + # The current TS client admits at most 1 MiB of control JSON. + _require(len(_json(execution)) <= 1024 * 1024, "resource_limit") + capture = dict(execution=execution, resources=resources) + _unexpired(expiry) + complete = True + return capture + except MaterializationError: + raise + except Exception: + raise MaterializationError("materialization_failed") from None + finally: + if not complete: + _discard(created) + + +def materialize_sqlite(database, trusted_sql, **options): + """Execute trusted, already-authorized/compiled SQL on an existing read-only DB. + + No public HTTP request is accepted here. Connections/cursors are always closed. + The reader-only materialize() entry point also works with an existing ADBC cursor. + """ + from adbc_driver_sqlite import dbapi + + capture, complete = None, False + try: + _require(_text(trusted_sql)) + uri = Path(database).resolve().as_uri() + "?mode=ro" + with dbapi.connect(uri=uri) as connection: + with connection.cursor() as cursor: + try: + cursor.execute("PRAGMA query_only = ON") + cursor.execute(trusted_sql) + capture = materialize(cursor.fetch_record_batch(), **options) + except Exception: + # SQLite finalize repeats a failed step's error. Clear it before + # DBAPI close (ADBC 1.12); prepare does not execute/retry the query. + with suppress(dbapi.Error): + cursor.adbc_statement.set_sql_query("") + cursor.adbc_statement.prepare() + raise + _unexpired(capture["execution"]["result"]["expires_at"]) + complete = True + return capture + except MaterializationError: + raise + except Exception: + raise MaterializationError("materialization_failed") from None + finally: + if capture is not None and not complete: + _discard(Path(options["output"]) / r["file"] for r in capture["resources"]) diff --git a/examples/execution-adbc/pyproject.toml b/examples/execution-adbc/pyproject.toml new file mode 100644 index 000000000..34e7212e7 --- /dev/null +++ b/examples/execution-adbc/pyproject.toml @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[project] +name = "ossie-execution-adbc-example" +version = "0.0.0" +requires-python = ">=3.11" +dependencies = [ + "adbc-driver-sqlite>=1.9,<2", + "pyarrow>=19,<25", +] + +[dependency-groups] +dev = [ + "jsonschema>=4.23,<5", + "pytest>=8,<10", +] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["."] diff --git a/examples/execution-adbc/tests/test_materialize.py b/examples/execution-adbc/tests/test_materialize.py new file mode 100644 index 000000000..706d4a045 --- /dev/null +++ b/examples/execution-adbc/tests/test_materialize.py @@ -0,0 +1,331 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import copy +import json +import sqlite3 +import traceback +from contextlib import closing +from datetime import date, datetime, time, timezone +from decimal import Decimal +from pathlib import Path + +import jsonschema +import pyarrow as pa +import pytest +from adbc_driver_sqlite import dbapi + +from demo import IDENTITY, SQL, seed_database +from materialize import COLUMN_KEY, EXECUTION_KEY, MaterializationError, materialize, materialize_sqlite + +SCHEMA = json.loads((Path(__file__).resolve().parents[3] / "docs" / "proposals" / + "execution-http" / "execution.schema.json").read_text()) + + +def validate(value, definition): + schema = {**SCHEMA, "$ref": f"#/$defs/{definition}"} + jsonschema.Draft202012Validator(schema, format_checker=jsonschema.FormatChecker()).validate(value) + + +@pytest.fixture +def database(tmp_path): + path = tmp_path / "database.sqlite" + seed_database(path) + return path + + +def read_capture(capture, output): + execution = capture["execution"] + validate(execution, "Execution") + manifest = execution["result"] + validate(manifest, "ResultManifest") + pages = manifest.get("pages") + tables = [] + total_rows = total_bytes = 0 + for i, resource in enumerate(capture["resources"]): + path = output / resource["file"] + assert path.name == resource["file"] + with pa.ipc.open_stream(path) as reader: + table = reader.read_all() + tables.append(table) + metadata = json.loads(table.schema.metadata[EXECUTION_KEY]) + validate(metadata, "ExecutionMetadata") + expected = {key: execution[key] for key in + ("contract_version", "execution_id", "data_source_id", "model")} + expected["result_id"] = manifest["result_id"] + if pages is not None: + expected.update(page_index=i, row_offset=total_rows) + assert pages[i] == dict(page_index=i, row_offset=total_rows, + row_count=table.num_rows, byte_count=path.stat().st_size, + url=resource["url"]) + else: + assert len(capture["resources"]) == 1 and resource["url"] == manifest["url"] + assert metadata == expected + for field, column in zip(table.schema, manifest["columns"]): + assert field.name == column["name"] and field.nullable == column["nullable"] + field_metadata = json.loads(field.metadata[COLUMN_KEY]) + validate(field_metadata, "ColumnMetadata") + assert field_metadata == {"column_id": column["column_id"]} + total_rows += table.num_rows + total_bytes += path.stat().st_size + assert total_rows == manifest["row_count"] + assert total_bytes == manifest["byte_count"] > 0 + assert manifest["completeness"] == "complete" + return tables + + +@pytest.mark.parametrize("page_rows", [None, 1, 2, 3, 20]) +def test_real_adbc_sqlite_reader(database, tmp_path, page_rows): + # Use the actual ADBC cursor reader, not a Python SQLite row-list conversion. + with dbapi.connect(uri=database.resolve().as_uri() + "?mode=ro") as connection: + with connection.cursor() as cursor: + cursor.execute(SQL) + capture = materialize(cursor.fetch_record_batch(), output=tmp_path, + page_rows=page_rows, **IDENTITY) + tables = read_capture(capture, tmp_path) + assert [value for table in tables for value in table.column(0).to_pylist()] == [1, 2, 3] + assert [value for table in tables for value in table.column(1).to_pylist()] == [9007199254740993, None, -7] + assert all(table.schema.field(1).type == pa.int64() for table in tables) + assert capture["execution"]["result"]["columns"] == IDENTITY["columns"] + + +@pytest.mark.parametrize("page_rows", [None, 2]) +def test_zero_rows_keep_real_adbc_schema(database, tmp_path, page_rows): + capture = materialize_sqlite(database, SQL.replace("ORDER BY", "WHERE 0 ORDER BY"), + output=tmp_path, page_rows=page_rows, **IDENTITY) + tables = read_capture(capture, tmp_path) + assert len(tables) == 1 and tables[0].num_rows == 0 + assert tables[0].schema.field(1).type == pa.int64() + + +def test_read_only_backend_rejects_writes_and_sanitizes_errors(database, tmp_path): + sentinel = tmp_path / "unrelated.arrow" + sentinel.write_bytes(b"leave me alone") + for sql in ("INSERT INTO sample VALUES (4, 10)", + "SELECT secret_provider_token FROM private_customer_table"): + with pytest.raises(MaterializationError) as caught: + materialize_sqlite(database, sql, output=tmp_path, **IDENTITY) + assert caught.value.category == "materialization_failed" + assert "private_customer" not in json.dumps(caught.value.public_detail()) + assert "secret_provider_token" not in "".join(traceback.format_exception(caught.value)) + with closing(sqlite3.connect(database)) as connection: + assert connection.execute("SELECT count(*) FROM sample").fetchone()[0] == 3 + assert sentinel.read_bytes() == b"leave me alone" + assert set(p.name for p in tmp_path.iterdir()) == {"database.sqlite", "unrelated.arrow"} + + +def test_exact_budgets_and_no_success_shaped_truncation(database, tmp_path): + reference = materialize_sqlite(database, SQL, output=tmp_path, page_rows=1, **IDENTITY) + exact_bytes = reference["execution"]["result"]["byte_count"] + original = {p.name: p.read_bytes() for p in tmp_path.iterdir()} + for limits in ({"max_rows": 2}, {"max_bytes": 1}, {"max_bytes": exact_bytes - 1}): + with pytest.raises(MaterializationError, match="resource limit") as caught: + materialize_sqlite(database, SQL, output=tmp_path, page_rows=1, **limits, **IDENTITY) + assert caught.value.category == "resource_limit" + assert {p.name: p.read_bytes() for p in tmp_path.iterdir()} == original + capture = materialize_sqlite(database, SQL, output=tmp_path, page_rows=1, + max_rows=3, max_bytes=exact_bytes, **IDENTITY) + read_capture(capture, tmp_path) + + +class ObservedReader: + def __init__(self, schema, batches, failure=False): + self.schema, self.batches, self.failure = schema, batches, failure + self.closed = self.exhausted = False + + def __iter__(self): + yield from self.batches + if self.failure: + raise RuntimeError("provider password=secret db=private_customer") + self.exhausted = True + + def close(self): + self.closed = True + + +def reader_for(table, **kwargs): + return ObservedReader(table.schema, table.to_batches(max_chunksize=1), **kwargs) + + +def test_exhaustion_cleanup_and_input_snapshots(tmp_path): + table = pa.table({"sequence": pa.array([1, 2, 3]), "amount": pa.array([9, None, -7])}) + reader = reader_for(table) + identity = copy.deepcopy(IDENTITY) + capture = materialize(reader, output=tmp_path, page_rows=1, **identity) + assert reader.closed and reader.exhausted + identity["model"]["revision"] = "changed" + identity["columns"][0]["label"] = "changed" + assert capture["execution"]["model"]["revision"] == "1" + assert "label" not in capture["execution"]["result"]["columns"][0] + original = {p.name: p.read_bytes() for p in tmp_path.iterdir()} + reader = reader_for(table, failure=True) + with pytest.raises(MaterializationError) as caught: + materialize(reader, output=tmp_path, page_rows=1, **IDENTITY) + assert reader.closed and not reader.exhausted + assert "password" not in json.dumps(caught.value.public_detail()) + assert "password" not in "".join(traceback.format_exception(caught.value)) + assert {p.name: p.read_bytes() for p in tmp_path.iterdir()} == original + + +@pytest.mark.parametrize("arrow_type,logical,value", [ + (pa.string(), "String", "text"), + (pa.int8(), "Integer", -3), + (pa.uint64(), "Integer", 18446744073709551615), + (pa.float32(), "Float", 1.5), + (pa.float64(), "Float", 2.5), + (pa.bool_(), "Boolean", True), + (pa.decimal128(20, 3), "Decimal", Decimal("123.456")), + (pa.decimal256(50, 6), "Decimal", Decimal("123.456789")), + (pa.date32(), "Date", date(2025, 1, 1)), + (pa.time32("s"), "Time", time(12, 30)), + (pa.time64("us"), "Time", time(12, 30, 1, 123456)), + (pa.timestamp("ns"), "DateTime", datetime(2025, 1, 1)), + (pa.timestamp("us", "UTC"), "DateTimeTz", datetime(2025, 1, 1, tzinfo=timezone.utc)), +]) +def test_primitive_preservation_and_metadata_mapping(tmp_path, arrow_type, logical, value): + field = pa.field("value", arrow_type, metadata={b"provider.secret": b"do not forward"}) + schema = pa.schema([field], metadata={b"provider.secret": b"do not forward"}) + table = pa.Table.from_arrays([pa.array([value, None], type=arrow_type)], schema=schema) + column = dict(column_id="c", name="value", logical_type=logical, nullable=True, role="field", + model_ref="model.value", label="Value", description="Example", unit="units", currency="USD") + capture = materialize(reader_for(table), output=tmp_path, page_rows=1, + **{**IDENTITY, "columns": [column]}) + tables = read_capture(capture, tmp_path) + assert pa.concat_tables(tables).column(0).equals(table.column(0)) + assert all(t.schema.field(0).type == arrow_type for t in tables) + assert all(set(t.schema.metadata) == {EXECUTION_KEY} for t in tables) + assert all(set(t.schema.field(0).metadata) == {COLUMN_KEY} for t in tables) + assert capture["execution"]["result"]["columns"] == [column] + + +@pytest.mark.parametrize("arrow_type", [ + pa.binary(), pa.large_string(), pa.list_(pa.int64()), pa.null(), pa.date64(), + pa.float16(), pa.dictionary(pa.int8(), pa.string()), +]) +def test_unsupported_types_close_reader_without_output(tmp_path, arrow_type): + schema = pa.schema([pa.field("value", arrow_type)]) + reader = ObservedReader(schema, []) + column = dict(column_id="c", name="value", logical_type="String", nullable=True) + with pytest.raises(MaterializationError) as caught: + materialize(reader, output=tmp_path, **{**IDENTITY, "columns": [column]}) + assert caught.value.category == "unsupported_type" + assert reader.closed and not list(tmp_path.iterdir()) + + +def test_extension_metadata_and_schema_drift_are_refused(tmp_path): + table = pa.table({"sequence": [1, 2], "amount": [3, 4]}) + field = table.schema.field(0).with_metadata({b"ARROW:extension:name": b"opaque"}) + schema = table.schema.set(0, field) + reader = ObservedReader(schema, []) + with pytest.raises(MaterializationError): + materialize(reader, output=tmp_path, **IDENTITY) + assert reader.closed + batches = table.to_batches(max_chunksize=1) + reader = ObservedReader(table.schema, [batches[0], batches[1].replace_schema_metadata({b"x": b"y"})]) + with pytest.raises(MaterializationError): + materialize(reader, output=tmp_path, page_rows=1, **IDENTITY) + assert reader.closed and not list(tmp_path.iterdir()) + + +@pytest.mark.parametrize("options", [ + {"expires_at": "2000-01-01T00:00:00Z"}, + {"max_rows": -1}, {"max_bytes": 0}, {"page_rows": 0}, + {"max_rows": 9007199254740992}, {"max_rows": True}, + {"model": {"id": "model", "revision": " "}}, + {"columns": [{**c, "column_id": "duplicate"} for c in IDENTITY["columns"]]}, +]) +def test_invalid_input_closes_reader(tmp_path, options): + reader = reader_for(pa.table({"sequence": [1], "amount": [2]})) + with pytest.raises(MaterializationError): + materialize(reader, output=tmp_path, **{**IDENTITY, **options}) + assert reader.closed and not list(tmp_path.iterdir()) + + +def test_ids_never_choose_files_and_existing_files_are_not_overwritten(database, tmp_path): + existing = tmp_path / "capture.json" + existing.write_text("unrelated") + capture = materialize_sqlite(database, SQL, output=tmp_path, **{ + **IDENTITY, "execution_id": "../../query", "result_id": r"..\capture.json"}) + read_capture(capture, tmp_path) + assert existing.read_text() == "unrelated" + assert all(r["file"].startswith("page-") for r in capture["resources"]) + + +def test_real_adbc_error_during_fetch_discards_partial_pages(tmp_path): + database = tmp_path / "mixed.sqlite" + with closing(sqlite3.connect(database)) as connection, connection: + connection.execute("CREATE TABLE mixed (value)") + connection.executemany("INSERT INTO mixed VALUES (?)", [(1,), (2,), ("private-value",)]) + with dbapi.connect(uri=database.resolve().as_uri() + "?mode=ro") as connection: + with connection.cursor(adbc_stmt_kwargs={"adbc.sqlite.query.batch_rows": "1"}) as cursor: + cursor.execute("SELECT value FROM mixed ORDER BY rowid") + reader = cursor.fetch_record_batch() + column = dict(column_id="c", name="value", logical_type="Integer", nullable=True) + with pytest.raises(MaterializationError) as caught: + materialize(reader, output=tmp_path, page_rows=1, **{**IDENTITY, "columns": [column]}) + assert caught.value.category == "materialization_failed" + assert "private-value" not in json.dumps(caught.value.public_detail()) + assert list(tmp_path.iterdir()) == [database] + + +def test_exclusive_creation_never_removes_colliding_file(tmp_path, monkeypatch): + import materialize as adapter + + class FixedName: + hex = "a" * 32 + + monkeypatch.setattr(adapter, "uuid4", FixedName) + path = tmp_path / ("page-" + "a" * 32 + ".arrow") + path.write_bytes(b"unrelated") + reader = reader_for(pa.table({"sequence": [1], "amount": [2]})) + with pytest.raises(MaterializationError): + materialize(reader, output=tmp_path, **IDENTITY) + assert reader.closed + assert path.read_bytes() == b"unrelated" + + +def test_duplicate_names_use_column_identity_not_name(tmp_path): + schema = pa.schema([pa.field("value", pa.int64()), pa.field("value", pa.int64())]) + table = pa.Table.from_arrays([pa.array([1]), pa.array([2])], schema=schema) + columns = [dict(column_id=f"c{i}", name="value", nullable=True, logical_type="Integer") + for i in range(2)] + capture = materialize(reader_for(table), output=tmp_path, **{**IDENTITY, "columns": columns}) + tables = read_capture(capture, tmp_path) + assert tables[0].column(0).to_pylist() == [1] and tables[0].column(1).to_pylist() == [2] + + +def test_expiry_rechecked_after_reader_closes(tmp_path, monkeypatch): + import materialize as adapter + + now = [datetime(2098, 1, 1, tzinfo=timezone.utc)] + + class Clock(datetime): + @classmethod + def now(cls, tz=None): + return now[0] + + class ExpiringReader(ObservedReader): + def close(self): + super().close() + now[0] = datetime(2100, 1, 1, tzinfo=timezone.utc) + + monkeypatch.setattr(adapter, "datetime", Clock) + table = pa.table({"sequence": [1], "amount": [2]}) + reader = ExpiringReader(table.schema, table.to_batches()) + with pytest.raises(MaterializationError): + materialize(reader, output=tmp_path, **IDENTITY) + assert reader.closed and not list(tmp_path.iterdir()) diff --git a/examples/execution-adbc/uv.lock b/examples/execution-adbc/uv.lock new file mode 100644 index 000000000..4743f8d88 --- /dev/null +++ b/examples/execution-adbc/uv.lock @@ -0,0 +1,414 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "adbc-driver-manager" +version = "1.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9c/f8/ed6475b49a7cf35ea888d5c95e7d4bc9dc6568f9d741f14c0573d622cc1e/adbc_driver_manager-1.12.0.tar.gz", hash = "sha256:45991f0c2de369d330c6a211ca2edbcce6389c5dc81cde70461bdeb6f8f7b268", size = 217579, upload-time = "2026-07-28T00:43:03.512Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/2d/e41ea911f9486c497534ae181dfdab19adca21f71abc8a1fcaf2c27251a8/adbc_driver_manager-1.12.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:3c0c73670c8aa6fe42de1d5e71a0b329c4b37f7c55c560c23f6f3a1609200c1f", size = 600030, upload-time = "2026-07-28T00:41:54.753Z" }, + { url = "https://files.pythonhosted.org/packages/10/ea/1a8b51999785d7dce17079dd635c9ee2372c75ec7328423ce862741cb503/adbc_driver_manager-1.12.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6943c7adcf3c7c9f7c4b5bdb7589c331027a347e3c77471eb3f656b1a881e351", size = 611058, upload-time = "2026-07-28T00:41:56.306Z" }, + { url = "https://files.pythonhosted.org/packages/85/a2/5ede53173a420742fa71d6c26792e2295fc73f25cf39055f912a5385b1f0/adbc_driver_manager-1.12.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78c9936adb280e2c10e90632e41b58aa23be358e1136d8fb3c52862b72818a95", size = 4663735, upload-time = "2026-07-28T00:41:58.793Z" }, + { url = "https://files.pythonhosted.org/packages/1c/1a/781561d0f55e05a0b884244ed563dab14b165b4dd74abd8af3f8efd95e3d/adbc_driver_manager-1.12.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:30d96ab4a2594b4109496fb4913646f41a5bf1ecce79b4313847d240a2a62db3", size = 4747090, upload-time = "2026-07-28T00:42:00.746Z" }, + { url = "https://files.pythonhosted.org/packages/c6/fa/47c755a74ea4887968c52a968e02736007da4042fc0820292dc8c6827a94/adbc_driver_manager-1.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:67419b92c286646944426992069f56fed90c2ceac83521f6d66d7d3cbf6c17ea", size = 760952, upload-time = "2026-07-28T00:42:02.432Z" }, + { url = "https://files.pythonhosted.org/packages/de/8c/cd3fe16df716719116a6c79e64a768fe994f6ded55d5a8f091bb4f42d6f0/adbc_driver_manager-1.12.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:fd02364c65b8b376c5627e3b77410f457fcbbf983e52e8d15ca099da3a7ae314", size = 599054, upload-time = "2026-07-28T00:42:04.072Z" }, + { url = "https://files.pythonhosted.org/packages/49/4a/2f060ff6bd61420ea1613670e1f85a22a8714934c235186dc3803de8ddac/adbc_driver_manager-1.12.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d8dcf62621090e8d9c8216e08dfc4043f16331872522186af61a5de9478e9c63", size = 609964, upload-time = "2026-07-28T00:42:05.82Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f1/0746db149828ae91e4a6cf49f8d0e49210eec20c03ad80044454139c8240/adbc_driver_manager-1.12.0-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:efa5dbbf101962d212b176f25e6fc509dacf07afd4cf70b5027d81ec6871bdec", size = 4685726, upload-time = "2026-07-28T00:42:08.1Z" }, + { url = "https://files.pythonhosted.org/packages/b9/c3/f8e9c5157b19e986df719259eb3502dad1268df9f7a1034f65ca220ab2ea/adbc_driver_manager-1.12.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8b340679a005a8adf6b0b58754dbc638dff00db7b2559c140406a1d92678b48c", size = 4768774, upload-time = "2026-07-28T00:42:10.359Z" }, + { url = "https://files.pythonhosted.org/packages/92/51/f8e625af691e6b4c54945790854524356a02a0a69063e888f7cfee1b2e50/adbc_driver_manager-1.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:47f428a922d224fd486b661deeaf9520e5faec558b3d144832bed09a080cac88", size = 760087, upload-time = "2026-07-28T00:42:11.871Z" }, + { url = "https://files.pythonhosted.org/packages/9a/f9/674c5bbc5093617d72c4f58a5dab67982710b2320cc9aa826050a6aaa131/adbc_driver_manager-1.12.0-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:c42ca4d9caa22b3a5ce76bde8729169f403bb7393e3671734b9416634c207125", size = 596815, upload-time = "2026-07-28T00:42:13.64Z" }, + { url = "https://files.pythonhosted.org/packages/56/5f/c1d888d787330801edae282d2a9def3765e8157547cc20e71154ff38c1bb/adbc_driver_manager-1.12.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c894117c8f5c484b902c8b070bcfd9d31d90efe0288b2b58a3ddab97c80f66e7", size = 608277, upload-time = "2026-07-28T00:42:15.643Z" }, + { url = "https://files.pythonhosted.org/packages/06/4b/ee799babf171e39690ef45560451096f869d9e7387bc0e5a754bb243ed2a/adbc_driver_manager-1.12.0-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:214f80f9b65562f08b4d1c52a756b5db557530e3c0652f587c43aaa80039579a", size = 4667230, upload-time = "2026-07-28T00:42:17.97Z" }, + { url = "https://files.pythonhosted.org/packages/00/c6/a35e38ef5e0db391be79e0e14c019ce378b87d9d7e31d1dfcd451e9d291f/adbc_driver_manager-1.12.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:532ab290b3d923ce0a75bca21dc6e13f55835625f78808e1664755939f3ebdf6", size = 4745299, upload-time = "2026-07-28T00:42:20.189Z" }, + { url = "https://files.pythonhosted.org/packages/16/e2/62bacd6844859036d79ea229401b5200056fb5050c82dc3a2e28b08ff49b/adbc_driver_manager-1.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:034da82c1a6e195d67ca1f0c97a1a517046037ec3029ab9a0ea8f7ccb14056e4", size = 758878, upload-time = "2026-07-28T00:42:21.598Z" }, + { url = "https://files.pythonhosted.org/packages/50/ea/f53b434fe36d0f138d147fc10a95784c8c0eeea1bec1f3f31eee5ec8bdb5/adbc_driver_manager-1.12.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a740d634118722f42af31176374fddbad3846fa2e6536f497bac145e9511cecc", size = 597579, upload-time = "2026-07-28T00:42:23.216Z" }, + { url = "https://files.pythonhosted.org/packages/ba/57/6208e66d9256550c2aff75db4a323a855a0d5d2d1bd639526f825d3e08b4/adbc_driver_manager-1.12.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8a77ae39832e67946009816d83c321e540a3024aad1419ccba24ddeb7b6a01f4", size = 610337, upload-time = "2026-07-28T00:42:25.051Z" }, + { url = "https://files.pythonhosted.org/packages/1d/cd/f5ea3f08191af5ae15041821fcb52bf35837dce1a9ac16fa039b3bfe308c/adbc_driver_manager-1.12.0-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:690f140ca67d49f995afac59f85441c3d5e896cd2fc8fd381423fe900e51f1f7", size = 4664297, upload-time = "2026-07-28T00:42:27.474Z" }, + { url = "https://files.pythonhosted.org/packages/df/81/823a71a515078545eab8a4be8381206887129e11b91e9bf51ca2a9eea44d/adbc_driver_manager-1.12.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fd568c94874c0586d82f99de2bb5d2c02b4fa9c5bafe3d0d8ab353bddf9d2fd6", size = 4733739, upload-time = "2026-07-28T00:42:29.814Z" }, + { url = "https://files.pythonhosted.org/packages/cf/f7/7612d078d935344aee679a44a6283de6aae9008eb8e0ef80e475dd12dffa/adbc_driver_manager-1.12.0-cp314-cp314-win_amd64.whl", hash = "sha256:57f5101fb2a853b1ffb81ff807b5e29a51ba14c64032eb0038b8dfd433b6d533", size = 777952, upload-time = "2026-07-28T00:42:40.881Z" }, + { url = "https://files.pythonhosted.org/packages/b0/ad/2478338aaece38b8b72259dbfd4d4c84d9a038421e25bbc283e510d47555/adbc_driver_manager-1.12.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:bb9db6e4a3bcd73153435a900b5ae40ad36f5875df93a8faf784d9fcf6833983", size = 615694, upload-time = "2026-07-28T00:42:31.932Z" }, + { url = "https://files.pythonhosted.org/packages/bc/a0/0592c85e653f005aa28de7733b3c3c4f0282238301694f76806e5f3cc1e1/adbc_driver_manager-1.12.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:07cae26bd5ccee6caa4227f817c0fd57f9ac131c2dd98e0c5d7fecfef61819c7", size = 628341, upload-time = "2026-07-28T00:42:33.481Z" }, + { url = "https://files.pythonhosted.org/packages/9d/00/65705a72f768bc2dda82623a74cf816609dfdff56f3ad22b073d4a1ea7f8/adbc_driver_manager-1.12.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:442ed2ee8ea62c475bf3478385555bb4f0b25d9d551087ffe40c73b91bf5431e", size = 4730268, upload-time = "2026-07-28T00:42:35.661Z" }, + { url = "https://files.pythonhosted.org/packages/44/b9/60ecde5d9dde5acc5576cb0ba5ffa34e154464e07fa295c57cd975ea27c7/adbc_driver_manager-1.12.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c2aa05c5dc52164692284b2df27fba5680dbc967b8e3ca704aabf5399667996", size = 4777527, upload-time = "2026-07-28T00:42:37.709Z" }, + { url = "https://files.pythonhosted.org/packages/ac/76/6749e0c0c437219780c65487cff67dc09a556c1fccf577a2b27f7b92a704/adbc_driver_manager-1.12.0-cp314-cp314t-win_amd64.whl", hash = "sha256:cfa08f8c7c63e3fa92eb4e26ef4d8a9520cf92a39281cd011821f6f16a963080", size = 793451, upload-time = "2026-07-28T00:42:39.222Z" }, +] + +[[package]] +name = "adbc-driver-sqlite" +version = "1.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "adbc-driver-manager" }, + { name = "importlib-resources" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f5/02/2dc143bdd2a62c52d103d4b0ae491a347944aaed25b3d40fb11797750c70/adbc_driver_sqlite-1.12.0.tar.gz", hash = "sha256:18466a2f0c14f94cb0b17818157cc14ed6b93aef0a48ef648de945e9bac1540d", size = 12849, upload-time = "2026-07-28T00:43:05.408Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a1/f7/c35740269d3a5e3aa07b9ab155d4e943a7f5267f64d8f7396d5e14184a02/adbc_driver_sqlite-1.12.0-py3-none-macosx_10_15_x86_64.whl", hash = "sha256:2d5b3e9d0b5dbc66324b0ccf2ded886e3781f901be986892d319529b05536d3b", size = 1413592, upload-time = "2026-07-28T00:42:53.523Z" }, + { url = "https://files.pythonhosted.org/packages/e6/31/5d1d637e6ae76fcc57d5116d537aa78d2ab687354d5a0b2d527e085b61d4/adbc_driver_sqlite-1.12.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:5a81f53791e4aec69afbf8f77dac6acf48749fd84684e86601eafdd36d2eb7c3", size = 1357479, upload-time = "2026-07-28T00:42:55.194Z" }, + { url = "https://files.pythonhosted.org/packages/6c/99/415bf90eb912403d2d5d0c31baa1cedf200bd510f40027ee8fd3421c4c02/adbc_driver_sqlite-1.12.0-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:c987d03e3f4850e57f218c8a0b9d224209123af642469ee1f36901c5a51725bd", size = 1501753, upload-time = "2026-07-28T00:42:57.442Z" }, + { url = "https://files.pythonhosted.org/packages/69/10/a3156f19fadd254a4f58a328a8aa9472c981ff93bb4d23f3c22a4341796e/adbc_driver_sqlite-1.12.0-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:3005a80bedf6624c6856da98037ea943a791aa8e82dad458259e0558be32912c", size = 1548999, upload-time = "2026-07-28T00:42:59.163Z" }, + { url = "https://files.pythonhosted.org/packages/f4/d9/3245d741936100365ea77c434f84a0985523467bd812e5e11bb9b36d7152/adbc_driver_sqlite-1.12.0-py3-none-win_amd64.whl", hash = "sha256:0982bfc06158c2140b5c490b1a1325019c827b158f8432a30d49c8a0c18533ad", size = 1522964, upload-time = "2026-07-28T00:43:00.917Z" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "importlib-resources" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e4/06/b56dfa750b44e86157093bc8fca0ab81dccbf5260510de4eaf1cb69b5b99/importlib_resources-7.1.0.tar.gz", hash = "sha256:0722d4c6212489c530f2a145a34c0a7a3b4721bc96a15fada5930e2a0b760708", size = 44985, upload-time = "2026-04-12T16:36:09.232Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/db/55a262f3606bebcae07cc14095338471ad7c0bbcaa37707e6f0ee49725b7/importlib_resources-7.1.0-py3-none-any.whl", hash = "sha256:1bd7b48b4088eddb2cd16382150bb515af0bd2c70128194392725f82ad2c96a1", size = 37232, upload-time = "2026-04-12T16:36:08.219Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/e1/2069291243c926a2ff1cd706c7f3eeb9b62144bf60f77c9fb9ff2fb26bd3/iniconfig-2.3.1.tar.gz", hash = "sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960", size = 21209, upload-time = "2026-10-06T22:48:38.076Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/43/4ca9e49d27a1fcf6bece6f6aec0ea46bb9112489b93d4b688fb415457bdb/iniconfig-2.3.1-py3-none-any.whl", hash = "sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7", size = 7552, upload-time = "2026-10-06T22:48:36.959Z" }, +] + +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + +[[package]] +name = "ossie-execution-adbc-example" +version = "0.0.0" +source = { virtual = "." } +dependencies = [ + { name = "adbc-driver-sqlite" }, + { name = "pyarrow" }, +] + +[package.dev-dependencies] +dev = [ + { name = "jsonschema" }, + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "adbc-driver-sqlite", specifier = ">=1.9,<2" }, + { name = "pyarrow", specifier = ">=19,<25" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "jsonschema", specifier = ">=4.23,<5" }, + { name = "pytest", specifier = ">=8,<10" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pyarrow" +version = "24.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/91/13/13e1069b351bdc3881266e11147ffccf687505dbb0ea74036237f5d454a5/pyarrow-24.0.0.tar.gz", hash = "sha256:85fe721a14dd823aca09127acbb06c3ca723efbd436c004f16bca601b04dcc83", size = 1180261, upload-time = "2026-04-21T10:51:25.837Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/c9/a47ab7ece0d86cbe6678418a0fbd1ac4bb493b9184a3891dfa0e7f287ae0/pyarrow-24.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:b0e131f880cda8d04e076cee175a46fc0e8bc8b65c99c6c09dff6669335fde74", size = 35068898, upload-time = "2026-04-21T10:46:36.599Z" }, + { url = "https://files.pythonhosted.org/packages/d1/bc/8db86617a9a58008acf8913d6fed68ea2a46acb6de928db28d724c891a68/pyarrow-24.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:1b2fe7f9a5566401a0ef2571f197eb92358925c1f0c8dba305d6e43ea0871bb3", size = 36679915, upload-time = "2026-04-21T10:46:42.602Z" }, + { url = "https://files.pythonhosted.org/packages/eb/8e/fb178720400ef69db251eb4a9c3ccf4af269bc1feb5055529b8fc87170d1/pyarrow-24.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:0b3537c00fb8d384f15ac1e79b6eb6db04a16514c8c1d22e59a9b95c8ba42868", size = 45697931, upload-time = "2026-04-21T10:46:48.403Z" }, + { url = "https://files.pythonhosted.org/packages/f3/27/99c42abe8e21b44f4917f62631f3aa31404882a2c41d8a4cd5c110e13d52/pyarrow-24.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:14e31a3c9e35f1ab6356c6378f6f72830e6d2d5f1791df3774a7b097d18a6a1e", size = 48837449, upload-time = "2026-04-21T10:46:55.329Z" }, + { url = "https://files.pythonhosted.org/packages/36/b6/333749e2666e9032891125bf9c691146e92901bece62030ac1430e2e7c88/pyarrow-24.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b7d9a514e73bc42711e6a35aaccf3587c520024fe0a25d830a1a8a27c15f4f57", size = 49395949, upload-time = "2026-04-21T10:47:01.869Z" }, + { url = "https://files.pythonhosted.org/packages/17/25/c5201706a2dd374e8ba6ee3fd7a8c89fb7ffc16eed5217a91fd2bd7f7626/pyarrow-24.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b196eb3f931862af3fa84c2a253514d859c08e0d8fe020e07be12e75a5a9780c", size = 51912986, upload-time = "2026-04-21T10:47:09.872Z" }, + { url = "https://files.pythonhosted.org/packages/f8/d2/4d1bbba65320b21a49678d6fbdc6ff7c649251359fdcfc03568c4136231d/pyarrow-24.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:35405aecb474e683fb36af650618fd5340ee5471fc65a21b36076a18bbc6c981", size = 27255371, upload-time = "2026-04-21T10:47:15.943Z" }, + { url = "https://files.pythonhosted.org/packages/b4/a9/9686d9f07837f91f775e8932659192e02c74f9d8920524b480b85212cc68/pyarrow-24.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:6233c9ed9ab9d1db47de57d9753256d9dcffbf42db341576099f0fd9f6bf4810", size = 34981559, upload-time = "2026-04-21T10:47:22.17Z" }, + { url = "https://files.pythonhosted.org/packages/80/b6/0ddf0e9b6ead3474ab087ae598c76b031fc45532bf6a63f3a553440fb258/pyarrow-24.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:f7616236ec1bc2b15bfdec22a71ab38851c86f8f05ff64f379e1278cf20c634a", size = 36663654, upload-time = "2026-04-21T10:47:28.315Z" }, + { url = "https://files.pythonhosted.org/packages/7c/3b/926382efe8ce27ba729071d3566ade6dfb86bdf112f366000196b2f5780a/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:1617043b99bd33e5318ae18eb2919af09c71322ef1ca46566cdafc6e6712fb66", size = 45679394, upload-time = "2026-04-21T10:47:34.821Z" }, + { url = "https://files.pythonhosted.org/packages/b3/7a/829f7d9dfd37c207206081d6dad474d81dde29952401f07f2ba507814818/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6165461f55ef6314f026de6638d661188e3455d3ec49834556a0ebbdbace18bb", size = 48863122, upload-time = "2026-04-21T10:47:42.056Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e8/f88ce625fe8babaae64e8db2d417c7653adb3019b08aae85c5ed787dc816/pyarrow-24.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3b13dedfe76a0ad2d1d859b0811b53827a4e9d93a0bcb05cf59333ab4980cc7e", size = 49376032, upload-time = "2026-04-21T10:47:48.967Z" }, + { url = "https://files.pythonhosted.org/packages/36/7a/82c363caa145fff88fb475da50d3bf52bb024f61917be5424c3392eaf878/pyarrow-24.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:25ea65d868eb04015cd18e6df2fbe98f07e5bda2abefabcb88fce39a947716f6", size = 51929490, upload-time = "2026-04-21T10:47:55.981Z" }, + { url = "https://files.pythonhosted.org/packages/66/1c/e3e72c8014ad2743ca64a701652c733cc5cbcee15c0463a32a8c55518d9e/pyarrow-24.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:295f0a7f2e242dabd513737cf076007dc5b2d59237e3eca37b05c0c6446f3826", size = 27355660, upload-time = "2026-04-21T10:48:01.718Z" }, + { url = "https://files.pythonhosted.org/packages/6f/d3/a1abf004482026ddc17f4503db227787fa3cfe41ec5091ff20e4fea55e57/pyarrow-24.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:02b001b3ed4723caa44f6cd1af2d5c86aa2cf9971dacc2ffa55b21237713dfba", size = 34976759, upload-time = "2026-04-21T10:48:07.258Z" }, + { url = "https://files.pythonhosted.org/packages/4f/4a/34f0a36d28a2dd32225301b79daad44e243dc1a2bb77d43b60749be255c4/pyarrow-24.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:04920d6a71aabd08a0417709efce97d45ea8e6fb733d9ca9ecffb13c67839f68", size = 36658471, upload-time = "2026-04-21T10:48:13.347Z" }, + { url = "https://files.pythonhosted.org/packages/1f/78/543b94712ae8bb1a6023bcc1acf1a740fbff8286747c289cd9468fced2a5/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a964266397740257f16f7bb2e4f08a0c81454004beab8ff59dd531b73610e9f2", size = 45675981, upload-time = "2026-04-21T10:48:20.201Z" }, + { url = "https://files.pythonhosted.org/packages/84/9f/8fb7c222b100d314137fa40ec050de56cd8c6d957d1cfff685ce72f15b17/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f066b179d68c413374294bc1735f68475457c933258df594443bb9d88ddc2a0", size = 48859172, upload-time = "2026-04-21T10:48:27.541Z" }, + { url = "https://files.pythonhosted.org/packages/a7/d3/1ea72538e6c8b3b475ed78d1049a2c518e655761ea50fe1171fc855fcab7/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1183baeb14c5f587b1ec52831e665718ce632caab84b7cd6b85fd44f96114495", size = 49385733, upload-time = "2026-04-21T10:48:34.7Z" }, + { url = "https://files.pythonhosted.org/packages/c3/be/c3d8b06a1ba35f2260f8e1f771abbee7d5e345c0937aab90675706b1690a/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:806f24b4085453c197a5078218d1ee08783ebbba271badd153d1ae22a3ee804f", size = 51934335, upload-time = "2026-04-21T10:48:42.099Z" }, + { url = "https://files.pythonhosted.org/packages/9c/62/89e07a1e7329d2cde3e3c6994ba0839a24977a2beda8be6005ea3d860b99/pyarrow-24.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:e4505fc6583f7b05ab854934896bcac8253b04ac1171a77dfb73efef92076d91", size = 27271748, upload-time = "2026-04-21T10:49:42.532Z" }, + { url = "https://files.pythonhosted.org/packages/17/1a/cff3a59f80b5b1658549d46611b67163f65e0664431c076ad728bf9d5af4/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:1a4e45017efbf115032e4475ee876d525e0e36c742214fbe405332480ecd6275", size = 35238554, upload-time = "2026-04-21T10:48:48.526Z" }, + { url = "https://files.pythonhosted.org/packages/a8/99/cce0f42a327bfef2c420fb6078a3eb834826e5d6697bf3009fe11d2ad051/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:7986f1fa71cee060ad00758bcc79d3a93bab8559bf978fab9e53472a2e25a17b", size = 36782301, upload-time = "2026-04-21T10:48:55.181Z" }, + { url = "https://files.pythonhosted.org/packages/2a/66/8e560d5ff6793ca29aca213c53eec0dd482dd46cb93b2819e5aab52e4252/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d3e0b61e8efb24ed38898e5cdc5fffa9124be480008d401a1f8071500494ae42", size = 45721929, upload-time = "2026-04-21T10:49:03.676Z" }, + { url = "https://files.pythonhosted.org/packages/27/0c/a26e25505d030716e078d9f16eb74973cbf0b33b672884e9f9da1c83b871/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:55a3bc1e3df3b5567b7d27ef551b2283f0c68a5e86f1cd56abc569da4f31335b", size = 48825365, upload-time = "2026-04-21T10:49:11.714Z" }, + { url = "https://files.pythonhosted.org/packages/5f/eb/771f9ecb0c65e73fe9dccdd1717901b9594f08c4515d000c7c62df573811/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:641f795b361874ac9da5294f8f443dfdbee355cf2bd9e3b8d97aaac2306b9b37", size = 49451819, upload-time = "2026-04-21T10:49:21.474Z" }, + { url = "https://files.pythonhosted.org/packages/48/da/61ae89a88732f5a785646f3ec6125dbb640fa98a540eb2b9889caa561403/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8adc8e6ce5fccf5dc707046ae4914fd537def529709cc0d285d37a7f9cd442ca", size = 51909252, upload-time = "2026-04-21T10:49:31.164Z" }, + { url = "https://files.pythonhosted.org/packages/cb/1a/8dd5cafab7b66573fa91c03d06d213356ad4edd71813aa75e08ce2b3a844/pyarrow-24.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:9b18371ad2f44044b81a8d23bc2d8a9b6a6226dca775e8e16cfee640473d6c5d", size = 27388127, upload-time = "2026-04-21T10:49:37.334Z" }, + { url = "https://files.pythonhosted.org/packages/ad/80/d022a34ff05d2cbedd8ccf841fc1f532ecfa9eb5ed1711b56d0e0ea71fc9/pyarrow-24.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:1cc9057f0319e26333b357e17f3c2c022f1a83739b48a88b25bfd5fa2dc18838", size = 35007997, upload-time = "2026-04-21T10:49:48.796Z" }, + { url = "https://files.pythonhosted.org/packages/1a/ff/f01485fda6f4e5d441afb8dd5e7681e4db18826c1e271852f5d3957d6a80/pyarrow-24.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e6f1278ee4785b6db21229374a1c9e54ec7c549de5d1efc9630b6207de7e170b", size = 36678720, upload-time = "2026-04-21T10:49:55.858Z" }, + { url = "https://files.pythonhosted.org/packages/9e/c2/2d2d5fea814237923f71b36495211f20b43a1576f9a4d6da7e751a64ec6f/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:adbbedc55506cbdabb830890444fb856bfb0060c46c6f8026c6c2f2cf86ae795", size = 45741852, upload-time = "2026-04-21T10:50:04.624Z" }, + { url = "https://files.pythonhosted.org/packages/8e/3a/28ba9c1c1ebdbb5f1b94dfebb46f207e52e6a554b7fe4132540fde29a3a0/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:ae8a1145af31d903fa9bb166824d7abe9b4681a000b0159c9fb99c11bc11ad26", size = 48889852, upload-time = "2026-04-21T10:50:12.293Z" }, + { url = "https://files.pythonhosted.org/packages/df/51/4a389acfd31dca009f8fb82d7f510bb4130f2b3a8e18cf00194d0687d8ac/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d7027eba1df3b2069e2e8d80f644fa0918b68c46432af3d088ddd390d063ecde", size = 49445207, upload-time = "2026-04-21T10:50:20.677Z" }, + { url = "https://files.pythonhosted.org/packages/19/4b/0bab2b23d2ae901b1b9a03c0efd4b2d070256f8ce3fc43f6e58c167b2081/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e56a1ffe9bf7b727432b89104cc0849c21582949dd7bdcb34f17b2001a351a76", size = 51954117, upload-time = "2026-04-21T10:50:29.14Z" }, + { url = "https://files.pythonhosted.org/packages/29/88/f4e9145da0417b3d2c12035a8492b35ff4a3dbc653e614fcfb51d9dedb38/pyarrow-24.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:38be1808cdd068605b787e6ca9119b27eb275a0234e50212c3492331680c3b1e", size = 28001155, upload-time = "2026-04-21T10:51:22.337Z" }, + { url = "https://files.pythonhosted.org/packages/79/4f/46a49a63f43526da895b1a45bbb51d5baf8e4d77159f8528fc3e5490007f/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:418e48ce50a45a6a6c73c454677203a9c75c966cb1e92ca3370959185f197a05", size = 35250387, upload-time = "2026-04-21T10:50:35.552Z" }, + { url = "https://files.pythonhosted.org/packages/a0/da/d5e0cd5ef00796922404806d5f00325cdadc3441ce2c13fe7115f2df9a64/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:2f16197705a230a78270cdd4ea8a1d57e86b2fdcbc34a1f6aebc72e65c986f9a", size = 36797102, upload-time = "2026-04-21T10:50:42.417Z" }, + { url = "https://files.pythonhosted.org/packages/34/c7/5904145b0a593a05236c882933d439b5720f0a145381179063722fbfc123/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:fb24ac194bfc5e86839d7dcd52092ee31e5fe6733fe11f5e3b06ef0812b20072", size = 45745118, upload-time = "2026-04-21T10:50:49.324Z" }, + { url = "https://files.pythonhosted.org/packages/13/d3/cca42fe166d1c6e4d5b80e530b7949104d10e17508a90ae202dac205ce2a/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:9700ebd9a51f5895ce75ff4ac4b3c47a7d4b42bc618be8e713e5d56bacf5f931", size = 48844765, upload-time = "2026-04-21T10:50:55.579Z" }, + { url = "https://files.pythonhosted.org/packages/b0/49/942c3b79878ba928324d1e17c274ed84581db8c0a749b24bcf4cbdf15bd3/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d8ddd2768da81d3ee08cfea9b597f4abb4e8e1dc8ae7e204b608d23a0d3ab699", size = 49471890, upload-time = "2026-04-21T10:51:02.439Z" }, + { url = "https://files.pythonhosted.org/packages/76/97/ff71431000a75d84135a1ace5ca4ba11726a231a8007bbb320a4c54075d5/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:61a3d7eaa97a14768b542f3d284dc6400dd2470d9f080708b13cd46b6ae18136", size = 51932250, upload-time = "2026-04-21T10:51:10.576Z" }, + { url = "https://files.pythonhosted.org/packages/51/be/6f79d55816d5c22557cf27533543d5d70dfe692adfbee4b99f2760674f38/pyarrow-24.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:c91d00057f23b8d353039520dc3a6c09d8608164c692e9f59a175a42b2ae0c19", size = 28131282, upload-time = "2026-04-21T10:51:16.815Z" }, +] + +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, +] + +[[package]] +name = "rpds-py" +version = "2026.9.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/42/68/3bd46b8a5e01d3c2ebdf9c5e9497912e3fe0cde02bac21a7130ca866e403/rpds_py-2026.9.1.tar.gz", hash = "sha256:4793ef7f78268b124b73fa933440f01d258bbae01de9fa53e9080c9ab0425a12", size = 63948, upload-time = "2026-10-04T16:32:36.469Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f5/6d/ddb8dfcf41892ba9c672d27988592c038d30db61708230313831b6780a1b/rpds_py-2026.9.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:2711d29b653b3bce48a63d18b9c6b53274669e6d6c4094dddeb4d9a0e45128b2", size = 344947, upload-time = "2026-10-04T16:28:55.207Z" }, + { url = "https://files.pythonhosted.org/packages/9e/b2/632c3d034961149eff606e3e357547620a9ab37b849ae5815f67838753fd/rpds_py-2026.9.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3231c4c0e521dafa5be0c9f114ee2c2ad46650836f2d72caa86801950c3e7044", size = 346770, upload-time = "2026-10-04T16:28:56.884Z" }, + { url = "https://files.pythonhosted.org/packages/6d/45/4df346410bb873d03ec60c3b891d1fceb8cabde1971d29844bbb625cc296/rpds_py-2026.9.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e01b3c878c8641913e688edd1b3f08658c6783d29cf6b826bd3c0d1ae7a1ffaa", size = 377795, upload-time = "2026-10-04T16:28:58.193Z" }, + { url = "https://files.pythonhosted.org/packages/ba/b8/a70f302207e4f29f873c01a986fcac5def49b9c8224328cef1ad7da6b7a9/rpds_py-2026.9.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3e524c7874ac72884d28e16dd5b8d839fd09e0fe76b020d3fbca23212a7b8c52", size = 383137, upload-time = "2026-10-04T16:28:59.411Z" }, + { url = "https://files.pythonhosted.org/packages/13/e3/3aa28a48a01169d0e20728bf0e5ebbea3ae2774285b40853ae49b3ef332a/rpds_py-2026.9.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:761fdae6728ceb99ab182fad2f0cc1e262f610834dc891aea1d1a2a2e634776f", size = 496789, upload-time = "2026-10-04T16:29:01.011Z" }, + { url = "https://files.pythonhosted.org/packages/3e/a7/414e8d088602d66ad6f47c65bf07fe5090992a8574cdd4571c1a83b647ed/rpds_py-2026.9.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6b723eb406dec5bc9ec516c73ab9c3239a3284e017f7eb89ee2b3258bd504fb7", size = 395929, upload-time = "2026-10-04T16:29:02.465Z" }, + { url = "https://files.pythonhosted.org/packages/2d/5c/aac3b960fd55d37b49f77c0eebbe043eaf36762ebd48e7b46951e59c1c05/rpds_py-2026.9.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:136a1c3fe4402b7008bc81cb62ee538481795b61a7e83df88dff3b3f02b726ff", size = 375426, upload-time = "2026-10-04T16:29:03.944Z" }, + { url = "https://files.pythonhosted.org/packages/d3/80/77987a41c40be7202a78d9aed4a650bfb05792237cfd3b9e22638d990fe5/rpds_py-2026.9.1-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:839dde845559254f34885267c6878f60d61d5205180226d976fe488d45fa128e", size = 381497, upload-time = "2026-10-04T16:29:05.308Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d0/640b02eb1ecd71a913c1178b6235287b31006266bca9a26862e80e497b54/rpds_py-2026.9.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:56cd8b3f77d7b6812f533b662186a1f28316931166ddc00fb893b1b0db7e9888", size = 405740, upload-time = "2026-10-04T16:29:06.578Z" }, + { url = "https://files.pythonhosted.org/packages/3e/8c/d034195d45d215625a23f2ae85dbbdb86a49a93a51272061c97c80b9d316/rpds_py-2026.9.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:074a4d198bc34d9a8ea425114fc3ded6d11ec01f6a314a8db67454a5152d8834", size = 555850, upload-time = "2026-10-04T16:29:08.115Z" }, + { url = "https://files.pythonhosted.org/packages/de/f8/532f1addfc86207bc3cf23749638bb0d457bce891595012ed8a08d0543b8/rpds_py-2026.9.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6beb738155fe8ab8091afdfa5a3226b21c2b1593f1e50ebb90eb25b44dbc0391", size = 618661, upload-time = "2026-10-04T16:29:09.554Z" }, + { url = "https://files.pythonhosted.org/packages/50/5b/b0f2c9833a36a5b1f6ac5cb0e635fabacf713266f46ceefba2102da0bf51/rpds_py-2026.9.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:57492a550a1d88d29d003247e5f78dd8cf04a701fac0e4c8db8745a6d2504e0a", size = 583530, upload-time = "2026-10-04T16:29:11.072Z" }, + { url = "https://files.pythonhosted.org/packages/bf/20/464ed4b98d3c8c6a334f3b5e24d3886de04b05658a95fc40a37e0e6e4775/rpds_py-2026.9.1-cp311-cp311-win32.whl", hash = "sha256:d95a354e02393eada6d7351184671aced9d4cce109dabf927cb7aa99624352a1", size = 208125, upload-time = "2026-10-04T16:29:12.408Z" }, + { url = "https://files.pythonhosted.org/packages/df/c9/384a9b62f8cd89c8e3e609b8c9052c140e8c01d807261e0517c904b8bf29/rpds_py-2026.9.1-cp311-cp311-win_amd64.whl", hash = "sha256:ce4d4f52e2a4324396caddbd45a97d8d7be5f42edd25d2355282a9c34f9b2f7f", size = 226045, upload-time = "2026-10-04T16:29:13.751Z" }, + { url = "https://files.pythonhosted.org/packages/4f/3b/0a691f0dcb2301fa9c736346da2d0d4991778373283301c5c12b8f952f3d/rpds_py-2026.9.1-cp311-cp311-win_arm64.whl", hash = "sha256:fdcd198979b4ecffcc1beba366a7fbcf4eb41243691a82fe52ceb0b902f09c12", size = 222636, upload-time = "2026-10-04T16:29:14.989Z" }, + { url = "https://files.pythonhosted.org/packages/5d/34/a828586ea3329fbb50895b50e9cf98ca3d924a9f41a0b26d443bff1b3794/rpds_py-2026.9.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:50906f5aea24b5a865cbd0a589698288631d9f3a54c3a937c83aefa95a0d14af", size = 346822, upload-time = "2026-10-04T16:29:16.258Z" }, + { url = "https://files.pythonhosted.org/packages/90/81/ac6a0d064982251856ce009c9e1dd51a34110b3c055aa7d1aad18b4899a3/rpds_py-2026.9.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e21c1429e205828ea886a2293a4a2c8e01f4c25d9893ca330e97a6cf73f52e7b", size = 341380, upload-time = "2026-10-04T16:29:17.706Z" }, + { url = "https://files.pythonhosted.org/packages/42/ff/bf7d54f362748fd6a49277b9d6531c394074110fedf50ad791ec59133fbb/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2693b2728bbcc48d09a981a356954b0c47c53ff25b545856f28a889ea619f69a", size = 373053, upload-time = "2026-10-04T16:29:19.063Z" }, + { url = "https://files.pythonhosted.org/packages/60/de/74b0ccdbbd28687b9b5fdb34c1cabed88352182facaced9d6f5486b8b9ed/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8601470267d938bcb7f3ab1a336100af51a4fd5b6ed030ef52461bb3ef5e7e07", size = 377742, upload-time = "2026-10-04T16:29:20.543Z" }, + { url = "https://files.pythonhosted.org/packages/c2/6d/b979775a3057b2a5c26d75ecbff60a20a24ef081db6dd76836fca3f20247/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3890a6aa36e6baa53d5258a2a25d3ef8b37ad165a6ab27a892d7c3e3a432cd69", size = 487289, upload-time = "2026-10-04T16:29:21.958Z" }, + { url = "https://files.pythonhosted.org/packages/1d/5d/7c34734ce3ece943d9d6ee0122a74a21bbc75b47acb5b7053c83f6efb1ed/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6b5b393eda5ea42cca1c1a6665f2a4882b4fd5d1777e41ce0545a107fb008c9d", size = 390710, upload-time = "2026-10-04T16:29:23.378Z" }, + { url = "https://files.pythonhosted.org/packages/1d/6d/b26eb1e75395925b3a142ed351cbed2ec8212f5c9d937aee3df32c701baa/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:addeda51556dac7c1a2f14cda62db8b621cd12afba3091d03a96c72932387eab", size = 371261, upload-time = "2026-10-04T16:29:25.1Z" }, + { url = "https://files.pythonhosted.org/packages/7c/98/b2fdfe10301a9e27af21e634337fbba7baac0bc17fe44619a17c26bba2cb/rpds_py-2026.9.1-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:d9edf30457d74eebfd76b045535e36f1cd89062566a128a0db2145ca042d787e", size = 376075, upload-time = "2026-10-04T16:29:26.748Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b1/c4b8d954e49c3c69cf8063c0c9e0919c99f3d2cecc46311606a1cbe835dc/rpds_py-2026.9.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:815d26356930846a40c7bc1366e7b1b0320ab8a063e66c11298a208bed0fd237", size = 399358, upload-time = "2026-10-04T16:29:28.282Z" }, + { url = "https://files.pythonhosted.org/packages/b3/59/9559a7293c97dff0cbb293efffbd36644103883ecc27741af8ef90c84ca8/rpds_py-2026.9.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3b5a6f40f0a1486b4b36c888123afc67acdbd9f33235927acf5ff295429a0ba3", size = 550267, upload-time = "2026-10-04T16:29:29.851Z" }, + { url = "https://files.pythonhosted.org/packages/6c/9d/6dc60e49511de4c8b00e74f9c1d43aed4d27f712cdb1ade92f2c199cbf64/rpds_py-2026.9.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b5b8b0753718d258fd454283fbd57e14545d3b40583fa672e27cb4f987626bcc", size = 614018, upload-time = "2026-10-04T16:29:31.54Z" }, + { url = "https://files.pythonhosted.org/packages/fb/bc/92a4ecf27301b886232a413360f6b348a81d55ffef654f2ebda5970ebc3d/rpds_py-2026.9.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:46d80bc76b51a6c24f9944368c28d38b8bcbcea1da4f2f8d3ebc31a67e8c6ec6", size = 578509, upload-time = "2026-10-04T16:29:33.01Z" }, + { url = "https://files.pythonhosted.org/packages/08/53/efb97f2e6589b7ab8394385a73870e0b2ab2a195281c117b21fd6d7d4855/rpds_py-2026.9.1-cp312-cp312-win32.whl", hash = "sha256:befc2d6a953e563f8a7bfd87a42c22ebf8a3e980dcb7b6a4d17b70b0e914e8a3", size = 205401, upload-time = "2026-10-04T16:29:34.451Z" }, + { url = "https://files.pythonhosted.org/packages/1c/84/1700cc748d0eaa747486e4e82882d2575224e6e9a3148df583058850c629/rpds_py-2026.9.1-cp312-cp312-win_amd64.whl", hash = "sha256:5ce8943f79c2210f7abcc28e86367b03b28d95027fd01c46d2472373ae70c86f", size = 222923, upload-time = "2026-10-04T16:29:35.709Z" }, + { url = "https://files.pythonhosted.org/packages/c3/89/0302215373c2f8b4408b0fdfee955368cc378a98ff59508d47a7e0dc2dbf/rpds_py-2026.9.1-cp312-cp312-win_arm64.whl", hash = "sha256:501909f2e4a1e2dee528ef766fe3c469060ebc17e54a8383d404ba07a81a6f02", size = 218415, upload-time = "2026-10-04T16:29:37.263Z" }, + { url = "https://files.pythonhosted.org/packages/83/ea/ee88fd9e756ff93fb6b1182a47ec09504a242620e33ce1d20679efefe841/rpds_py-2026.9.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:a36b70596407634ca82d4b989a3729074a008537a0522e4c8046a67c729103e9", size = 346229, upload-time = "2026-10-04T16:29:38.82Z" }, + { url = "https://files.pythonhosted.org/packages/57/71/a097d6552f837500fc36e6b23d09cfb9890c3cc47531f9ca64e149799615/rpds_py-2026.9.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eba5d173f7d5708b22a93815017a4611873ed54db9f268077c0dd1ed99cfc858", size = 340514, upload-time = "2026-10-04T16:29:40.405Z" }, + { url = "https://files.pythonhosted.org/packages/bd/b7/497e85768bf4e0d8ddbaa096a4cac31d1509251dee2728a8490aa367e0b5/rpds_py-2026.9.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:457866b85daf5034296666168b84a69e0b2e89dc4f1af102b46f6448a60b9063", size = 372611, upload-time = "2026-10-04T16:29:41.778Z" }, + { url = "https://files.pythonhosted.org/packages/52/4b/74ab4108916250b6e198e0d3af05bc6835eb046315f22f7a0ceb49667c5a/rpds_py-2026.9.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a3a52a3ba86436ab3aef510fbe21512abc2ddd1993005dfe50514bd2284ef025", size = 377712, upload-time = "2026-10-04T16:29:43.242Z" }, + { url = "https://files.pythonhosted.org/packages/0c/8e/067e77d9d7b3cc793c9d909b7e97e7aadbbb1fb094093b6876902cc96d38/rpds_py-2026.9.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d7841166b7fa64c9c56404617ae4341448847482d45933b13135d26c130519e5", size = 488106, upload-time = "2026-10-04T16:29:44.692Z" }, + { url = "https://files.pythonhosted.org/packages/3c/b4/c5aae6c2dde269bf955f6b7d35065c655a57e47750d9668052ad67e74dda/rpds_py-2026.9.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:926bdd3e3b5998ddf70cc64bc8cf57209571f9044542913afb673799fec77dd0", size = 390795, upload-time = "2026-10-04T16:29:46.129Z" }, + { url = "https://files.pythonhosted.org/packages/a0/36/76fab39973ee11e7f9f357c55138197bb01c86f6502cb76487e3b4f42db0/rpds_py-2026.9.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7868b85224291c6cb6759f9b5adb9745f486d226f62b16a614dd5a2a5ab2b35b", size = 370794, upload-time = "2026-10-04T16:29:47.603Z" }, + { url = "https://files.pythonhosted.org/packages/3d/fe/cd2a80e6d7b871937a60e935c5d507aa390d143f4ff3636f640b9733d5df/rpds_py-2026.9.1-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:3cd182d7291d29b92c521a0069d9c01ba6193628a9a105531d11b40a6d731a33", size = 375673, upload-time = "2026-10-04T16:29:49.223Z" }, + { url = "https://files.pythonhosted.org/packages/6c/18/7464a9953724e55a3b3206062fa0ffdeaa519584c6aabf65d3956d94f131/rpds_py-2026.9.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e6ea1cda8d8c688278430e4268a42f5e5da3bdd74578dfadc0820c3f1766ce83", size = 398758, upload-time = "2026-10-04T16:29:50.601Z" }, + { url = "https://files.pythonhosted.org/packages/c0/86/1534b436700fd49ff411063b7c4d7e938adfabf90895b6cf1622d5a7d1f6/rpds_py-2026.9.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5943980471829f6de242a20b109de3111ba6b77e3af0ffc587028ac854b05e6c", size = 550370, upload-time = "2026-10-04T16:29:52.002Z" }, + { url = "https://files.pythonhosted.org/packages/57/1c/e1fa82a8a01e3c5820f3ba98a8b2673f642128eb368fa88871b01dd2c909/rpds_py-2026.9.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:76d3af9732d2dab69f28179b40ba2d87e2f1d5824b4a694780aa787d685e8f36", size = 613106, upload-time = "2026-10-04T16:29:53.655Z" }, + { url = "https://files.pythonhosted.org/packages/29/55/b20b8c4c3dde8755bfcd5b08492a02d0cd2e26929aedf6199ca2a377d42a/rpds_py-2026.9.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:78326f4cb4427a56ba4996c0762b63be45f06b85f086526420d2b3a66e40f84d", size = 578235, upload-time = "2026-10-04T16:29:55.157Z" }, + { url = "https://files.pythonhosted.org/packages/55/42/df3f7bbc3f7ab37a8a9db8d6c2ff2c985422899f1f7926afbf7ec3c0b8b4/rpds_py-2026.9.1-cp313-cp313-win32.whl", hash = "sha256:172e47169583f46ce118cbec68e6795d0da0f4606b488b6434f8276bca0a058c", size = 205293, upload-time = "2026-10-04T16:29:56.669Z" }, + { url = "https://files.pythonhosted.org/packages/31/9c/ba5a9569d719bfdd6ce863df4133ac6a1658cf1b07cc3534c31db729fbbc/rpds_py-2026.9.1-cp313-cp313-win_amd64.whl", hash = "sha256:3e93b2cd69a9830be33e03945cd7cda940a0a8bfcfbff41d6144f0cb0d3d8bd9", size = 222409, upload-time = "2026-10-04T16:29:58.049Z" }, + { url = "https://files.pythonhosted.org/packages/35/72/f28ca566f6c23c35bbf7445f65eb0364577b25a026305995e24f78b83d94/rpds_py-2026.9.1-cp313-cp313-win_arm64.whl", hash = "sha256:d151e148117294133bf8af7eeace085e7e87432db15ab6adf640330298a47f6f", size = 217681, upload-time = "2026-10-04T16:29:59.449Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f2/67b94be1532767803415c1c5a1fd88ea487643d74a673cec1ba140af77bb/rpds_py-2026.9.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:c9d1aca01f49170fdcf5c92761b1fafe97f554b721ca4570c5949fff778f0d4b", size = 347178, upload-time = "2026-10-04T16:30:00.865Z" }, + { url = "https://files.pythonhosted.org/packages/04/37/b751de2b59b0197a1d92a5dd491de88e8a5e928c2e6562581974f1e85263/rpds_py-2026.9.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3f0e9ac28fc067d4d34b88ae43c48e9489455c97fee9633d851f7eeed5a05d35", size = 341878, upload-time = "2026-10-04T16:30:02.564Z" }, + { url = "https://files.pythonhosted.org/packages/72/e2/5873bc4643c250db9e05d48dc0c93763d4aa68bc3b81164cb1af3b45b284/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07deecbfce94c78473018bc7d10b337cc651d12df87a1eb2cb3e4024bc9c33d0", size = 373963, upload-time = "2026-10-04T16:30:04.026Z" }, + { url = "https://files.pythonhosted.org/packages/51/03/5acf7632158247f3f6386ff0af3a1ee48167d575037e8d0920594b76d92b/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:821b2755db9194409254012f429c56643416fb96ef9be090be82ec8826b7f477", size = 377975, upload-time = "2026-10-04T16:30:05.555Z" }, + { url = "https://files.pythonhosted.org/packages/e6/00/63fda451b8bffa5808fc8bb311ee7c073b340974b09f273c7b2a145d3d62/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3c91c210ae7645626c608400e3519b4a642f837cce09ca830db3beb2e9f274d4", size = 490347, upload-time = "2026-10-04T16:30:07.156Z" }, + { url = "https://files.pythonhosted.org/packages/a1/ae/c093ffd070ba0fb02f76c565d06fecc65ad6e4afdbae78f7031076d3cdac/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:54ac2158a6f96cfbabff0b2eedaf94b90c5ec7ca8317fcadc61e1c2b2e0ff6ef", size = 392962, upload-time = "2026-10-04T16:30:08.77Z" }, + { url = "https://files.pythonhosted.org/packages/22/9d/d08a1128ab199b2f0cf25bfeb0639bd05119fff4b7c47bec24ef9a8ec23f/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eac2f5dbafd585dfe31f86a23ebf0d3ba480a9d49ebc87947267b5608d4ea0cd", size = 371623, upload-time = "2026-10-04T16:30:10.501Z" }, + { url = "https://files.pythonhosted.org/packages/53/c7/4758ddcbb75609414bbccfcb11d612436f9b3ee821bd2f33f0f1604ee648/rpds_py-2026.9.1-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:8aa5dda18d39b6143eb24809d158f9252c88f402749b6f1b62a506cc7d96cc35", size = 376997, upload-time = "2026-10-04T16:30:12.124Z" }, + { url = "https://files.pythonhosted.org/packages/87/e4/947bd7f608ff60faf46dc9d389c3dffd0e3d767d78a0be19978448ef0ce7/rpds_py-2026.9.1-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5c90e7fa02e8f5de0d10c17595c568ada48c5302e749462c0ea1a4c362111a86", size = 400169, upload-time = "2026-10-04T16:30:13.804Z" }, + { url = "https://files.pythonhosted.org/packages/4b/35/fe93e020a0543b5670472c18d7e6af3197c08da571240ce1965c84f85c0f/rpds_py-2026.9.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e6d198bad4e49dd6732fbd636e2fc5c082f45c8cad0b4acb756b00c82c76072e", size = 551615, upload-time = "2026-10-04T16:30:15.332Z" }, + { url = "https://files.pythonhosted.org/packages/0d/4f/5d2a0136bb03b2a56a39dc6ff92d58a6e3e53a2e17079238b86228882f16/rpds_py-2026.9.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:96beca19ec79de272e8668585380ff9092c47077c1d7a1e098e00bbd921f4785", size = 614655, upload-time = "2026-10-04T16:30:16.92Z" }, + { url = "https://files.pythonhosted.org/packages/09/1c/3f1025aaf70d9bf7272cc41f8b64ee76b48bf01726248138430e16f23b38/rpds_py-2026.9.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a5cf77eb04f20b720be95265a3e00eb2a14814074255cc27069c551b2db53118", size = 579602, upload-time = "2026-10-04T16:30:18.555Z" }, + { url = "https://files.pythonhosted.org/packages/53/0d/5c72e6204f76610608706da32b6b7e11ef7e10317558a7bb15a122e008dc/rpds_py-2026.9.1-cp314-cp314-win32.whl", hash = "sha256:a03d57b86d2a51d0a66c92177e2be154ad015f357791d306e714569999cdb4cc", size = 206315, upload-time = "2026-10-04T16:30:20.05Z" }, + { url = "https://files.pythonhosted.org/packages/a4/0b/489d48abbcc7d70cf3fbf662d9d22abf1f4650761c0a9ae05260800800d3/rpds_py-2026.9.1-cp314-cp314-win_amd64.whl", hash = "sha256:837c6b305e26fe0f75b15c92cf3b2ba29e0ae19dc40b1c557b026cb426347d0c", size = 222852, upload-time = "2026-10-04T16:30:21.604Z" }, + { url = "https://files.pythonhosted.org/packages/91/16/bbb05a7e6a10cf79ba639be7f799d770ee15f64175cc61d081b218dd402a/rpds_py-2026.9.1-cp314-cp314-win_arm64.whl", hash = "sha256:fce4b85234a0cbad67bf8e6e1201ee815d172c9aebad75f25645bc4d834f8e31", size = 218709, upload-time = "2026-10-04T16:30:23.036Z" }, + { url = "https://files.pythonhosted.org/packages/22/ac/ac507a0a4ec478ca470440a09583db4be5259ba7670aeba0620822f1e57a/rpds_py-2026.9.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3a72c11530d71abfb66c8d7696a2f86c43e63fca8b948f1a784ac490f4ec688e", size = 349596, upload-time = "2026-10-04T16:30:24.558Z" }, + { url = "https://files.pythonhosted.org/packages/e9/f2/817a46b658d5070f477f722c298ee9a24525b0e4017347964146ef5fdd0e/rpds_py-2026.9.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:068c37bba854ec2fe42f7365c640af11dd9895890ccbf2df5070d0c059bd7f96", size = 343114, upload-time = "2026-10-04T16:30:26.048Z" }, + { url = "https://files.pythonhosted.org/packages/6c/42/6ade976b13ac1b4cb3bf2eb603f1be2fe74df19a29988d4c2b386be59d6f/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fca4eb6df565e2a928f1c7dad92d27db8f9df0f449e76423ed5d7e713ed445", size = 374197, upload-time = "2026-10-04T16:30:27.699Z" }, + { url = "https://files.pythonhosted.org/packages/d9/70/77cdf1d3f1a07faabe936016ae623aec7981f73108a8fe7a203ed2e21998/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c933c6678c6f116ff8af47a4c6db0868b8ace74af0343016c0ef00f00272ea69", size = 378025, upload-time = "2026-10-04T16:30:29.451Z" }, + { url = "https://files.pythonhosted.org/packages/3f/6b/18a44a3beaa9b7931acb04af7bd9539836477c630a794452d4826d6185d4/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:028ad274ea951dac64491b5d1e65712a4aeabfdbdb9fccf797b57bd899b0c495", size = 491969, upload-time = "2026-10-04T16:30:30.995Z" }, + { url = "https://files.pythonhosted.org/packages/73/27/fb39cfd6bddaf741b024f813890374578ff8ac1f1adc473659c048b03b05/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:740d0a99cf9de0b17a3943388e9294a59becf75e7c43421f387bd3c7a9901f7c", size = 393905, upload-time = "2026-10-04T16:30:32.628Z" }, + { url = "https://files.pythonhosted.org/packages/ed/71/0fa7bb77b57af0d710273964180d11b503f62b8a5c358eb2d8c3f62feff6/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0da298fb372dc192610a4b9ecbc68a0cd8b675bbbd1fc519d01b41cfd658333e", size = 374569, upload-time = "2026-10-04T16:30:34.257Z" }, + { url = "https://files.pythonhosted.org/packages/54/22/f41cfac269af3b449513ef1bc3d7f32fde52abbbd2d01c7e76b47743acd9/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:eb61be926bb81567c1f48bdc8aa22b9855048dc2efd53871f9f7e6e9a5632346", size = 377482, upload-time = "2026-10-04T16:30:35.997Z" }, + { url = "https://files.pythonhosted.org/packages/b4/fc/312b49006e7f8f9ca5f96647577b8aa6f3df30519c46bd448f5c425af0b2/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:42e75466f83cd43f6026c81eab74246efb2bdadafb307b85700632d06c68f299", size = 400040, upload-time = "2026-10-04T16:30:37.76Z" }, + { url = "https://files.pythonhosted.org/packages/cf/a6/18cca7a878dc7fa95165a83343fd4d7b65643fd22e54e47340a451121d5c/rpds_py-2026.9.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:617f59cde379b4f648a09797b7f683d04b90a46344cddab85639da5aff0f5531", size = 551849, upload-time = "2026-10-04T16:30:39.443Z" }, + { url = "https://files.pythonhosted.org/packages/e4/6d/1f5685e20f39604691bdc3c05aaa6b8bd2f954e9e996477adf2376768e33/rpds_py-2026.9.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:3edae8c5ddfdb6985d49ae9d150516e5076888879022f91a26c2de9276ce0bdb", size = 614842, upload-time = "2026-10-04T16:30:41.231Z" }, + { url = "https://files.pythonhosted.org/packages/c6/25/98652109fd9f7e10268dd4571aa52b81987001b806f37ef1a18260de714a/rpds_py-2026.9.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0f045bb053c9057720d72c56dffe30dffdc05997b2897a827b9325f0ab6623fa", size = 582606, upload-time = "2026-10-04T16:30:43.345Z" }, + { url = "https://files.pythonhosted.org/packages/19/03/11ca09099bab5f53373a80a334c424ec917c13d050760a59499d2af5171e/rpds_py-2026.9.1-cp314-cp314t-win32.whl", hash = "sha256:bf35d0568abda97233239ce32896d3ad53fccc537832c104e30c94aa5fb93569", size = 203341, upload-time = "2026-10-04T16:30:44.954Z" }, + { url = "https://files.pythonhosted.org/packages/6f/8a/88909e3ffd9f47f5b58211473875d8c3c09079f0058c46fb72c55a702a26/rpds_py-2026.9.1-cp314-cp314t-win_amd64.whl", hash = "sha256:1e8d4d79d828299bf44a55db22a9388ab967b49d17132c88eab0f4360b48da8e", size = 222843, upload-time = "2026-10-04T16:30:46.486Z" }, + { url = "https://files.pythonhosted.org/packages/f6/b7/a662f367d4896dd0a10cef2fc91f10b7f08af1c10858e287e019563f338d/rpds_py-2026.9.1-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:1d77b649e6f7cdf12ca5c2a98dad0ad37f9ea9b6f960408a92f0cb12bb3d04d9", size = 347463, upload-time = "2026-10-04T16:30:48.203Z" }, + { url = "https://files.pythonhosted.org/packages/ae/3f/ad45d03df4f84ebae5439577ee81f3999d182711e82235c8037b6528890e/rpds_py-2026.9.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:00ba2d8c7dd4ee537978ddf4b3fbd712bef2d8751603f7f3146b3f4287768e25", size = 342051, upload-time = "2026-10-04T16:30:49.872Z" }, + { url = "https://files.pythonhosted.org/packages/7e/31/3dcd68c13d4bcc59c1f7eb33ac8e80698f06f3eb0d8a1e06419836071c20/rpds_py-2026.9.1-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec450527cbf485e13c8d3602a54f428ab0432fdade0ede75efd74b735421c871", size = 374143, upload-time = "2026-10-04T16:30:51.508Z" }, + { url = "https://files.pythonhosted.org/packages/cf/0d/68c1f058a250fbd1380ebda9fc227cbf50117adf8ffe8161ef383ea79f68/rpds_py-2026.9.1-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:306ee1850d8105b5baf977e78d45fcadd12c1a54678d614c9baf217708446e91", size = 378604, upload-time = "2026-10-04T16:30:53.206Z" }, + { url = "https://files.pythonhosted.org/packages/d7/d6/2d4c59b85397cb4800594fadf692688ccf5ce556adc930e7a5bf21061a5e/rpds_py-2026.9.1-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ef6b65b03247c54692ad4fd9ee97cb772781927db72e3cb05e70b3db6d1ff14f", size = 490790, upload-time = "2026-10-04T16:30:54.925Z" }, + { url = "https://files.pythonhosted.org/packages/da/04/7e05dc3aebaf52f4e026766bd668fdd09a9d0e23f64a14686b36b3501892/rpds_py-2026.9.1-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a575404ebc9cf2e91edd32eaf570ec1430eb900d4f56724ba7dd4bc1fc9c176d", size = 393146, upload-time = "2026-10-04T16:30:56.625Z" }, + { url = "https://files.pythonhosted.org/packages/57/ca/e2e9a0a46a74ed51a0498ba1fe10f4ea6b2d9a155f372a2f91e51f18cf10/rpds_py-2026.9.1-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c16ab111bc27c646ba8aa005d0527754edc538ebb636f0b1bf8e244b48d1945", size = 372022, upload-time = "2026-10-04T16:30:58.295Z" }, + { url = "https://files.pythonhosted.org/packages/ba/cb/8f8774df5134e23424372838bcc5c7ed4127d723e1ff52f7bebd4dcb2563/rpds_py-2026.9.1-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:7664419f27db41d4f1c43a78dccda7dd6e8ef2428df3ee01d0c2a07a6b071297", size = 377092, upload-time = "2026-10-04T16:30:59.984Z" }, + { url = "https://files.pythonhosted.org/packages/90/02/8d7095d73bf9114219be40230baa5df00611e0a82ed9517779ff9c19f82b/rpds_py-2026.9.1-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4b26b03d9d2658ee2fa234f8f4f19f38a09773fe5261028025032e26d4d35af0", size = 400830, upload-time = "2026-10-04T16:31:01.721Z" }, + { url = "https://files.pythonhosted.org/packages/ec/02/8206856f8f363cd042a8315dc86b3912f5dcb6d3c66bbb24b69c2bfb0775/rpds_py-2026.9.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:be3e47e2d91aa3942ff9bf4077a505226005abfc39b6f7554a91c1b9393986b9", size = 551478, upload-time = "2026-10-04T16:31:03.472Z" }, + { url = "https://files.pythonhosted.org/packages/41/6b/36211f1bb1f0b0313f496d92f5905b74ea107f27cb16fa3355a82f04575e/rpds_py-2026.9.1-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:6307a0da524939decb8ca4a3933b8ab62525794411d6984fca6726e732804af6", size = 615173, upload-time = "2026-10-04T16:31:05.281Z" }, + { url = "https://files.pythonhosted.org/packages/35/77/cda0c4a6f055446b692f0ed5692f73707cafd3dff82672ede31b1a9b59de/rpds_py-2026.9.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:159a7aab5c5e8b112c8830f54717ce56da1252ebdbb526f5be2df2309280b9e7", size = 579653, upload-time = "2026-10-04T16:31:07.065Z" }, + { url = "https://files.pythonhosted.org/packages/74/ec/d8385f446240aed643b9e92a5055cff3015cc04a13c61f73b2883c478ed5/rpds_py-2026.9.1-cp315-cp315-win32.whl", hash = "sha256:dbc2673f9223d420c91145599b3ba45a8a50c207d1976908e5fb5ddb0c9b9429", size = 206498, upload-time = "2026-10-04T16:31:08.989Z" }, + { url = "https://files.pythonhosted.org/packages/6d/a5/71b5cd00e0521e3b6b81828baea368c62b6b700ebdd9554cd7d41ddf12fa/rpds_py-2026.9.1-cp315-cp315-win_amd64.whl", hash = "sha256:75c38c50ab9aca840225d9a9a3810bf11d04bd5c1f186cabbb8aee56db3e9b15", size = 223121, upload-time = "2026-10-04T16:31:10.84Z" }, + { url = "https://files.pythonhosted.org/packages/33/58/dba857c3bc8221b31b62eb170a3080f4f191de79f047200389b7ed1b06a7/rpds_py-2026.9.1-cp315-cp315-win_arm64.whl", hash = "sha256:a431156bb41865fc14cd5d79bb9d7bbed83110b0159e34e62ae30951f96c0009", size = 218741, upload-time = "2026-10-04T16:31:12.592Z" }, + { url = "https://files.pythonhosted.org/packages/5b/d0/320ab28ccc1415eeb509d68682b0014fb74690cd49f1c2d29a232475af50/rpds_py-2026.9.1-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:ef0d8c843e2827d6c120ab4687e9423fb1d893db1df27b7c1506615bcb9734a0", size = 349719, upload-time = "2026-10-04T16:31:14.48Z" }, + { url = "https://files.pythonhosted.org/packages/56/88/f5b12f1358f443c08b7ce3cc8391d82d335f4872580e2e097847fd36087a/rpds_py-2026.9.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:45bc6bccf78b20fd834237d18db64965d7ee68ba7f60440a26c7ab71e7b8d51a", size = 343267, upload-time = "2026-10-04T16:31:16.827Z" }, + { url = "https://files.pythonhosted.org/packages/f7/0c/c765b0059d532acb3b9c45d781ccc22f15a96dbe443d00903f643ba9df10/rpds_py-2026.9.1-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d55198263bb51f557550c6ed2e6d1cb6a6fed6eb5c9120b741c5926bef8a45d", size = 374555, upload-time = "2026-10-04T16:31:18.931Z" }, + { url = "https://files.pythonhosted.org/packages/6b/8a/cafddfda77564a10cd21184640c3bffda6a2b8d20972fb5dedd5e0166328/rpds_py-2026.9.1-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a8763f20692da7df39b0afdd1ba3042b004c50a45994f76c2d9a25641f7673db", size = 378346, upload-time = "2026-10-04T16:31:20.75Z" }, + { url = "https://files.pythonhosted.org/packages/b9/01/5e626016eff72c183bf6c96539240cace15d402468647a453ec08415b2fc/rpds_py-2026.9.1-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e43d4a1f673e8a1cbd8533e809e02b4bf9d4f2280269bb640436556312121250", size = 492373, upload-time = "2026-10-04T16:31:22.614Z" }, + { url = "https://files.pythonhosted.org/packages/3b/9c/15a2469e9389242f46896b3f0a01d68caea8a5a35c011fcb05ef333aae73/rpds_py-2026.9.1-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea394a937f17a54c51239348bdbe2e3518124c8d4a8951ba04a311d3095bd18f", size = 394371, upload-time = "2026-10-04T16:31:24.768Z" }, + { url = "https://files.pythonhosted.org/packages/63/f5/c100ff77e1e6366e947c75969c258fdfc4b7bc5bbfe7351e62ffbf2a1228/rpds_py-2026.9.1-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdeaa99ce822dca76cfb1b993e9120c5ea212f2eb66d48950ad63c349668a018", size = 375132, upload-time = "2026-10-04T16:31:26.588Z" }, + { url = "https://files.pythonhosted.org/packages/dd/f4/fe0269c9de253e99c81cabc12b8971a5feaa083debdaff1221e06264d9e3/rpds_py-2026.9.1-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:b4f062343e7ad3fa94f2c66e5ae667dee47ee74dd41a9057c4fbe163236a123d", size = 377642, upload-time = "2026-10-04T16:31:28.677Z" }, + { url = "https://files.pythonhosted.org/packages/05/65/b34a7b257baccff8f4a24a722933166d4941d5ebdc9c3f4bc4ffcd5ce4f4/rpds_py-2026.9.1-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:22ffd29a63d71fb1b81552c21f2c2b734949b7ac751a9be70675a939a900839b", size = 400855, upload-time = "2026-10-04T16:31:30.802Z" }, + { url = "https://files.pythonhosted.org/packages/98/32/844e54176b6071b90b38a564e6940bc6eb8f97b2890dc709c19db9dec0f4/rpds_py-2026.9.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:08dae4a4095150a7c4545a1fb40b98e1ab1744fbc2770d92c977b9dadaa49ab6", size = 552573, upload-time = "2026-10-04T16:31:32.709Z" }, + { url = "https://files.pythonhosted.org/packages/17/73/6041d20729dffbfdf155c02d65be58bc225a1c1fb548fd87c23ef306138f/rpds_py-2026.9.1-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:9a0460d43603d1fd9ef59c30278531e15d78581721ddb538fa560aa7817ea4ad", size = 615840, upload-time = "2026-10-04T16:31:34.565Z" }, + { url = "https://files.pythonhosted.org/packages/af/9e/418094adaee6b056ce199051b255448ed872829051e341b2294c80da0977/rpds_py-2026.9.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:1c2d1f6da5128eabf34e963d7163a818846075a52568250d006c4c953b40f903", size = 582764, upload-time = "2026-10-04T16:31:36.665Z" }, + { url = "https://files.pythonhosted.org/packages/3d/9b/1698ebf6b840ddfe8b472198abbed6ace35c6e398faeecd6c47dae742a4e/rpds_py-2026.9.1-cp315-cp315t-win32.whl", hash = "sha256:5c6ee90dee3e85e055ddfd502d611643d9b0fd94c818220bda84ec3dacd9b27b", size = 203795, upload-time = "2026-10-04T16:31:38.53Z" }, + { url = "https://files.pythonhosted.org/packages/8a/e2/91f70d804c61f8eac39a417e82aa1024f655ab9e8bdd7393b196242bc41b/rpds_py-2026.9.1-cp315-cp315t-win_amd64.whl", hash = "sha256:fe5ad0664ec772b02c45859041aa17655709cced7a31005817fbbbd988c25567", size = 223037, upload-time = "2026-10-04T16:31:40.468Z" }, + { url = "https://files.pythonhosted.org/packages/f3/ad/f0eba548e297987964ef8088403db875b333963bf8189f3f563b6936b20e/rpds_py-2026.9.1-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c0d2cb595a420b34d5086db0add011e26e2c09d6a024afbac4228bf8f863a30", size = 384112, upload-time = "2026-10-04T16:31:42.643Z" }, + { url = "https://files.pythonhosted.org/packages/33/89/891fa4d81f1eae7fe262ecd0aba2c51de94a116c7f3ccfb034fa75f4c537/rpds_py-2026.9.1-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f3d6ed6a98cfd19155996605474982cc470d7601746a6439078f1a5a3fa8b050", size = 389662, upload-time = "2026-10-04T16:31:44.642Z" }, + { url = "https://files.pythonhosted.org/packages/4f/46/912cf623b2263f9c9c2d04e8508fc38dbe89983faeba5fb36ca4558ae550/rpds_py-2026.9.1-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8171b44a054e5c67fd748ada04187f1250bf35b95f85e52ab64bcf3331a923bb", size = 504637, upload-time = "2026-10-04T16:31:46.667Z" }, + { url = "https://files.pythonhosted.org/packages/9b/5c/7f3cfa7c5f01765c9f47c8398210ed0c1eb686c0e99c223cb1c6776d3fb9/rpds_py-2026.9.1-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fda1d96e542c37b6c804547dbf489c129fe7c97183a76a5ec275909ba1a063df", size = 402607, upload-time = "2026-10-04T16:31:49.068Z" }, + { url = "https://files.pythonhosted.org/packages/f0/76/2e915d4115ee453c33e6af17bc1df7e22ddc52974e178eeeb92a78c09983/rpds_py-2026.9.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:270bdcdaac5d5b6f73c5e22e7e135c7f2a50e789f71d9e241d5be8d90026e19a", size = 382762, upload-time = "2026-10-04T16:31:51.24Z" }, + { url = "https://files.pythonhosted.org/packages/0d/e5/844382925d7b84a4074ad79fc2b9d05eb6231ea243b34f247e68e519b606/rpds_py-2026.9.1-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:84a6ecc0c940169190d2c23bd969debd48c94dbc855acd60188a68d71d421608", size = 389563, upload-time = "2026-10-04T16:31:53.549Z" }, + { url = "https://files.pythonhosted.org/packages/81/82/d6e3951602dc2e24790ef9d299aa24bf5e2493df5043f2c6da00480cc447/rpds_py-2026.9.1-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ab4b2fda7c2b542f7f9d886cc6a838c5079d2b76f72e6081411faba11adde2c9", size = 416842, upload-time = "2026-10-04T16:31:55.76Z" }, + { url = "https://files.pythonhosted.org/packages/ea/1c/8d248e4ea671d525ab7daa0703ad81cc2175076fa0ccd7ef088facbd2b01/rpds_py-2026.9.1-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:44b32a7c4f0da3d28af31c259e38ddcff096f855e205ed0671d02fcf44f1ea1c", size = 562434, upload-time = "2026-10-04T16:31:57.874Z" }, + { url = "https://files.pythonhosted.org/packages/ad/38/e7ac0db898120d15dea25e3661b1242a40cb3b23d1f4db86c15a01fe0ba1/rpds_py-2026.9.1-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:a3dbc5ed9514908d5046107d7b1346bde71eea61de6e0e4919c19354f97e769f", size = 627909, upload-time = "2026-10-04T16:31:59.977Z" }, + { url = "https://files.pythonhosted.org/packages/03/c8/37e555514932cb1ee36c75349187aa882a98dee5d5654c761699ed8a0d27/rpds_py-2026.9.1-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:6eae33003518fd4cb4f83a218d5371469dd3001aa3b87128c005b07762f7fe5e", size = 593658, upload-time = "2026-10-04T16:32:02.14Z" }, + { url = "https://files.pythonhosted.org/packages/12/5c/5f69a91add94a8a33acea68a2b034a464946a654ff7296af9d957ef0d6a5/rpds_py-2026.9.1-pp311-pypy311_pp80-macosx_10_12_x86_64.whl", hash = "sha256:0483515261947e4e8b8e1375bf7463e7eb6ccfb3d86e7b554d90cd5285f20f32", size = 351769, upload-time = "2026-10-04T16:32:04.675Z" }, + { url = "https://files.pythonhosted.org/packages/41/22/35ea8bf3b2682b90ab687c74e4828418ff12298a5b0b0a0c033f61c86a30/rpds_py-2026.9.1-pp311-pypy311_pp80-macosx_11_0_arm64.whl", hash = "sha256:4cfaf02209061880210819934de2f4f6aa83dc04dafe6770276acc240a56da31", size = 351719, upload-time = "2026-10-04T16:32:06.834Z" }, + { url = "https://files.pythonhosted.org/packages/a3/90/a9ba81408369d34c1aca73319c15adc91ed45c1d21e9e8ee4832ffb7fe0b/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6cdc537c8633d7fd92a82e2e0d2ab74320a3f63d5e59fb9cf08711e08fe151c4", size = 381109, upload-time = "2026-10-04T16:32:08.994Z" }, + { url = "https://files.pythonhosted.org/packages/b3/44/5192a0bed94cec86bd2a5e1cc5a1bb8f7eec3aec907eacd2deb13e87e326/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:10e208f2425d973938afcd56e28a7c4be32e27b6a60b5d381f49fb9d8acf9759", size = 388222, upload-time = "2026-10-04T16:32:11.321Z" }, + { url = "https://files.pythonhosted.org/packages/92/cd/5356549711448f18b52f7a11ffbe90f1228774996fbf3c6cd1b18d22abaf/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6c0dbbcc19735fe5f8b0a54c07659d154a9e69f47e15d0a6ab7299215daf62cb", size = 500384, upload-time = "2026-10-04T16:32:13.478Z" }, + { url = "https://files.pythonhosted.org/packages/7a/88/ddda9d28adfe33119c75b2e733d4ba7e326f41d7e1b1093951771768ca48/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:684fd492fff4fead00587544e059be2bbcb6f93454f21fa2a91b66fc7508be82", size = 401661, upload-time = "2026-10-04T16:32:15.653Z" }, + { url = "https://files.pythonhosted.org/packages/32/c3/bb59ba16a6b4a57995d15b8c04c00f28df3b938c4dcdde48fbe0f73edb0c/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1028417bb44037eb3069c1009bd7b7277212876cda22fbe565b0bca9fab6d2c", size = 380748, upload-time = "2026-10-04T16:32:17.727Z" }, + { url = "https://files.pythonhosted.org/packages/f5/b8/0580faa1c5a32ddc160130dd7ae54d2d007272a89a40a6b5db3e50f9f585/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:492e5e428cbe126221611f47e068f01660352feec4ad18bc0f5ea9b2ae88fb14", size = 385731, upload-time = "2026-10-04T16:32:19.855Z" }, + { url = "https://files.pythonhosted.org/packages/bf/70/f73564642bbe3322c7eeef2c2f258cf040b71c41a430b532e59d04a1b838/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:88b5268892fde430d5531f95bc560b6efbbd67c929662c586afd729a96e7461c", size = 412131, upload-time = "2026-10-04T16:32:22.285Z" }, + { url = "https://files.pythonhosted.org/packages/8c/5d/17fff2e1f8f68721a2afb5cb48f7e442c751bb6b4883157c5abaf618cd08/rpds_py-2026.9.1-pp312-pypy312_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:01445c8d194aa032a08e944f16567672da1c62dbdbefd8b6d0693032e290cf68", size = 558193, upload-time = "2026-10-04T16:32:24.4Z" }, + { url = "https://files.pythonhosted.org/packages/2a/d0/869ab6fb08531f97aae4a780b4d61b4190d8aef48fe054ee28bcf7114466/rpds_py-2026.9.1-pp312-pypy312_pp73-musllinux_1_2_i686.whl", hash = "sha256:eef6a03b0b6d08d0835ccfa8ec8d1bc70525e3801387567137b50c557695e6da", size = 622448, upload-time = "2026-10-04T16:32:26.844Z" }, + { url = "https://files.pythonhosted.org/packages/ba/a5/9672e532fe02cd3b92c78cfd8743abca939d96853006c63835009a1ecb6f/rpds_py-2026.9.1-pp312-pypy312_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:6b9bf3135b4ad5981df9a73d71a35272d650a2985ae9c2746357b24d59de2448", size = 591126, upload-time = "2026-10-04T16:32:29.274Z" }, + { url = "https://files.pythonhosted.org/packages/80/ef/3a9f8e4279c7920af561deed4cb7ebebed2794a8f608cf404d5eef14a105/rpds_py-2026.9.1-pp312-pypy312_pp80-macosx_10_12_x86_64.whl", hash = "sha256:56c6952a9b15047466d0c2347c446a761d4527f89976156341e68f0ce5cc08b0", size = 357108, upload-time = "2026-10-04T16:32:31.641Z" }, + { url = "https://files.pythonhosted.org/packages/b7/55/4b2fa381a583760aea5c92841e4e928a358e1c6511b129219e9c2826a226/rpds_py-2026.9.1-pp312-pypy312_pp80-macosx_11_0_arm64.whl", hash = "sha256:b242c27c8f836305a4a72df9cdd564386ac57b807bd252a063223331c9316b37", size = 346877, upload-time = "2026-10-04T16:32:34.061Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] diff --git a/examples/execution-adbc/verify-client.mjs b/examples/execution-adbc/verify-client.mjs new file mode 100644 index 000000000..1da49b714 --- /dev/null +++ b/examples/execution-adbc/verify-client.mjs @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from "node:assert/strict"; +import { readFile, realpath } from "node:fs/promises"; +import path from "node:path"; +import { OssieClient, ARROW_MEDIA_TYPE } from "../../typescript/execution/dist/index.js"; + +assert.equal(process.argv.length, 3, "Usage: node verify-client.mjs "); +const root = await realpath(process.argv[2]); +const capture = JSON.parse(await readFile(path.join(root, "capture.json"), "utf8")); +assert.deepEqual(Object.keys(capture).sort(), ["execution", "resources"]); +assert.equal(capture.execution.state, "succeeded"); +assert.ok(Array.isArray(capture.resources) && capture.resources.length > 0); +const resources = new Map(); +for (const resource of capture.resources) { + assert.deepEqual(Object.keys(resource).sort(), ["file", "url"]); + assert.match(resource.file, /^page-[a-f0-9]{32}\.arrow$/); + assert.equal(resource.url, `https://example.test/results/${resource.file}`); + assert.ok(!resources.has(resource.url)); + const file = await realpath(path.join(root, resource.file)); + assert.equal(path.dirname(file), root, "Capture resources must remain inside the output directory"); + resources.set(resource.url, await readFile(file)); +} +const execution = capture.execution; +const descriptors = execution.result.pages ?? [{ + url: execution.result.url, row_offset: 0, row_count: execution.result.row_count, +}]; +assert.deepEqual([...resources.keys()], descriptors.map(page => page.url)); +const expected = [[1n, 9007199254740993n], [2n, null], [3n, -7n]]; +const rows = table => Array.from({ length: table.numRows }, (_, i) => + [table.getChildAt(0).get(i), table.getChildAt(1).get(i)]); +const fields = execution.result.columns.map(column => column.model_ref); +assert.equal(fields.length, 2, "The demo capture must have two columns"); +assert.ok(fields.every(field => typeof field === "string" && field.trim().length > 0)); +const request = { + data_source_id: execution.data_source_id, + query: { + fields, + order_by: [{ field: fields[0], direction: "ASC" }], + }, +}; +let submissions = 0; +const api = new OssieClient({ + baseUrl: "https://example.test/", + fetch: async (input, init) => { + const url = String(input); + if (init.method === "POST") { + assert.equal(url, "https://example.test/queries"); + assert.deepEqual(JSON.parse(init.body), request); + submissions++; + return new Response(JSON.stringify(execution), { headers: { "Content-Type": "application/json" } }); + } + assert.equal(init.method, "GET"); + assert.ok(resources.has(url), "Fetch must only read the captured IPC resources"); + return new Response(resources.get(url), { headers: { "Content-Type": ARROW_MEDIA_TYPE } }); + }, +}); + +// Exercise selected-page validation, then full exhaustion and materialization. +for (let i = descriptors.length - 1; i >= 0; i--) { + const page = descriptors[i]; + assert.deepEqual(rows(await api.readPage(execution, i)), + expected.slice(page.row_offset, page.row_offset + page.row_count)); +} +const consumed = []; +for await (const batch of api.readBatches(execution)) consumed.push(...rows(batch)); +assert.deepEqual(consumed, expected); +const result = await api.execute(request); +assert.deepEqual(rows(result.table), expected); +assert.deepEqual(result.execution, execution); +assert.equal(submissions, 1); +console.log("ADBC SQLite -> PyArrow IPC -> existing TS client verified (synthetic Fetch, not live HTTP)."); diff --git a/typescript/execution/README.md b/typescript/execution/README.md index b2888d40c..27a2f71ff 100644 --- a/typescript/execution/README.md +++ b/typescript/execution/README.md @@ -29,6 +29,10 @@ The package does not implement an execution server, semantic evaluator, authentication system, Flight transport, or vendor adapters. Local tests exercise the protocol against fixtures; they do not prove production vendor interoperability, Foundation expression semantics, or row-level security. +The companion [ADBC SQLite example](../../examples/execution-adbc/README.md) +exercises a real backend reader and produces payloads for this client. It is +separate from the npm package and supplies neither a generic Layer 3 compiler +nor a live HTTP service. ## HTTP versions @@ -54,6 +58,8 @@ with Flight have not been demonstrated. - [OpenAPI 3.1](../../docs/proposals/execution-http/openapi.json), referencing those same definitions. - [Public TypeScript types](src/types.ts). +- [Flight SQL / ADBC backend mapping](../../docs/proposals/execution-http/backend-mapping.md) + and [materialization example](../../examples/execution-adbc/README.md). The draft pins the still-open [MCP #529][mcp] at `b5418ee3aa19f117a1c464733de21f5a0dc0326b` (query profile `0.4-draft`) and @@ -392,14 +398,17 @@ uses Arrow, or that two systems implement the same query semantics. | [Power BI REST Execute Queries][powerbi-rest] | DAX request against a semantic model | JSON; limits include 100,000 rows or 1,000,000 values, 15 MB; some limited-data/error responses are HTTP `200` | No Arrow result in this REST contract. Applications must inspect the body, not just status. | | [Power BI/Analysis Services XMLA with ADOMD][xmla] | A separate Analysis Services client/protocol surface | XMLA text or binary XML, with transport compression options | Binary XML is not evidence of Arrow. Arrow on this path has not been verified. | | [Cube REST API][cube] | Cube JSON semantic query, including `/v1/load` | JSON data plus annotations such as title, short title, and type | Prior art for semantic result descriptors; its JSON “columnar” layout is not Arrow IPC. | -| [Arrow Flight / Flight SQL][flight] | Arrow-oriented RPC; Flight SQL adds a SQL service protocol | Arrow record batches with tickets/endpoints | A distinct transport, not implemented by this browser HTTP client. A future mapping requires separate design. | -| [ADBC][adbc] | Database driver API with statements and driver-dependent capabilities | Arrow-oriented result consumption | An API abstraction, not an HTTP wire protocol or implemented adapter here. | +| [Arrow Flight / Flight SQL][flight] | Arrow-oriented RPC; Flight SQL adds a SQL service protocol | Arrow record batches with tickets/endpoints | The backend mapping documents reuse and gaps; no live Flight adapter is included. | +| [ADBC][adbc] | Database driver API with statements and driver-dependent capabilities | Arrow-oriented result consumption | A separate SQLite materialization example produces this profile's results; the browser package is not an ADBC driver or general backend adapter. | The [existing Ossie REST discussion][rest-thread] addresses model/catalog interchange; [Polaris #4961][polaris] implements semantic-model CRUD. Neither establishes this execution endpoint. The draft narrows familiar patterns into a proposed browser contract; it does not claim novel transport mechanisms or parity with a native vendor client. +This is a complementary browser binding, not a Flight SQL or ADBC replacement. +Two independent backend implementations are a proposed adoption criterion, +not a claim already met by the fixtures or the single ADBC example. [mcp]: https://github.com/apache/ossie/pull/529 [foundation]: https://github.com/apache/ossie/pull/246 From 4514a622421e27f836697a98d55ade10a9408c40 Mon Sep 17 00:00:00 2001 From: Markus Cozowicz Date: Mon, 12 Oct 2026 23:31:15 +0200 Subject: [PATCH 4/4] feat: add negotiated live Arrow execution with terminal verification --- docs/proposals/execute-query-http.md | 41 +- .../execution-http/backend-mapping.md | 42 +- .../execution-http/live-streaming.md | 231 ++++++ .../proposals/execution-http/live.schema.json | 108 +++ docs/proposals/execution-http/openapi.json | 54 +- examples/execution-adbc/README.md | 6 + typescript/execution/README.md | 105 ++- typescript/execution/src/arrow.ts | 47 +- typescript/execution/src/client.ts | 309 +++++++- typescript/execution/src/errors.ts | 16 +- typescript/execution/src/transport.ts | 2 +- typescript/execution/src/types.ts | 42 + typescript/execution/src/validation.ts | 82 +- .../execution/test/browser/smoke.test.mjs | 86 ++- typescript/execution/test/live-fixtures.mjs | 84 ++ typescript/execution/test/live.test.mjs | 721 ++++++++++++++++++ typescript/execution/test/schema.test.mjs | 84 +- 17 files changed, 1981 insertions(+), 79 deletions(-) create mode 100644 docs/proposals/execution-http/live-streaming.md create mode 100644 docs/proposals/execution-http/live.schema.json create mode 100644 typescript/execution/test/live-fixtures.mjs create mode 100644 typescript/execution/test/live.test.mjs diff --git a/docs/proposals/execute-query-http.md b/docs/proposals/execute-query-http.md index 6381d2834..358c95659 100644 --- a/docs/proposals/execute-query-http.md +++ b/docs/proposals/execute-query-http.md @@ -17,14 +17,17 @@ under the License. --> -# Ossie HTTP execution — DRAFT 0.1 +# Ossie HTTP execution — materialized base and live extension -**Status:** proposal `0.1-draft`, not an adopted Apache Ossie standard. +**Status:** materialized proposal `0.1-draft` with an opt-in +[`0.2-draft` live extension](execution-http/live-streaming.md), not an adopted Apache Ossie standard. The associated TypeScript package is a client and local protocol-validation prototype, not a server, semantic engine, or vendor adapter. This proposal binds structured Ossie query submission and status to HTTP/JSON, -and completed tabular results to Apache Arrow IPC streams. It targets browser +and tabular results to Apache Arrow IPC streams. The base materialized mode +offers completed replayable results; the negotiated live extension permits +downloading while execution is running. It targets browser applications without requiring gRPC. It does not define a new query language. It is a complementary access binding, not a replacement for Flight SQL or ADBC. Engines may retain those interfaces behind an HTTP adapter; Layer 3 @@ -48,6 +51,9 @@ JSON Schema (draft 2020-12). [openapi.json](execution-http/openapi.json) describes the HTTP operations using references to those definitions, not a second set of object schemas. These documents specify structure; the rules below additionally specify behavior. +The separate [live schema](execution-http/live.schema.json) reuses the base +query/column definitions and defines versioned live request and response shapes. +The base schema and materialized method contracts are not reinterpreted. The lowercase `expression`, `projection`, `predicate`, and `query` definitions are copied unchanged from the [pinned MCP query schema][query-schema]. @@ -55,14 +61,17 @@ The public `Query` definition adds only a portable integer bound to `limit`. All wire property names are case-sensitive lower snake case; schema definition names such as `Execution` are not wire keys. -In scope: one read-only query, immutable model binding, bounded synchronous wait, +In scope for the materialized base: one read-only query, immutable model binding, bounded synchronous wait, polling, cancellation, errors, and one materialized result delivered as a single stream or immutable pages. Out of scope: model/catalog CRUD, native-query escape, query compilation, semantic evaluation, sessions, transactions, mutations, SSE/progress subscriptions, multi-result or -live-result delivery, cursor APIs, result-link renewal, and automatic repair or +live-result delivery without the negotiated extension, cursor APIs, result-link renewal, and automatic repair or re-execution. Flight is a possible future mapping, not implemented or negotiated by this draft. +Live delivery is an alternative to materialization, not a change to Arrow IPC. +Materialized delivery does not require keeping the whole result in memory: +an implementation can spool to disk, as the ADBC example does. ### HTTP versions and transport behavior @@ -123,10 +132,13 @@ stricter than the pinned MCP schema's unbounded nonnegative `limit`. } ``` -The three version values MUST match this draft exactly. `query_shapes` is a +The three base version values MUST match this draft exactly. `query_shapes` is a nonempty unique list; `features` is a possibly empty unique list. These advertise support, not semantic-conformance certificates. `max_wait_seconds` MUST be an integer from 0 through 60. +An endpoint supporting the live extension additionally advertises +`live-stream-v0.2` in `features`. The base `contract_version` here remains +`0.1-draft`; the explicitly opted-in live executions use `0.2-draft`. `GET /bindings/{id}` returns a `Binding`; `{id}` is its `data_source_id`: @@ -152,6 +164,9 @@ Paths are relative to a deployment-selected base URL. Control requests and successful responses use `application/json`. Servers MUST NOT redirect these operations. A non-2xx response MUST use `application/problem+json` with `ProblemDetails`; this draft uses 4xx/5xx for failures, not redirects. +The following table describes materialized executions. Live requests use the +same control paths with the separate shapes and readiness/error rules in +[the live extension](execution-http/live-streaming.md). | Operation | Successful response | | --- | --- | @@ -203,7 +218,7 @@ POST beyond `N` seconds waiting for execution completion. It returns `200` if terminal or `202` if pending, without cancelling the execution at wait expiry. Clients MAY discover/cache the advertised bound or validate only the protocol range locally and rely on the server to enforce its advertised maximum. -The TypeScript prototype does not automatically preflight capabilities or +The materialized TypeScript methods do not automatically preflight capabilities or bindings and does not cache discovery results for submission-time enforcement. Its explicit getters return validated data; applications can use that data to choose requests, while the server enforces its advertised restrictions. @@ -296,7 +311,7 @@ executions for inspection; `wait` and `execute` throw for those states. ## 4. Results and Arrow contract -A successful result is one immutable materialization, available only after +In materialized mode a successful result is one immutable materialization, available only after execution succeeds. Its `ResultManifest` MUST provide **exactly one** delivery form: `url` for one self-contained Arrow IPC stream, or a nonempty `pages` array of `ResultPage` descriptors, each pointing to a self-contained Arrow IPC stream. @@ -610,10 +625,12 @@ that criterion. There are no confirmed independent engine commitments recorded by this proposal. Keeping the PR experimental avoids treating a prototype as community adoption; adoption remains subject to the project's normal process. -A future streaming-partition mode, different metadata attachment rules, or -renewable access references should be motivated by adapter evidence and reviewed -as protocol changes. They are not implicit relaxations of this draft's exact -counts, immutable manifests, page identity, or expiry requirements. +The separately negotiated live extension defers final counts and starts a +forward-only stream before completion; it does not relax materialized counts, +immutable manifests, page identity, or replay guarantees. It is tested with +controlled HTTP producers, not a live AS, Flight SQL, or ADBC engine. +Live pagination, different metadata attachment rules, and renewable access +references remain follow-up protocol changes requiring evidence and review. ## 8. Prototype implementation notes (non-normative) diff --git a/docs/proposals/execution-http/backend-mapping.md b/docs/proposals/execution-http/backend-mapping.md index bc5c6bb27..027caf69a 100644 --- a/docs/proposals/execution-http/backend-mapping.md +++ b/docs/proposals/execution-http/backend-mapping.md @@ -24,6 +24,9 @@ additional wire contract or adopted standard.** Flight/ADBC and a browser HTTP binding are complementary, not replacements for each other. Reusing established backend execution/data APIs can reduce gateway work without pretending that they already provide Ossie semantic planning or the HTTP draft's guarantees. +The main table concerns materialized delivery; the +[negotiated live extension](live-streaming.md) offers an alternative described +below without weakening those materialized guarantees. ## Concrete boundaries @@ -79,7 +82,32 @@ through. Even when some Arrow buffers can be shared in-process, metadata rewriting and IPC serialization still incur work: no zero-copy, live-streaming, or Flight performance-parity claim follows. -## Evidence first, then possible extensions +## Live delivery without materialization + +The opt-in `0.2-draft` live mode can consume a Flight or ADBC reader as batches +become available, serialize plain IPC directly to one forward-only HTTP stream, +and compute row/decoded-byte counts during output rather than before download. +The schema and Ossie result identity must be established before data starts. +Each emitted batch must contain final values with the required ordering; no +correction/retraction protocol is defined. A blocking physical operator may +still delay the first batch. + +The gateway exposes stream readiness on a running execution and concurrently +retains status. Backend or serialization failure after data headers have been +sent is published as terminal `failed` with a structured error, then the data +stream terminates. Final success requires production to finish and publishes +exact counts for the same result id. The browser validates both the stream and +that terminal record; EOF alone is not success. ADBC iterator exhaustion or +Flight RPC completion must not be conflated with successful delivery to a +particular downstream browser. + +No replay or indexed page retrieval is promised for this mode. The gateway +still owns authorization, state retention, cancellation handling, descriptor +identity, and IPC framing; it is not a transparent byte proxy and no zero-copy +claim follows. A server-side live gateway over Flight/ADBC is not implemented +by the current example. + +## Evidence and possible further extensions The [ADBC SQLite example](../../../examples/execution-adbc/README.md) implements one narrow path today: trusted SQL or an existing ADBC reader into immutable, @@ -89,12 +117,12 @@ validators through synthetic Fetch. It does not run a Flight SQL service or a live HTTP engine, implement a planner/job gateway, or prove multiple vendors adopt the profile. -This makes the existing materialized contract testable before weakening it. -A future streaming-partition mode could expose evolving endpoints/unknown -totals; a future optional-metadata mode could reduce annotation requirements. -Both would change observable guarantees and need explicit negotiation, -completeness/ordering/expiry rules, and evidence from additional integrations. -They are follow-up questions, **not changes made by this mapping or example**. +This makes the existing materialized contract testable without changing it. +Controlled HTTP producers exercise the separate live client's early-batch and +late-error paths; these do not prove live AS or Flight SQL interoperability. +Multiple evolving live partitions, optional embedded identity metadata, and +renewable result references remain future protocol changes requiring explicit +negotiation and evidence. The live extension does not add them implicitly. [flight]: https://arrow.apache.org/docs/format/Flight.html [flight-sql]: https://arrow.apache.org/docs/format/FlightSql.html diff --git a/docs/proposals/execution-http/live-streaming.md b/docs/proposals/execution-http/live-streaming.md new file mode 100644 index 000000000..c83daaad7 --- /dev/null +++ b/docs/proposals/execution-http/live-streaming.md @@ -0,0 +1,231 @@ + + +# Live execution extension - DRAFT 0.2 + +**Proposed, opt-in, and not an adopted standard.** This extension adds early +result delivery to the [materialized HTTP base](../execute-query-http.md). +It does not change the meaning or representation of existing `0.1-draft` +materialized requests, results, or immutable pages. The capitalized requirement +words have the same BCP 14 meaning as in that document. + +## 1. Delivery mode and negotiation + +Materialization is a delivery choice, not a requirement of Arrow IPC and not +a requirement that an engine buffer the whole result in RAM. The base adapter +spools completed results to disk. Live delivery instead permits a consumer to +read final-valued result batches while the backend is still producing them, +without first retaining the complete result. + +The endpoint MUST advertise `live-stream-v0.2` in the base capabilities +`features` array before a client selects live delivery. The client MUST request +it explicitly using `LiveRequest`, defined by [live.schema.json](live.schema.json): + +```json +{ + "contract_version": "0.2-draft", + "delivery": "live", + "data_source_id": "sales-v7", + "query": { + "fields": ["orders.order_id", "orders.amount"], + "limit": 100 + } +} +``` + +`POST /queries` otherwise follows the existing media type, authorization, +bounded-wait, and `200` terminal / `202` pending rules. The response MUST be a +`LiveExecution` with matching version, delivery mode, and data source. A server +unable to implement live delivery MUST refuse it; neither side may silently +downgrade to materialized delivery or resubmit the query. The Layer 3 query, +Foundation revision, fixed model binding, and semantic refusal codes are +unchanged. A capability advertisement is not proof of supported query semantics. + +Legacy materialized submission remains the default. An old client cannot +consume this new schema by treating a running execution as a completed result. +The TypeScript live methods perform capability discovery before their one +submission; ordinary materialized methods do not gain automatic preflight. + +## 2. Execution state and stream readiness + +`LiveExecution` carries `contract_version`, `delivery`, `execution_id`, +`data_source_id`, `model`, `state`, `warnings`, and optional `extensions`. +It follows the base execution-state transitions with these payload rules: + +| State | Stream / completion | +| --- | --- | +| `queued` | No stream or completion; optional progress | +| `running`, `cancelling` | Optional stream once ready; optional progress; no completion | +| `succeeded` | Stream and final completion required; no progress or error | +| `failed` | Error required; stream retained if it was previously advertised; no completion or progress | +| `cancelled` | Stream retained if previously advertised; no completion, error or progress | + +Readiness is independent of completion. Once the schema is fixed and a result +can be streamed, the server publishes a `LiveStream` descriptor: + +```json +{ + "contract_version": "0.2-draft", + "delivery": "live", + "execution_id": "q-live", + "data_source_id": "sales-v7", + "model": { "id": "sales", "revision": "7" }, + "state": "running", + "warnings": [], + "stream": { + "result_id": "r-live", + "format": "arrow-stream", + "url": "https://api.example.test/streams/r-live", + "expires_at": "2099-01-01T00:00:00Z", + "replayable": false, + "columns": [ + { "column_id": "id", "name": "order_id", "logical_type": "Integer", "nullable": false }, + { "column_id": "amount", "name": "amount", "logical_type": "Decimal", "nullable": true } + ] + } +} +``` + +The consumer MUST start the stream without waiting for `succeeded`. A pending +POST MAY return as soon as the stream is ready even when its wait preference +has time remaining. Servers MUST NOT make completion a precondition of live +stream access. A blocking aggregation, sort, or other physical operator may +still delay the first valid batch: this extension does not override execution +semantics or require speculative values. Already emitted values MUST NOT +require later corrections or retractions. + +Execution/model identity is fixed. After first advertisement the stream +descriptor MUST NOT disappear or change, including on terminal failure. +Terminal representations are immutable. Final values describe the same +`result_id`; a new execution or replacement result cannot stand in for it. + +## 3. Plain IPC data, separate status + +`GET stream.url` returns `application/vnd.apache.arrow.stream`. It contains +only ordinary Arrow IPC messages, not JSON events, sentinel rows, or errors +disguised as data. The base flat physical-type subset, predecode safety, +schema stability, logical-type mapping, and column metadata requirements apply. + +The schema's `org.apache.ossie.execution` value is `LiveExecutionMetadata`: +`contract_version: "0.2-draft"`, `execution_id`, `result_id`, `data_source_id`, +and `model`. Page coordinates are forbidden. Column identifiers and semantic +descriptors remain identical to those of the stream descriptor. + +Counts are deliberately absent while running; missing means unknown, not zero. +Consumers MUST enforce actual received-byte and row limits during streaming, +not rely on a promised future total. Produced bytes are counted after HTTP +decompression and include every IPC message, padding, and end marker. +Client read buffering, network backpressure and engine buffering remain +distinct; no end-to-end zero-copy or heap-size guarantee is introduced. + +### Mid-stream failure + +Before the data response starts, failures can use HTTP Problem Details. +After headers have been sent, a server cannot change that HTTP response code. +On execution or serialization failure it MUST: + +1. Publish terminal `failed` with the structured `QueryError` on the execution + status resource, preserving the advertised stream and result identity. +2. Stop producing data and terminate or abort the data response. + +For confirmed cancellation the same rule applies with terminal `cancelled`. +`GET /queries/{id}` remains authoritative for execution outcome. The client +monitors it while reading and stops local consumption on failure/cancellation. +After a stream error the client can inspect status, within its existing +deadline, to obtain a recorded engine error. A data transport or validation +failure MUST NOT be converted into success if the status says `succeeded`. +Neither HTTP trailers nor a nonstandard Arrow error message are required. + +### Final verification + +The server MUST NOT mark an execution `succeeded` until both evaluation and +production/IPC serialization have completed successfully. It publishes a +`LiveCompletion` in that terminal record: + +```json +{ + "result_id": "r-live", + "row_count": 100, + "byte_count": 8192, + "completeness": "complete" +} +``` + +This example uses hypothetical counts. Exact terminal counts MUST match the +same stream the client consumed. `completeness: "truncated"` additionally +requires a nonblank `truncation_reason`; accepting it is an explicit client +choice, not an excuse for a broken stream or backend failure. + +Verified success requires **both** successful full IPC consumption and a +successful terminal record with matching identity, schema and final counts. +HTTP `200`, EOF, an IPC end marker, or a progress fraction alone is insufficient. +If EOF arrives while execution is still pending, the client waits for terminal +status within its original deadline. If terminal status is unavailable, +completion remains unverified and the operation fails. An unavailable status +channel cannot be silently treated as an empty or successful result. + +Batches exposed before final verification are provisional. On any later +failure the application MUST discard them or clearly label them incomplete. +The `executeLive` convenience operation returns a table only after verification; +the streaming iterator yields incrementally and validates before successful +exhaustion. A later downstream network failure after server production succeeded +does not retroactively mutate a successful execution: that is a delivery failure. + +## 4. Lifetime, cancellation and compatibility + +`replayable: false` means a forward-only, single-consumer stream. The server +MUST reject repeated stream acquisition; the client MUST NOT automatically +retry, resume, reread, or resubmit. A fresh client instance is not a way to +claim replay support. Retrying a lost live download requires a new explicit +user decision, not a hidden query retry. + +`expires_at` is the latest time to **start** the live data request. An already +opened stream may finish after that time subject to its operation deadline; +expiry is not an instruction to discard an active stream or invent success. +The server MUST retain status while a stream is active and for at least 60 +seconds after termination, and at least until `expires_at`. Operational +retention beyond this minimum remains a deployment choice. Clients need a +bounded final verification, not indefinite job retention. + +The existing explicit cancellation endpoint still distinguishes request +acceptance from confirmed backend cancellation. Local abort, deadline or early +iterator return stops local data/status work and releases readers; it does not +claim backend cancellation. A client may separately call `cancelLive`. + +HTTPS, caller authorization, exact model binding, origin allowlisting, +credential isolation, redirect refusal, and CORS rules apply to data and status +requests. Structured backend errors remain untrusted reference data and must +not leak private credentials or paths. + +Materialized single-stream and immutable-page results retain their original +replay, upfront count, expiry, and completeness rules. Live mode does not offer +indexed pagination, live partition discovery, renewable links, or replay. +Supporting early delivery, arbitrary page rereads and no retained result +storage simultaneously is not a promise of this extension. + +## 5. Evidence and backend alignment + +Flight/ADBC readers can feed IPC production without spooling solely to discover +final counts. A gateway still owns execution identity, lifecycle/error +publication, authorization, and final byte accounting. Those interfaces do not +provide Ossie semantic planning automatically. + +The existing ADBC SQLite example remains a materialized adapter. Controlled +HTTP producers exercise the live client and browser flow; they do not establish +live AS/Flight SQL interoperability or a general engine implementation. diff --git a/docs/proposals/execution-http/live.schema.json b/docs/proposals/execution-http/live.schema.json new file mode 100644 index 000000000..87c1aa82d --- /dev/null +++ b/docs/proposals/execution-http/live.schema.json @@ -0,0 +1,108 @@ +{ + "$comment": "Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See NOTICE for copyright ownership. Licensed under the Apache License, Version 2.0; http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, distributed on an AS IS BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND.", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:apache:ossie:execution:http:live:0.2-draft", + "title": "Ossie optional LIVE HTTP execution extension 0.2-draft", + "description": "Explicit opt-in advertised by live-stream-v0.2 in unchanged 0.1 capabilities. Not an adopted standard. References use the locally registered 0.1 schema URN, never a remote fetch. Cross-observation immutability, single consumption and terminal count equality require runtime validation.", + "$ref": "#/$defs/LiveRequest", + "$defs": { + "LiveRequest": { + "type": "object", + "additionalProperties": false, + "required": ["contract_version", "delivery", "data_source_id", "query"], + "properties": { + "contract_version": { "const": "0.2-draft" }, + "delivery": { "const": "live" }, + "data_source_id": { "$ref": "urn:apache:ossie:execution:http:0.1-draft#/$defs/Identifier" }, + "query": { "$ref": "urn:apache:ossie:execution:http:0.1-draft#/$defs/Query" } + } + }, + "LiveExecution": { + "type": "object", + "additionalProperties": false, + "required": ["contract_version", "delivery", "execution_id", "data_source_id", "model", "state", "warnings"], + "properties": { + "contract_version": { "const": "0.2-draft" }, + "delivery": { "const": "live" }, + "execution_id": { "$ref": "urn:apache:ossie:execution:http:0.1-draft#/$defs/Identifier" }, + "data_source_id": { "$ref": "urn:apache:ossie:execution:http:0.1-draft#/$defs/Identifier" }, + "model": { "$ref": "urn:apache:ossie:execution:http:0.1-draft#/$defs/ModelIdentity" }, + "state": { "$ref": "urn:apache:ossie:execution:http:0.1-draft#/$defs/State" }, + "warnings": { "type": "array", "items": { "$ref": "urn:apache:ossie:execution:http:0.1-draft#/$defs/Warning" } }, + "extensions": { "$ref": "urn:apache:ossie:execution:http:0.1-draft#/$defs/Extensions" }, + "progress": { "$ref": "urn:apache:ossie:execution:http:0.1-draft#/$defs/Progress" }, + "stream": { "$ref": "#/$defs/LiveStream" }, + "completion": { "$ref": "#/$defs/LiveCompletion" }, + "error": { "$ref": "urn:apache:ossie:execution:http:0.1-draft#/$defs/QueryError" } + }, + "oneOf": [ + { + "properties": { "state": { "const": "queued" } }, + "not": { "anyOf": [{ "required": ["stream"] }, { "required": ["completion"] }, { "required": ["error"] }] } + }, + { + "properties": { "state": { "enum": ["running", "cancelling"] } }, + "not": { "anyOf": [{ "required": ["completion"] }, { "required": ["error"] }] } + }, + { + "properties": { "state": { "const": "succeeded" } }, + "required": ["stream", "completion"], + "not": { "anyOf": [{ "required": ["progress"] }, { "required": ["error"] }] } + }, + { + "properties": { "state": { "const": "failed" } }, + "required": ["error"], + "not": { "anyOf": [{ "required": ["progress"] }, { "required": ["completion"] }] } + }, + { + "properties": { "state": { "const": "cancelled" } }, + "not": { "anyOf": [{ "required": ["progress"] }, { "required": ["completion"] }, { "required": ["error"] }] } + } + ], + "description": "Version, delivery, execution, source and model identities are fixed. Once advertised, stream is immutable and remains present even on failed/cancelled terminal states. All terminal representations are immutable. Status remains readable through terminal verification under the server's documented retention window." + }, + "LiveStream": { + "type": "object", + "additionalProperties": false, + "required": ["result_id", "format", "url", "expires_at", "columns", "replayable"], + "properties": { + "result_id": { "$ref": "urn:apache:ossie:execution:http:0.1-draft#/$defs/Identifier" }, + "format": { "const": "arrow-stream" }, + "url": { "$ref": "urn:apache:ossie:execution:http:0.1-draft#/$defs/ResultUrl" }, + "expires_at": { "type": "string", "format": "date-time", "description": "Latest instant to START consumption; an already-open stream may finish within the operation deadline." }, + "columns": { "type": "array", "minItems": 1, "items": { "$ref": "urn:apache:ossie:execution:http:0.1-draft#/$defs/Column" } }, + "replayable": { "const": false } + }, + "description": "One single-consumer non-replayable Arrow IPC stream. No pages, upfront counts, completeness, range, resume or retries. Columns are ordered and column_id values unique. Batches are provisional until complete IPC and matching terminal success are both verified." + }, + "LiveCompletion": { + "type": "object", + "additionalProperties": false, + "required": ["result_id", "row_count", "byte_count", "completeness"], + "properties": { + "result_id": { "$ref": "urn:apache:ossie:execution:http:0.1-draft#/$defs/Identifier" }, + "row_count": { "$ref": "urn:apache:ossie:execution:http:0.1-draft#/$defs/Counter" }, + "byte_count": { "$ref": "urn:apache:ossie:execution:http:0.1-draft#/$defs/Counter" }, + "completeness": { "enum": ["complete", "truncated"] }, + "truncation_reason": { "$ref": "urn:apache:ossie:execution:http:0.1-draft#/$defs/NonEmptyString" } + }, + "if": { "properties": { "completeness": { "const": "truncated" } }, "required": ["completeness"] }, + "then": { "required": ["truncation_reason"] }, + "else": { "not": { "required": ["truncation_reason"] } }, + "description": "Only terminal success, after serialization is finalized without error. result_id matches stream.result_id. Exact row and raw IPC byte counts after HTTP decompression must match full consumption; EOF/EOS or HTTP 200 alone is not success." + }, + "LiveExecutionMetadata": { + "type": "object", + "additionalProperties": false, + "required": ["contract_version", "execution_id", "result_id", "data_source_id", "model"], + "properties": { + "contract_version": { "const": "0.2-draft" }, + "execution_id": { "$ref": "urn:apache:ossie:execution:http:0.1-draft#/$defs/Identifier" }, + "result_id": { "$ref": "urn:apache:ossie:execution:http:0.1-draft#/$defs/Identifier" }, + "data_source_id": { "$ref": "urn:apache:ossie:execution:http:0.1-draft#/$defs/Identifier" }, + "model": { "$ref": "urn:apache:ossie:execution:http:0.1-draft#/$defs/ModelIdentity" } + }, + "description": "JSON schema metadata under org.apache.ossie.execution. Matches the live execution/stream identities; no delivery or page metadata fields. Column metadata is the unchanged 0.1 ColumnMetadata definition." + } + } +} diff --git a/docs/proposals/execution-http/openapi.json b/docs/proposals/execution-http/openapi.json index 84dccc7cd..438bf50e7 100644 --- a/docs/proposals/execution-http/openapi.json +++ b/docs/proposals/execution-http/openapi.json @@ -5,8 +5,8 @@ }, "info": { "title": "Ossie HTTP execution proposal", - "version": "0.1-draft", - "description": "DRAFT, not an adopted standard or a deployed service. Browser-oriented HTTP/JSON control plane and one materialized result delivered as a single Arrow IPC stream or immutable self-contained pages. This binding uses HTTP semantics, not HTTP/1.1-specific framing: HTTP/1.1, HTTP/2, or HTTP/3 may carry it where supported, with version negotiation left to the HTTP stack and deployment. Supporting every version is not required. Multiplexing does not imply parallel page fetching or Flight-equivalent flow control; the prototype traverses pages sequentially. Current loopback integration fixtures cover HTTP/1.1, not HTTP/2/3 interoperability. All schemas are defined in execution.schema.json. Authentication is deployment supplied; this document does not implement an identity system.", + "version": "0.2-draft", + "description": "DRAFT, not an adopted standard or deployed service. Unchanged materialized 0.1 HTTP/JSON execution and immutable single-stream/paged Arrow results, plus explicit optional LIVE 0.2 streaming. Schemas are execution.schema.json and live.schema.json. This binding uses HTTP semantics, not HTTP/1.1-specific framing: HTTP/1.1, HTTP/2, or HTTP/3 may carry it where supported, with negotiation left to the stack and deployment. Supporting every version is not required. Multiplexing does not imply parallel pages or Flight-equivalent flow control; materialized pages are traversed sequentially. Synthetic loopback integration fixtures cover HTTP/1.1, not HTTP/2/3 or a real engine. Authentication is deployment supplied; this document does not implement an identity system.", "license": { "name": "Apache-2.0", "identifier": "Apache-2.0" @@ -20,7 +20,7 @@ "summary": "Discover the caller-visible contract and query capabilities", "responses": { "200": { - "description": "Capabilities for this authorized caller; declarations do not prove semantic conformance.", + "description": "Unchanged 0.1 capabilities for this authorized caller; optional features token live-stream-v0.2 advertises explicit 0.2 LIVE opt-in. Clients preflight it before live submission, without fallback. Declarations do not prove semantic conformance.", "content": { "application/json": { "schema": { "$ref": "execution.schema.json#/$defs/Capabilities" } @@ -53,7 +53,7 @@ "post": { "operationId": "submit", "summary": "Submit one read-only structured query", - "description": "Submit one execution job, not separate sync/async jobs. Omitting Prefer means wait=0: return immediately with 202 unless already terminal (200). wait=N holds POST at most N seconds, returning terminal 200 or pending 202 for the same execution_id; the server may return earlier. Bounded wait optimizes fast completion without unbounded synchronous connections or mandatory polling round trips. It is neither an execution deadline nor a download timeout and never cancels the query. No automatic retries, query repair, native SQL escape, or idempotency guarantee. Pagination partitions the completed result, not the query: no page-size/offset clause; limit remains a semantic cap.", + "description": "Submit one execution job, not separate sync/async jobs. The unchanged Request selects materialized 0.1; only LiveRequest with contract_version=0.2-draft and delivery=live selects the advertised LIVE extension. Reject unsupported live requests, never fall back or resubmit. Omitting Prefer means wait=0: return immediately with 202 unless already terminal (200). wait=N holds POST at most N seconds, returning terminal 200 or pending 202 for the same execution_id; the server may return earlier, particularly when a live stream is ready. Bounded wait is not an execution deadline or download timeout and never cancels the query. No automatic retries, query repair, native SQL escape, or idempotency guarantee. Materialized pagination partitions the completed result, not the query; LIVE has no pages. limit remains a semantic cap.", "parameters": [ { "name": "Prefer", @@ -67,7 +67,10 @@ "required": true, "content": { "application/json": { - "schema": { "$ref": "execution.schema.json#/$defs/Request" }, + "schema": { "oneOf": [ + { "$ref": "execution.schema.json#/$defs/Request" }, + { "$ref": "live.schema.json#/$defs/LiveRequest" } + ] }, "example": { "data_source_id": "sales-v7", "query": { "dimensions": ["region"], "measures": ["revenue"], "limit": 100 } @@ -77,10 +80,13 @@ }, "responses": { "200": { - "description": "Terminal Execution: succeeded, failed, or cancelled. HTTP success alone does not imply query success. Success includes the entire immutable result manifest: exactly one of url or pages, subject to the client's 1 MiB control-JSON limit.", + "description": "Terminal Execution or LiveExecution: succeeded, failed, or cancelled. HTTP success alone does not imply query success. Materialized success includes the immutable result manifest. LIVE success includes immutable stream and completion, with exact decoded IPC counts and matching result_id. The control-JSON limit is 1 MiB.", "content": { "application/json": { - "schema": { "$ref": "execution.schema.json#/$defs/Execution" }, + "schema": { "oneOf": [ + { "$ref": "execution.schema.json#/$defs/Execution" }, + { "$ref": "live.schema.json#/$defs/LiveExecution" } + ] }, "examples": { "single-stream": { "summary": "Existing single-stream delivery (illustrative IPC counts)", @@ -136,10 +142,13 @@ } }, "202": { - "description": "Pending Execution: queued, running, or cancelling. Poll using execution_id.", + "description": "Pending Execution or LiveExecution: queued, running, or cancelling. Poll using execution_id. LIVE running/cancelling may advertise the immutable single-use stream: open immediately and poll concurrently rather than waiting for success.", "content": { "application/json": { - "schema": { "$ref": "execution.schema.json#/$defs/Execution" } + "schema": { "oneOf": [ + { "$ref": "execution.schema.json#/$defs/Execution" }, + { "$ref": "live.schema.json#/$defs/LiveExecution" } + ] } } } }, @@ -154,10 +163,13 @@ "summary": "Read the current execution state", "responses": { "200": { - "description": "An Execution in any state; terminal states and manifests are immutable.", + "description": "Execution or LiveExecution in any state. Version/delivery and source/model identity never change; terminal representations and advertised live descriptors are immutable. Live descriptors remain on failure/cancellation. Status must remain readable through terminal verification for the documented server retention window. Concurrent polling is the live failure channel, including while a body stalls: errors are never embedded as JSON in Arrow or dependent on HTTP trailers.", "content": { "application/json": { - "schema": { "$ref": "execution.schema.json#/$defs/Execution" } + "schema": { "oneOf": [ + { "$ref": "execution.schema.json#/$defs/Execution" }, + { "$ref": "live.schema.json#/$defs/LiveExecution" } + ] } } } }, @@ -176,7 +188,10 @@ "description": "Terminal Execution: cancelled, succeeded, or failed.", "content": { "application/json": { - "schema": { "$ref": "execution.schema.json#/$defs/Execution" } + "schema": { "oneOf": [ + { "$ref": "execution.schema.json#/$defs/Execution" }, + { "$ref": "live.schema.json#/$defs/LiveExecution" } + ] } } } }, @@ -184,7 +199,10 @@ "description": "Pending Execution: cancelling or another currently observed pending state.", "content": { "application/json": { - "schema": { "$ref": "execution.schema.json#/$defs/Execution" } + "schema": { "oneOf": [ + { "$ref": "execution.schema.json#/$defs/Execution" }, + { "$ref": "live.schema.json#/$defs/LiveExecution" } + ] } } } }, @@ -223,6 +241,16 @@ "media-type": "application/vnd.apache.arrow.stream", "description": "Dynamic absolute same-origin or explicitly allowlisted HTTPS URL, not a fixed API path or cursor API. Every download is one immutable self-contained IPC stream with required Ossie schema/field metadata; no concatenation, redirects, or Range requests. Server-selected page_index and row_offset are contiguous from zero; page row/decoded-byte sums equal root totals, including repeated schemas. No empty page except the sole page of a zero-row result; page bytes are positive. Every page has identical fields and actual physical types (including decimal scale and timestamp unit/timezone) and metadata except ExecutionMetadata page_index/row_offset, which must appear together and match its descriptor; both are forbidden unpaged. Cross-origin fetch omits account authorization and cookies. The same authorization, origin, and whole-result expires_at protections apply to each page. HTTP content compression is allowed; Arrow IPC buffer compression is not. Decode bytes before comparing counts. Selected-page browsing does not establish complete-result consumption; completeness describes the whole result. Explicit repeated reads before expiry are allowed, but no automatic retries, resubmission, link renewal, live execution rows, or Flight are defined." }, + "x-live-download": { + "method": "GET", + "url": "LiveExecution.stream.url, as soon as advertised (never while queued)", + "schema": { "$ref": "live.schema.json#/$defs/LiveStream" }, + "completion-schema": { "$ref": "live.schema.json#/$defs/LiveCompletion" }, + "metadata-schema": { "$ref": "live.schema.json#/$defs/LiveExecutionMetadata" }, + "success-status": 200, + "media-type": "application/vnd.apache.arrow.stream", + "description": "Explicit optional LIVE only. Single consumer enforced by server, non-replayable, no page/range/resume/retry/fallback. Same origin, token, CORS and IPC safety rules as materialized results. Expiry is the latest start time, not an already-open stream deadline. Consume provisional Arrow batches concurrently with status polling; failed/cancelled status stops the reader. Status errors fail verification. Terminal success never aborts a still-open body: require valid complete IPC, matching result_id and exact row/decoded-byte counts. A network or malformed-body error cannot be cured by success status; one bounded status read may preserve an already-published engine error. EOF/EOS alone is not success. No JSON control messages inside Arrow, HTTP trailer dependency, row retractions or Flight. Blocking operators may delay batches." + }, "x-browser-cors": { "allow-methods": ["GET", "POST", "OPTIONS"], "allow-headers": ["Authorization", "Content-Type", "Accept", "Prefer"], diff --git a/examples/execution-adbc/README.md b/examples/execution-adbc/README.md index b6babe355..5174089d4 100644 --- a/examples/execution-adbc/README.md +++ b/examples/execution-adbc/README.md @@ -32,6 +32,12 @@ trusted backend SQL -> ADBC SQLite execute/fetch_record_batch -> PyArrow IPC materialization -> existing TypeScript client ``` +This example intentionally exercises the replayable materialized `0.1-draft` +mode and writes files incrementally; it does not retain the entire result in +RAM. The [live `0.2-draft` extension](../../docs/proposals/execution-http/live-streaming.md) +permits early delivery without that spool, but a live HTTP/status gateway over +this reader is not implemented here. + The CLI seeds a synthetic SQLite fixture, then opens it read-only through ADBC. It queries deliberately unsorted input with `ORDER BY sequence`, preserving `9007199254740993`, a null, and `-7` exactly. There is no public SQL submission diff --git a/typescript/execution/README.md b/typescript/execution/README.md index 27a2f71ff..153e66cce 100644 --- a/typescript/execution/README.md +++ b/typescript/execution/README.md @@ -20,9 +20,11 @@ # Ossie execution client — DRAFT A browser-oriented TypeScript client prototype for -[Ossie HTTP execution `0.1-draft`](../../docs/proposals/execute-query-http.md): +[Ossie HTTP execution](../../docs/proposals/execute-query-http.md): HTTP/JSON submission, status and cancellation, followed by materialized -Apache Arrow IPC results, either in one stream or immutable pages. +`0.1-draft` Apache Arrow IPC results, either in one stream or immutable pages. +An explicitly opted-in [`0.2-draft` live extension](../../docs/proposals/execution-http/live-streaming.md) +also delivers batches while execution is running. **This is a proposal, not an adopted standard or a published-service promise.** The package does not implement an execution server, semantic evaluator, @@ -58,6 +60,8 @@ with Flight have not been demonstrated. - [OpenAPI 3.1](../../docs/proposals/execution-http/openapi.json), referencing those same definitions. - [Public TypeScript types](src/types.ts). +- [Live extension](../../docs/proposals/execution-http/live-streaming.md) and + its [separate schema](../../docs/proposals/execution-http/live.schema.json). - [Flight SQL / ADBC backend mapping](../../docs/proposals/execution-http/backend-mapping.md) and [materialization example](../../examples/execution-adbc/README.md). @@ -93,6 +97,93 @@ Unless noted otherwise, methods accept `RequestOptions` with an optional `signal`. Types, `OssieClient`, default-limit constants, and error classes are exported from the package entry point. +### Opt-in live API + +The existing methods above remain materialized `0.1-draft`. Live delivery uses +separate methods so an old call cannot silently acquire different completeness, +replay or pagination semantics. + +| Live method | Purpose / return type | +| --- | --- | +| `submitLive(request, options?)` | `Promise`; verifies the `live-stream-v0.2` capability and sends one explicitly versioned live POST. Accepts `SubmitOptions`. | +| `getLiveExecution(id, options?)` | `Promise`; inspect status without downloading. Accepts `RequestOptions`. | +| `cancelLive(id, options?)` | `Promise`; explicit best-effort backend cancellation. Accepts `RequestOptions`. | +| `stream(request, options?)` | `AsyncGenerator`; submit once, start downloading when ready, and monitor status concurrently. Accepts `LiveOptions`. | +| `readLiveBatches(execution, options?)` | Attach to an already submitted live execution without another POST or capability preflight. Same iterator return type; accepts `LiveReadOptions`. | +| `executeLive(request, options?)` | `Promise<{ execution: LiveSuccessfulExecution; table: Table }>`; uses the same live workflow but buffers within limits and returns only verified data. | + +`request` still has the familiar `{ data_source_id, query }` client shape. +The submitting live methods add the wire version/delivery fields themselves. +`LiveReadOptions` provides `signal`, `allowTruncated` and an optional +`onProgress(LiveExecution)` callback (which may return a promise). +`LiveOptions` also includes `waitSeconds`. Capability discovery, submission, +status monitoring and download share one overall client deadline. + +```ts +const request = { + data_source_id: "sales-v7", + query: { fields: ["orders.order_id", "orders.amount"], limit: 100 }, +}; +const iterator = client.stream(request, { waitSeconds: 0 }); +try { + while (true) { + const step = await iterator.next(); + if (step.done) { + // Verified terminal success; step.value includes exact completion counts. + console.log(step.value.completion.row_count); + break; + } + // step.value is a provisional Arrow RecordBatch, not a completed answer. + } +} finally { + // Early return releases local reader/poller resources; it does not cancel the engine. + await iterator.return(undefined as never); +} +``` + +Do not call `executeLive` after this example to finish it: that would submit a +second execution. `executeLive` is an alternative consumption style. +After early return or any failure, discard or explicitly mark previously seen +batches incomplete. A `for await` loop can consume batches too; only normal +exhaustion validates the result, and an early `break` does not. + +Live readiness is not terminal success. Once the schema is ready, an execution +can publish its stream while still `running`; the reader must not wait for +completion before opening it. Counts are absent initially, never fake zeroes. +The server records a final completion only after evaluation and serialization +finish. The client checks that record's result id and exact row/decoded-byte +counts against the fully consumed stream. HTTP `200`, EOF, an Arrow end marker, +or a succeeded status by itself does not certify delivery. + +The data response stays plain Arrow IPC. Once its headers have been sent, late +engine errors are reported by the execution status resource and the stream +terminates. The client monitors status concurrently and throws +`LiveQueryExecutionError` (preserving the Foundation/server code and `.error`) +or `LiveExecutionCancelledError`; unavailable status, malformed data and +transport errors also fail the operation. No HTTP trailer support is needed. +Terminal success may arrive before the last buffered network bytes; the client +still reads and validates them. EOF may arrive before terminal status; the +client still waits for that status within the original deadline. + +Live streams are single-use (`replayable: false`), not immutable pages: +no indexed browsing, resume, implicit retry or repeated acquisition is +promised. A server must enforce single-consumer access. The prototype also +retains at most 64 live-consumption fences per client and fails closed with +`LIVE_STREAM_LIMIT` when full, rather than forgetting a consumed id. +`stream` and `executeLive` reserve capacity before capability discovery or +submission, including concurrent calls, so local saturation never creates +an execution they cannot consume. Submission-only `submitLive` remains +separate and does not reserve a future reader slot. +`LIVE_STREAM_CONSUMED` rejects a repeated attempt and `forgetExecution` does +not clear those fences. A new client does not grant permission to replay an +old server stream. These are prototype safeguards, not protocol capacity limits. + +An advertised live `expires_at` bounds when a download may **start**, not an +already-open stream's completion. The operation deadline and actual received +row/byte limits still apply. Truncation is decided from the terminal completion; +`allowTruncated` permits only an explicitly declared truncated answer, never +invalid IPC or a backend failure. Native Arrow buffers/types remain intact. + ### Example Once built and linked as a local dependency, the package imports as @@ -143,7 +234,7 @@ not how long the query may run. The server may return earlier. | Bounded wait (`waitSeconds: 5`) | Wait up to five seconds. Return `200` if terminal, otherwise `202` with the **same running execution**. | | Unbounded synchronous | Hold the connection until completion regardless of duration. This draft does not offer that submission mode. | -For a 200 ms query with `waitSeconds: 5`, the submission can return success +For a 200 ms materialized query with `waitSeconds: 5`, the submission can return success without polling. For a 20 s query, the submission returns pending at or before five seconds and the client polls the same id. It does not cancel, restart or submit another query. Results are still downloaded separately after success. @@ -208,9 +299,11 @@ Per-request `signal` supports local abort. `SubmitOptions.waitSeconds` controls the submission's `Prefer: wait=N`. The client checks the protocol range `0..60`; the server enforces its advertised lower `max_wait_seconds` and binding support. `getCapabilities` and `getBinding` are explicit validated -reads, not a cached preflight policy. Submission performs no automatic discovery +reads, not a cached preflight policy. Materialized submission performs no automatic discovery or cached capability/binding enforcement. Applications can use explicit discovery results to choose a supported shape and wait preference. +Live submission performs its mandatory capability preflight and checks the +advertised wait bound; the server remains responsible for semantic support. `WaitOptions.onProgress` receives execution observations. `ExecuteOptions.allowTruncated` explicitly accepts a server-declared truncated materialization without waiving count, identity, framing, or resource checks. @@ -260,7 +353,7 @@ Both shapes support `order_by` entries with `field`, optional `ASC`/`DESC` are refused locally; the server enforces advertised shape support. The client does not repair the query or substitute a native query. -## What result validation means +## What materialized result validation means - Result URLs exist only on `succeeded`. Each identifies one immutable, self-contained Arrow IPC **stream**, not live rows produced during execution. @@ -391,7 +484,7 @@ uses Arrow, or that two systems implement the same query semantics. | Interface | Submission/lifecycle | Result representation | Difference from this package | | --- | --- | --- | --- | -| **This draft/client prototype** | Structured Ossie JSON; bounded wait, handle/poll/cancel | Single stream or indexed immutable Arrow pages; per-page identity/count checks, full-result schema/total checks | Local protocol implementation only; no query server or vendor semantic proof. | +| **This draft/client prototype** | Structured Ossie JSON; bounded wait, handle/poll/cancel | Default materialized stream/pages, or opt-in live stream with unknown initial counts and mandatory terminal verification | Live batches may arrive while running; no live replay/pages. No query server or vendor semantic proof. | | [Databricks Statement Execution][databricks] | SQL over HTTP; bounded synchronous wait, handles, polling, cancellation, manifests | `ARROW_STREAM` **only with `EXTERNAL_LINKS`**; inline supports `JSON_ARRAY`, not Arrow | Closest lifecycle/download prior art; additional formats and fresh link resolution remain beyond this client. External downloads must omit account authorization. | | [dbt Semantic Layer GraphQL][dbt-graphql] | `createQuery`, then poll `query` with the id | Paginated base64-encoded Arrow in `arrowResult`; JSON is also available | Semantic query API, but different request schema and delivery encoding. [Its JDBC interface][dbt-jdbc] separately uses Arrow Flight SQL. | | [Snowflake SQL API][snowflake-sql] | SQL HTTP submission, status handles, cancellation | Partitioned JSON `ResultSet`/`jsonv2` | Not an Arrow REST result API in the cited contract. [Python connector Arrow fetch APIs][snowflake-python] are a separate connector surface. | diff --git a/typescript/execution/src/arrow.ts b/typescript/execution/src/arrow.ts index cccafdf4a..e89e96269 100644 --- a/typescript/execution/src/arrow.ts +++ b/typescript/execution/src/arrow.ts @@ -22,8 +22,12 @@ import type { RecordBatch, Schema } from "apache-arrow"; import { OssieError, ProtocolError } from "./errors.js"; import { BodySource, Operation } from "./transport.js"; import { COLUMN_METADATA_KEY, EXECUTION_METADATA_KEY } from "./types.js"; -import type { Column, ResultPage, SuccessfulExecution } from "./types.js"; -import { assert, count, equivalent, object, validateMetadata } from "./validation.js"; +import type { Column, LiveExecution, LiveStream, ResultPage, SuccessfulExecution } from "./types.js"; +import { assert, count, equivalent, object, validateLiveMetadata, validateMetadata } from "./validation.js"; + +type ArrowExecution = SuccessfulExecution | (LiveExecution & { stream: LiveStream }); +export interface ArrowStats { rows: number; bytes: number } +const descriptor = (execution: ArrowExecution) => "stream" in execution ? execution.stream : execution.result; const timeUnits = [TimeUnit.SECOND, TimeUnit.MILLISECOND, TimeUnit.MICROSECOND, TimeUnit.NANOSECOND]; @@ -69,19 +73,21 @@ function metadataJson(value: string | undefined): unknown { catch { throw new ProtocolError("Ossie Arrow metadata is not valid JSON."); } } -function schemaMatches(schema: Schema, execution: SuccessfulExecution, page?: ResultPage): void { - const metadata = validateMetadata(metadataJson(schema.metadata.get(EXECUTION_METADATA_KEY))); +function schemaMatches(schema: Schema, execution: ArrowExecution, page?: ResultPage): void { + const result = descriptor(execution); + const validate = "stream" in execution ? validateLiveMetadata : validateMetadata; + const metadata = validate(metadataJson(schema.metadata.get(EXECUTION_METADATA_KEY))); assert(equivalent(metadata, { contract_version: execution.contract_version, execution_id: execution.execution_id, - result_id: execution.result.result_id, + result_id: result.result_id, data_source_id: execution.data_source_id, model: execution.model, ...(page === undefined ? {} : { page_index: page.page_index, row_offset: page.row_offset }), }), "Arrow execution metadata does not match the execution."); - assert(schema.fields.length === execution.result.columns.length, "Arrow column count mismatch."); + assert(schema.fields.length === result.columns.length, "Arrow column count mismatch."); schema.fields.forEach((field, index) => { - const column = execution.result.columns[index]!; + const column = result.columns[index]!; assert(field.name === column.name && field.nullable === column.nullable, "Arrow column name, order or nullability mismatch."); const metadata = object(metadataJson(field.metadata.get(COLUMN_METADATA_KEY)), ["column_id"], ["column_id"]); @@ -343,15 +349,18 @@ function validateBatchBody(header: BatchHeader, schema: Schema, body: Uint8Array } /** Validates IPC framing before Arrow allocates or silently accepts schema replacement. */ -async function* frames(source: BodySource, operation: Operation, execution: SuccessfulExecution, - maxRows: number, maxBytes: number, page?: ResultPage): AsyncGenerator { +async function* frames(source: BodySource, operation: Operation, execution: ArrowExecution, + maxRows: number, maxBytes: number, page?: ResultPage, stats?: ArrowStats): AsyncGenerator { let schema: Schema | undefined; let rows = 0; - const counts = page ?? execution.result; + const counts = "stream" in execution ? undefined : page ?? execution.result; const finish = (): void => { assert(schema !== undefined, "The IPC stream has no schema."); - assert(source.bytes === counts.byte_count, "Decoded IPC byte count mismatch."); - assert(rows === counts.row_count, "Arrow row count mismatch."); + if (counts) { + assert(source.bytes === counts.byte_count, "Decoded IPC byte count mismatch."); + assert(rows === counts.row_count, "Arrow row count mismatch."); + } + if (stats) { stats.rows = rows; stats.bytes = source.bytes; } }; while (true) { operation.check(); @@ -375,7 +384,7 @@ async function* frames(source: BodySource, operation: Operation, execution: Succ assert(length > 0 && length <= Math.min(maxBytes, 1024 * 1024), "Invalid Arrow metadata length."); assert((length + prefix.byteLength) % 8 === 0, "Unaligned Arrow metadata."); const bytes = (await source.exact(length))!; - new MetadataVerifier(bytes, operation, maxBytes).verify(schema, execution.result.columns.length); + new MetadataVerifier(bytes, operation, maxBytes).verify(schema, descriptor(execution).columns.length); const message = Message.decode(bytes); assert([MetadataVersion.V4, MetadataVersion.V5].includes(message.version), "Unsupported Arrow metadata version."); count(message.bodyLength); @@ -393,7 +402,7 @@ async function* frames(source: BodySource, operation: Operation, execution: Succ if (!Number.isSafeInteger(rows) || rows > maxRows) { throw new OssieError("RESULT_LIMIT_EXCEEDED", "The result exceeded the configured row limit."); } - assert(rows <= counts.row_count, "Arrow row count exceeds the manifest."); + if (counts) assert(rows <= counts.row_count, "Arrow row count exceeds the manifest."); } // Retain at most one complete IPC message, not the whole result. const body = (await source.exact(message.bodyLength))!; @@ -402,14 +411,18 @@ async function* frames(source: BodySource, operation: Operation, execution: Succ yield prefix; yield bytes; if (body.byteLength) yield body; + // Arrow 21's async adapter pulls once more when demand exactly equals its + // buffered bytes. A zero-byte boundary releases this validated frame without + // reading the next network frame (which may depend on consumer progress). + if ("stream" in execution) yield new Uint8Array(0); } } -export async function* arrowBatches(response: Response, execution: SuccessfulExecution, +export async function* arrowBatches(response: Response, execution: ArrowExecution, operation: Operation, maxBytes: number, maxRows: number, page?: ResultPage, - remainingBytes = maxBytes): AsyncGenerator { + remainingBytes = maxBytes, stats?: ArrowStats): AsyncGenerator { const source = new BodySource(response, operation, Math.min(maxBytes, remainingBytes)); - const input = frames(source, operation, execution, maxRows, maxBytes, page); + const input = frames(source, operation, execution, maxRows, maxBytes, page, stats); let reader: RecordBatchReader | undefined; try { reader = await operation.race(() => RecordBatchReader.from(input)); diff --git a/typescript/execution/src/client.ts b/typescript/execution/src/client.ts index f4eb4c67d..5992e7f62 100644 --- a/typescript/execution/src/client.ts +++ b/typescript/execution/src/client.ts @@ -20,13 +20,15 @@ import { Schema, Table } from "apache-arrow"; import type { RecordBatch } from "apache-arrow"; import { arrowBatches } from "./arrow.js"; -import { ExecutionCancelledError, HttpProblemError, OssieError, ProtocolError, QueryExecutionError } from "./errors.js"; -import { ARROW_MEDIA_TYPE, EXECUTION_METADATA_KEY } from "./types.js"; +import { ExecutionCancelledError, HttpProblemError, LiveExecutionCancelledError, LiveQueryExecutionError, + OssieError, ProtocolError, QueryExecutionError } from "./errors.js"; +import { ARROW_MEDIA_TYPE, EXECUTION_METADATA_KEY, LIVE_CONTRACT_VERSION, LIVE_FEATURE } from "./types.js"; import type { Binding, Capabilities, ClientOptions, ExecuteOptions, ExecuteRequest, Execution, + LiveExecution, LiveOptions, LiveReadOptions, LiveStream, LiveSuccessfulExecution, RequestOptions, ResultPage, SubmitOptions, SuccessfulExecution, WaitOptions } from "./types.js"; import { Operation, cancelBody, mediaType, readJson } from "./transport.js"; import { assert, count, equivalent, terminal, text, transition, validateBinding, validateCapabilities, - validateExecution, validateProblem, validateRequest } from "./validation.js"; + validateExecution, validateLiveExecution, validateProblem, validateRequest } from "./validation.js"; export const DEFAULT_TIMEOUT_MS = 60_000; export const DEFAULT_POLL_INTERVAL_MS = 250; @@ -38,6 +40,19 @@ interface Observation { previous: Execution | undefined; fallback?: Execution; } +interface LiveObservation { + previous: LiveExecution | undefined; + fallback?: LiveExecution; +} +interface LiveConsumption { + acquire(id: string): void; + release(): void; +} + +function liveFailure(execution: LiveExecution): void { + if (execution.state === "failed") throw new LiveQueryExecutionError(execution); + if (execution.state === "cancelled") throw new LiveExecutionCancelledError(execution); +} function integerOption(value: unknown, fallback: number, minimum: number, maximum = Number.MAX_SAFE_INTEGER): number { if (value === undefined) return fallback; @@ -113,6 +128,10 @@ export class OssieClient { private readonly maxResultBytes: number; private readonly maxResultRows: number; private readonly executions = new Map(); + private readonly liveExecutions = new Map(); + // Fail closed at the bound rather than evicting a single-use consumption fence. + private readonly consumedLive = new Set(); + private reservedLive = 0; constructor(options: ClientOptions) { assert(options !== null && typeof options === "object", "Client options are required."); @@ -191,10 +210,98 @@ export class OssieClient { executionId, undefined, observation)); } - /** Drops local lifecycle history only; does not cancel or delete server data. */ + /** Explicit opt-in: preflight 0.1 capabilities, then exactly one 0.2 live POST. */ + async submitLive(request: ExecuteRequest, options: SubmitOptions = {}): Promise { + const body = this.liveRequestBody(request); + const prefer = waitPreference(options); + return this.run(options, operation => this.submitLiveInternal(body, prefer, operation)); + } + + async getLiveExecution(id: string, options: RequestOptions = {}): Promise { + idPath(id); + return this.run(options, operation => this.getLiveInternal(id, operation)); + } + + async cancelLive(id: string, options: RequestOptions = {}): Promise { + const path = `queries/${idPath(id)}/cancel`; + const observation: LiveObservation = { previous: this.liveExecutions.get(id) }; + return this.run(options, async operation => this.rememberLive(validateLiveExecution( + await this.control(path, "POST", operation, [200, 202], undefined, undefined, "live")), + id, undefined, observation)); + } + + /** Provisional batches from one execution. Only the generator's return value + * certifies success, after both complete IPC and terminal count verification. + */ + async *stream(request: ExecuteRequest, options: LiveOptions = {}): + AsyncGenerator { + const body = this.liveRequestBody(request); + const prefer = waitPreference(options); + this.validateLiveOptions(options); + const operation = new Operation(options.signal, this.timeoutMs); + let consumption: LiveConsumption | undefined; + try { + operation.check(); + consumption = this.reserveLiveConsumption(); + const execution = await this.submitLiveInternal(body, prefer, operation); + return yield* this.readLiveInternal(execution, options, operation, consumption); + } finally { + consumption?.release(); + operation.controller.abort(new OssieError("ABORTED", "The live reader was closed.")); + operation.close(); + } + } + + /** Attach without POST or preflight. Never wait for success before opening an + * advertised stream. Early return closes local resources, not the backend job. + */ + async *readLiveBatches(execution: LiveExecution, options: LiveReadOptions = {}): + AsyncGenerator { + validateLiveExecution(execution); + const known = this.liveExecutions.get(execution.execution_id); + if (known) this.sameLiveIdentity(known, execution); + const snapshot = this.rememberLive(known && !terminal(execution) ? known : execution); + this.validateLiveOptions(options); + const operation = new Operation(options.signal, this.timeoutMs); + let consumption: LiveConsumption | undefined; + try { + operation.check(); + consumption = this.reserveLiveConsumption(snapshot.execution_id); + return yield* this.readLiveInternal(snapshot, options, operation, consumption); + } finally { + consumption?.release(); + operation.controller.abort(new OssieError("ABORTED", "The live reader was closed.")); + operation.close(); + } + } + + /** Bounded materialization through exactly the same live streaming workflow. */ + async executeLive(request: ExecuteRequest, options: LiveOptions = {}): + Promise<{ execution: LiveSuccessfulExecution; table: Table }> { + const iterator = this.stream(request, options); + const batches: RecordBatch[] = []; + try { + while (true) { + const next = await iterator.next(); + if (next.done) { + assert(batches.length > 0, "The Arrow reader did not preserve the result schema."); + return { execution: next.value, table: new Table(batches[0]!.schema, batches) }; + } + batches.push(next.value); + } + } finally { + // A completed generator ignores the return argument. + await iterator.return(undefined as never); + } + } + + /** Drops local lifecycle history only; does not cancel/delete server data or + * clear single-use live consumption fences (bounded to 64 per client). + */ forgetExecution(executionId: string): void { idPath(executionId); this.executions.delete(executionId); + this.liveExecutions.delete(executionId); } /** Streams batches from one completed immutable result. Full counts are only @@ -271,6 +378,18 @@ export class OssieClient { if (error instanceof OssieError) throw error; throw new ProtocolError("The query request is not serializable."); } + + } + + private liveRequestBody(request: ExecuteRequest): string { + return JSON.stringify({ ...(JSON.parse(this.requestBody(request)) as ExecuteRequest), + contract_version: LIVE_CONTRACT_VERSION, delivery: "live" }); + } + + private validateLiveOptions(options: LiveReadOptions): void { + assert(options.onProgress === undefined || typeof options.onProgress === "function", + "onProgress must be a function."); + this.validateTruncated(options); } private validateProgressCallback(options: WaitOptions): void { @@ -295,6 +414,7 @@ export class OssieClient { } private remember(execution: Execution, id?: string, source?: string, observation?: Observation): Execution { + assert(!this.liveExecutions.has(execution.execution_id), "Execution delivery changed."); if (id !== undefined) assert(execution.execution_id === id, "Execution identifier mismatch."); if (source !== undefined) assert(execution.data_source_id === source, "Execution data-source mismatch."); const previous = this.executions.get(execution.execution_id); @@ -319,6 +439,183 @@ export class OssieClient { return snapshot; } + private sameLiveIdentity(previous: LiveExecution, next: LiveExecution): void { + assert(previous.contract_version === next.contract_version && previous.delivery === next.delivery + && previous.execution_id === next.execution_id && previous.data_source_id === next.data_source_id + && equivalent(previous.model, next.model), "Execution identity changed."); + if (previous.stream && next.stream) { + assert(equivalent(previous.stream, next.stream), "The live stream descriptor changed."); + } + } + + private rememberLive(execution: LiveExecution, id?: string, source?: string, + observation?: LiveObservation): LiveExecution { + assert(!this.executions.has(execution.execution_id), "Execution delivery changed."); + if (id !== undefined) assert(execution.execution_id === id, "Execution identifier mismatch."); + if (source !== undefined) assert(execution.data_source_id === source, "Execution data-source mismatch."); + const previous = this.liveExecutions.get(execution.execution_id); + if (observation?.previous) transition(observation.previous, execution); + if (observation?.fallback) transition(observation.fallback, execution); + if (previous) this.sameLiveIdentity(previous, execution); + if (observation && previous && previous !== observation.previous && !terminal(execution)) { + this.liveExecutions.delete(previous.execution_id); + this.liveExecutions.set(previous.execution_id, previous); + return structuredClone(previous); + } + if (previous) transition(previous, execution); + const snapshot = structuredClone(execution); + this.liveExecutions.delete(execution.execution_id); + this.liveExecutions.set(execution.execution_id, structuredClone(snapshot)); + if (this.liveExecutions.size > MAX_TRACKED_EXECUTIONS) { + this.liveExecutions.delete(this.liveExecutions.keys().next().value!); + } + return snapshot; + } + + private async submitLiveInternal(body: string, prefer: string | undefined, operation: Operation): Promise { + const capabilities = validateCapabilities(await this.control("capabilities", "GET", operation, [200])); + if (!capabilities.features.includes(LIVE_FEATURE)) { + throw new OssieError("LIVE_UNSUPPORTED", "The endpoint does not advertise live execution support."); + } + assert(prefer === undefined || Number(prefer.slice("wait=".length)) <= capabilities.max_wait_seconds, + "waitSeconds exceeds the endpoint's advertised maximum wait."); + const request = JSON.parse(body) as ExecuteRequest; + return this.rememberLive(validateLiveExecution( + await this.control("queries", "POST", operation, [200, 202], body, prefer, "live")), + undefined, request.data_source_id); + } + + private async getLiveInternal(id: string, operation: Operation, fallback?: LiveExecution): Promise { + const observation: LiveObservation = { + previous: this.liveExecutions.get(id), ...(fallback === undefined ? {} : { fallback }), + }; + return this.rememberLive(validateLiveExecution( + await this.control(`queries/${idPath(id)}`, "GET", operation, [200])), id, undefined, observation); + } + + private reserveLiveConsumption(id?: string): LiveConsumption { + if (id !== undefined && this.consumedLive.has(id)) { + throw new OssieError("LIVE_STREAM_CONSUMED", "This client already attempted to consume the live execution."); + } + if (this.consumedLive.size + this.reservedLive >= MAX_TRACKED_EXECUTIONS) { + throw new OssieError("LIVE_STREAM_LIMIT", "This client's bounded live consumption registry is full."); + } + this.reservedLive++; + let reserved = true; + const release = (): void => { + if (reserved) { + reserved = false; + this.reservedLive--; + } + }; + return { + acquire: executionId => { + assert(reserved, "Live consumption reservation was already released."); + if (this.consumedLive.has(executionId)) { + throw new OssieError("LIVE_STREAM_CONSUMED", "This client already attempted to consume the live execution."); + } + this.consumedLive.add(executionId); + release(); + }, + release, + }; + } + + private async *readLiveInternal(initial: LiveExecution, options: LiveReadOptions, operation: Operation, + consumption: LiveConsumption): + AsyncGenerator { + let execution = initial; + while (true) { + operation.check(); + liveFailure(execution); + if (options.onProgress) await operation.race(() => options.onProgress!(structuredClone(execution))); + if (execution.stream) break; + await operation.sleep(this.pollIntervalMs); + execution = await this.getLiveInternal(execution.execution_id, operation, execution); + } + const opened = execution as LiveExecution & { stream: LiveStream }; + const stream = opened.stream; + if (Date.parse(stream.expires_at) <= Date.now()) { + throw new OssieError("RESULT_EXPIRED", "The live stream start window has expired."); + } + const target = this.resultUrl(stream.url); + consumption.acquire(execution.execution_id); + const dataOperation = new Operation(operation.signal, this.timeoutMs); + const monitorOperation = new Operation(operation.signal, this.timeoutMs); + let monitorFailed = false; + let monitorError: unknown; + const monitor = (async (): Promise => { + try { + let current = opened as LiveExecution; + while (!terminal(current)) { + await monitorOperation.sleep(this.pollIntervalMs); + current = await this.getLiveInternal(current.execution_id, monitorOperation, current); + liveFailure(current); + if (options.onProgress) await monitorOperation.race(() => options.onProgress!(structuredClone(current))); + } + assert(current.state === "succeeded", "Live execution did not succeed."); + return current; + } catch (error) { + if (!monitorOperation.signal.aborted) { + monitorFailed = true; + monitorError = error; + dataOperation.controller.abort(error); + } + throw error; + } finally { + monitorOperation.close(); + } + })(); + // Monitoring must remain active even while the consumer is paused at yield. + // Attach a rejection handler immediately; consumption may end early. + void monitor.catch(() => {}); + try { + const stats = { rows: 0, bytes: 0 }; + try { + const response = await this.fetchResponse(target, "GET", dataOperation, ARROW_MEDIA_TYPE, + undefined, undefined, Date.parse(stream.expires_at)); + try { + if (response.status >= 400) await this.problem(response, dataOperation); + assert(response.status === 200 && mediaType(response) === ARROW_MEDIA_TYPE, + "Expected a successful Arrow IPC stream response."); + yield* arrowBatches(response, opened, dataOperation, this.maxResultBytes, + this.maxResultRows, undefined, this.maxResultBytes, stats); + } finally { + cancelBody(response); + } + } catch (error) { + operation.check(); + if (monitorFailed) throw monitorError; + monitorOperation.controller.abort(new OssieError("ABORTED", "The live data reader failed.")); + await monitor.catch(() => {}); + const known = this.liveExecutions.get(execution.execution_id); + if (known) liveFailure(known); + // One bounded status inspection can preserve a published engine failure. + // Neither success nor unavailable status can repair invalid/missing IPC. + let final: LiveExecution | undefined; + try { final = await this.getLiveInternal(execution.execution_id, operation, execution); } + catch { /* Keep the precise original data-plane failure. */ } + if (final) liveFailure(final); + throw error; + } + const completed = await operation.race(() => monitor); + operation.check(); + assert(completed.completion.result_id === stream.result_id, "Live completion result identity mismatch."); + assert(completed.completion.row_count === stats.rows, "Live Arrow row count mismatch."); + assert(completed.completion.byte_count === stats.bytes, "Live decoded IPC byte count mismatch."); + if (completed.completion.completeness === "truncated" && options.allowTruncated !== true) { + throw new OssieError("TRUNCATED_RESULT", "The result is truncated; explicit acceptance is required."); + } + return completed; + } finally { + const closed = new OssieError("ABORTED", "The live reader was closed."); + dataOperation.controller.abort(closed); + monitorOperation.controller.abort(closed); + dataOperation.close(); + monitorOperation.close(); + } + } + private async submitInternal(body: string, prefer: string | undefined, operation: Operation): Promise { const request = JSON.parse(body) as ExecuteRequest; const result = await this.control("queries", "POST", operation, [200, 202], body, prefer, true); @@ -407,7 +704,7 @@ export class OssieClient { } private async control(path: string, method: string, operation: Operation, statuses: readonly number[], - body?: string, prefer?: string, lifecyclePost = false): Promise { + body?: string, prefer?: string, lifecyclePost: boolean | "live" = false): Promise { const response = await this.fetchResponse(new URL(path, this.base), method, operation, "application/json", body, prefer); try { if (response.status >= 400) await this.problem(response, operation); @@ -415,7 +712,7 @@ export class OssieClient { assert(mediaType(response) === "application/json", "Expected an application/json response."); const result = await readJson(response, operation); if (lifecyclePost) { - const execution = validateExecution(result); + const execution = lifecyclePost === "live" ? validateLiveExecution(result) : validateExecution(result); assert((response.status === 200) === terminal(execution), "HTTP status does not match execution state."); } return result; diff --git a/typescript/execution/src/errors.ts b/typescript/execution/src/errors.ts index 120c4fa41..af06dad9c 100644 --- a/typescript/execution/src/errors.ts +++ b/typescript/execution/src/errors.ts @@ -17,7 +17,7 @@ * under the License. */ -import type { Execution, ProblemDetails, QueryError } from "./types.js"; +import type { Execution, LiveExecution, ProblemDetails, QueryError } from "./types.js"; /** Base error. Messages deliberately exclude URLs, credentials and server bodies. * Structured server details are untrusted data and must not be logged blindly. @@ -49,6 +49,20 @@ export class ExecutionCancelledError extends OssieError { } } +export class LiveQueryExecutionError extends OssieError { + readonly error: QueryError; + constructor(public readonly execution: Extract) { + super(execution.error.code, "The live query execution failed."); + this.error = execution.error; + } +} + +export class LiveExecutionCancelledError extends OssieError { + constructor(public readonly execution: Extract) { + super("EXECUTION_CANCELLED", "The live query execution was cancelled."); + } +} + export class HttpProblemError extends OssieError { readonly status: number; constructor(public readonly problem: ProblemDetails) { diff --git a/typescript/execution/src/transport.ts b/typescript/execution/src/transport.ts index 6273b7397..1d4712eca 100644 --- a/typescript/execution/src/transport.ts +++ b/typescript/execution/src/transport.ts @@ -78,7 +78,7 @@ export function cancelBody(response: Response): void { /** Reads decoded fetch bytes, never Content-Length (which can describe compressed HTTP data). */ export class BodySource { private readonly reader: ReadableStreamDefaultReader; - private readonly abort = (): void => { void this.reader.cancel().catch(() => {}); }; + private readonly abort = (): void => { this.close(); }; private closed = false; private pending: Uint8Array = new Uint8Array(0); private offset = 0; diff --git a/typescript/execution/src/types.ts b/typescript/execution/src/types.ts index b6bb8e254..70019d808 100644 --- a/typescript/execution/src/types.ts +++ b/typescript/execution/src/types.ts @@ -18,6 +18,8 @@ */ export const CONTRACT_VERSION = "0.1-draft" as const; +export const LIVE_CONTRACT_VERSION = "0.2-draft" as const; +export const LIVE_FEATURE = "live-stream-v0.2" as const; export const FOUNDATION_REVISION = "cc0d07099a3ef31e85af1fd93458a52186fae2c6" as const; export const QUERY_SCHEMA_VERSION = "mcp-0.4-draft" as const; export const ARROW_MEDIA_TYPE = "application/vnd.apache.arrow.stream" as const; @@ -182,3 +184,43 @@ export interface WaitOptions extends RequestOptions { export interface ExecuteOptions extends SubmitOptions, WaitOptions { allowTruncated?: boolean; } + +export interface LiveRequest extends ExecuteRequest { + contract_version: typeof LIVE_CONTRACT_VERSION; + delivery: "live"; +} +export interface LiveStream { + result_id: string; + format: "arrow-stream"; + url: string; + expires_at: string; + columns: readonly Column[]; + replayable: false; +} +export interface LiveCompletion { + result_id: string; + row_count: number; + byte_count: number; + completeness: "complete" | "truncated"; + truncation_reason?: string; +} +interface LiveExecutionBase extends Omit { + contract_version: typeof LIVE_CONTRACT_VERSION; + delivery: "live"; +} +export type LiveExecution = LiveExecutionBase & ( + | { state: "queued"; progress?: Progress; stream?: never; completion?: never; error?: never } + | { state: "running" | "cancelling"; progress?: Progress; stream?: LiveStream; completion?: never; error?: never } + | { state: "succeeded"; stream: LiveStream; completion: LiveCompletion; progress?: never; error?: never } + | { state: "failed"; error: QueryError; stream?: LiveStream; completion?: never; progress?: never } + | { state: "cancelled"; stream?: LiveStream; completion?: never; progress?: never; error?: never } +); +export type LiveSuccessfulExecution = Extract; +export interface LiveExecutionMetadata extends Omit { + contract_version: typeof LIVE_CONTRACT_VERSION; +} +export interface LiveReadOptions extends RequestOptions { + onProgress?: (execution: LiveExecution) => void | Promise; + allowTruncated?: boolean; +} +export interface LiveOptions extends SubmitOptions, LiveReadOptions {} diff --git a/typescript/execution/src/validation.ts b/typescript/execution/src/validation.ts index f2f6fbf67..7ae0df111 100644 --- a/typescript/execution/src/validation.ts +++ b/typescript/execution/src/validation.ts @@ -18,8 +18,9 @@ */ import { ProtocolError } from "./errors.js"; -import { CONTRACT_VERSION, FOUNDATION_REVISION, QUERY_SCHEMA_VERSION } from "./types.js"; +import { CONTRACT_VERSION, FOUNDATION_REVISION, LIVE_CONTRACT_VERSION, QUERY_SCHEMA_VERSION } from "./types.js"; import type { Binding, Capabilities, ExecuteRequest, Execution, ExecutionMetadata, + LiveCompletion, LiveExecution, LiveExecutionMetadata, LiveStream, ProblemDetails, Query, QueryError, ResultManifest } from "./types.js"; type ObjectValue = Record; @@ -261,9 +262,15 @@ export function validateManifest(value: unknown): ResultManifest { assert(manifest.completeness === "truncated", "A complete result cannot have a truncation reason."); } timestamp(manifest.expires_at); - assert(Array.isArray(manifest.columns) && manifest.columns.length > 0, "Expected result columns."); + columns(manifest.columns); + extensions(manifest); + return value as ResultManifest; +} + +function columns(value: unknown): void { + assert(Array.isArray(value) && value.length > 0, "Expected result columns."); const ids = new Set(); - for (const item of manifest.columns) { + for (const item of value) { const column = object(item, ["column_id", "name", "logical_type", "nullable", "role", "model_ref", "label", "description", "unit", "currency", "extensions"], ["column_id", "name", "logical_type", "nullable"]); text(column.column_id); @@ -279,8 +286,6 @@ export function validateManifest(value: unknown): ResultManifest { } extensions(column); } - extensions(manifest); - return value as ResultManifest; } export function validateExecution(value: unknown): Execution { @@ -333,7 +338,7 @@ export function validateMetadata(value: unknown): ExecutionMetadata { return value as ExecutionMetadata; } -export function terminal(execution: Execution): boolean { +export function terminal(execution: Execution | LiveExecution): boolean { return ["succeeded", "failed", "cancelled"].includes(execution.state); } @@ -347,9 +352,14 @@ export function equivalent(left: unknown, right: unknown): boolean { return keys.length === Object.keys(b).length && keys.every(key => own(b, key) && equivalent(a[key], b[key])); } -export function transition(previous: Execution, next: Execution): void { - assert(previous.execution_id === next.execution_id && previous.data_source_id === next.data_source_id +export function transition(previous: Execution | LiveExecution, next: Execution | LiveExecution): void { + assert(previous.contract_version === next.contract_version + && previous.execution_id === next.execution_id && previous.data_source_id === next.data_source_id && equivalent(previous.model, next.model), "Execution identity changed."); + if ("delivery" in previous) { + assert("delivery" in next && next.delivery === previous.delivery, "Execution delivery changed."); + if (previous.stream) assert(equivalent(previous.stream, next.stream), "The live stream descriptor changed."); + } const allowed: Record = { queued: ["queued", "running", "cancelling", "succeeded", "failed", "cancelled"], running: ["running", "cancelling", "succeeded", "failed", "cancelled"], @@ -361,3 +371,59 @@ export function transition(previous: Execution, next: Execution): void { assert(equivalent(previous, next), "An immutable terminal execution changed."); } } + +export function validateLiveStream(value: unknown): LiveStream { + const stream = object(value, ["result_id", "format", "url", "expires_at", "columns", "replayable"], + ["result_id", "format", "url", "expires_at", "columns", "replayable"]); + text(stream.result_id); + enumeration(stream.format, ["arrow-stream"]); + resultUrl(stream.url); + timestamp(stream.expires_at); + columns(stream.columns); + assert(stream.replayable === false, "A live stream must be single-use."); + return value as LiveStream; +} + +export function validateLiveCompletion(value: unknown): LiveCompletion { + const completion = object(value, ["result_id", "row_count", "byte_count", "completeness", "truncation_reason"], + ["result_id", "row_count", "byte_count", "completeness"]); + text(completion.result_id); + count(completion.row_count); + count(completion.byte_count); + enumeration(completion.completeness, ["complete", "truncated"]); + assert(own(completion, "truncation_reason") === (completion.completeness === "truncated"), + "Only a truncated live completion must include a truncation reason."); + if (own(completion, "truncation_reason")) text(completion.truncation_reason); + return value as LiveCompletion; +} + +export function validateLiveExecution(value: unknown): LiveExecution { + const execution = object(value, ["contract_version", "delivery", "execution_id", "data_source_id", + "model", "warnings", "extensions", "state", "progress", "stream", "completion", "error"], + ["contract_version", "delivery", "execution_id", "data_source_id", "model", "warnings", "state"]); + enumeration(execution.contract_version, [LIVE_CONTRACT_VERSION]); + enumeration(execution.delivery, ["live"]); + // Reuse the unchanged 0.1 common-field/state validator, not its result contract. + const { delivery: _delivery, stream: _stream, completion: _completion, ...common } = execution; + validateExecution({ ...common, contract_version: CONTRACT_VERSION, + ...(execution.state === "succeeded" ? { state: "cancelled" } : {}) }); + if (own(execution, "stream")) { + assert(execution.state !== "queued", "Queued live executions cannot advertise a stream."); + validateLiveStream(execution.stream); + } + if (execution.state === "succeeded") { + assert(own(execution, "stream"), "Live success requires a stream."); + const completion = validateLiveCompletion(execution.completion); + assert(completion.result_id === (execution.stream as LiveStream).result_id, + "Live completion result identity mismatch."); + } else assert(!own(execution, "completion"), "Only live success can include completion."); + return value as LiveExecution; +} + +export function validateLiveMetadata(value: unknown): LiveExecutionMetadata { + const metadata = object(value, ["contract_version", "execution_id", "result_id", "data_source_id", "model"], + ["contract_version", "execution_id", "result_id", "data_source_id", "model"]); + enumeration(metadata.contract_version, [LIVE_CONTRACT_VERSION]); + validateMetadata({ ...metadata, contract_version: CONTRACT_VERSION }); + return value as LiveExecutionMetadata; +} diff --git a/typescript/execution/test/browser/smoke.test.mjs b/typescript/execution/test/browser/smoke.test.mjs index 8eca9365a..ee4d15a20 100644 --- a/typescript/execution/test/browser/smoke.test.mjs +++ b/typescript/execution/test/browser/smoke.test.mjs @@ -27,8 +27,9 @@ import { delimiter, join } from "node:path"; import { fileURLToPath } from "node:url"; import test from "node:test"; import { build } from "esbuild"; -import { ARROW_MEDIA_TYPE } from "../../dist/index.js"; +import { ARROW_MEDIA_TYPE, LIVE_CONTRACT_VERSION, LIVE_FEATURE } from "../../dist/index.js"; import { BINDING, CAPABILITIES, ERROR, REQUEST, arrowFixture, execution, pagedFixture } from "../fixtures.mjs"; +import { deferred, firstBatchEnd, liveFixture } from "../live-fixtures.mjs"; const packageDirectory = fileURLToPath(new URL("../../", import.meta.url)); const browserCandidates = process.env.OSSIE_BROWSER_BINARY ? [process.env.OSSIE_BROWSER_BINARY] : [ @@ -91,7 +92,7 @@ async function bundle(contents) { }); } -test("real browser validates synthetic HTTP Arrow execution, polling, CORS and late failure", { +test("real browser validates materialized and LIVE HTTP Arrow, observation gating, CORS and late failure", { timeout: 40_000, }, async () => { assert.ok(browser, "Set OSSIE_BROWSER_BINARY to an installed Chromium/Chrome/Edge executable"); @@ -99,6 +100,9 @@ test("real browser validates synthetic HTTP Arrow execution, polling, CORS and l const paginated = pagedFixture(); const requests = []; const polls = { asynchronous: 0, "late-failure": 0 }; + const live = Object.fromEntries(["live", "live-failure"].map(mode => [mode, { + fixture: liveFixture(), observed: false, body: undefined, closed: deferred(), polls: 0, + }])); let pageOrigin; let bundleText = ""; const pageServer = createServer((request, response) => { @@ -118,6 +122,7 @@ test("real browser validates synthetic HTTP Arrow execution, polling, CORS and l path: request.url, method: request.method, origin: request.headers.origin, authorization: request.headers.authorization, requestedHeaders: request.headers["access-control-request-headers"], + httpVersion: request.httpVersion, }; requests.push(observation); const denied = request.url.startsWith("/cors-denied/"); @@ -131,6 +136,29 @@ test("real browser validates synthetic HTTP Arrow execution, polling, CORS and l response.end(); return; } + const liveMode = request.url.split("/")[1]; + const liveState = live[liveMode]; + if (liveState && request.url.endsWith("/results/result-1")) { + assert.equal(liveState.body, undefined, "The browser must not retry the single-use stream"); + assert.equal(liveState.observed, false, "No terminal state before first-batch observation"); + liveState.body = response; + response.writeHead(200, { "Content-Type": ARROW_MEDIA_TYPE }); + response.write(liveState.fixture.bytes.slice(0, firstBatchEnd(liveState.fixture.bytes))); + response.on("close", () => liveState.closed.resolve()); + return; + } + if (liveState && request.url.endsWith("/observed")) { + assert.ok(liveState.body, "Browser acknowledges only after opening and decoding the stream"); + assert.equal(liveState.observed, false); + liveState.observed = true; + if (liveMode === "live") { + liveState.body.end(liveState.fixture.bytes.slice(firstBatchEnd(liveState.fixture.bytes))); + } + // Failure deliberately leaves the body stalled; only status can stop it. + response.writeHead(204); + response.end(); + return; + } if (request.url === "/results/result-1") { response.writeHead(200, { "Content-Type": ARROW_MEDIA_TYPE }); response.end(fixture.bytes); @@ -147,6 +175,19 @@ test("real browser validates synthetic HTTP Arrow execution, polling, CORS and l for await (const chunk of request) bodyParts.push(chunk); observation.body = Buffer.concat(bodyParts).toString("utf8"); const mode = request.url.split("/")[1]; + if (liveState) { + response.writeHead(request.method === "POST" ? 202 : 200, { "Content-Type": "application/json" }); + if (request.url.endsWith("/capabilities")) { + response.end(JSON.stringify({ ...CAPABILITIES, features: [LIVE_FEATURE] })); + } else if (request.method === "POST") { + response.end(JSON.stringify(liveState.fixture.running)); + } else { + liveState.polls++; + response.end(JSON.stringify(!liveState.observed ? liveState.fixture.running + : mode === "live" ? liveState.fixture.succeeded : liveState.fixture.live("failed", { error: ERROR }))); + } + return; + } const pendingSubmission = request.method === "POST" && !["immediate", "paginated"].includes(mode); response.writeHead(pendingSubmission ? 202 : 200, { "Content-Type": "application/json" }); if (request.url.endsWith("/capabilities")) response.end(JSON.stringify(CAPABILITIES)); @@ -166,6 +207,9 @@ test("real browser validates synthetic HTTP Arrow execution, polling, CORS and l } pageOrigin = `http://127.0.0.1:${pageServer.address().port}`; const apiOrigin = `http://127.0.0.1:${apiServer.address().port}`; + for (const [mode, state] of Object.entries(live)) { + state.fixture.stream.url = `${apiOrigin}/${mode}/results/result-1`; + } fixture.succeeded.result.url = `${apiOrigin}/results/result-1`; for (const stream of paginated.streams) { stream.descriptor.url = `${apiOrigin}/results/result-1/pages/${stream.descriptor.page_index}`; @@ -173,7 +217,7 @@ test("real browser validates synthetic HTTP Arrow execution, polling, CORS and l const profile = await mkdtemp(join(packageDirectory, "test", "browser", ".browser-profile-")); try { const result = await bundle(` - import { OssieClient, QueryExecutionError } from './dist/index.js'; + import { OssieClient, QueryExecutionError, LiveQueryExecutionError } from './dist/index.js'; const check = (condition, label) => { if (!condition) throw new Error(label); }; (async () => { const createClient = mode => new OssieClient({ @@ -212,6 +256,31 @@ test("real browser validates synthetic HTTP Arrow execution, polling, CORS and l check(failure instanceof QueryExecutionError, 'late failure classification'); check(failure.code === 'INVALID_QUERY', 'late failure error code'); check(failure.error.replacement_query.measures[0] === 'revenue', 'replacement query retained'); + for (const mode of ['live', 'live-failure']) { + const iterator = createClient(mode).stream(${JSON.stringify(REQUEST)}); + const first = await iterator.next(); + check(!first.done && first.value.numRows === 1, 'LIVE first batch before terminal status'); + check(first.value.getChildAt(1).get(0) === 9007199254740993n, 'LIVE exact int64'); + // This acknowledgement is the ONLY gate that releases the last batch or + // publishes terminal status. Timer-only chunking cannot satisfy this test. + const acknowledged = await fetch(${JSON.stringify(apiOrigin)} + '/' + mode + '/observed', { + headers: { Authorization: 'Bearer SYNTHETIC_BROWSER_TOKEN' } + }); + check(acknowledged.ok, 'first batch observation gate'); + if (mode === 'live') { + const second = await iterator.next(); + check(!second.done && second.value.getChildAt(0).get(0) === null, 'LIVE last batch'); + const end = await iterator.next(); + check(end.done && end.value.state === 'succeeded' && end.value.completion.row_count === 2, + 'LIVE verified generator return'); + } else { + let liveFailure; + try { await iterator.next(); } catch (error) { liveFailure = error; } + check(liveFailure instanceof LiveQueryExecutionError, 'LIVE status interrupts stalled body'); + check(liveFailure.error.code === 'INVALID_QUERY' && liveFailure.execution.stream.replayable === false, + 'LIVE terminal failure retains result identity'); + } + } const denied = new OssieClient({ baseUrl: ${JSON.stringify(apiOrigin)} + '/cors-denied', allowInsecureLocalhost: true, timeoutMs: 10000 @@ -280,6 +349,17 @@ test("real browser validates synthetic HTTP Arrow execution, polling, CORS and l assert.equal(request.authorization, "Bearer SYNTHETIC_BROWSER_TOKEN"); } assert.equal(requests.filter(request => request.path === "/cors-denied/capabilities").length, 1); + for (const [mode, state] of Object.entries(live)) { + assert.equal(state.observed, true, "Real browser decoded a provisional first batch"); + assert.ok(state.polls > 0, "Live status polling accompanied streaming"); + await Promise.race([state.closed.promise, delay(3000).then(() => { throw new Error("Live body was not closed"); })]); + const submissions = requests.filter(r => r.method === "POST" && r.path === `/${mode}/queries`); + assert.equal(submissions.length, 1); + assert.deepEqual(JSON.parse(submissions[0].body), + { ...REQUEST, contract_version: LIVE_CONTRACT_VERSION, delivery: "live" }); + assert.equal(requests.filter(r => r.method === "GET" && r.path === `/${mode}/results/result-1`).length, 1); + } + assert.ok(requests.every(r => r.httpVersion === "1.1"), "Synthetic harness verifies loopback HTTP/1.1 only"); } finally { if (cdp) { await cdp.send("Browser.close").catch(() => {}); diff --git a/typescript/execution/test/live-fixtures.mjs b/typescript/execution/test/live-fixtures.mjs new file mode 100644 index 000000000..410e9293f --- /dev/null +++ b/typescript/execution/test/live-fixtures.mjs @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Message } from "apache-arrow"; +import { ARROW_MEDIA_TYPE, LIVE_CONTRACT_VERSION } from "../dist/index.js"; +import { MODEL, arrowFixture, execution } from "./fixtures.mjs"; + +export function liveFixture(options = {}) { + const fixture = arrowFixture({ + batchRows: [[["West", 9007199254740993n]], [[null, null]]], + metadata: { + contract_version: LIVE_CONTRACT_VERSION, execution_id: "exec-1", + result_id: "result-1", data_source_id: "sales", model: MODEL, + }, + ...options, + }); + const result = fixture.succeeded.result; + const stream = { + result_id: result.result_id, format: result.format, url: result.url, + expires_at: result.expires_at, columns: result.columns, replayable: false, + }; + const live = (state, extra = {}) => ({ + ...execution(state), contract_version: LIVE_CONTRACT_VERSION, delivery: "live", + ...(state === "queued" ? {} : { stream }), ...extra, + }); + const completion = { + result_id: result.result_id, row_count: result.row_count, byte_count: result.byte_count, + completeness: result.completeness, + ...(result.truncation_reason === undefined ? {} : { truncation_reason: result.truncation_reason }), + }; + return { ...fixture, stream, completion, live, running: live("running"), + succeeded: live("succeeded", { completion }) }; +} + +export function firstBatchEnd(bytes) { + let offset = 0; + while (offset < bytes.length) { + const view = new DataView(bytes.buffer, bytes.byteOffset + offset); + const prefix = view.getInt32(0, true) === -1 ? 8 : 4; + const length = view.getInt32(prefix - 4, true); + if (length === 0) break; + const message = Message.decode(bytes.subarray(offset + prefix, offset + prefix + length)); + offset += prefix + length + message.bodyLength; + if (message.isRecordBatch()) return offset; + } + throw new Error("No record batch"); +} + +export function deferred() { + let resolve, reject; + const promise = new Promise((yes, no) => { resolve = yes; reject = no; }); + return { promise, resolve, reject }; +} + +export function gatedBody(bytes, split = firstBatchEnd(bytes)) { + let controller; + let cancelled = false; + const response = new Response(new ReadableStream({ + start(value) { controller = value; value.enqueue(bytes.slice(0, split)); }, + cancel() { cancelled = true; }, + }, { highWaterMark: 0 }), { headers: { "Content-Type": ARROW_MEDIA_TYPE } }); + return { + response, + finish() { controller.enqueue(bytes.slice(split)); controller.close(); }, + fail() { controller.error(new Error("SYNTHETIC_BODY_FAILURE")); }, + get cancelled() { return cancelled; }, + }; +} diff --git a/typescript/execution/test/live.test.mjs b/typescript/execution/test/live.test.mjs new file mode 100644 index 000000000..684e171ac --- /dev/null +++ b/typescript/execution/test/live.test.mjs @@ -0,0 +1,721 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from "node:assert/strict"; +import test from "node:test"; +import { Decimal, Field, TimestampNanosecond, makeData } from "apache-arrow"; +import { + COLUMN_METADATA_KEY, LIVE_CONTRACT_VERSION, LIVE_FEATURE, MAX_TRACKED_EXECUTIONS, + LiveExecutionCancelledError, LiveQueryExecutionError, OssieClient, ProtocolError, +} from "../dist/index.js"; +import { CAPABILITIES, ERROR, MODEL, ORIGIN, REQUEST, arrowResponse, collect, json, mockFetch } from "./fixtures.mjs"; +import { deferred, gatedBody, liveFixture } from "./live-fixtures.mjs"; + +const delay = ms => new Promise(resolve => setTimeout(resolve, ms)); +const code = expected => error => error.code === expected; +const capabilities = { ...CAPABILITIES, features: [LIVE_FEATURE] }; + +function setup({ fixture = liveFixture(), status, data, initial, options = {}, preflight } = {}) { + const mock = mockFetch(call => { + if (call.url.pathname === "/capabilities") return preflight?.(call) ?? json(capabilities); + if (call.url.pathname === "/queries" && call.method === "POST") { + const value = initial ?? fixture.running; + return json(value, ["succeeded", "failed", "cancelled"].includes(value.state) ? 200 : 202); + } + if (call.url.pathname.startsWith("/results/")) return data?.(call) ?? arrowResponse(fixture.bytes); + return status?.(call) ?? json(fixture.succeeded); + }); + const api = new OssieClient({ baseUrl: ORIGIN, fetch: mock.fetch, pollIntervalMs: 1, timeoutMs: 1500, ...options }); + return { ...mock, api, fixture }; +} + +async function drain(iterator) { + const batches = []; + while (true) { + const next = await iterator.next(); + if (next.done) return { batches, execution: next.value }; + batches.push(next.value); + } +} + +test("LIVE yields before backend completion and returns verified terminal success after exhaustion", async () => { + const fixture = liveFixture(); + const body = gatedBody(fixture.bytes); + let observed = false; + const { api, calls } = setup({ + fixture, data: () => body.response, status: () => json(observed ? fixture.succeeded : fixture.running), + }); + const iterator = api.stream(REQUEST); + const first = await iterator.next(); + assert.equal(first.done, false); + assert.equal(first.value.getChildAt(1).get(0), 9007199254740993n); + assert.equal(observed, false); + observed = true; + body.finish(); + const result = await drain(iterator); + assert.equal(result.batches.length, 1); + assert.deepEqual(result.execution, fixture.succeeded); + assert.equal(calls.filter(call => call.method === "POST").length, 1); + assert.equal(calls.filter(call => call.url.pathname.startsWith("/results/")).length, 1); + assert.deepEqual(JSON.parse(calls.find(call => call.method === "POST").body), + { ...REQUEST, contract_version: LIVE_CONTRACT_VERSION, delivery: "live" }); + assert.equal(calls[0].url.pathname, "/capabilities"); +}); + +test("LIVE EOF while running awaits terminal status; EOS alone cannot succeed", async () => { + const fixture = liveFixture(); + const polled = deferred(); + const terminal = deferred(); + const { api } = setup({ fixture, status: () => { polled.resolve(); return terminal.promise; } }); + let done = false; + const result = api.executeLive(REQUEST).then(value => { done = true; return value; }); + await polled.promise; + await delay(10); + assert.equal(done, false); + terminal.resolve(json(fixture.succeeded)); + assert.equal((await result).table.numRows, 2); +}); + +test("LIVE terminal success does not abort delayed IPC EOF", async () => { + const fixture = liveFixture(); + const body = gatedBody(fixture.bytes, fixture.bytes.length); + const succeeded = deferred(); + const { api } = setup({ fixture, data: () => body.response }); + let done = false; + const result = api.executeLive(REQUEST, { onProgress: e => { if (e.state === "succeeded") succeeded.resolve(); } }) + .then(value => { done = true; return value; }); + await succeeded.promise; + assert.equal(body.cancelled, false); + assert.equal(done, false); + body.finish(); + assert.equal((await result).table.numRows, 2); +}); + +test("LIVE terminal success before the last batch preserves still-undelivered bytes", async () => { + const fixture = liveFixture(); + const body = gatedBody(fixture.bytes); + const succeeded = deferred(); + const { api } = setup({ fixture, data: () => body.response }); + const iterator = api.stream(REQUEST, { onProgress: e => { + if (e.state === "succeeded") succeeded.resolve(); + } }); + assert.equal((await iterator.next()).value.numRows, 1); + await succeeded.promise; + assert.equal(body.cancelled, false); + body.finish(); + const result = await drain(iterator); + assert.equal(result.batches.length, 1); + assert.equal(result.batches[0].getChildAt(0).get(0), null); + assert.deepEqual(result.execution, fixture.succeeded); +}); + +test("LIVE succeeded status and data headers alone cannot certify a result", async () => { + const fixture = liveFixture(); + const body = gatedBody(fixture.bytes, 0); + const { api } = setup({ fixture, initial: fixture.succeeded, data: () => body.response, + options: { timeoutMs: 40 }, + }); + await assert.rejects(api.executeLive(REQUEST), code("TIMEOUT")); + assert.equal(body.cancelled, true); +}); + +test("LIVE schema-only zero-row stream preserves its schema", async () => { + const fixture = liveFixture({ batchRows: [] }); + const { api } = setup({ fixture }); + const result = await api.executeLive(REQUEST); + assert.equal(result.table.numRows, 0); + assert.equal(result.table.numCols, 2); + assert.equal(result.execution.completion.row_count, 0); +}); + +for (const state of ["failed", "cancelled"]) { + test(`LIVE ${state} status promptly stops a stalled body and preserves typed terminal shape`, async () => { + const fixture = liveFixture(); + const body = gatedBody(fixture.bytes); + let failed = false; + const terminal = fixture.live(state, state === "failed" ? { error: ERROR } : {}); + const { api, calls } = setup({ + fixture, data: () => body.response, status: () => json(failed ? terminal : fixture.running), + }); + const iterator = api.stream(REQUEST); + assert.equal((await iterator.next()).done, false); + failed = true; + await assert.rejects(iterator.next(), error => { + assert.ok(error instanceof (state === "failed" ? LiveQueryExecutionError : LiveExecutionCancelledError)); + assert.deepEqual(error.execution, terminal); + if (state === "failed") assert.deepEqual(error.error.replacement_query, ERROR.replacement_query); + return true; + }); + assert.equal(body.cancelled, true); + assert.equal(body.response.body.locked, false); + assert.equal(calls.filter(call => call.method === "POST").length, 1); + }); +} + +test("LIVE status failure while iterator is paused releases the body and has no unhandled rejection", async () => { + const fixture = liveFixture(); + const body = gatedBody(fixture.bytes); + const statusFailed = deferred(); + let fail = false; + const { api } = setup({ fixture, data: () => body.response, status: () => { + if (!fail) return json(fixture.running); + statusFailed.resolve(); + throw new Error("SYNTHETIC_STATUS_FAILURE"); + } }); + const iterator = api.stream(REQUEST); + await iterator.next(); + fail = true; + await statusFailed.promise; + await delay(10); + assert.equal(body.cancelled, true); + assert.equal(body.response.body.locked, false); + await assert.rejects(iterator.next(), code("TRANSPORT_ERROR")); +}); + +for (const final of ["failed", "succeeded", "unavailable"]) { + test(`LIVE body failure with ${final} final inspection is never cured or retried`, async () => { + const fixture = liveFixture(); + const body = gatedBody(fixture.bytes); + let broken = false; + let finalReads = 0; + const { api, calls } = setup({ fixture, data: () => body.response, status: () => { + if (!broken) return json(fixture.running); + finalReads++; + if (final === "unavailable") throw new Error("SYNTHETIC_STATUS_FAILURE"); + return json(final === "failed" ? fixture.live("failed", { error: ERROR }) : fixture.succeeded); + } }); + const iterator = api.stream(REQUEST); + await iterator.next(); + broken = true; + body.fail(); + await assert.rejects(iterator.next(), final === "failed" ? LiveQueryExecutionError : code("TRANSPORT_ERROR")); + assert.equal(finalReads, 1); + assert.equal(calls.filter(call => call.url.pathname.startsWith("/results/")).length, 1); + assert.equal(calls.filter(call => call.method === "POST").length, 1); + }); +} + +test("LIVE status unavailable after EOF rejects rather than accepting a complete Arrow stream", async () => { + const { api } = setup({ status: () => { throw new Error("SYNTHETIC_UNAVAILABLE"); } }); + await assert.rejects(api.executeLive(REQUEST), code("TRANSPORT_ERROR")); +}); + +test("LIVE body failure racing an in-flight failed status preserves the backend error", async () => { + const fixture = liveFixture(); + const body = gatedBody(fixture.bytes); + const started = deferred(), pending = deferred(); + const failed = fixture.live("failed", { error: ERROR }); + let reads = 0; + const { api, calls } = setup({ fixture, data: () => body.response, status: () => { + if (++reads === 1) { started.resolve(); return pending.promise; } + return json(failed); + } }); + const iterator = api.stream(REQUEST); + await iterator.next(); + await started.promise; + const rejected = assert.rejects(iterator.next(), error => { + assert.ok(error instanceof LiveQueryExecutionError); + assert.equal(error.code, ERROR.code); + assert.deepEqual(error.execution, failed); + return true; + }); + body.fail(); + pending.resolve(json(failed)); + await rejected; + assert.ok(reads <= 2, "Only the in-flight poll and at most one recovery inspection"); + assert.equal(calls.filter(c => c.method === "POST").length, 1); + assert.equal(calls.filter(c => c.url.pathname.startsWith("/results/")).length, 1); +}); + +for (const key of ["row_count", "byte_count", "result_id"]) { + test(`LIVE terminal ${key} mismatch rejects`, async () => { + const fixture = liveFixture(); + const wrong = structuredClone(fixture.succeeded); + wrong.completion[key] = key === "result_id" ? "different" : wrong.completion[key] + 1; + const { api } = setup({ fixture, status: () => json(wrong) }); + await assert.rejects(api.executeLive(REQUEST), ProtocolError); + }); +} + +for (const mutate of [ + e => { e.stream.url += "?changed"; }, + e => { e.stream.result_id = "different"; }, + e => { e.stream.columns[0].name = "different"; }, + e => { delete e.stream; }, + e => { e.model.revision = "different"; }, + e => { e.data_source_id = "different"; }, + e => { e.contract_version = "0.1-draft"; }, + e => { e.delivery = "materialized"; }, + e => { e.state = "queued"; delete e.stream; }, +]) { + test(`LIVE rejects mutation: ${mutate}`, async () => { + const fixture = liveFixture(); + const changed = structuredClone(fixture.running); + mutate(changed); + const { api } = setup({ fixture, status: () => json(changed) }); + const submitted = await api.submitLive(REQUEST); + assert.equal(submitted.state, "running"); + await assert.rejects(api.getLiveExecution(submitted.execution_id), ProtocolError); + }); +} + +test("LIVE descriptor must persist even on terminal failure; terminal states are immutable", async () => { + const fixture = liveFixture(); + let value = fixture.live("failed", { error: ERROR, stream: undefined }); + const { api } = setup({ fixture, status: () => json(value) }); + await api.submitLive(REQUEST); + await assert.rejects(api.getLiveExecution("exec-1"), ProtocolError); + value = fixture.live("failed", { error: ERROR }); + await api.getLiveExecution("exec-1"); + value = { ...value, warnings: [{ code: "LATE", message: "Changed" }] }; + await assert.rejects(api.getLiveExecution("exec-1"), ProtocolError); +}); + +test("LIVE unsupported preflight rejects before POST; existing materialized remains selectable", async () => { + const { api, calls } = setup({ preflight: () => json(CAPABILITIES) }); + await assert.rejects(api.submitLive(REQUEST), code("LIVE_UNSUPPORTED")); + assert.deepEqual(calls.map(call => call.method), ["GET"]); +}); + +for (const maximum of [0, 30]) { + test(`LIVE preflight rejects waitSeconds above advertised maximum ${maximum} before POST`, async () => { + const { api, calls } = setup({ preflight: () => json({ ...capabilities, max_wait_seconds: maximum }) }); + await assert.rejects(api.submitLive(REQUEST, { waitSeconds: maximum + 1 }), ProtocolError); + assert.deepEqual(calls.map(call => call.method), ["GET"]); + }); +} + +test("LIVE preflight accepts waitSeconds zero when advertised maximum is zero", async () => { + const { api, calls } = setup({ preflight: () => json({ ...capabilities, max_wait_seconds: 0 }) }); + assert.equal((await api.submitLive(REQUEST, { waitSeconds: 0 })).state, "running"); + assert.equal(calls.find(call => call.method === "POST").headers.get("Prefer"), "wait=0"); +}); + +test("LIVE inspection/cancel remain explicit and do not preflight, submit, or open streams", async () => { + const fixture = liveFixture(); + const { api, calls } = setup({ fixture, status: call => json(call.method === "POST" + ? fixture.live("cancelled") : fixture.running) }); + assert.deepEqual(await api.getLiveExecution("exec-1"), fixture.running); + assert.equal((await api.cancelLive("exec-1")).state, "cancelled"); + assert.deepEqual(calls.map(c => [c.method, c.url.pathname]), + [["GET", "/queries/exec-1"], ["POST", "/queries/exec-1/cancel"]]); +}); + +test("LIVE readLiveBatches attaches to submission without a second POST or preflight", async () => { + const { api, calls } = setup(); + const submitted = await api.submitLive(REQUEST); + const result = await drain(api.readLiveBatches(submitted)); + assert.equal(result.execution.state, "succeeded"); + assert.equal(calls.filter(c => c.method === "POST").length, 1); + assert.equal(calls.filter(c => c.url.pathname === "/capabilities").length, 1); + await assert.rejects(collect(api.readLiveBatches(result.execution)), code("LIVE_STREAM_CONSUMED")); + api.forgetExecution("exec-1"); + await assert.rejects(collect(api.readLiveBatches(result.execution)), code("LIVE_STREAM_CONSUMED")); +}); + +test("LIVE active concurrent readers cannot open a second stream", async () => { + const fixture = liveFixture(); + const body = gatedBody(fixture.bytes); + const { api, calls } = setup({ fixture, data: () => body.response, status: () => json(fixture.running) }); + const iterator = api.readLiveBatches(fixture.running); + await iterator.next(); + await assert.rejects(collect(api.readLiveBatches(fixture.running)), code("LIVE_STREAM_CONSUMED")); + await iterator.return(); + assert.equal(calls.filter(c => c.url.pathname.startsWith("/results/")).length, 1); +}); + +test("LIVE consumption registry is bounded and fails closed without evicting a consumption fence", async () => { + const fixture = liveFixture(); + const { api, calls } = setup({ fixture, initial: { ...fixture.running, execution_id: "submit-only" } }); + for (let i = 0; i < MAX_TRACKED_EXECUTIONS; i++) { + const value = { ...fixture.succeeded, execution_id: `unique-${i}` }; + // The mismatching IPC identity fails after claiming the one-use fence. + await assert.rejects(collect(api.readLiveBatches(value)), ProtocolError); + } + await assert.rejects(collect(api.readLiveBatches(fixture.succeeded)), code("LIVE_STREAM_LIMIT")); + assert.equal(calls.filter(c => c.url.pathname.startsWith("/results/")).length, MAX_TRACKED_EXECUTIONS); + const count = calls.length; + await assert.rejects(api.executeLive(REQUEST), code("LIVE_STREAM_LIMIT")); + await assert.rejects(collect(api.stream(REQUEST)), code("LIVE_STREAM_LIMIT")); + assert.equal(calls.length, count, "Predictable saturation must not make capability or submission requests"); + assert.equal((await api.submitLive(REQUEST)).state, "running", "Submission-only remains a separate explicit operation"); + assert.equal(calls.filter(c => c.method === "POST").length, 1); +}); + +test("LIVE consuming submissions reserve remaining capacity before concurrent preflight", async () => { + const ready = deferred(), preflight = deferred(); + const { api, calls } = setup({ preflight: () => { ready.resolve(); return preflight.promise; } }); + for (let index = 0; index < MAX_TRACKED_EXECUTIONS - 1; index++) api.consumedLive.add(`older-${index}`); + const first = api.executeLive(REQUEST); + await ready.promise; + await assert.rejects(api.executeLive(REQUEST), code("LIVE_STREAM_LIMIT")); + assert.equal(calls.length, 1, "Only the admitted call starts capability discovery"); + preflight.resolve(json(capabilities)); + assert.equal((await first).table.numRows, 2); + assert.equal(api.reservedLive, 0); + assert.equal(calls.filter(c => c.method === "POST").length, 1); +}); + +test("LIVE failed capability preflight releases its reserved reader capacity", async () => { + let attempts = 0; + const { api } = setup({ preflight: () => json(attempts++ === 0 ? CAPABILITIES : capabilities) }); + for (let index = 0; index < MAX_TRACKED_EXECUTIONS - 1; index++) api.consumedLive.add(`older-${index}`); + await assert.rejects(api.executeLive(REQUEST), code("LIVE_UNSUPPORTED")); + assert.equal(api.reservedLive, 0); + assert.equal((await api.executeLive(REQUEST)).table.numRows, 2); + assert.equal(api.reservedLive, 0); +}); + +test("LIVE terminal failure before acquisition releases capacity without clearing consumed ids", async () => { + const fixture = liveFixture(); + const { api, calls } = setup({ fixture, initial: fixture.live("failed", { stream: undefined, error: ERROR }) }); + for (let index = 0; index < MAX_TRACKED_EXECUTIONS - 1; index++) api.consumedLive.add(`older-${index}`); + await assert.rejects(api.executeLive(REQUEST), LiveQueryExecutionError); + assert.equal(api.reservedLive, 0); + assert.equal(api.consumedLive.size, MAX_TRACKED_EXECUTIONS - 1); + assert.equal(calls.filter(c => c.url.pathname.startsWith("/results/")).length, 0); +}); + +test("LIVE observation history stays bounded independently of single-use consumption fences", async () => { + const fixture = liveFixture(); + const { api, calls } = setup({ fixture, status: call => json(fixture.live("cancelled", { + execution_id: decodeURIComponent(call.url.pathname.split("/").at(-1)), + })) }); + for (let index = 0; index < MAX_TRACKED_EXECUTIONS * 2; index++) { + await api.getLiveExecution(`history-${index}`); + } + assert.equal(api.liveExecutions.size, MAX_TRACKED_EXECUTIONS); + assert.equal(api.liveExecutions.has("history-0"), false); + assert.equal(api.consumedLive.size, 0); + const count = calls.length; + api.forgetExecution(`history-${MAX_TRACKED_EXECUTIONS * 2 - 1}`); + assert.equal(api.liveExecutions.size, MAX_TRACKED_EXECUTIONS - 1); + assert.equal(calls.length, count); +}); + +for (const [options, expected] of [ + [{ maxResultRows: 0 }, "RESULT_LIMIT_EXCEEDED"], + [{ maxResultBytes: 32 }, "RESULT_LIMIT_EXCEEDED"], +]) { + test(`LIVE incremental configured bounds: ${JSON.stringify(options)}`, async () => { + const { api } = setup({ options }); + await assert.rejects(api.executeLive(REQUEST), code(expected)); + }); +} + +test("LIVE expiry is checked before obtaining result credentials", async () => { + const fixture = liveFixture(); + fixture.stream.expires_at = "2000-01-01T00:00:00Z"; + let tokens = 0; + const { api, calls } = setup({ fixture, options: { getAccessToken: () => { tokens++; return "SYNTHETIC"; } } }); + await assert.rejects(collect(api.readLiveBatches(fixture.running)), code("RESULT_EXPIRED")); + assert.equal(tokens, 0); + assert.equal(calls.length, 0); +}); + +test("LIVE already-open body may finish after stream-start expiry", async () => { + const fixture = liveFixture(); + fixture.stream.expires_at = new Date(Date.now() + 200).toISOString(); + const body = gatedBody(fixture.bytes); + const { api } = setup({ fixture, data: () => body.response }); + const iterator = api.stream(REQUEST); + await iterator.next(); + await delay(210); + body.finish(); + assert.equal((await drain(iterator)).execution.state, "succeeded"); +}); + +for (const stage of ["preflight", "queued", "body"]) { + test(`LIVE caller abort during ${stage} cancels locally without backend cancel`, async () => { + const fixture = liveFixture(); + const gate = deferred(); + const abort = new AbortController(); + const body = gatedBody(fixture.bytes); + const { api, calls } = setup({ + fixture, ...(stage === "queued" ? { initial: fixture.live("queued") } : {}), + preflight: stage === "preflight" ? () => { gate.resolve(); return new Promise(() => {}); } : undefined, + data: () => body.response, status: () => { + gate.resolve(); + return json(stage === "queued" ? fixture.live("queued") : fixture.running); + }, + }); + const iterator = api.stream(REQUEST, { signal: abort.signal }); + const first = iterator.next(); + const rejected = assert.rejects(stage === "body" ? first.then(() => iterator.next()) : first, code("ABORTED")); + if (stage === "body") await first; + else await gate.promise; + abort.abort(); + await rejected; + assert.ok(calls.every(c => !c.url.pathname.endsWith("/cancel"))); + if (stage === "body") assert.equal(body.cancelled, true); + }); +} + +test("LIVE one overall deadline includes preflight, POST, and a stalled body", async () => { + const fixture = liveFixture(); + const body = gatedBody(fixture.bytes); + const { api, calls } = setup({ fixture, data: () => body.response, options: { timeoutMs: 60 }, + preflight: async () => { await delay(30); return json(capabilities); }, + status: () => json(fixture.running), + }); + await assert.rejects(api.executeLive(REQUEST), code("TIMEOUT")); + assert.equal(body.cancelled, true); + assert.equal(calls.filter(c => c.method === "POST").length, 1); +}); + +test("LIVE early return aborts in-flight status, releases body and does not cancel backend", async () => { + const fixture = liveFixture(); + const body = gatedBody(fixture.bytes); + const polling = deferred(); + const { api, calls } = setup({ fixture, data: () => body.response, status: call => { + polling.resolve(call.signal); + return new Promise(() => {}); + } }); + const iterator = api.stream(REQUEST); + await iterator.next(); + const statusSignal = await polling.promise; + await iterator.return(); + assert.equal(statusSignal.aborted, true); + assert.equal(body.cancelled, true); + assert.equal(body.response.body.locked, false); + const count = calls.length; + await delay(10); + assert.equal(calls.length, count); + assert.ok(calls.every(c => !c.url.pathname.endsWith("/cancel"))); +}); + +test("LIVE async progress mutations cannot rewrite identity; callback failure cleans up", async () => { + const fixture = liveFixture(); + const body = gatedBody(fixture.bytes); + const { api } = setup({ fixture, data: () => body.response, status: () => json(fixture.running) }); + let progress = 0; + const boom = new Error("SYNTHETIC_CALLBACK_FAILURE"); + const iterator = api.stream(REQUEST, { onProgress: async e => { + e.model.revision = "caller mutation"; + await Promise.resolve(); + if (++progress > 1) throw boom; + } }); + await iterator.next(); + await assert.rejects(iterator.next(), error => error === boom); + assert.equal(body.cancelled, true); + assert.equal((await api.getLiveExecution("exec-1")).model.revision, MODEL.revision); +}); + +test("LIVE concurrent cancel wins over older pending poll without regressing history", async () => { + const fixture = liveFixture(); + const pending = deferred(), started = deferred(); + let delayed = true; + const { api } = setup({ fixture, status: call => { + if (call.method === "POST" || !delayed) return json(fixture.live("cancelled")); + started.resolve(); + return pending.promise; + } }); + await api.submitLive(REQUEST); + const poll = api.getLiveExecution("exec-1"); + await started.promise; + assert.equal((await api.cancelLive("exec-1")).state, "cancelled"); + delayed = false; + pending.resolve(json(fixture.running)); + assert.equal((await poll).state, "cancelled"); + assert.equal((await api.getLiveExecution("exec-1")).state, "cancelled"); +}); + +test("LIVE exact decimal and nanosecond buffers are preserved", async () => { + const columns = [ + { column_id: "decimal", name: "decimal", logical_type: "Decimal", nullable: false }, + { column_id: "timestamp", name: "timestamp", logical_type: "DateTime", nullable: false }, + ]; + const fields = [new Decimal(9, 30, 128), new TimestampNanosecond()].map((type, i) => + new Field(columns[i].name, type, false, new Map([[COLUMN_METADATA_KEY, JSON.stringify({ column_id: columns[i].column_id })]]))); + const decimal = new Uint32Array([0xffffffff, 0x12345678, 0xabcdef12, 0]); + const timestamp = new BigInt64Array([1700000000123456789n]); + const fixture = liveFixture({ columns, fields, batchRows: [[[null, null]]], columnData: [ + makeData({ type: fields[0].type, length: 1, data: decimal }), + makeData({ type: fields[1].type, length: 1, data: timestamp }), + ] }); + const { api } = setup({ fixture }); + const { table } = await api.executeLive(REQUEST); + assert.deepEqual(table.getChildAt(0).data[0].values, decimal); + assert.deepEqual(table.getChildAt(1).data[0].values, timestamp); +}); + +for (const accepted of [false, true]) { + test(`LIVE truncation is checked only after completion; acceptance=${accepted}`, async () => { + const fixture = liveFixture({ manifest: { completeness: "truncated", truncation_reason: "SERVER_LIMIT" } }); + const { api } = setup({ fixture }); + const result = api.executeLive(REQUEST, { allowTruncated: accepted }); + if (accepted) assert.equal((await result).table.numRows, 2); + else await assert.rejects(result, code("TRUNCATED_RESULT")); + }); +} + +test("LIVE queued then running without a stream polls only until a descriptor is available", async () => { + const fixture = liveFixture(); + const pending = [fixture.live("running", { stream: undefined }), fixture.running, fixture.succeeded]; + const { api, calls } = setup({ fixture, initial: fixture.live("queued"), status: () => json(pending.shift()) }); + const result = await api.executeLive(REQUEST); + assert.equal(result.table.numRows, 2); + const paths = calls.map(c => c.url.pathname); + assert.ok(paths.indexOf("/results/result-1") < paths.lastIndexOf("/queries/exec-1"), + "Download begins before the final status read"); +}); + +for (const phase of ["submit", "download"]) { + test(`LIVE ${phase} HTTP failure is not retried or silently materialized`, async () => { + const problem = () => json({ type: "about:blank", title: "Unavailable", status: 503, + error: { ...ERROR, code: "UNAVAILABLE", retryable: true } }, 503, + { "Content-Type": "application/problem+json", "Retry-After": "0" }); + const fixture = liveFixture(); + const mock = mockFetch(call => { + if (call.url.pathname === "/capabilities") return json(capabilities); + if (call.url.pathname === "/queries") return phase === "submit" ? problem() : json(fixture.running, 202); + if (call.url.pathname.startsWith("/results/")) return problem(); + return json(fixture.succeeded); + }); + const api = new OssieClient({ baseUrl: ORIGIN, fetch: mock.fetch }); + await assert.rejects(api.executeLive(REQUEST), code("UNAVAILABLE")); + assert.equal(mock.calls.filter(c => c.method === "POST").length, 1); + assert.equal(mock.calls.filter(c => c.url.pathname.startsWith("/results/")).length, phase === "submit" ? 0 : 1); + }); +} + +test("LIVE malformed IPC cannot be cured by already succeeded status", async () => { + const fixture = liveFixture(); + const { api } = setup({ fixture, initial: fixture.succeeded, + data: () => arrowResponse(fixture.bytes.slice(0, -9)), + }); + await assert.rejects(api.executeLive(REQUEST), error => { + assert.ok(error instanceof ProtocolError); + assert.match(error.message, /inside a message/); + return true; + }); +}); + +for (const patch of [ + { contract_version: "0.1-draft" }, + { result_id: "wrong" }, + { model: { id: "sales-model", revision: "wrong" } }, + { delivery: "live" }, + { page_index: 0, row_offset: 0 }, +]) { + test(`LIVE rejects incorrect Arrow execution metadata: ${JSON.stringify(patch)}`, async () => { + const fixture = liveFixture({ metadata: { + contract_version: LIVE_CONTRACT_VERSION, execution_id: "exec-1", result_id: "result-1", + data_source_id: "sales", model: MODEL, ...patch, + } }); + const { api } = setup({ fixture }); + await assert.rejects(api.executeLive(REQUEST), ProtocolError); + }); +} + +test("LIVE status failure stops a body that never supplies even a schema", async () => { + const fixture = liveFixture(); + const body = gatedBody(fixture.bytes, 0); + const { api } = setup({ fixture, data: () => body.response, + status: () => { throw new Error("SYNTHETIC_STATUS_FAILURE"); }, + }); + await assert.rejects(api.executeLive(REQUEST), code("TRANSPORT_ERROR")); + assert.equal(body.cancelled, true); + assert.equal(body.response.body.locked, false); +}); + +test("LIVE timeout during preflight never POSTs", async () => { + const { api, calls } = setup({ preflight: () => new Promise(() => {}), options: { timeoutMs: 20 } }); + await assert.rejects(api.executeLive(REQUEST), code("TIMEOUT")); + assert.equal(calls.length, 1); + assert.equal(calls[0].signal.aborted, true); +}); + +test("LIVE original deadline remains TIMEOUT while verifying status after EOF", async () => { + const { api } = setup({ status: () => new Promise(() => {}), options: { timeoutMs: 40 } }); + await assert.rejects(api.executeLive(REQUEST), code("TIMEOUT")); +}); + +test("LIVE original caller abort remains ABORTED while verifying status after EOF", async () => { + const polled = deferred(); + const abort = new AbortController(); + const { api } = setup({ status: () => { polled.resolve(); return new Promise(() => {}); } }); + const result = api.executeLive(REQUEST, { signal: abort.signal }); + const rejected = assert.rejects(result, code("ABORTED")); + await polled.promise; + abort.abort(); + await rejected; +}); + +test("LIVE data protocol failure retains precise error when final status is unavailable", async () => { + const fixture = liveFixture(); + const { api } = setup({ fixture, initial: fixture.succeeded, + data: () => arrowResponse(fixture.bytes.slice(0, -9)), + status: () => { throw new Error("SYNTHETIC_UNAVAILABLE"); }, + }); + await assert.rejects(api.executeLive(REQUEST), error => { + assert.ok(error instanceof ProtocolError); + assert.match(error.message, /inside a message/); + return true; + }); +}); + +test("LIVE expiry is rechecked after delayed token acquisition without a result GET", async () => { + const fixture = liveFixture(); + fixture.stream.expires_at = new Date(Date.now() + 50).toISOString(); + const { api, calls } = setup({ fixture, options: { getAccessToken: async () => { + await delay(60); + return "SYNTHETIC"; + } } }); + await assert.rejects(collect(api.readLiveBatches(fixture.succeeded)), code("RESULT_EXPIRED")); + assert.equal(calls.filter(c => c.url.pathname.startsWith("/results/")).length, 0); +}); + +test("LIVE explicit cross-origin stream omits account tokens and cookies", async () => { + const fixture = liveFixture(); + fixture.stream.url = "https://results.example/results/result-1"; + const { api, calls } = setup({ fixture, options: { + allowedResultOrigins: ["https://results.example"], getAccessToken: () => "SYNTHETIC", + } }); + await api.executeLive(REQUEST); + const download = calls.find(c => c.url.origin === "https://results.example"); + assert.equal(download.headers.has("authorization"), false); + assert.equal(download.credentials, "omit"); + assert.equal(download.redirect, "error"); + assert.equal(download.headers.has("range"), false); +}); + +test("LIVE IPC byte counts include HTTP-decoded bytes, not Content-Length", async () => { + const fixture = liveFixture(); + const { api } = setup({ fixture, data: () => arrowResponse(fixture.bytes, { + "Content-Length": "10", "Content-Encoding": "gzip", + }) }); + assert.equal((await api.executeLive(REQUEST)).execution.completion.byte_count, fixture.bytes.length); +}); + +test("LIVE pending late poll cannot replace an advertised immutable descriptor", async () => { + const fixture = liveFixture(); + const pending = deferred(), started = deferred(); + let reads = 0; + const { api } = setup({ fixture, initial: fixture.live("queued"), status: () => { + if (++reads === 1) { started.resolve(); return pending.promise; } + return json(fixture.running); + } }); + await api.submitLive(REQUEST); + const stale = api.getLiveExecution("exec-1"); + await started.promise; + await api.getLiveExecution("exec-1"); + pending.resolve(json(fixture.live("running", { stream: undefined }))); + assert.deepEqual((await stale).stream, fixture.stream); +}); diff --git a/typescript/execution/test/schema.test.mjs b/typescript/execution/test/schema.test.mjs index 813279247..7da817773 100644 --- a/typescript/execution/test/schema.test.mjs +++ b/typescript/execution/test/schema.test.mjs @@ -28,13 +28,17 @@ import { import { BINDING, CAPABILITIES, ERROR, REQUEST, arrowFixture, execution, pagedFixture, } from "./fixtures.mjs"; +import { liveFixture } from "./live-fixtures.mjs"; const schema = JSON.parse(await readFile(new URL("../../../docs/proposals/execution-http/execution.schema.json", import.meta.url), "utf8")); +const liveSchema = JSON.parse(await readFile(new URL("../../../docs/proposals/execution-http/live.schema.json", import.meta.url), "utf8")); const openapi = JSON.parse(await readFile(new URL("../../../docs/proposals/execution-http/openapi.json", import.meta.url), "utf8")); const ajv = new Ajv2020({ allErrors: true, strict: false }); addFormats(ajv); -ajv.addSchema(schema); +ajv.addSchema(schema, "execution.schema.json"); +ajv.addSchema(liveSchema, "live.schema.json"); const validate = name => ajv.getSchema(`${schema.$id}#/$defs/${name}`); +const validateLive = name => ajv.getSchema(`${liveSchema.$id}#/$defs/${name}`); test("independent JSON Schema validators accept every fixture envelope and Arrow metadata document", () => { const fixture = arrowFixture(); @@ -111,6 +115,11 @@ test("OpenAPI defines lifecycle and references only available execution schema d assert.ok(schema.$defs[name], `Unresolved schema reference ${name}`); references++; } + if (typeof value.$ref === "string" && value.$ref.includes("live.schema.json#/$defs/")) { + const name = value.$ref.split("#/$defs/")[1]; + assert.ok(liveSchema.$defs[name], `Unresolved live schema reference ${name}`); + references++; + } for (const item of Object.values(value)) walk(item); } walk(openapi); @@ -121,9 +130,8 @@ test("OpenAPI media-type examples validate against their referenced contract def let examples = 0; function walk(value) { if (!value || typeof value !== "object") return; - const reference = value.schema?.$ref; - if (typeof reference === "string" && reference.includes("execution.schema.json#/$defs/")) { - const validator = validate(reference.split("#/$defs/")[1]); + if (value.schema) { + const validator = ajv.compile(value.schema); const values = [ ...(Object.hasOwn(value, "example") ? [value.example] : []), ...Object.values(value.examples ?? {}).filter(example => Object.hasOwn(example, "value")).map(example => example.value), @@ -153,7 +161,10 @@ test("normative draft JSON and HTTP-body examples conform structurally to the sh : "model" in value && "query_shapes" in value ? "Binding" : undefined; assert.ok(name, "Every documented JSON object must identify a known contract definition"); - const validator = validate(name); + const liveName = value.contract_version === "0.2-draft" + ? { Execution: "LiveExecution", Request: "LiveRequest", ExecutionMetadata: "LiveExecutionMetadata" }[name] + : undefined; + const validator = liveName ? validateLive(liveName) : validate(name); assert.ok(validator(value), `${name}: ${ajv.errorsText(validator.errors)}`); if (language === "http" && name === "ProblemDetails") { assert.equal(value.status, Number(/^HTTP\/1\.1 (\d+)/.exec(contents)?.[1])); @@ -163,3 +174,66 @@ test("normative draft JSON and HTTP-body examples conform structurally to the sh assert.deepEqual([...definitions].sort(), ["Binding", "Capabilities", "Execution", "ExecutionMetadata", "ProblemDetails", "Request"]); }); + +test("LIVE JSON Schema accepts opt-in envelopes and rejects materialized/live shape mixing", () => { + const fixture = liveFixture(); + const request = { ...REQUEST, contract_version: "0.2-draft", delivery: "live" }; + const examples = [ + ["LiveRequest", request], + ["LiveStream", fixture.stream], + ["LiveCompletion", fixture.completion], + ["LiveExecutionMetadata", JSON.parse(fixture.schema.metadata.get(EXECUTION_METADATA_KEY))], + ["LiveExecution", fixture.live("queued")], + ["LiveExecution", fixture.running], + ["LiveExecution", fixture.live("cancelling")], + ["LiveExecution", fixture.live("failed", { error: ERROR })], + ["LiveExecution", fixture.live("cancelled")], + ["LiveExecution", fixture.succeeded], + ]; + for (const [name, value] of examples) { + const validator = validateLive(name); + assert.ok(validator(value), `${name}: ${ajv.errorsText(validator.errors)}`); + } + const invalid = [ + ["LiveRequest", REQUEST], + ["LiveRequest", { ...request, contract_version: "0.1-draft" }], + ["LiveRequest", { ...request, query: { fields: ["x"], measures: ["y"] } }], + ["LiveStream", { ...fixture.stream, row_count: 2 }], + ["LiveStream", { ...fixture.stream, completeness: "complete" }], + ["LiveStream", { ...fixture.stream, replayable: true }], + ["LiveStream", { ...fixture.stream, pages: [] }], + ["LiveCompletion", { ...fixture.completion, row_count: Number.MAX_SAFE_INTEGER + 1 }], + ["LiveCompletion", { ...fixture.completion, completeness: "truncated" }], + ["LiveCompletion", { ...fixture.completion, truncation_reason: "unexpected" }], + ["LiveExecution", { ...fixture.running, result: arrowFixture().succeeded.result }], + ["LiveExecution", { ...fixture.running, state: "queued" }], + ["LiveExecution", { ...fixture.running, completion: fixture.completion }], + ["LiveExecution", { ...fixture.succeeded, completion: undefined }], + ["LiveExecution", { ...fixture.succeeded, stream: undefined }], + ["LiveExecution", fixture.live("failed")], + ["LiveExecution", fixture.live("cancelled", { progress: {} })], + ["LiveExecutionMetadata", { ...JSON.parse(fixture.schema.metadata.get(EXECUTION_METADATA_KEY)), page_index: 0, row_offset: 0 }], + ["LiveExecutionMetadata", { ...JSON.parse(fixture.schema.metadata.get(EXECUTION_METADATA_KEY)), delivery: "live" }], + ]; + for (const [name, value] of invalid) assert.equal(validateLive(name)(value), false, name); + assert.equal(validate("Request")(request), false); + assert.equal(validate("Execution")(fixture.running), false); +}); + +test("LIVE companion proposal JSON examples conform to the separate live schema", async () => { + const document = await readFile(new URL("../../../docs/proposals/execution-http/live-streaming.md", import.meta.url), "utf8"); + const definitions = new Set(); + for (const [, contents] of document.matchAll(/```json\r?\n([\s\S]*?)\r?\n```/g)) { + const value = JSON.parse(contents); + const name = "state" in value ? "LiveExecution" + : "query" in value ? "LiveRequest" + : "execution_id" in value ? "LiveExecutionMetadata" + : "url" in value ? "LiveStream" : "LiveCompletion"; + const validator = validateLive(name); + assert.ok(validator(value), `${name}: ${ajv.errorsText(validator.errors)}`); + definitions.add(name); + } + for (const required of ["LiveRequest", "LiveExecution", "LiveCompletion"]) { + assert.ok(definitions.has(required), `Missing documented ${required} example`); + } +});