diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d16690f729..18abebc02f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -125,6 +125,65 @@ jobs: timeout-minutes: 45 run: cargo test + sql-bridge: + # fluree-sql-bridge is a standalone workspace (it links sqlx + three + # database drivers, none of which belong in the fluree binary), so the + # workspace gates above never reach it. + runs-on: ubuntu-latest + # SQLite exercises the protocol in-process, but it is the one backend whose + # string literals cannot misbehave. The escaping rule the bridge enforces + # (NO_BACKSLASH_ESCAPES on MySQL) is only observable against a real server, + # so both are supplied here. `server_backends_are_configured_in_ci` fails if + # these ever stop being set, so a skipped test cannot read as a pass. + services: + mysql: + image: mysql:8 + env: + MYSQL_ROOT_PASSWORD: fluree + MYSQL_DATABASE: bridge_test + ports: + - 3306:3306 + options: >- + --health-cmd="mysqladmin ping -h 127.0.0.1 -uroot -pfluree" + --health-interval=5s + --health-timeout=5s + --health-retries=30 + postgres: + image: postgres:16 + env: + POSTGRES_PASSWORD: fluree + POSTGRES_DB: bridge_test + ports: + - 5432:5432 + options: >- + --health-cmd=pg_isready + --health-interval=5s + --health-timeout=5s + --health-retries=30 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@1.97.0 + - uses: rui314/setup-mold@v1 + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + workspaces: "fluree-sql-bridge -> target" + + - name: Format + working-directory: fluree-sql-bridge + run: cargo fmt --all -- --check + + - name: Clippy + working-directory: fluree-sql-bridge + run: cargo clippy --all-targets -- -D warnings + + - name: Test (SQLite, MySQL and Postgres backed protocol tests) + working-directory: fluree-sql-bridge + env: + FLUREE_BRIDGE_MYSQL_URL: mysql://root:fluree@127.0.0.1:3306/bridge_test + FLUREE_BRIDGE_POSTGRES_URL: postgres://postgres:fluree@127.0.0.1:5432/bridge_test + run: cargo test + bench-paths: # Cost gate for bench-compare below, which is a ~30-minute job. GitHub's # `paths:` filter is workflow-scoped and the rest of this workflow must run diff --git a/Cargo.lock b/Cargo.lock index 653e623f2b..3478e11ea1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2650,6 +2650,7 @@ dependencies = [ "fluree-db-shacl", "fluree-db-sparql", "fluree-db-spatial", + "fluree-db-sql", "fluree-db-storage-aws", "fluree-db-storage-ipfs", "fluree-db-tabular", @@ -2686,6 +2687,7 @@ dependencies = [ "tracing", "tracing-subscriber", "wasm-bindgen-futures", + "wiremock", "xxhash-rust", "zstd", ] @@ -3413,6 +3415,26 @@ dependencies = [ "zstd", ] +[[package]] +name = "fluree-db-sql" +version = "4.1.6" +dependencies = [ + "async-stream", + "async-trait", + "base64 0.22.1", + "chrono", + "fluree-db-iceberg", + "fluree-db-tabular", + "futures", + "reqwest", + "serde", + "serde_json", + "thiserror 1.0.69", + "tokio", + "tracing", + "wiremock", +] + [[package]] name = "fluree-db-storage-aws" version = "4.1.6" diff --git a/Cargo.toml b/Cargo.toml index 92b2131dfe..30007f8cf0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,7 @@ members = [ "fluree-db-iceberg", "fluree-db-tabular", "fluree-db-r2rml", + "fluree-db-sql", "fluree-db-server", "fluree-db-bolt", "fluree-db-peer", @@ -48,7 +49,7 @@ members = [ "fluree-db-consensus", "fluree-raft-core", ] -exclude = ["testsuite-sparql", "testsuite-shacl", "scripts/local/load"] +exclude = ["testsuite-sparql", "testsuite-shacl", "scripts/local/load", "fluree-sql-bridge"] [workspace.package] version = "4.1.6" diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 917cf09d4d..fc3caa6d84 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -50,6 +50,7 @@ - [mcp](cli/mcp.md) - [docs](cli/docs.md) - [iceberg](cli/iceberg.md) + - [sql](cli/sql.md) - [bm25](cli/bm25.md) - [materialize](cli/materialize.md) - [completions](cli/completions.md) @@ -174,6 +175,7 @@ - [Overview](graph-sources/overview.md) - [Iceberg / Parquet](graph-sources/iceberg.md) - [R2RML](graph-sources/r2rml.md) + - [SQL endpoints (Trino / bridge)](graph-sources/sql.md) - [BM25 graph source](graph-sources/bm25.md) - [Fluree for AI and agents](ai/README.md) diff --git a/docs/api/endpoints.md b/docs/api/endpoints.md index 92fdbaf481..0bf0fca98b 100644 --- a/docs/api/endpoints.md +++ b/docs/api/endpoints.md @@ -1750,7 +1750,7 @@ A flat array of ledgers and graph sources. Retracted entries are omitted. {"name": "mydb", "branch": "dev", "type": "Ledger", "t": 3}, {"name": "docsearch", "branch": "main", "type": "BM25", "t": 5, "dependencies": ["mydb:main"]}, - {"name": "warehouse", "branch": "main", "type": "Iceberg", "t": 0, + {"name": "orders-db", "branch": "main", "type": "Iceberg", "t": 0, "dependencies": ["mydb:main"]} ] ``` @@ -3042,6 +3042,73 @@ By default the server does not sync on commit, so an index only advances when so See also the CLI equivalent: [fluree bm25 sync](../cli/bm25.md#fluree-bm25-sync). +### POST {api_base_url}/sql/map + +Map tables behind a SQL endpoint as an R2RML graph source. The endpoint speaks the Trino client protocol (Trino, Starburst, PrestoDB, or a `fluree-sql-bridge` sidecar). Admin-protected — requires the admin Bearer token when an admin token is configured. Available only when the server is built with the `sql` feature (on by default). See [SQL graph sources](../graph-sources/sql.md). + +**URL:** +``` +POST {api_base_url}/sql/map +``` + +**Request Body:** + +```json +{ + "name": "orders-db", + "endpoint": "https://trino.example.com:8443", + "r2rml": "@prefix rr: . ...", + "r2rml_type": "text/turtle", + "branch": "main", + "dialect": "trino", + "protocol": "trino", + "catalog": "hive", + "schema": "sales", + "user": "fluree", + "auth_bearer": "…", + "session": { "query_max_run_time": "5m" } +} +``` + +| Field | Type | Description | +|-------|------|-------------| +| `name` | string | Graph source name (required) | +| `endpoint` | string | Statement endpoint base URL (required); `/v1/statement` is appended. Loopback/private hosts are allowed; the link-local/metadata range is refused. | +| `r2rml` | string | Inline R2RML mapping (required). `rr:tableName` and `rr:sqlQuery` logical tables are both accepted. | +| `r2rml_type` | string | Media type of `r2rml` (`text/turtle`, `application/ld+json`) | +| `branch` | string | Branch name (default: `main`) | +| `dialect` | string | `trino` (default), `postgres`, `mysql`, `sqlite` — the engine behind a bridge | +| `protocol` | string | `trino` (default, `X-Trino-*` headers) or `presto` | +| `catalog`, `schema` | string | Defaults for unqualified table names | +| `user` | string | Protocol user header (default `fluree`) | +| `auth_bearer` | string | Static bearer token | +| `oauth2_token_url`, `oauth2_client_id`, `oauth2_client_secret`, `oauth2_scope`, `oauth2_audience` | string | OAuth2 client-credentials flow (refreshes); `oauth2_token_url` is guarded against internal hosts | +| `session` | object | Session properties sent as `X-Trino-Session` | + +**Response:** + +```json +{ + "graph_source_id": "orders-db:main", + "endpoint": "https://trino.example.com:8443", + "connection_tested": true, + "mapping_source": "bafy…", + "triples_map_count": 3, + "table_count": 2, + "table_names": ["sales.customers", "sales.orders"], + "mapping_validated": true +} +``` + +`connection_tested` reports whether `SELECT 1` succeeded against the endpoint; a failure does not block registration. + +**Status Codes:** +- `201 Created` — graph source created +- `400 Bad Request` — invalid body, unknown `dialect`/`protocol`, endpoint refused by the SSRF guard, or an invalid mapping +- `401 Unauthorized` — admin token required + +--- + ### POST {api_base_url}/iceberg/materialize Materialize a graph source into a native ledger (so BM25 / vector / reasoning can run over it). Reads incrementally from a per-`(source, target, table)` watermark persisted in a shared `fluree_materialize_state:main` ledger, or fully with `force_full`. `target` may be a template that fans out into one ledger per partition (see the field table). Admin-protected; `iceberg` feature only. See [Materialization](../graph-sources/iceberg.md#materialization-into-a-native-ledger). diff --git a/docs/cli/README.md b/docs/cli/README.md index 078f525921..6fe7669413 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -75,6 +75,7 @@ fluree query 'SELECT ?name WHERE { ?s ?name }' | [`reindex`](reindex.md) | Full reindex from commit history | | [`sweep`](sweep.md) | Reclaim index artifacts no index chain references | | [`iceberg`](iceberg.md) | Map and manage Iceberg tables as graph sources (map, list, info, drop) | +| [`sql`](sql.md) | Map and manage SQL tables as graph sources through a Trino-protocol endpoint (map, list, info, drop) | | [`materialize`](materialize.md) | Build a native ledger twin from a virtual (Iceberg/R2RML) graph source | | [`bm25`](bm25.md) | Manage BM25 full-text search indexes (create, list, sync, drop) | diff --git a/docs/cli/sql.md b/docs/cli/sql.md new file mode 100644 index 0000000000..99c9386631 --- /dev/null +++ b/docs/cli/sql.md @@ -0,0 +1,113 @@ +# fluree sql + +Manage SQL graph sources — R2RML mappings over tables reached through a +Trino-protocol endpoint (Trino, Starburst, PrestoDB, or a `fluree-sql-bridge` +sidecar). See [SQL graph sources](../graph-sources/sql.md). + +## Subcommands + +| Subcommand | Description | +|------------|-------------| +| `map` | Map tables behind a SQL endpoint as a graph source | +| `list` | List mapped graph sources (SQL, Iceberg and R2RML) | +| `info` | Show details for a mapped graph source | +| `drop` | Drop a mapped graph source | + +`list`, `info` and `drop` are shared with [`fluree iceberg`](iceberg.md): both +commands operate on the same family of mapped sources. + +## fluree sql map + +### Usage + +```bash +fluree sql map --endpoint --r2rml [OPTIONS] +``` + +### Arguments + +| Argument | Description | +|----------|-------------| +| `` | Graph source name (e.g., "orders-db") | + +### Options + +**Endpoint:** + +| Option | Description | +|--------|-------------| +| `--endpoint ` | Statement endpoint base URL (required), e.g. `https://trino.example.com:8443` or `http://localhost:8080` for a sidecar | +| `--dialect ` | SQL rendering dialect: `trino` (default), `postgres`, `mysql`, `sqlite`. Use the engine behind a bridge. | +| `--protocol ` | Header family: `trino` (default) or `presto` | +| `--catalog ` | Default catalog for unqualified table names | +| `--schema ` | Default schema for unqualified table names | +| `--user ` | Protocol user (`X-Trino-User`); defaults to `fluree` | +| `--session KEY=VALUE` | Session property (repeatable), e.g. `--session query_max_run_time=5m` | + +**R2RML mapping:** + +| Option | Description | +|--------|-------------| +| `--r2rml ` | Mapping file (required). Each `rr:tableName` names a table reachable through the endpoint; `rr:sqlQuery` is also accepted. | +| `--r2rml-type ` | Mapping media type (e.g., `text/turtle`); inferred from extension if omitted | + +**Authentication:** + +| Option | Description | +|--------|-------------| +| `--auth-bearer ` | Static bearer token | +| `--oauth2-token-url ` | OAuth2 client-credentials token endpoint | +| `--oauth2-client-id ` | OAuth2 client ID | +| `--oauth2-client-secret ` | OAuth2 client secret | +| `--oauth2-scope ` | OAuth2 scope | +| `--oauth2-audience ` | OAuth2 audience | + +**General:** + +| Option | Description | +|--------|-------------| +| `--branch ` | Branch name (defaults to `main`) | +| `--remote ` | Execute against a remote server | + +### Examples + +```bash +# Trino with a bearer token; tables are qualified inside hive.sales +fluree sql map orders-db \ + --endpoint https://trino.example.com:8443 \ + --catalog hive --schema sales \ + --auth-bearer "$TRINO_TOKEN" \ + --r2rml mappings/orders.ttl + +# A bridge sidecar in front of Postgres +fluree sql map crm \ + --endpoint http://localhost:8080 \ + --dialect postgres --schema public \ + --r2rml mappings/crm.ttl +``` + +### Output + +``` +Mapped SQL endpoint as graph source 'orders-db:main' + Endpoint: https://trino.example.com:8443 + R2RML: bafy… + TriplesMaps: 3 + Tables: 2 (sales.orders, sales.customers) + Connection: verified + Mapping: validated +``` + +`Connection: not tested` means the `SELECT 1` probe failed; the source is +still registered and the first query reports the underlying error. + +## fluree sql list / info / drop + +```bash +fluree sql list +fluree sql info orders-db +fluree sql drop orders-db --force +``` + +Behave exactly as the [`fluree iceberg`](iceberg.md) equivalents; SQL sources +show the type `SQL`. diff --git a/docs/concepts/graph-sources.md b/docs/concepts/graph-sources.md index e015becde8..71c24e26d6 100644 --- a/docs/concepts/graph-sources.md +++ b/docs/concepts/graph-sources.md @@ -138,6 +138,22 @@ WHERE { See the [R2RML documentation](../graph-sources/r2rml.md) for details. +### SQL Endpoints + +**Differentiator**: The R2RML mapping runs over a live relational database or warehouse through a Trino-protocol HTTP endpoint — no copy, and no database driver inside Fluree. One Trino coordinator reaches Postgres, MySQL, SQL Server, Oracle, Snowflake, BigQuery and more; a small `fluree-sql-bridge` sidecar covers a single Postgres/MySQL/SQLite database without a JVM. + +**Use Cases:** +- A virtual graph over an operational database +- Federating a ledger with warehouse tables in one query +- Serverless deployments — every scan is a stateless HTTP request + +**Key Features:** +- Typed filter pushdown and exact `COUNT` per table; joins in the engine +- `rr:sqlQuery` logical tables +- Reads the current table state (no snapshots or time travel) + +See [SQL graph sources](../graph-sources/sql.md) for details. + ## Graph Source Lifecycle ### Creation diff --git a/docs/graph-sources/README.md b/docs/graph-sources/README.md index adc0fab1ee..f98c08f29e 100644 --- a/docs/graph-sources/README.md +++ b/docs/graph-sources/README.md @@ -31,6 +31,14 @@ Relational database mapping: - Join optimization - Supported databases (PostgreSQL, MySQL, etc.) +### [SQL endpoints](sql.md) + +Relational databases and warehouses through a Trino-protocol endpoint: +- Trino / Starburst / PrestoDB, or the `fluree-sql-bridge` sidecar +- R2RML mappings with `rr:tableName` and `rr:sqlQuery` +- Typed filter pushdown and exact `COUNT` +- No database drivers in the Fluree binary + ### [BM25 Graph Source](bm25.md) Full-text search as graph source: @@ -233,6 +241,19 @@ See [Iceberg / Parquet](iceberg.md). See [R2RML](r2rml.md). +### SQL Endpoints + +**Purpose:** Query relational databases and warehouses as RDF, live + +**Backend:** Any Trino-protocol endpoint — Trino/Starburst in front of Postgres, MySQL, SQL Server, Oracle, Snowflake, BigQuery, …, or the `fluree-sql-bridge` sidecar for a single database + +**Use Cases:** +- Virtual graph over an operational database, no copy +- One SPARQL query spanning a ledger and a warehouse table +- Lambda deployments (every scan is a stateless HTTP request) + +See [SQL endpoints](sql.md). + ## Architecture ### Graph Source Registry diff --git a/docs/graph-sources/overview.md b/docs/graph-sources/overview.md index 012487b413..0e429a145b 100644 --- a/docs/graph-sources/overview.md +++ b/docs/graph-sources/overview.md @@ -159,6 +159,23 @@ See [Iceberg / Parquet](iceberg.md) for full configuration details and examples. } ``` +### 4. SQL Endpoints + +**Backend:** Tables behind a Trino-protocol HTTP endpoint (Trino / Starburst / PrestoDB, or the `fluree-sql-bridge` sidecar), via R2RML mapping + +**Purpose:** Virtual graph over a relational database or warehouse, read live + +SQL sources use the same [R2RML mapping](r2rml.md) as Iceberg sources and additionally accept `rr:sqlQuery`. The engine pushes one typed single-table `SELECT` per triples map and performs joins itself. No database driver is linked into Fluree; the endpoint holds the connections. + +See [SQL endpoints](sql.md) for configuration, pushdown rules and the bridge. + +**Query:** +```sparql +PREFIX ex: +SELECT ?name ?total FROM +WHERE { ?o ex:customer ?c ; ex:total ?total . ?c ex:name ?name . FILTER(?total > 100) } +``` + ## Creating Graph Sources ### Via Rust API @@ -201,22 +218,27 @@ SELECT ?s ?p ?o FROM WHERE { ?s ?p ?o } LIMIT 10 ```json { "from": "mydb:main", + "from-named": ["warehouse-orders:main"], "select": ["?customer", "?orderId", "?total"], "where": [ { "@id": "?customer", "schema:name": "?name" }, { "@id": "?customer", "ex:customerId": "?custId" }, - { - "graph": "warehouse-orders:main", - "where": [ - { "@id": "?order", "ex:customerId": "?custId" }, - { "@id": "?order", "ex:orderId": "?orderId" }, - { "@id": "?order", "ex:total": "?total" } - ] - } + ["graph", "warehouse-orders:main", { + "@id": "?order", + "ex:customerId": "?custId", + "ex:orderId": "?orderId", + "ex:total": "?total" + }] ] } ``` +A graph pattern is the **array form** `["graph", , ]` — an object +with a `"graph"` key is parsed as an ordinary node pattern and fails. The graph +source must also be part of the dataset (`"from-named"` here, `FROM NAMED` in +SPARQL), or the block matches nothing. Address a graph source by its full id: +dataset-local `fromNamed` aliases do not currently resolve to graph sources. + Iceberg graph sources use R2RML mappings to define how table rows become RDF triples. See [Iceberg / Parquet](iceberg.md) and [R2RML](r2rml.md) for details. ### Query Patterns a Graph Source Cannot Evaluate diff --git a/docs/graph-sources/sql.md b/docs/graph-sources/sql.md new file mode 100644 index 0000000000..651ee2f70f --- /dev/null +++ b/docs/graph-sources/sql.md @@ -0,0 +1,286 @@ +# SQL Graph Sources + +Query tables in a relational database or data warehouse as RDF, through an +[R2RML mapping](r2rml.md), without loading the data into a ledger. A SQL graph +source reaches its tables over HTTP through any engine that speaks the **Trino +client protocol** — so nothing in Fluree holds a database connection, no JDBC +or native driver is compiled into the binary, and the same source works from a +long-running server and from a Lambda. + +## Where the SQL runs + +Fluree does not talk to Postgres, MySQL, Snowflake or Oracle directly. It sends +one `POST /v1/statement` per table scan to an endpoint and pages through the +result. Anything that implements that protocol works: + +| Endpoint | When to use it | +|----------|----------------| +| **Trino / Starburst** | The general answer. One Trino coordinator fronts Postgres, MySQL, SQL Server, Oracle, Snowflake, BigQuery, Redshift, Iceberg, Delta and dozens more through its connectors, and its client protocol is plain HTTP + JSON. | +| **PrestoDB** | Same protocol with the older `X-Presto-*` headers (`"protocol": "presto"`). | +| **`fluree-sql-bridge`** | A small sidecar for a single Postgres, MySQL or SQLite database when running a JVM is not wanted. It speaks the same protocol, so Fluree treats it exactly like Trino. See [Running the bridge](#running-the-bridge). | + +Every page of a result is one stateless HTTP request carrying its own +credentials; dropping a result stream cancels the statement server-side. + +## Registering a source + +=== CLI + +```bash +fluree sql map orders-db \ + --endpoint https://trino.example.com:8443 \ + --catalog hive --schema sales \ + --auth-bearer "$TRINO_TOKEN" \ + --r2rml mappings/orders.ttl +``` + +=== HTTP + +```bash +curl -X POST http://localhost:8090/v1/fluree/sql/map \ + -H "Authorization: Bearer $ADMIN_TOKEN" \ + -H "Content-Type: application/json" \ + -d @- <<'JSON' +{ + "name": "orders-db", + "endpoint": "https://trino.example.com:8443", + "catalog": "hive", + "schema": "sales", + "auth_bearer": "…", + "r2rml": "@prefix rr: . …" +} +JSON +``` + +=== Rust + +```rust +use fluree_db_api::{FlureeBuilder, SqlCreateConfig}; + +let fluree = FlureeBuilder::memory().build_memory(); +let mut config = SqlCreateConfig::new("orders-db", "https://trino.example.com:8443", MAPPING_TTL); +config.catalog = Some("hive".into()); +config.schema = Some("sales".into()); +fluree.create_sql_graph_source(config).await?; +``` + +Registration compiles the mapping, stores it in content-addressed storage, and +probes the endpoint with `SELECT 1`. A failed probe is reported +(`connection_tested: false`) but does not block registration — credentials can +be fixed later; the first query surfaces the real error. + +### Configuration + +| Field | Default | Meaning | +|-------|---------|---------| +| `endpoint` | — | Base URL; `/v1/statement` is appended | +| `dialect` | `trino` | How identifiers and literals are rendered: `trino`, `postgres`, `mysql`, `sqlite`. Use the engine *behind* a bridge. | +| `protocol` | `trino` | Header family: `trino` (`X-Trino-*`) or `presto` (`X-Presto-*`) | +| `catalog`, `schema` | — | Defaults for unqualified `rr:tableName`s | +| `user` | `fluree` | The protocol's user header; required even with a bearer token | +| `auth` | none | `bearer` (static token) or `oauth2_client_credentials`; values accept the same `env_var` / `secret_ref` indirection as Iceberg catalog auth | +| `session` | `{}` | Session properties, e.g. `{"query_max_run_time": "5m"}` | +| `request_timeout_secs` | `120` | Per page fetch | + +Table names in the mapping are dotted and quoted part by part: +`rr:tableName "sales.orders"` becomes `"sales"."orders"`; with `catalog` +set, an unqualified name resolves inside it. + +### `rr:sqlQuery` + +Unlike Iceberg sources, a SQL source accepts the R2RML `rr:sqlQuery` logical +table. The query is scanned as a derived table, with Fluree's projection and +pushed filters applied on top of it: + +```turtle +<#OpenOrders> a rr:TriplesMap ; + rr:logicalTable [ rr:sqlQuery "SELECT id, total FROM sales.orders WHERE status = 'open'" ] ; + rr:subjectMap [ rr:template "http://example.org/order/{id}" ; rr:class ex:Order ] ; + rr:predicateObjectMap [ rr:predicate ex:total ; rr:objectMap [ rr:column "total" ] ] . +``` + +```sql +-- what the engine sends for ?o ex:total ?total +SELECT "id", "total" FROM (SELECT id, total FROM sales.orders WHERE status = 'open') AS "__fluree_q" +``` + +The query text is trusted as written — a mapping author already has +root-equivalent read access to the source, exactly as with `rr:tableName`. + +## Querying + +A SQL source is queried like any other mapped source — as a `from` target, in +`FROM <…>`, or inside `GRAPH`: + +```sparql +PREFIX ex: +SELECT ?name ?total +FROM +WHERE { + ?o a ex:Order ; ex:customer ?c ; ex:total ?total . + ?c ex:name ?name . + FILTER(?total > 100) +} +``` + +### Joining with a ledger + +Put the ledger in `FROM`, the SQL source in `FROM NAMED`, and address it with +`GRAPH`. The join runs in the engine over the rows the source returns: + +```sparql +PREFIX ex: +SELECT ?name ?team +FROM +FROM NAMED +WHERE { + ?p ex:team ?team . + GRAPH { ?p ex:name ?name } +} +``` + +Without the `FROM NAMED`, the `GRAPH` block resolves to nothing and the join is +empty — the same dataset rule that applies to Iceberg sources. + +### What is pushed to SQL + +The query engine asks the source for **one table at a time** — a projection, +conjunctive filters, and nothing else — and does joins, `OPTIONAL`, `UNION`, +property paths and aggregation itself over the returned rows. So each triples +map touched by a query becomes one statement of the shape: + +```sql +SELECT "id", "customer_id", "total" FROM "sales"."orders" WHERE "total" > 1E2 +``` + +Pushed as `WHERE`: + +- `FILTER` comparisons and `IN` / single-variable `VALUES` on a mapped column +- constant objects (`?o ex:status "open"`) +- a bound subject (` ex:total ?t`), reversed + through the subject template to the key column + +Every predicate is rendered **against the column's type**, learned from a +cached `SELECT * FROM … LIMIT 0` probe. A literal that cannot be compared +safely with the column — a string against a `bigint`, a naive timestamp +against a `timestamp with time zone`, a NaN, or (on `dialect: mysql`) a string +containing a backslash — is simply not pushed. The in-engine `FILTER` remains +the authority in every case, so a declined push costs I/O, never correctness. + +`COUNT` over a single triples map is answered by an exact +`SELECT COUNT(*) … WHERE IS NOT NULL` — exact where the Iceberg +source can only use manifest statistics. Note the trade: this is a real query +against the endpoint, so cardinality on a large table costs an aggregate scan, +where an Iceberg source answers from metadata. Most engines optimize +`COUNT(*)`, but plan for it if a query shape asks for cardinality repeatedly. + +**Not pushed:** `ORDER BY … LIMIT`. A NULL in a key or required column would +consume `LIMIT` slots for rows the mapping drops, so the engine's own sort +runs over the full scan. Joins between triples maps on the same source are +also performed in the engine (a whole-query SQL rewrite in the Ontop style is +a possible later optimization, not a v1 requirement). + +### Types + +Trino's column types map onto Fluree's tabular types; the R2RML datatype +rules then apply as for any source. + +| SQL / Trino type | Fluree column | RDF datatype (default) | +|------------------|---------------|------------------------| +| `boolean` | Boolean | `xsd:boolean` | +| `tinyint`, `smallint`, `integer` | Int32 | `xsd:integer` | +| `bigint` | Int64 | `xsd:integer` | +| `real` | Float32 | `xsd:float` | +| `double` | Float64 | `xsd:double` | +| `decimal(p,s)` | Decimal | `xsd:decimal` (exact) | +| `varchar`, `char`, `json`, `uuid`, … | String | `xsd:string` | +| `varbinary` | Bytes | `xsd:base64Binary` | +| `date` | Date | `xsd:date` | +| `timestamp(p)` | Timestamp | `xsd:dateTime` | +| `timestamp(p) with time zone` | TimestampTz | `xsd:dateTime` (UTC) | +| `array`, `map`, `row` | String (Trino's JSON rendering) | `xsd:string` | + +Zoned timestamps are selected `AT TIME ZONE 'UTC'` on the Trino dialect, so a +value stored in a named region never has to be decoded client-side. Fractional +seconds beyond microseconds are truncated. + +## Freshness and materialization + +A SQL source has no snapshot: every query reads the tables as they are at that +moment. Consequently + +- `as-of` time travel is not available on a SQL source; +- [materialization](iceberg.md#materialization) into a twin ledger and + `fluree iceberg track` are **not yet supported** for SQL sources — both are + built on Iceberg snapshot windows, and a mutable table has no delta between + two reads. Attempting either returns a clear error naming the source. A + full-rebuild materialization is a planned follow-up. + +## Security + +- The endpoint is admin-configured, never query-supplied. The server route + requires the admin token like `/iceberg/map`. +- Outbound requests follow no redirects and refuse the link-local / + cloud-metadata range (`169.254.0.0/16`, `fe80::/10`) both up front and at + DNS resolution. Loopback and private hosts are **allowed** — a sidecar or a + Trino on the same network is the normal deployment — which is the same + posture as the Iceberg S3 `endpoint` override. +- Filter literals are rendered with proper quoting and typed against the + probed schema; identifiers are quoted per dialect. `rr:sqlQuery` text is + the mapping author's, not a query author's. +- String literals are escaped by the standard-SQL rule — `''` is an escaped + quote, and a backslash is an ordinary character. That holds on Trino, + Postgres and SQLite. MySQL's default `sql_mode` treats a backslash as live + inside a literal, so on `dialect: mysql` a value containing one is **not + pushed down** at all; the in-engine `FILTER` applies it instead. The bridge + additionally pins the rule on the sessions it opens — `NO_BACKSLASH_ESCAPES` + on MySQL, `standard_conforming_strings = on` on Postgres (already the + default there, set explicitly so a server-, database- or role-level override + cannot change it). If you point a source at some other Trino-protocol + endpoint, ensure the equivalent holds there. +- Credentials can be indirected (`{"env_var": "TRINO_TOKEN"}` or + `{"secret_ref": "…"}`) rather than stored inline — but only in a + graph-source record whose config JSON is authored directly. Both + `POST /v1/fluree/sql/map` and `fluree sql map` store what they are given as + a literal, so a secret supplied to either lives at rest in the record, which + should be protected accordingly. This matches how the Iceberg REST catalog + registers; accepting a `secret_ref` through those paths is a follow-up. + +## Running the bridge + +`fluree-sql-bridge` is a separate small binary (not part of `fluree`) that +exposes a Postgres, MySQL or SQLite database through the Trino client +protocol. Run it next to the database: + +```bash +fluree-sql-bridge --listen 0.0.0.0:8080 --database postgres://app:secret@db:5432/crm +``` + +then register the source with the engine's dialect: + +```bash +fluree sql map crm --endpoint http://bridge:8080 --dialect postgres --schema public --r2rml crm.ttl +``` + +The bridge holds the connection pool; Fluree holds nothing. It answers +`POST /v1/statement` with the same paged JSON Trino returns, reporting column +types in Trino's names, so everything on this page applies unchanged. + +## Comparison with Iceberg sources + +| | Iceberg source | SQL source | +|-|----------------|------------| +| Reads | Parquet files directly (S3/GCS/local) | SQL through an endpoint | +| Filters | file/row-group pruning by min/max stats | exact `WHERE` | +| `COUNT` | manifest stats, when provably exact | exact `COUNT(*)` | +| `ORDER BY … LIMIT` | top-k file ordering | not pushed | +| Snapshots / time travel | pinned per query, incremental twins | none; full rebuilds | +| `rr:sqlQuery` | refused | supported | +| Extra infrastructure | none | Trino, or a bridge sidecar | + +## See also + +- [R2RML mappings](r2rml.md) +- [`fluree sql` CLI](../cli/sql.md) +- [`POST /sql/map`](../api/endpoints.md#post-api_base_urlsqlmap) +- [Iceberg / Parquet sources](iceberg.md) diff --git a/docs/reference/crate-map.md b/docs/reference/crate-map.md index 4667381a08..00d0dcc24f 100644 --- a/docs/reference/crate-map.md +++ b/docs/reference/crate-map.md @@ -51,7 +51,8 @@ fluree-db/ ├── Graph Sources │ ├── fluree-db-tabular/ # Tabular column batch types │ ├── fluree-db-iceberg/ # Apache Iceberg integration -│ └── fluree-db-r2rml/ # R2RML mapping support +│ ├── fluree-db-r2rml/ # R2RML mapping support +│ └── fluree-db-sql/ # SQL graph sources (Trino-protocol HTTP) │ ├── Search │ ├── fluree-search-protocol/ # Search service protocol types @@ -560,6 +561,20 @@ crate takes plain IRIs, which is what makes it testable without a ledger. - fluree-db-tabular - fluree-vocab +### fluree-db-sql + +**Purpose:** SQL graph sources — R2RML scans over a Trino-protocol HTTP endpoint + +**Responsibilities:** +- Typed rendering of single-table scans (`SELECT … WHERE …`) against a probed schema +- The statement/page protocol client (streaming, retry, cancel-on-drop) +- Trino type names and JSON page values → column batches + +**Dependencies:** +- fluree-db-tabular +- fluree-db-iceberg (base: shared `ConfigValue` / auth / secret resolution) +- reqwest + ## Search Crates ### fluree-search-protocol diff --git a/docs/reference/vocabulary.md b/docs/reference/vocabulary.md index f81380a67d..e78c9b9abe 100644 --- a/docs/reference/vocabulary.md +++ b/docs/reference/vocabulary.md @@ -302,6 +302,7 @@ Nameservice records use `@type` to classify what kind of graph source a record r | `f:GeoIndex` | `https://ns.flur.ee/db#GeoIndex` | Geospatial index | | `f:IcebergMapping` | `https://ns.flur.ee/db#IcebergMapping` | Iceberg-mapped database | | `f:R2rmlMapping` | `https://ns.flur.ee/db#R2rmlMapping` | R2RML relational mapping | +| `f:SqlMapping` | `https://ns.flur.ee/db#SqlMapping` | R2RML mapping over a SQL (Trino-protocol) endpoint | --- diff --git a/fluree-db-api/Cargo.toml b/fluree-db-api/Cargo.toml index 135ec8db0d..2a5353dc17 100644 --- a/fluree-db-api/Cargo.toml +++ b/fluree-db-api/Cargo.toml @@ -17,6 +17,10 @@ aws = ["fluree-db-connection/aws", "dep:fluree-db-storage-aws", "dep:aws-sdk-sts # Arrow columnar reader is the single graph-source read path (native predicate # pushdown: row-group skipping + exact row filtering). iceberg = ["dep:fluree-db-iceberg", "fluree-db-iceberg/aws"] +# SQL graph sources (R2RML over a Trino-protocol endpoint). Stacks on `iceberg` +# because the graph-source provider dispatch lives there; the crate itself adds +# only an HTTP client, no database drivers. +sql = ["iceberg", "dep:fluree-db-sql"] # Opt-in LocalStack-backed S3/DynamoDB tests (auto-start via testcontainers) aws-testcontainers = [ "dep:fluree-db-storage-aws", @@ -42,7 +46,7 @@ vector = ["fluree-db-query/vector"] # IPFS-backed storage (via Kubo HTTP RPC) ipfs = ["dep:fluree-db-storage-ipfs"] # Convenience bundle (excludes vector, aws, and test-only features) -full = ["native", "credential", "iceberg", "shacl", "ipfs", "graphql"] +full = ["native", "credential", "iceberg", "sql", "shacl", "ipfs", "graphql"] [dependencies] fluree-db-core = { path = "../fluree-db-core" } @@ -61,6 +65,7 @@ fluree-db-cypher = { path = "../fluree-db-cypher" } fluree-db-graphql = { path = "../fluree-db-graphql", optional = true } fluree-db-iceberg = { path = "../fluree-db-iceberg", default-features = false, optional = true, features = ["aws"] } fluree-db-r2rml = { path = "../fluree-db-r2rml", features = ["turtle"] } +fluree-db-sql = { path = "../fluree-db-sql", optional = true } fluree-db-tabular = { path = "../fluree-db-tabular" } fluree-db-storage-aws = { path = "../fluree-db-storage-aws", optional = true } fluree-db-storage-ipfs = { path = "../fluree-db-storage-ipfs", optional = true } @@ -139,6 +144,7 @@ fluree-bench-support = { path = "../fluree-bench-support" } fluree-bench-alloc = { path = "../fluree-bench-alloc" } rand = { workspace = true } fluree-db-nameservice-sync = { path = "../fluree-db-nameservice-sync" } +wiremock = { workspace = true } # Residency-mode read arms for the wasm read-path recovery tests # (it_residency_retry): dev-only feature unification — production native # builds keep the feature off and compile the pre-residency read path. @@ -180,6 +186,13 @@ name = "it_iceberg_warehouse_root" path = "tests/it_iceberg_warehouse_root.rs" required-features = ["iceberg", "native"] +# End-to-end over a SQL graph source against a fake Trino-protocol endpoint +# (wiremock), through registration, the R2RML query path and pushdown. +[[test]] +name = "it_sql_graph_source" +path = "tests/it_sql_graph_source.rs" +required-features = ["sql", "native"] + [[test]] name = "grp_index" path = "tests/grp_index.rs" diff --git a/fluree-db-api/src/admin.rs b/fluree-db-api/src/admin.rs index 391e19b6ab..9a2b208a17 100644 --- a/fluree-db-api/src/admin.rs +++ b/fluree-db-api/src/admin.rs @@ -1524,11 +1524,9 @@ impl crate::Fluree { #[cfg(feature = "iceberg")] if matches!(mode, DropMode::Hard) { if let Some(ref record) = record { - // Try to delete the CAS-stored mapping blob - if let Ok(iceberg_config) = - fluree_db_iceberg::IcebergGsConfig::from_json(&record.config) + // Try to delete the CAS-stored mapping blob (Iceberg, R2RML or SQL record) { - if let Some(mapping) = &iceberg_config.mapping { + if let Some(mapping) = &crate::graph_source::mapping_source_of(record) { if let Ok(cid) = mapping.source.parse::() { // Resolve CID to storage path and delete let path = fluree_db_core::content_path( diff --git a/fluree-db-api/src/graph_source/cache.rs b/fluree-db-api/src/graph_source/cache.rs index ed0548defd..3b16953a27 100644 --- a/fluree-db-api/src/graph_source/cache.rs +++ b/fluree-db-api/src/graph_source/cache.rs @@ -23,6 +23,8 @@ use super::catalog_session::CachedLoadTable; use fluree_db_iceberg::catalog::RestCatalogClient; #[cfg(feature = "iceberg")] use fluree_db_iceberg::{io::parquet::ParquetFooterCache, metadata::TableMetadata, DataFile}; +#[cfg(feature = "sql")] +use fluree_db_sql::TrinoClient; #[cfg(feature = "iceberg")] use std::time::Duration; @@ -139,6 +141,13 @@ pub struct R2rmlCache { /// skip the ~1.3–3s catalog GET. #[cfg(feature = "iceberg")] rest_load_tables: SyncCache>, + + /// Process-wide SQL endpoint clients keyed like `rest_clients` (id + raw + /// config fingerprint), sharing its TTL rationale. Each client also holds + /// the per-table schema probes, so reuse across queries skips the + /// `LIMIT 0` round trip. + #[cfg(feature = "sql")] + sql_clients: SyncCache>, } // moka::sync::Cache is Send+Sync but doesn't implement Debug @@ -190,6 +199,11 @@ impl R2rmlCache { .max_capacity(metadata_cap) .time_to_live(Duration::from_secs(rest_loadtable_ttl_secs())) .build(), + #[cfg(feature = "sql")] + sql_clients: SyncCache::builder() + .max_capacity(64) + .time_to_live(Duration::from_secs(rest_client_ttl_secs())) + .build(), } } @@ -296,6 +310,16 @@ impl R2rmlCache { self.rest_clients.insert(fingerprint, client); } + #[cfg(feature = "sql")] + pub(crate) fn sql_client(&self, key: &str) -> Option> { + self.sql_clients.get(key) + } + + #[cfg(feature = "sql")] + pub(crate) fn put_sql_client(&self, key: String, client: Arc) { + self.sql_clients.insert(key, client); + } + /// Get a cross-query `loadTable` response if cached, within TTL, and its /// vended credentials are not near expiry; otherwise `None` (an expired /// entry is invalidated). Returns `None` when caching or the cross-query @@ -336,6 +360,8 @@ impl R2rmlCache { self.rest_clients.invalidate_all(); self.rest_load_tables.invalidate_all(); } + #[cfg(feature = "sql")] + self.sql_clients.invalidate_all(); } /// Get cache statistics. diff --git a/fluree-db-api/src/graph_source/catalog_session.rs b/fluree-db-api/src/graph_source/catalog_session.rs index fc730921fa..a50ddc81bc 100644 --- a/fluree-db-api/src/graph_source/catalog_session.rs +++ b/fluree-db-api/src/graph_source/catalog_session.rs @@ -130,9 +130,54 @@ pub(crate) struct IcebergCatalogSession { /// build (not once per table). Always cached (independent of the loadTable /// cache toggle) — the listing is stable for the build. warehouse_listings: Mutex>>>, + /// Graph sources this session has scanned that are SQL-backed. A SQL source + /// has no snapshot to pin, so the loadTable-cache precondition in + /// `verify_build_snapshot_integrity` does not apply to it. + sql_sources: Mutex>, + /// Per-query memo of the SQL-dispatch decision: `None` = Iceberg-backed. + /// Without it every `scan_table` / `table_row_count` would repeat the + /// nameservice lookup (two object reads on a storage-backed nameservice) + /// just to learn the source family. + #[cfg(feature = "sql")] + sql_dispatch: Mutex>>>, } impl IcebergCatalogSession { + pub(crate) fn mark_sql_source(&self, graph_source_id: &str) { + self.sql_sources + .lock() + .unwrap() + .insert(graph_source_id.to_string()); + } + + pub(crate) fn is_sql_source(&self, graph_source_id: &str) -> bool { + self.sql_sources.lock().unwrap().contains(graph_source_id) + } + + #[cfg(feature = "sql")] + pub(crate) fn sql_dispatch( + &self, + graph_source_id: &str, + ) -> Option>> { + self.sql_dispatch + .lock() + .unwrap() + .get(graph_source_id) + .cloned() + } + + #[cfg(feature = "sql")] + pub(crate) fn memo_sql_dispatch( + &self, + graph_source_id: &str, + decision: Option>, + ) { + self.sql_dispatch + .lock() + .unwrap() + .insert(graph_source_id.to_string(), decision); + } + /// Cache key for a `loadTable` response: source id + fully-qualified table. pub(crate) fn load_table_key(graph_source_id: &str, namespace: &str, table: &str) -> String { format!("{graph_source_id}\u{1f}{namespace}.{table}") diff --git a/fluree-db-api/src/graph_source/mod.rs b/fluree-db-api/src/graph_source/mod.rs index 9318ccb575..b7877e423a 100644 --- a/fluree-db-api/src/graph_source/mod.rs +++ b/fluree-db-api/src/graph_source/mod.rs @@ -141,6 +141,12 @@ mod ephemeral; #[cfg(feature = "iceberg")] mod r2rml_materialize; +#[cfg(feature = "sql")] +mod sql; + +#[cfg(feature = "sql")] +pub use sql::{SqlCreateConfig, SqlCreateResult}; + // Re-export configuration types pub use config::Bm25CreateConfig; @@ -168,7 +174,7 @@ pub use iceberg_sample::{sample_column_values, sample_iceberg_rows}; #[cfg(feature = "iceberg")] pub(crate) use iceberg_catalog::table_schema_from_metadata; #[cfg(feature = "iceberg")] -pub(crate) use r2rml::rest_client_cache_key; +pub(crate) use r2rml::{mapping_source_of, rest_client_cache_key}; #[cfg(feature = "iceberg")] pub use iceberg_generate::{ diff --git a/fluree-db-api/src/graph_source/r2rml.rs b/fluree-db-api/src/graph_source/r2rml.rs index 24461d9486..89b972f8e7 100644 --- a/fluree-db-api/src/graph_source/r2rml.rs +++ b/fluree-db-api/src/graph_source/r2rml.rs @@ -93,6 +93,56 @@ fn iceberg_scan_concurrency(num_files: usize) -> usize { /// reference, so rotating the underlying secret leaves the fingerprint unchanged /// — the client cache's TTL (see `cache::DEFAULT_REST_CLIENT_TTL_SECS`), not this /// fingerprint, is what bounds staleness in that case. +/// The R2RML mapping reference carried by a mapped graph-source record, per +/// source family. `None` for a non-mapped type, an unparseable config, or a +/// record registered without a mapping. +/// The Iceberg-only paths (snapshot pinning, incremental materialization, +/// tracking) parse the record as `IcebergGsConfig`; a SQL record would fail +/// that parse with a misleading message. Refuse it by name instead. +fn require_iceberg_backed( + record: &fluree_db_nameservice::GraphSourceRecord, + graph_source_id: &str, +) -> QueryResult<()> { + #[cfg(feature = "sql")] + if record.source_type == GraphSourceType::Sql { + return Err(QueryError::InvalidQuery(format!( + "Graph source '{graph_source_id}' is SQL-backed: snapshot pinning and \ + materialization are not available for SQL graph sources (they read the \ + live tables); query it directly instead" + ))); + } + let _ = (record, graph_source_id); + Ok(()) +} + +pub(crate) fn mapping_source_of( + record: &fluree_db_nameservice::GraphSourceRecord, +) -> Option { + match record.source_type { + GraphSourceType::R2rml | GraphSourceType::Iceberg => { + IcebergGsConfig::from_json(&record.config) + .ok() + .and_then(|c| c.mapping) + } + #[cfg(feature = "sql")] + GraphSourceType::Sql => super::sql::mapping_source(record), + _ => None, + } +} + +/// An Iceberg-backed source scans tables, never queries: refuse a mapping with +/// `rr:sqlQuery` at registration rather than at first query. +fn reject_sql_queries(compiled: &CompiledR2rmlMapping) -> Result<()> { + if compiled.has_sql_queries() { + return Err(crate::ApiError::Config( + "rr:sqlQuery logical tables are only supported by SQL graph sources; \ + use rr:tableName for Iceberg-backed mappings" + .to_string(), + )); + } + Ok(()) +} + fn config_fingerprint(config: &str) -> u64 { use std::hash::{Hash, Hasher}; let mut h = std::collections::hash_map::DefaultHasher::new(); @@ -551,6 +601,7 @@ impl crate::Fluree { // CID address, which is also extensionless). let compiled = Self::compile_r2rml_content(content, config.mapping_media_type.as_deref(), "")?; + reject_sql_queries(&compiled)?; let count = compiled.len(); let tables = Self::sorted_table_names(&compiled); let gs_id = config.graph_source_id(); @@ -684,7 +735,7 @@ impl crate::Fluree { /// `media_type` is given. Format selection goes through the shared /// [`fluree_db_r2rml::loader::MappingFormat`] resolver (default Turtle) so /// registration and query time can never disagree (issue #1397). - fn compile_r2rml_content( + pub(crate) fn compile_r2rml_content( content: &str, media_type: Option<&str>, source: &str, @@ -734,7 +785,7 @@ impl crate::Fluree { /// Collect the distinct logical table names referenced by a compiled /// mapping, sorted for deterministic reporting. - fn sorted_table_names(compiled: &CompiledR2rmlMapping) -> Vec { + pub(crate) fn sorted_table_names(compiled: &CompiledR2rmlMapping) -> Vec { let mut names: Vec = compiled .table_names() .into_iter() @@ -782,6 +833,35 @@ impl<'a> FlureeR2rmlProvider<'a> { } } + /// The SQL source behind `graph_source_id`, or `None` when it is + /// Iceberg-backed. Decided once per query session: the nameservice lookup + /// is not free on a storage-backed nameservice, and a query scans a source + /// once per triples map it touches. + #[cfg(feature = "sql")] + async fn sql_source( + &self, + graph_source_id: &str, + ) -> QueryResult>> { + if let Some(decision) = self.session.sql_dispatch(graph_source_id) { + return Ok(decision); + } + let record = self + .fluree + .nameservice() + .lookup_graph_source(graph_source_id) + .await + .map_err(|e| QueryError::Internal(format!("Nameservice error: {e}")))?; + let decision = match record { + Some(r) if r.source_type == GraphSourceType::Sql => Some(Arc::new( + super::sql::SqlSource::open(self.fluree, &r).await?, + )), + _ => None, + }; + self.session + .memo_sql_dispatch(graph_source_id, decision.clone()); + Ok(decision) + } + /// Resolve a graph source's storage backend, parsed table metadata, and /// metadata-location — the shared setup behind both full and incremental /// scans (REST/Direct × GCS/S3 × credentials × caching). @@ -805,6 +885,7 @@ impl<'a> FlureeR2rmlProvider<'a> { QueryError::InvalidQuery(format!("Graph source '{graph_source_id}' not found")) })?; + require_iceberg_backed(&record, graph_source_id)?; let iceberg_config = IcebergGsConfig::from_json(&record.config).map_err(|e| { QueryError::Internal(format!( "Failed to parse Iceberg graph source config for '{graph_source_id}': {e}" @@ -1096,6 +1177,7 @@ impl<'a> FlureeR2rmlProvider<'a> { .ok_or_else(|| { QueryError::InvalidQuery(format!("Graph source '{graph_source_id}' not found")) })?; + require_iceberg_backed(&record, graph_source_id)?; let config = IcebergGsConfig::from_json(&record.config).map_err(|e| { QueryError::Internal(format!( "Failed to parse Iceberg graph source config for '{graph_source_id}': {e}" @@ -1402,21 +1484,7 @@ impl R2rmlProvider for FlureeR2rmlProvider<'_> { .lookup_graph_source(graph_source_id) .await { - Ok(Some(record)) => { - // First check if this is an R2RML or Iceberg graph source type - if !matches!( - record.source_type, - GraphSourceType::R2rml | GraphSourceType::Iceberg - ) { - return false; - } - - // Parse into typed config to stay aligned with real config schema - match IcebergGsConfig::from_json(&record.config) { - Ok(config) => config.mapping.is_some(), - Err(_) => false, - } - } + Ok(Some(record)) => mapping_source_of(&record).is_some(), Ok(None) => false, Err(_) => false, } @@ -1441,29 +1509,23 @@ impl R2rmlProvider for FlureeR2rmlProvider<'_> { QueryError::InvalidQuery(format!("Graph source '{graph_source_id}' not found")) })?; - // Verify it's an R2RML or Iceberg graph source - if !matches!( - record.source_type, - GraphSourceType::R2rml | GraphSourceType::Iceberg - ) { + if !record + .source_type + .kind() + .eq(&fluree_db_nameservice::GraphSourceKind::Mapped) + { return Err(QueryError::InvalidQuery(format!( "Graph source '{}' is not an R2RML graph source (type: {:?})", graph_source_id, record.source_type ))); } - // Parse into typed config - let iceberg_config = IcebergGsConfig::from_json(&record.config).map_err(|e| { - QueryError::Internal(format!( - "Failed to parse graph source config for '{graph_source_id}': {e}" - )) - })?; - - let mapping_config = iceberg_config.mapping.as_ref().ok_or_else(|| { + let mapping_config = mapping_source_of(&record).ok_or_else(|| { QueryError::InvalidQuery(format!( "Graph source '{graph_source_id}' is missing 'mapping' in config" )) })?; + let mapping_config = &mapping_config; let mapping_source = &mapping_config.source; let media_type = mapping_config.media_type.as_deref(); @@ -1587,7 +1649,12 @@ impl R2rmlProvider for FlureeR2rmlProvider<'_> { graph_source_id: &str, ) -> std::result::Result<(), fluree_db_r2rml::R2rmlError> { use fluree_db_r2rml::R2rmlError; - if !super::catalog_session::cache_enabled() { + // A SQL source pins nothing (its watermark is endpoint+table+time), so + // the loadTable-cache precondition is meaningless for it. Known only + // once a scan has run, which is fine: the up-front check at build start + // still applies to a mixed session's Iceberg sources. + if !super::catalog_session::cache_enabled() && !self.session.is_sql_source(graph_source_id) + { return Err(R2rmlError::BuildSnapshotIntegrity( "the loadTable metadata cache is disabled (FLUREE_ICEBERG_LOADTABLE_CACHE=0), so \ Iceberg snapshot pinning is a no-op and the twin's stamped watermark cannot be \ @@ -1966,6 +2033,13 @@ impl FlureeR2rmlProvider<'_> { non_null_cols: &[String], _as_of_t: Option, ) -> QueryResult> { + #[cfg(feature = "sql")] + if let Some(sql) = self.sql_source(graph_source_id).await? { + let mapping = self.compiled_mapping(graph_source_id, None).await?; + return sql + .row_count(&self.session, &mapping, table_name, non_null_cols) + .await; + } // Same pinned context as the scan: one Iceberg snapshot per query (the // shared `self.session` pin), so a count and a scan cannot disagree. // GREP: r2rml-as-of-t — `as_of_t` is ignored here exactly as the scan path @@ -2180,6 +2254,12 @@ impl FlureeR2rmlProvider<'_> { graph_source_id: &str, table_name: &str, ) -> QueryResult<(Arc>, Arc, String)> { + if fluree_db_r2rml::mapping::LogicalTable::is_sql_query_alias(table_name) { + return Err(QueryError::InvalidQuery(format!( + "Graph source '{graph_source_id}': rr:sqlQuery logical tables are only \ + supported by SQL graph sources" + ))); + } // Look up the graph source record to get Iceberg connection info let record = self .fluree @@ -2192,6 +2272,7 @@ impl FlureeR2rmlProvider<'_> { })?; // Parse the Iceberg graph source config + require_iceberg_backed(&record, graph_source_id)?; let iceberg_config = IcebergGsConfig::from_json(&record.config).map_err(|e| { QueryError::Internal(format!( "Failed to parse Iceberg graph source config for '{graph_source_id}': {e}" @@ -2688,6 +2769,13 @@ impl FlureeR2rmlProvider<'_> { // `_as_of_t` is deliberately ignored. If as-of semantics ever land here, // `table_row_count_inner` MUST honor them identically (matching breadcrumb // there): a COUNT and a scan in one query must read the same snapshot. + #[cfg(feature = "sql")] + if let Some(sql) = self.sql_source(graph_source_id).await? { + let mapping = self.compiled_mapping(graph_source_id, None).await?; + return sql + .scan(&self.session, &mapping, table_name, projection, filters) + .await; + } info!( graph_source_id = %graph_source_id, table_name = %table_name, diff --git a/fluree-db-api/src/graph_source/sql.rs b/fluree-db-api/src/graph_source/sql.rs new file mode 100644 index 0000000000..263364a60e --- /dev/null +++ b/fluree-db-api/src/graph_source/sql.rs @@ -0,0 +1,490 @@ +//! SQL graph sources: an R2RML mapping over tables reached through a +//! Trino-protocol HTTP endpoint. +//! +//! Registration mirrors the Iceberg/R2RML path (mapping compiled and stored in +//! CAS, record published under `f:SqlMapping`), and scans are served through +//! the same [`super::FlureeR2rmlProvider`], which dispatches here when the +//! record's type is `Sql`. A SQL source has no snapshot to pin, so its build +//! watermark records the endpoint, table and first-touch time. + +use std::collections::BTreeMap; +use std::sync::Arc; + +use fluree_db_nameservice::{GraphSourceRecord, GraphSourceType}; +use fluree_db_query::error::{QueryError, Result as QueryResult}; +use fluree_db_query::r2rml::{ColumnBatchStream, ScanFilter, ScanValue, TableWatermark}; +use fluree_db_r2rml::mapping::CompiledR2rmlMapping; +use fluree_db_sql::{ + AuthConfig, CmpOp, Literal, LogicalSource, MappingSource, Predicate, ScanRequest, SqlDialect, + SqlError, SqlGsConfig, TrinoClient, WireProtocol, +}; +use futures::StreamExt; +use tracing::{debug, info, warn}; + +use super::config::R2rmlMappingInput; +use crate::graph_source::catalog_session::IcebergCatalogSession; + +/// Everything needed to register a SQL graph source. +#[derive(Debug, Clone)] +pub struct SqlCreateConfig { + /// Graph source name (e.g. `"warehouse-sql"`). + pub name: String, + /// Branch (defaults to `"main"`). + pub branch: Option, + /// Statement endpoint base URL. + pub endpoint: String, + pub dialect: SqlDialect, + pub protocol: WireProtocol, + pub catalog: Option, + pub schema: Option, + /// `X-Trino-User`; defaults to `fluree`. + pub user: Option, + pub auth: AuthConfig, + pub session: BTreeMap, + /// The R2RML mapping — inline content or a pre-existing address. + pub mapping: R2rmlMappingInput, + pub mapping_media_type: Option, +} + +impl SqlCreateConfig { + pub fn new( + name: impl Into, + endpoint: impl Into, + mapping_content: impl Into, + ) -> Self { + Self { + name: name.into(), + branch: None, + endpoint: endpoint.into(), + dialect: SqlDialect::default(), + protocol: WireProtocol::default(), + catalog: None, + schema: None, + user: None, + auth: AuthConfig::default(), + session: BTreeMap::new(), + mapping: R2rmlMappingInput::Content(mapping_content.into()), + mapping_media_type: None, + } + } + + pub fn effective_branch(&self) -> &str { + self.branch.as_deref().unwrap_or("main") + } + + pub fn graph_source_id(&self) -> String { + format!("{}:{}", self.name, self.effective_branch()) + } + + /// The persisted config, with the mapping's CAS address filled in. + pub fn to_gs_config(&self, mapping_address: &str) -> SqlGsConfig { + let mut cfg = SqlGsConfig::new(self.endpoint.clone()); + cfg.dialect = self.dialect; + cfg.protocol = self.protocol; + cfg.catalog = self.catalog.clone(); + cfg.schema = self.schema.clone(); + if let Some(u) = &self.user { + cfg.user = u.clone(); + } + cfg.auth = self.auth.clone(); + cfg.session = self.session.clone(); + let media_type = self.mapping_media_type.clone().unwrap_or_else(|| { + fluree_db_r2rml::loader::MappingFormat::resolve(None, mapping_address) + .media_type() + .to_string() + }); + cfg.mapping = Some(MappingSource { + source: mapping_address.to_string(), + media_type: Some(media_type), + }); + cfg + } + + pub fn validate(&self) -> crate::Result<()> { + if self.name.trim().is_empty() { + return Err(crate::ApiError::Config( + "graph source name must not be empty".to_string(), + )); + } + if self.name.contains(':') { + return Err(crate::ApiError::Config(format!( + "graph source name '{}' may not contain ':'", + self.name + ))); + } + self.to_gs_config("") + .validate() + .map_err(|e| crate::ApiError::Config(e.to_string())) + } +} + +/// What `create_sql_graph_source` reports back. +#[derive(Debug, Clone, serde::Serialize)] +pub struct SqlCreateResult { + pub graph_source_id: String, + pub endpoint: String, + pub mapping_source: String, + pub triples_map_count: usize, + pub table_count: usize, + pub table_names: Vec, + /// Whether `SELECT 1` succeeded against the endpoint. A failure is logged, + /// not fatal: the record is still created (credentials may arrive later). + pub connection_tested: bool, + pub mapping_validated: bool, +} + +impl crate::Fluree { + /// Register a SQL graph source. Compiles the mapping, stores it in CAS, + /// probes the endpoint, and publishes the record. + pub async fn create_sql_graph_source( + &self, + config: SqlCreateConfig, + ) -> crate::Result { + let graph_source_id = config.graph_source_id(); + info!(graph_source_id = %graph_source_id, "Creating SQL graph source"); + config.validate()?; + + let (mapping_address, triples_map_count, table_names, mapping_validated) = match &config + .mapping + { + R2rmlMappingInput::Content(content) => { + let compiled = + Self::compile_r2rml_content(content, config.mapping_media_type.as_deref(), "")?; + let count = compiled.len(); + let tables = Self::sorted_table_names(&compiled); + let cid = self + .content_store(&graph_source_id) + .put( + fluree_db_core::ContentKind::GraphSourceMapping, + content.as_bytes(), + ) + .await + .map_err(|e| { + crate::ApiError::Config(format!("Failed to store R2RML mapping: {e}")) + })?; + (cid.to_string(), count, tables, true) + } + R2rmlMappingInput::Address(address) => { + let storage = self.admin_storage().ok_or_else(|| { + crate::ApiError::Config( + "address-based mappings are not supported on this backend".to_string(), + ) + })?; + let (count, tables, validated) = match storage.read_bytes(address).await { + Ok(bytes) => match String::from_utf8(bytes) + .map_err(|e| crate::ApiError::Config(e.to_string())) + .and_then(|content| { + Self::compile_r2rml_content( + &content, + config.mapping_media_type.as_deref(), + address, + ) + }) { + Ok(compiled) => (compiled.len(), Self::sorted_table_names(&compiled), true), + Err(e) => { + warn!(graph_source_id = %graph_source_id, error = %e, "Could not validate R2RML mapping from address"); + (0, Vec::new(), false) + } + }, + Err(e) => { + warn!(graph_source_id = %graph_source_id, error = %e, "Could not read R2RML mapping from address"); + (0, Vec::new(), false) + } + }; + (address.clone(), count, tables, validated) + } + }; + + let gs_config = config.to_gs_config(&mapping_address); + let connection_tested = match self.test_sql_connection(&gs_config).await { + Ok(()) => true, + Err(e) => { + warn!(graph_source_id = %graph_source_id, error = %e, "SQL endpoint connection test failed; registering anyway"); + false + } + }; + + let config_json = gs_config + .to_json() + .map_err(|e| crate::ApiError::Config(format!("Failed to serialize config: {e}")))?; + self.publisher()? + .publish_graph_source( + &config.name, + config.effective_branch(), + GraphSourceType::Sql, + &config_json, + &[], + ) + .await?; + + info!(graph_source_id = %graph_source_id, mapping_address = %mapping_address, "Created SQL graph source"); + Ok(SqlCreateResult { + graph_source_id, + endpoint: gs_config.endpoint, + mapping_source: mapping_address, + triples_map_count, + table_count: table_names.len(), + table_names, + connection_tested, + mapping_validated, + }) + } + + /// `SELECT 1` against the endpoint with the configured credentials. + pub async fn test_sql_connection(&self, config: &SqlGsConfig) -> crate::Result<()> { + let client = build_sql_client(config, self.secret_resolver()) + .await + .map_err(|e| crate::ApiError::Config(e.to_string()))?; + client + .execute_collect("SELECT 1") + .await + .map(|_| ()) + .map_err(|e| { + crate::ApiError::Config(format!("SQL endpoint connection test failed: {e}")) + }) + } +} + +/// Hydrate secrets, build the auth provider, and construct the client. +async fn build_sql_client( + config: &SqlGsConfig, + resolver: Option<&Arc>, +) -> Result { + let hydrated = config.hydrate(resolver).await?; + let auth = hydrated.auth.create_provider_arc()?; + TrinoClient::new(&hydrated, auth) +} + +/// One SQL source resolved from its nameservice record. +pub(crate) struct SqlSource { + pub(crate) graph_source_id: String, + pub(crate) config: SqlGsConfig, + pub(crate) client: Arc, +} + +impl SqlSource { + /// Resolve the record's config and the (process-cached) client. The cache + /// key is a fingerprint of the RAW config so a secret rotation behind an + /// env var / secret ref does not rebuild the client every query. + pub(crate) async fn open( + fluree: &crate::Fluree, + record: &GraphSourceRecord, + ) -> QueryResult { + let config = SqlGsConfig::from_json(&record.config).map_err(|e| { + QueryError::Internal(format!( + "Failed to parse SQL graph source config for '{}': {e}", + record.graph_source_id + )) + })?; + let cache = fluree.r2rml_cache(); + let key = super::r2rml::rest_client_cache_key(&record.graph_source_id, &record.config); + let client = match cache.sql_client(&key) { + Some(c) => c, + None => { + let c = Arc::new( + build_sql_client(&config, fluree.secret_resolver()) + .await + .map_err(|e| { + QueryError::Internal(format!( + "SQL graph source '{}': {e}", + record.graph_source_id + )) + })?, + ); + cache.put_sql_client(key, Arc::clone(&c)); + c + } + }; + Ok(Self { + graph_source_id: record.graph_source_id.clone(), + config, + client, + }) + } + + /// A table name, or the `rr:sqlQuery` text behind a query alias. + fn source(&self, mapping: &CompiledR2rmlMapping, table_name: &str) -> LogicalSource { + match mapping.sql_query_for_table(table_name) { + Some(sql) => LogicalSource::Query(sql.to_string()), + None => LogicalSource::Table(table_name.to_string()), + } + } + + /// Stamp this table into the build watermark on first touch. + fn record_watermark(&self, session: &IcebergCatalogSession, table_name: &str) { + session.mark_sql_source(&self.graph_source_id); + session.record_snapshot( + IcebergCatalogSession::snapshot_key(&self.graph_source_id, table_name), + TableWatermark { + metadata_location: format!( + "sql://{}/{}@{}", + self.config + .endpoint_base() + .trim_start_matches("https://") + .trim_start_matches("http://"), + table_name, + chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true) + ), + snapshot_id: None, + sequence_number: None, + }, + ); + } + + pub(crate) async fn scan( + &self, + session: &IcebergCatalogSession, + mapping: &CompiledR2rmlMapping, + table_name: &str, + projection: &[String], + filters: &[ScanFilter], + ) -> QueryResult { + let source = self.source(mapping, table_name); + let schema = self + .client + .schema(&source) + .await + .map_err(|e| sql_query_error(&self.graph_source_id, table_name, e))?; + self.record_watermark(session, table_name); + + let request = ScanRequest { + source, + projection: projection.to_vec(), + predicates: filters.iter().map(to_predicate).collect(), + }; + let rendered = + fluree_db_sql::dialect::render_scan(&request, &schema, self.client.dialect()) + .map_err(|e| sql_query_error(&self.graph_source_id, table_name, e))?; + if !rendered.declined_predicates.is_empty() { + debug!( + graph_source_id = %self.graph_source_id, + table_name, + declined = ?rendered.declined_predicates, + "SQL pushdown declined some predicates (in-engine FILTER enforces them)" + ); + } + info!( + graph_source_id = %self.graph_source_id, + table_name, + sql = %rendered.sql, + "SQL table scan" + ); + + let gs = self.graph_source_id.clone(); + let table = table_name.to_string(); + let stream = self + .client + .execute(rendered.sql) + .map(move |item| item.map_err(|e| sql_query_error(&gs, &table, e))); + Ok(Box::pin(stream)) + } + + pub(crate) async fn row_count( + &self, + session: &IcebergCatalogSession, + mapping: &CompiledR2rmlMapping, + table_name: &str, + non_null_cols: &[String], + ) -> QueryResult> { + let source = self.source(mapping, table_name); + self.record_watermark(session, table_name); + let n = self + .client + .count(&source, non_null_cols) + .await + .map_err(|e| sql_query_error(&self.graph_source_id, table_name, e))?; + Ok(Some(n)) + } +} + +fn sql_query_error(graph_source_id: &str, table_name: &str, e: SqlError) -> QueryError { + let msg = format!("SQL graph source '{graph_source_id}', table '{table_name}': {e}"); + match e { + SqlError::Config(_) | SqlError::Unsupported(_) => QueryError::InvalidQuery(msg), + _ => QueryError::Internal(msg), + } +} + +fn to_predicate(f: &ScanFilter) -> Predicate { + use fluree_db_query::r2rml::ScanCmpOp; + Predicate { + column: f.column.clone(), + op: match f.op { + ScanCmpOp::Eq => CmpOp::Eq, + ScanCmpOp::NotEq => CmpOp::NotEq, + ScanCmpOp::Lt => CmpOp::Lt, + ScanCmpOp::LtEq => CmpOp::LtEq, + ScanCmpOp::Gt => CmpOp::Gt, + ScanCmpOp::GtEq => CmpOp::GtEq, + ScanCmpOp::In => CmpOp::In, + }, + value: to_literal(&f.value), + } +} + +fn to_literal(v: &ScanValue) -> Literal { + match v { + ScanValue::Bool(b) => Literal::Bool(*b), + ScanValue::Int(i) => Literal::Int(*i), + ScanValue::Date(d) => Literal::Date(*d), + ScanValue::Str(s) => Literal::Str(s.clone()), + ScanValue::Double(d) => Literal::Double(*d), + ScanValue::Decimal { + unscaled, scale, .. + } => Literal::Decimal { + unscaled: *unscaled, + scale: *scale, + }, + ScanValue::TemplateKey(k) => Literal::TemplateKey(k.clone()), + ScanValue::Set(members) => Literal::Set(members.iter().map(to_literal).collect()), + ScanValue::Timestamp { micros, tz } => Literal::Timestamp { + micros: *micros, + tz: *tz, + }, + } +} + +/// The mapping reference of a SQL record, if it has one. +pub(crate) fn mapping_source(record: &GraphSourceRecord) -> Option { + SqlGsConfig::from_json(&record.config) + .ok() + .and_then(|c| c.mapping) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn create_config_round_trips_into_gs_config() { + let mut c = SqlCreateConfig::new("wh", "http://localhost:8080/", "@prefix rr: ."); + c.catalog = Some("pg".into()); + c.user = Some("svc".into()); + assert_eq!(c.graph_source_id(), "wh:main"); + let gs = c.to_gs_config("bafy123"); + assert_eq!(gs.endpoint_base(), "http://localhost:8080"); + assert_eq!(gs.catalog.as_deref(), Some("pg")); + assert_eq!(gs.user, "svc"); + let m = gs.mapping.unwrap(); + assert_eq!(m.source, "bafy123"); + assert_eq!(m.media_type.as_deref(), Some("text/turtle")); + c.validate().unwrap(); + c.name = "a:b".into(); + assert!(c.validate().is_err()); + } + + #[test] + fn scan_filters_convert() { + let f = ScanFilter { + column: "id".into(), + op: fluree_db_query::r2rml::ScanCmpOp::In, + value: ScanValue::Set(vec![ScanValue::Int(1), ScanValue::TemplateKey("2".into())]), + }; + let p = to_predicate(&f); + assert_eq!(p.op, CmpOp::In); + assert_eq!( + p.value, + Literal::Set(vec![Literal::Int(1), Literal::TemplateKey("2".into())]) + ); + } +} diff --git a/fluree-db-api/src/ledger_info.rs b/fluree-db-api/src/ledger_info.rs index 45095062ab..ff1fdf49a2 100644 --- a/fluree-db-api/src/ledger_info.rs +++ b/fluree-db-api/src/ledger_info.rs @@ -1184,6 +1184,7 @@ pub fn graph_source_type_label(source_type: &GraphSourceType) -> String { GraphSourceType::Geo => "Geo".to_string(), GraphSourceType::R2rml => "R2RML".to_string(), GraphSourceType::Iceberg => "Iceberg".to_string(), + GraphSourceType::Sql => "SQL".to_string(), GraphSourceType::Unknown(s) => format!("Unknown({s})"), } } diff --git a/fluree-db-api/src/lib.rs b/fluree-db-api/src/lib.rs index 91e5de4ad6..4cf88b1ed6 100644 --- a/fluree-db-api/src/lib.rs +++ b/fluree-db-api/src/lib.rs @@ -229,6 +229,14 @@ pub use graph_source::{ ValidateR2rmlResponse, }; +#[cfg(feature = "sql")] +pub use fluree_db_sql::{ + validate_sql_endpoint, AuthConfig as SqlAuthConfig, ConfigValue as SqlConfigValue, SqlDialect, + SqlGsConfig, WireProtocol, +}; +#[cfg(feature = "sql")] +pub use graph_source::{SqlCreateConfig, SqlCreateResult}; + /// Secret-resolution injection point for `ConfigValue::SecretRef` in Iceberg /// graph-source auth. The host constructs a [`SecretResolver`] with the tenant /// captured and injects it via [`Fluree::with_secret_resolver`]; db stays diff --git a/fluree-db-api/tests/it_sql_graph_source.rs b/fluree-db-api/tests/it_sql_graph_source.rs new file mode 100644 index 0000000000..be13a30f27 --- /dev/null +++ b/fluree-db-api/tests/it_sql_graph_source.rs @@ -0,0 +1,564 @@ +//! End-to-end over a SQL graph source: registration, the R2RML query path, +//! typed filter pushdown and the exact COUNT shortcut — against a fake +//! Trino-protocol endpoint, so the SQL the engine actually sends is asserted. + +#![cfg(all(feature = "sql", feature = "native"))] + +use fluree_db_api::{CommitOpts, FlureeBuilder, IndexConfig, SqlCreateConfig, TxnOpts}; +use serde_json::{json, Value}; +use wiremock::matchers::{body_string_contains, method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +const PEOPLE_R2RML: &str = r#" + @prefix rr: . + @prefix ex: . + + + a rr:TriplesMap ; + rr:logicalTable [ rr:tableName "sales.people" ] ; + rr:subjectMap [ + rr:template "http://example.org/person/{id}" ; + rr:class ex:Person + ] ; + rr:predicateObjectMap [ + rr:predicate ex:name ; + rr:objectMap [ rr:column "name" ] + ] ; + rr:predicateObjectMap [ + rr:predicate ex:score ; + rr:objectMap [ rr:column "score" ] + ] ; + rr:predicateObjectMap [ + rr:predicate ex:born ; + rr:objectMap [ rr:column "born" ] + ] . +"#; + +fn columns() -> Value { + json!([ + {"name": "id", "type": "bigint"}, + {"name": "name", "type": "varchar"}, + {"name": "score", "type": "double"}, + {"name": "born", "type": "date"} + ]) +} + +fn finished(data: Value) -> ResponseTemplate { + ResponseTemplate::new(200).set_body_json(json!({ + "id": "q", + "columns": columns(), + "data": data, + "stats": {"state": "FINISHED"} + })) +} + +/// The fake endpoint. Mocks are tried in priority order (lower first). +async fn fake_trino() -> MockServer { + let server = MockServer::start().await; + // SELECT 1 — the registration-time connection test. + Mock::given(method("POST")) + .and(path("/v1/statement")) + .and(body_string_contains("SELECT 1")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "t", "columns": [{"name": "_col0", "type": "integer"}], "data": [[1]], "stats": {"state": "FINISHED"} + }))) + .with_priority(1) + .mount(&server) + .await; + // Schema probe. + Mock::given(method("POST")) + .and(path("/v1/statement")) + .and(body_string_contains("LIMIT 0")) + .respond_with(finished(json!([]))) + .with_priority(2) + .mount(&server) + .await; + // Exact count. + Mock::given(method("POST")) + .and(path("/v1/statement")) + .and(body_string_contains("COUNT(*)")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "c", "columns": [{"name": "_col0", "type": "bigint"}], "data": [[3]], "stats": {"state": "FINISHED"} + }))) + .with_priority(3) + .mount(&server) + .await; + // A pushed equality on name. + Mock::given(method("POST")) + .and(path("/v1/statement")) + .and(body_string_contains(r#""name" = 'bob'"#)) + .respond_with(finished(json!([[2, "bob", 7.5, "1990-05-04"]]))) + .with_priority(4) + .mount(&server) + .await; + // Any other scan of the table: every row. + Mock::given(method("POST")) + .and(path("/v1/statement")) + .and(body_string_contains(r#"FROM "sales"."people""#)) + .respond_with(finished(json!([ + [1, "alice", 9.25, "1985-01-02"], + [2, "bob", 7.5, "1990-05-04"], + [3, null, null, null] + ]))) + .with_priority(5) + .mount(&server) + .await; + server +} + +/// SPARQL JSON results → the binding rows. +fn bindings(v: &Value) -> Vec { + v.pointer("/results/bindings") + .and_then(Value::as_array) + .cloned() + .unwrap_or_else(|| panic!("not SPARQL JSON results: {v}")) +} + +async fn statements(server: &MockServer) -> Vec { + server + .received_requests() + .await + .unwrap_or_default() + .iter() + .filter(|r| r.method == "POST") + .map(|r| String::from_utf8_lossy(&r.body).to_string()) + .collect() +} + +#[tokio::test] +async fn sql_graph_source_end_to_end() { + let server = fake_trino().await; + let fluree = FlureeBuilder::memory().build_memory(); + + // 1. Register. + let mut config = SqlCreateConfig::new("people-sql", server.uri(), PEOPLE_R2RML); + config.catalog = Some("pg".into()); + let created = fluree + .create_sql_graph_source(config) + .await + .expect("create sql graph source"); + assert_eq!(created.graph_source_id, "people-sql:main"); + assert!(created.connection_tested, "SELECT 1 probe succeeded"); + assert!(created.mapping_validated); + assert_eq!(created.table_names, vec!["sales.people".to_string()]); + assert_eq!(created.triples_map_count, 1); + + let info = fluree + .nameservice() + .lookup_graph_source("people-sql:main") + .await + .expect("lookup") + .expect("record"); + assert_eq!( + info.source_type, + fluree_db_nameservice::GraphSourceType::Sql + ); + + // 2. A plain scan. + let query = json!({ + "@context": {"ex": "http://example.org/"}, + "from": "people-sql:main", + "select": ["?name"], + "where": {"@id": "?s", "ex:name": "?name"}, + }); + let rows = fluree + .query_from() + .jsonld(&query) + .execute_formatted() + .await + .expect("query sql source"); + let names: Vec = rows + .as_array() + .expect("array") + .iter() + .map(std::string::ToString::to_string) + .collect(); + assert_eq!( + names.len(), + 2, + "the null-name row yields no ex:name triple: {names:?}" + ); + assert!(names.iter().any(|n| n.contains("alice")) && names.iter().any(|n| n.contains("bob"))); + + let sent = statements(&server).await; + let probe = sent + .iter() + .find(|s| s.contains("LIMIT 0")) + .expect("schema probe was issued"); + assert_eq!(probe, r#"SELECT * FROM "sales"."people" LIMIT 0"#); + let scan = sent + .iter() + .find(|s| s.starts_with("SELECT \"") && !s.contains("WHERE")) + .expect("scan statement"); + assert!( + scan.contains(r#""id""#) && scan.contains(r#""name""#), + "{scan}" + ); + assert!( + !scan.contains(r#""score""#), + "only mapped+needed columns are projected: {scan}" + ); + + // 3. A constant object is pushed as a typed WHERE. + let sparql = r#" + PREFIX ex: + SELECT ?s ?score FROM + WHERE { ?s ex:name "bob" ; ex:score ?score } + "#; + let rows = fluree + .query_from() + .sparql(sparql) + .execute_formatted() + .await + .expect("filtered query"); + let rows = bindings(&rows); + assert_eq!(rows.len(), 1, "{rows:?}"); + assert!(rows[0].to_string().contains("person/2"), "{rows:?}"); + let sent = statements(&server).await; + assert!( + sent.iter().any(|s| s.contains(r#"WHERE "name" = 'bob'"#)), + "equality pushed to SQL: {sent:?}" + ); + + // 4. Typed decoding: a date column round-trips as xsd:date. + let sparql = " + PREFIX ex: + SELECT ?born FROM + WHERE { ex:born ?born } + "; + let rows = fluree + .query_from() + .sparql(sparql) + .execute_formatted() + .await + .expect("date query"); + assert!(rows.to_string().contains("1985-01-02"), "{rows}"); + + // 5. COUNT over the class answers 3 whether the exact shortcut fired or + // the scan counted (the fake is consistent); record which. + let sparql = " + PREFIX ex: + SELECT (COUNT(?s) AS ?n) FROM + WHERE { ?s a ex:Person } + "; + let rows = fluree + .query_from() + .sparql(sparql) + .execute_formatted() + .await + .expect("count query"); + assert!(rows.to_string().contains('3'), "{rows}"); + let sent = statements(&server).await; + eprintln!( + "COUNT(*) shortcut fired: {}", + sent.iter().any(|s| s.contains("COUNT(*)")) + ); +} + +const ORDERS_SQLQUERY_R2RML: &str = r#" + @prefix rr: . + @prefix ex: . + + + a rr:TriplesMap ; + rr:logicalTable [ rr:sqlQuery "SELECT id, total FROM sales.orders WHERE status = 'open'" ] ; + rr:subjectMap [ + rr:template "http://example.org/order/{id}" ; + rr:class ex:Order + ] ; + rr:predicateObjectMap [ + rr:predicate ex:total ; + rr:objectMap [ rr:column "total" ] + ] . +"#; + +/// An `rr:sqlQuery` logical table is scanned as a derived table, with the +/// projection and pushed filters applied on top of the mapping's query. +#[tokio::test] +async fn sql_query_logical_table_is_scanned_as_a_derived_table() { + let server = MockServer::start().await; + let orders_columns = + json!([{"name": "id", "type": "bigint"}, {"name": "total", "type": "decimal(10,2)"}]); + Mock::given(method("POST")) + .and(path("/v1/statement")) + .and(body_string_contains("SELECT 1")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "t", "columns": [{"name": "_col0", "type": "integer"}], "data": [[1]], "stats": {"state": "FINISHED"} + }))) + .with_priority(1) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/v1/statement")) + .and(body_string_contains("LIMIT 0")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "p", "columns": orders_columns, "data": [], "stats": {"state": "FINISHED"} + }))) + .with_priority(2) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/v1/statement")) + .and(body_string_contains(r#"AS "__fluree_q""#)) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "s", "columns": orders_columns, "data": [[10, "99.50"], [11, "5.00"]], "stats": {"state": "FINISHED"} + }))) + .with_priority(3) + .mount(&server) + .await; + + let fluree = FlureeBuilder::memory().build_memory(); + let created = fluree + .create_sql_graph_source(SqlCreateConfig::new( + "orders-sql", + server.uri(), + ORDERS_SQLQUERY_R2RML, + )) + .await + .expect("create"); + assert_eq!(created.table_count, 1); + assert!( + created.table_names[0].starts_with("sqlQuery:"), + "{:?}", + created.table_names + ); + + let sparql = " + PREFIX ex: + SELECT ?o ?total FROM + WHERE { ?o ex:total ?total } ORDER BY ?o + "; + let rows = fluree + .query_from() + .sparql(sparql) + .execute_formatted() + .await + .expect("query over rr:sqlQuery"); + let rows = bindings(&rows); + assert_eq!(rows.len(), 2, "{rows:?}"); + assert!( + rows[0].to_string().contains("order/10") && rows[0].to_string().contains("99.50"), + "{rows:?}" + ); + + let sent = statements(&server).await; + let scan = sent + .iter() + .find(|s| s.contains(r#"AS "__fluree_q""#) && !s.contains("LIMIT 0")) + .expect("derived-table scan"); + assert_eq!( + scan, + r#"SELECT "id", "total" FROM (SELECT id, total FROM sales.orders WHERE status = 'open') AS "__fluree_q""# + ); +} + +/// The Iceberg-backed registration path refuses `rr:sqlQuery` up front. +#[tokio::test] +async fn iceberg_sources_refuse_sql_query_mappings() { + let fluree = FlureeBuilder::memory().build_memory(); + let config = fluree_db_api::R2rmlCreateConfig::new( + "ice", + "https://polaris.example.invalid", + "default.default", + ORDERS_SQLQUERY_R2RML, + ); + let err = fluree + .create_r2rml_graph_source(config) + .await + .expect_err("rr:sqlQuery is not for Iceberg"); + assert!(err.to_string().contains("rr:sqlQuery"), "{err}"); +} + +#[tokio::test] +async fn registration_survives_an_unreachable_endpoint() { + let fluree = FlureeBuilder::memory().build_memory(); + let config = SqlCreateConfig::new("dead-sql", "http://127.0.0.1:9", PEOPLE_R2RML); + let created = fluree + .create_sql_graph_source(config) + .await + .expect("registration does not require a live endpoint"); + assert!(!created.connection_tested); + assert!(created.mapping_validated); + + // Querying it surfaces the transport error rather than empty results. + let query = json!({ + "@context": {"ex": "http://example.org/"}, + "from": "dead-sql:main", + "select": ["?name"], + "where": {"@id": "?s", "ex:name": "?name"}, + }); + let err = fluree + .query_from() + .jsonld(&query) + .execute_formatted() + .await + .expect_err("unreachable endpoint fails the query"); + assert!(err.to_string().contains("dead-sql:main"), "{err}"); +} + +/// Against a live `fluree-sql-bridge` (or Trino) serving a `people(id, name, +/// score, born)` table — run with `FLUREE_SQL_BRIDGE_URL=http://127.0.0.1:8080` +/// and, for a bridge, `FLUREE_SQL_BRIDGE_DIALECT=sqlite|postgres|mysql`; +/// `FLUREE_SQL_BRIDGE_CATALOG` / `FLUREE_SQL_BRIDGE_SCHEMA` qualify the table. +/// Skips (loudly) when unset, so CI without a bridge does not silently pass it. +#[tokio::test] +async fn live_bridge_round_trip() { + let Ok(endpoint) = std::env::var("FLUREE_SQL_BRIDGE_URL") else { + eprintln!("SKIPPED live_bridge_round_trip: FLUREE_SQL_BRIDGE_URL not set"); + return; + }; + let mapping = PEOPLE_R2RML.replace("sales.people", "people"); + let fluree = FlureeBuilder::memory().build_memory(); + let mut config = SqlCreateConfig::new("live-sql", endpoint, mapping); + config.dialect = match std::env::var("FLUREE_SQL_BRIDGE_DIALECT").as_deref() { + Ok("sqlite") => fluree_db_api::SqlDialect::Sqlite, + Ok("postgres") => fluree_db_api::SqlDialect::Postgres, + Ok("mysql") => fluree_db_api::SqlDialect::Mysql, + _ => fluree_db_api::SqlDialect::Trino, + }; + config.catalog = std::env::var("FLUREE_SQL_BRIDGE_CATALOG").ok(); + config.schema = std::env::var("FLUREE_SQL_BRIDGE_SCHEMA").ok(); + let created = fluree + .create_sql_graph_source(config) + .await + .expect("create"); + assert!( + created.connection_tested, + "SELECT 1 against the live endpoint" + ); + + let sparql = " + PREFIX ex: + SELECT ?s ?name ?score ?born FROM + WHERE { ?s ex:name ?name . OPTIONAL { ?s ex:score ?score } OPTIONAL { ?s ex:born ?born } } + ORDER BY ?s + "; + let rows = fluree + .query_from() + .sparql(sparql) + .execute_formatted() + .await + .expect("live query"); + let rows = bindings(&rows); + assert_eq!(rows.len(), 2, "{rows:?}"); + let text = rows[0].to_string(); + assert!( + text.contains("alice") && text.contains("9.25") && text.contains("1985-01-02"), + "{text}" + ); + + let sparql = " + PREFIX ex: + SELECT (COUNT(?s) AS ?n) FROM WHERE { ?s a ex:Person } + "; + let rows = fluree + .query_from() + .sparql(sparql) + .execute_formatted() + .await + .expect("live count"); + assert!(rows.to_string().contains('3'), "{rows}"); + + let sparql = " + PREFIX ex: + SELECT ?s FROM WHERE { ?s ex:name \"bob\" } + "; + let rows = fluree + .query_from() + .sparql(sparql) + .execute_formatted() + .await + .expect("live pushed filter"); + let rows = bindings(&rows); + assert_eq!(rows.len(), 1, "{rows:?}"); + assert!(rows[0].to_string().contains("person/2"), "{rows:?}"); +} + +/// A ledger joined with a SQL source in one query: the ledger holds facts +/// about the same subjects the SQL rows mint, and the join happens in-engine. +#[tokio::test] +async fn ledger_and_sql_source_join_in_one_dataset() { + let server = fake_trino().await; + let fluree = FlureeBuilder::memory().build_memory(); + fluree + .create_sql_graph_source(SqlCreateConfig::new( + "people-sql", + server.uri(), + PEOPLE_R2RML, + )) + .await + .expect("create sql source"); + + let ledger = fluree.create_ledger("teams:main").await.expect("ledger"); + fluree + .insert_turtle_with_opts( + ledger, + "@prefix ex: .\n\ + ex:team \"red\" .\n\ + ex:team \"blue\" .", + TxnOpts::default(), + CommitOpts::default(), + &IndexConfig { + reindex_min_bytes: 5_000_000_000, + reindex_max_bytes: 5_000_000_000, + }, + None, + ) + .await + .expect("insert"); + + // Joining a ledger with a mapped source: the ledger is the default graph, + // the source is a GRAPH block — and on the dataset path it must be listed + // with FROM NAMED, exactly as for an Iceberg source (without it the GRAPH + // block resolves to nothing and the join is empty). + let sparql = " + PREFIX ex: + SELECT ?name ?team FROM FROM NAMED + WHERE { ?p ex:team ?team . GRAPH { ?p ex:name ?name } } + ORDER BY ?name + "; + let rows = fluree + .query_from() + .sparql(sparql) + .execute_formatted() + .await + .expect("join query"); + let rows = bindings(&rows); + assert_eq!(rows.len(), 2, "{rows:?}"); + let row = |i: usize, var: &str| rows[i][var]["value"].as_str().unwrap_or("").to_string(); + assert_eq!( + (row(0, "name"), row(0, "team")), + ("alice".into(), "red".into()), + "{rows:?}" + ); + assert_eq!( + (row(1, "name"), row(1, "team")), + ("bob".into(), "blue".into()), + "{rows:?}" + ); + + // The same join in JSON-LD: the graph pattern is the ARRAY form + // `["graph", , pattern]` (an object with a "graph" key is not a + // graph pattern), and the source enters the dataset via `fromNamed`. + let query = json!({ + "@context": {"ex": "http://example.org/"}, + "from": "teams:main", + "from-named": ["people-sql:main"], + "select": ["?name", "?team"], + "where": [ + {"@id": "?p", "ex:team": "?team"}, + ["graph", "people-sql:main", {"@id": "?p", "ex:name": "?name"}] + ], + "orderBy": "?name" + }); + let rows = fluree + .query_from() + .jsonld(&query) + .execute_formatted() + .await + .expect("jsonld join query"); + let rows = rows.as_array().expect("array").clone(); + assert_eq!(rows.len(), 2, "{rows:?}"); + assert_eq!(rows[0], json!(["alice", "red"]), "{rows:?}"); + assert_eq!(rows[1], json!(["bob", "blue"]), "{rows:?}"); +} diff --git a/fluree-db-cli/Cargo.toml b/fluree-db-cli/Cargo.toml index 12778522c1..80edc92995 100644 --- a/fluree-db-cli/Cargo.toml +++ b/fluree-db-cli/Cargo.toml @@ -20,9 +20,11 @@ path = "src/lib.rs" # reuses the dist binary) can use S3 storage + DynamoDB nameservice via # connection config. Dormant unless configured; ~+2 MB over the AWS SDK that # `iceberg` already pulls in. -default = ["server", "iceberg", "shacl", "aws", "graphql"] +default = ["server", "iceberg", "sql", "shacl", "aws", "graphql"] server = ["dep:fluree-db-server"] iceberg = ["fluree-db-api/iceberg"] +# SQL graph sources (R2RML over a Trino-protocol endpoint) +sql = ["iceberg", "fluree-db-api/sql", "fluree-db-server?/sql"] aws = ["fluree-db-server/aws", "fluree-db-nameservice-sync/aws"] # SHACL constraint validation at transaction time shacl = ["fluree-db-api/shacl"] diff --git a/fluree-db-cli/src/cli.rs b/fluree-db-cli/src/cli.rs index 2fcdfa90a6..fff879c78a 100644 --- a/fluree-db-cli/src/cli.rs +++ b/fluree-db-cli/src/cli.rs @@ -1348,6 +1348,12 @@ pub enum Commands { action: IcebergAction, }, + /// Manage SQL graph sources (R2RML over a Trino-protocol endpoint) + Sql { + #[command(subcommand)] + action: SqlAction, + }, + /// Materialize a native twin ledger from a virtual (R2RML-over-Iceberg) /// graph source: bulk-build every triple, verify it against the source, and /// write it as a native ledger or a .flpack pack (DEC-003 Deliverable 1). @@ -3033,6 +3039,127 @@ pub enum IcebergAction { }, } +#[derive(Debug, Clone, Subcommand)] +pub enum SqlAction { + /// Map tables behind a SQL endpoint as an R2RML graph source + /// + /// The endpoint speaks the Trino client protocol: Trino, Starburst, + /// PrestoDB, or a `fluree-sql-bridge` sidecar in front of Postgres, + /// MySQL or SQLite. + /// + /// Examples: + /// fluree sql map orders-db --endpoint https://trino.example.com:8443 --r2rml mappings/orders.ttl --auth-bearer $TOKEN + /// fluree sql map crm --endpoint http://localhost:8080 --catalog pg --schema public --r2rml crm.ttl + Map(Box), + + /// List mapped graph sources (SQL, Iceberg and R2RML) + List { + /// List graph sources on a remote server (by remote name, e.g., "origin") + #[arg(long)] + remote: Option, + }, + + /// Show details for a mapped graph source + Info { + /// Graph source name + name: String, + + /// Query a remote server (by remote name, e.g., "origin") + #[arg(long)] + remote: Option, + }, + + /// Drop a mapped graph source + Drop { + /// Graph source name + name: String, + + /// Required flag to confirm deletion + #[arg(long)] + force: bool, + + /// Execute against a remote server (by remote name, e.g., "origin") + #[arg(long)] + remote: Option, + }, +} + +/// Arguments for mapping a SQL endpoint as a graph source. +#[derive(Debug, Clone, clap::Args)] +pub struct SqlMapArgs { + /// Graph source name (e.g., "orders-db") + pub name: String, + + /// Execute against a remote server (by remote name, e.g., "origin") + #[arg(long)] + pub remote: Option, + + /// Statement endpoint base URL (e.g., "https://trino.example.com:8443") + #[arg(long)] + pub endpoint: String, + + /// R2RML mapping file. Each rr:tableName names a table reachable through + /// the endpoint; rr:sqlQuery is also accepted. + #[arg(long)] + pub r2rml: PathBuf, + + /// R2RML mapping media type (e.g., "text/turtle"); inferred from extension if omitted + #[arg(long)] + pub r2rml_type: Option, + + /// Branch name (defaults to "main") + #[arg(long)] + pub branch: Option, + + /// SQL rendering dialect: trino (default), postgres, mysql, sqlite + #[arg(long)] + pub dialect: Option, + + /// Header family: trino (default) or presto + #[arg(long)] + pub protocol: Option, + + /// Default catalog for unqualified table names + #[arg(long)] + pub catalog: Option, + + /// Default schema for unqualified table names + #[arg(long)] + pub schema: Option, + + /// Protocol user (X-Trino-User); defaults to "fluree" + #[arg(long)] + pub user: Option, + + /// Bearer token for endpoint authentication + #[arg(long)] + pub auth_bearer: Option, + + /// OAuth2 token URL for client credentials auth + #[arg(long)] + pub oauth2_token_url: Option, + + /// OAuth2 client ID + #[arg(long)] + pub oauth2_client_id: Option, + + /// OAuth2 client secret + #[arg(long)] + pub oauth2_client_secret: Option, + + /// OAuth2 scope + #[arg(long)] + pub oauth2_scope: Option, + + /// OAuth2 audience + #[arg(long)] + pub oauth2_audience: Option, + + /// Session property (repeatable): --session query_max_run_time=5m + #[arg(long = "session", value_name = "KEY=VALUE")] + pub session: Vec, +} + /// Arguments for mapping an Iceberg table as a graph source. #[derive(Debug, Clone, clap::Args)] pub struct IcebergMapArgs { diff --git a/fluree-db-cli/src/commands/iceberg.rs b/fluree-db-cli/src/commands/iceberg.rs index d90fc8d34e..f7db363e65 100644 --- a/fluree-db-cli/src/commands/iceberg.rs +++ b/fluree-db-cli/src/commands/iceberg.rs @@ -343,7 +343,7 @@ async fn run_iceberg_map_remote( /// `text/turtle` (the resolver's default), case-insensitively. Returns `None` /// only when the path has no extension at all, leaving the server to apply the /// same default. An explicit `--r2rml-type` still overrides this at the call site. -fn infer_mapping_media_type(path: &std::path::Path) -> Option { +pub(crate) fn infer_mapping_media_type(path: &std::path::Path) -> Option { use fluree_db_r2rml::loader::MappingFormat; // No extension means no signal to infer from — defer to the server default. path.extension()?; @@ -712,7 +712,7 @@ fn build_iceberg_config(args: &IcebergMapArgs) -> CliResult String { +pub(crate) fn format_table_summary(count: usize, names: &[String]) -> String { if names.is_empty() { count.to_string() } else { @@ -720,16 +720,19 @@ fn format_table_summary(count: usize, names: &[String]) -> String { } } +/// Mapped (R2RML-backed) graph sources: Iceberg, R2RML and SQL. `fluree sql` +/// and `fluree iceberg` share list/info/drop over this family. fn is_iceberg_family_source_type(st: &fluree_db_nameservice::GraphSourceType) -> bool { matches!( st, fluree_db_nameservice::GraphSourceType::Iceberg | fluree_db_nameservice::GraphSourceType::R2rml + | fluree_db_nameservice::GraphSourceType::Sql ) } fn is_iceberg_family_type_str(s: &str) -> bool { - matches!(s, "Iceberg" | "R2RML") + matches!(s, "Iceberg" | "R2RML" | "SQL") } #[cfg(test)] diff --git a/fluree-db-cli/src/commands/mod.rs b/fluree-db-cli/src/commands/mod.rs index 22689101c9..6905a6c132 100644 --- a/fluree-db-cli/src/commands/mod.rs +++ b/fluree-db-cli/src/commands/mod.rs @@ -39,6 +39,7 @@ pub mod remote; #[cfg(feature = "server")] pub mod server; pub mod show; +pub mod sql; pub mod sweep; pub mod sync; pub mod token; diff --git a/fluree-db-cli/src/commands/sql.rs b/fluree-db-cli/src/commands/sql.rs new file mode 100644 index 0000000000..282093a136 --- /dev/null +++ b/fluree-db-cli/src/commands/sql.rs @@ -0,0 +1,249 @@ +//! `fluree sql map` — register a SQL graph source. +//! +//! `fluree sql list|info|drop` share the mapped-source implementations in +//! [`super::iceberg`]. + +use crate::cli::SqlMapArgs; +use crate::error::{CliError, CliResult}; +use fluree_db_api::server_defaults::FlureeDir; + +pub async fn run_sql_map(args: SqlMapArgs, dirs: &FlureeDir, direct: bool) -> CliResult<()> { + if let Some(remote_name) = args.remote.as_deref() { + let client = crate::context::build_remote_client(remote_name, dirs).await?; + let result = run_sql_map_remote(&client, &args).await.map_err(|e| { + CliError::Remote(format!( + "failed to map SQL graph source on '{remote_name}': {e}" + )) + }); + crate::context::persist_refreshed_tokens(&client, remote_name, dirs).await; + return result; + } + + if !direct { + if let Some(client) = crate::context::try_server_route_client(dirs) { + return run_sql_map_remote(&client, &args) + .await + .map_err(|e| CliError::Remote(format!("failed to map SQL graph source: {e}"))); + } + } + + run_sql_map_local(args, dirs).await +} + +fn read_mapping(args: &SqlMapArgs) -> CliResult { + std::fs::read_to_string(&args.r2rml).map_err(|e| { + CliError::Input(format!( + "Failed to read R2RML mapping file '{}': {e}", + args.r2rml.display() + )) + }) +} + +fn mapping_media_type(args: &SqlMapArgs) -> Option { + args.r2rml_type + .clone() + .or_else(|| super::iceberg::infer_mapping_media_type(&args.r2rml)) +} + +fn session_pairs(args: &SqlMapArgs) -> CliResult> { + args.session + .iter() + .map(|kv| { + kv.split_once('=') + .map(|(k, v)| (k.trim().to_string(), v.trim().to_string())) + .filter(|(k, _)| !k.is_empty()) + .ok_or_else(|| CliError::Usage(format!("--session expects KEY=VALUE, got '{kv}'"))) + }) + .collect() +} + +fn args_to_json(args: &SqlMapArgs) -> CliResult { + let mut body = serde_json::json!({ + "name": args.name, + "endpoint": args.endpoint, + "r2rml": read_mapping(args)?, + }); + let obj = body.as_object_mut().unwrap(); + if let Some(v) = mapping_media_type(args) { + obj.insert("r2rml_type".into(), v.into()); + } + for (key, value) in [ + ("branch", &args.branch), + ("dialect", &args.dialect), + ("protocol", &args.protocol), + ("catalog", &args.catalog), + ("schema", &args.schema), + ("user", &args.user), + ("auth_bearer", &args.auth_bearer), + ("oauth2_token_url", &args.oauth2_token_url), + ("oauth2_client_id", &args.oauth2_client_id), + ("oauth2_client_secret", &args.oauth2_client_secret), + ("oauth2_scope", &args.oauth2_scope), + ("oauth2_audience", &args.oauth2_audience), + ] { + if let Some(v) = value { + obj.insert(key.into(), v.clone().into()); + } + } + let session = session_pairs(args)?; + if !session.is_empty() { + obj.insert("session".into(), serde_json::to_value(session).unwrap()); + } + Ok(body) +} + +async fn run_sql_map_remote( + client: &crate::remote_client::RemoteLedgerClient, + args: &SqlMapArgs, +) -> CliResult<()> { + let body = args_to_json(args)?; + let result = client.sql_map(&body).await?; + let get = |k: &str| { + result + .get(k) + .and_then(serde_json::Value::as_str) + .unwrap_or("-") + .to_string() + }; + let n = |k: &str| { + result + .get(k) + .and_then(serde_json::Value::as_u64) + .unwrap_or(0) + }; + let flag = |k: &str| { + result + .get(k) + .and_then(serde_json::Value::as_bool) + .unwrap_or(false) + }; + let tables: Vec = result + .get("table_names") + .and_then(|v| v.as_array()) + .map(|a| { + a.iter() + .filter_map(|t| t.as_str().map(String::from)) + .collect() + }) + .unwrap_or_default(); + print_created( + &get("graph_source_id"), + &get("endpoint"), + &get("mapping_source"), + n("triples_map_count") as usize, + n("table_count") as usize, + &tables, + flag("connection_tested"), + flag("mapping_validated"), + ); + Ok(()) +} + +#[cfg(feature = "sql")] +async fn run_sql_map_local(args: SqlMapArgs, dirs: &FlureeDir) -> CliResult<()> { + use fluree_db_api::{SqlAuthConfig, SqlConfigValue, SqlDialect, WireProtocol}; + + let fluree = crate::context::build_fluree(dirs)?; + let mapping = read_mapping(&args)?; + let mut config = fluree_db_api::SqlCreateConfig::new(&args.name, &args.endpoint, mapping); + config.branch = args.branch.clone(); + config.mapping_media_type = mapping_media_type(&args); + config.catalog = args.catalog.clone(); + config.schema = args.schema.clone(); + config.user = args.user.clone(); + config.session = session_pairs(&args)?; + if let Some(d) = &args.dialect { + config.dialect = match d.to_lowercase().as_str() { + "trino" => SqlDialect::Trino, + "postgres" | "postgresql" => SqlDialect::Postgres, + "mysql" => SqlDialect::Mysql, + "sqlite" => SqlDialect::Sqlite, + other => { + return Err(CliError::Usage(format!( + "unknown --dialect '{other}' (trino, postgres, mysql, sqlite)" + ))) + } + }; + } + if let Some(p) = &args.protocol { + config.protocol = match p.to_lowercase().as_str() { + "trino" => WireProtocol::Trino, + "presto" => WireProtocol::Presto, + other => { + return Err(CliError::Usage(format!( + "unknown --protocol '{other}' (trino, presto)" + ))) + } + }; + } + if let (Some(url), Some(secret)) = (&args.oauth2_token_url, &args.oauth2_client_secret) { + config.auth = SqlAuthConfig::OAuth2ClientCredentials { + token_url: url.clone(), + client_id: SqlConfigValue::Literal(args.oauth2_client_id.clone().unwrap_or_default()), + client_secret: SqlConfigValue::Literal(secret.clone()), + scope: args.oauth2_scope.clone(), + audience: args.oauth2_audience.clone(), + }; + } else if let Some(token) = &args.auth_bearer { + config.auth = SqlAuthConfig::Bearer { + token: SqlConfigValue::Literal(token.clone()), + }; + } + + let result = fluree.create_sql_graph_source(config).await?; + print_created( + &result.graph_source_id, + &result.endpoint, + &result.mapping_source, + result.triples_map_count, + result.table_count, + &result.table_names, + result.connection_tested, + result.mapping_validated, + ); + Ok(()) +} + +#[cfg(not(feature = "sql"))] +async fn run_sql_map_local(_args: SqlMapArgs, _dirs: &FlureeDir) -> CliResult<()> { + Err(CliError::Usage( + "SQL graph source support not compiled. Rebuild with `--features sql`.".into(), + )) +} + +#[allow(clippy::too_many_arguments)] +fn print_created( + graph_source_id: &str, + endpoint: &str, + mapping_source: &str, + triples_map_count: usize, + table_count: usize, + table_names: &[String], + connection_tested: bool, + mapping_validated: bool, +) { + println!("Mapped SQL endpoint as graph source '{graph_source_id}'"); + println!(" Endpoint: {endpoint}"); + println!(" R2RML: {mapping_source}"); + println!(" TriplesMaps: {triples_map_count}"); + println!( + " Tables: {}", + super::iceberg::format_table_summary(table_count, table_names) + ); + println!( + " Connection: {}", + if connection_tested { + "verified" + } else { + "not tested (endpoint unreachable or credentials rejected)" + } + ); + println!( + " Mapping: {}", + if mapping_validated { + "validated" + } else { + "not validated (check mapping source)" + } + ); +} diff --git a/fluree-db-cli/src/lib.rs b/fluree-db-cli/src/lib.rs index 896b0fc0e9..ac65a305f6 100644 --- a/fluree-db-cli/src/lib.rs +++ b/fluree-db-cli/src/lib.rs @@ -776,6 +776,42 @@ pub async fn run(cli: Cli) -> error::CliResult<()> { commands::memory::run(action, &fluree_dir).await } + Commands::Sql { action } => { + let fluree_dir = config::require_fluree_dir(config_path)?; + match action { + cli::SqlAction::Map(args) => { + commands::sql::run_sql_map(*args, &fluree_dir, direct).await + } + cli::SqlAction::List { remote } => { + commands::iceberg::run_iceberg_list(&fluree_dir, remote.as_deref(), direct) + .await + } + cli::SqlAction::Info { name, remote } => { + commands::iceberg::run_iceberg_info( + &name, + &fluree_dir, + remote.as_deref(), + direct, + ) + .await + } + cli::SqlAction::Drop { + name, + force, + remote, + } => { + commands::iceberg::run_iceberg_drop( + &name, + force, + &fluree_dir, + remote.as_deref(), + direct, + ) + .await + } + } + } + Commands::Iceberg { action } => { let fluree_dir = config::require_fluree_dir(config_path)?; match action { diff --git a/fluree-db-cli/src/remote_client.rs b/fluree-db-cli/src/remote_client.rs index 58870315a0..2c125b8968 100644 --- a/fluree-db-cli/src/remote_client.rs +++ b/fluree-db-cli/src/remote_client.rs @@ -2763,6 +2763,23 @@ impl RemoteLedgerClient { // Iceberg graph source operations // ========================================================================= + /// Map a SQL endpoint as a graph source on the remote server. + /// + /// Calls `POST {base_url}/sql/map`. + pub async fn sql_map( + &self, + body: &serde_json::Value, + ) -> Result { + let url = self.op_url_root("sql/map"); + self.send_json( + reqwest::Method::POST, + &url, + "application/json", + Some(RequestBody::Json(body)), + ) + .await + } + /// Map an Iceberg table as a graph source on the remote server. /// /// Calls `POST {base_url}/iceberg/map`. diff --git a/fluree-db-nameservice/src/lib.rs b/fluree-db-nameservice/src/lib.rs index 1c7ffd548f..93081b8ccd 100644 --- a/fluree-db-nameservice/src/lib.rs +++ b/fluree-db-nameservice/src/lib.rs @@ -268,6 +268,8 @@ pub enum GraphSourceType { R2rml, /// Apache Iceberg table Iceberg, + /// R2RML mapping over tables reached through a SQL endpoint + Sql, /// Unknown/custom graph source type Unknown(String), } @@ -279,7 +281,9 @@ impl GraphSourceType { GraphSourceType::Bm25 | GraphSourceType::Vector | GraphSourceType::Geo => { GraphSourceKind::Index } - GraphSourceType::R2rml | GraphSourceType::Iceberg => GraphSourceKind::Mapped, + GraphSourceType::R2rml | GraphSourceType::Iceberg | GraphSourceType::Sql => { + GraphSourceKind::Mapped + } GraphSourceType::Unknown(_) => GraphSourceKind::Index, // default assumption } } @@ -295,6 +299,7 @@ impl GraphSourceType { GraphSourceType::Geo => "f:GeoIndex".to_string(), GraphSourceType::R2rml => "f:R2rmlMapping".to_string(), GraphSourceType::Iceberg => "f:IcebergMapping".to_string(), + GraphSourceType::Sql => "f:SqlMapping".to_string(), GraphSourceType::Unknown(s) => s.clone(), } } @@ -311,12 +316,14 @@ impl GraphSourceType { "f:GeoIndex" => GraphSourceType::Geo, "f:R2rmlMapping" => GraphSourceType::R2rml, "f:IcebergMapping" => GraphSourceType::Iceberg, + "f:SqlMapping" => GraphSourceType::Sql, // Full IRI forms ns_types::BM25_INDEX => GraphSourceType::Bm25, ns_types::HNSW_INDEX => GraphSourceType::Vector, ns_types::GEO_INDEX => GraphSourceType::Geo, ns_types::R2RML_MAPPING => GraphSourceType::R2rml, ns_types::ICEBERG_MAPPING => GraphSourceType::Iceberg, + ns_types::SQL_MAPPING => GraphSourceType::Sql, _ => GraphSourceType::Unknown(s.to_string()), } } diff --git a/fluree-db-r2rml/src/loader/extractor.rs b/fluree-db-r2rml/src/loader/extractor.rs index 866d536fa7..52b667be32 100644 --- a/fluree-db-r2rml/src/loader/extractor.rs +++ b/fluree-db-r2rml/src/loader/extractor.rs @@ -140,14 +140,18 @@ impl<'a> MappingExtractor<'a> { } } - // Check for rr:sqlQuery (not supported) - if self - .find_object_optional(&table_triples, R2RML::SQL_QUERY) - .is_some() - { - return Err(R2rmlError::Unsupported( - "rr:sqlQuery is not supported for Iceberg graph sources".to_string(), - )); + // rr:sqlQuery — scanned as a derived table by SQL graph sources; + // Iceberg-backed sources refuse the alias at registration. + if let Some(query) = self.find_object_optional(&table_triples, R2RML::SQL_QUERY) { + if let Some(sql) = self.term_to_string(&query) { + if sql.trim().is_empty() { + return Err(R2rmlError::InvalidValue { + property: "rr:sqlQuery".to_string(), + message: "query text is empty".to_string(), + }); + } + return Ok(LogicalTable::sql_query(sql)); + } } Err(R2rmlError::MissingProperty( diff --git a/fluree-db-r2rml/src/loader/mod.rs b/fluree-db-r2rml/src/loader/mod.rs index e0b1da6640..88a7c6d338 100644 --- a/fluree-db-r2rml/src/loader/mod.rs +++ b/fluree-db-r2rml/src/loader/mod.rs @@ -259,3 +259,78 @@ mod tests { .is_some()); } } + +#[cfg(all(test, feature = "turtle"))] +mod sql_query_tests { + use super::R2rmlLoader; + use crate::mapping::LogicalTable; + + const MAPPING: &str = r#" + @prefix rr: . + @prefix ex: . + + a rr:TriplesMap ; + rr:logicalTable [ rr:sqlQuery """SELECT id, total FROM sales.orders WHERE status = 'open'""" ] ; + rr:subjectMap [ rr:template "http://example.org/order/{id}" ; rr:class ex:Order ] ; + rr:predicateObjectMap [ rr:predicate ex:total ; rr:objectMap [ rr:column "total" ] ] . + + a rr:TriplesMap ; + rr:logicalTable [ rr:tableName "sales.customers" ] ; + rr:subjectMap [ rr:template "http://example.org/customer/{id}" ] ; + rr:predicateObjectMap [ rr:predicate ex:name ; rr:objectMap [ rr:column "name" ] ] . + "#; + + #[test] + fn sql_query_compiles_to_a_stable_alias_that_resolves_back_to_the_query() { + let compiled = R2rmlLoader::from_turtle(MAPPING) + .unwrap() + .compile() + .unwrap(); + assert!(compiled.has_sql_queries()); + + let orders = compiled + .get("http://example.org/m#Orders") + .expect("orders map"); + let alias = orders + .table_name() + .expect("alias stands in for the table name"); + assert!(LogicalTable::is_sql_query_alias(alias), "{alias}"); + assert_eq!( + compiled.sql_query_for_table(alias), + Some("SELECT id, total FROM sales.orders WHERE status = 'open'") + ); + assert_eq!(compiled.sql_query_for_table("sales.customers"), None); + + // Same query text → same alias, so caches keyed on the name are stable. + let again = R2rmlLoader::from_turtle(MAPPING) + .unwrap() + .compile() + .unwrap(); + let alias_again = again + .triples_maps + .values() + .find(|tm| tm.iri.ends_with("#Orders")) + .and_then(|tm| tm.table_name()) + .unwrap(); + assert_eq!(alias, alias_again); + + // Both maps are reachable by table name. + assert_eq!(compiled.find_maps_for_table(alias).len(), 1); + assert_eq!(compiled.find_maps_for_table("sales.customers").len(), 1); + } + + #[test] + fn empty_sql_query_is_rejected() { + let mapping = r#" + @prefix rr: . + a rr:TriplesMap ; + rr:logicalTable [ rr:sqlQuery " " ] ; + rr:subjectMap [ rr:template "http://example.org/{id}" ] . + "#; + let err = R2rmlLoader::from_turtle(mapping) + .unwrap() + .compile() + .unwrap_err(); + assert!(err.to_string().contains("rr:sqlQuery"), "{err}"); + } +} diff --git a/fluree-db-r2rml/src/mapping/compiled.rs b/fluree-db-r2rml/src/mapping/compiled.rs index f4051139c3..5606281045 100644 --- a/fluree-db-r2rml/src/mapping/compiled.rs +++ b/fluree-db-r2rml/src/mapping/compiled.rs @@ -171,6 +171,22 @@ impl CompiledR2rmlMapping { } /// Get all unique table names referenced by the mapping + /// The `rr:sqlQuery` text behind a query alias returned by + /// [`TriplesMap::table_name`], if `table_name` is one. + pub fn sql_query_for_table(&self, table_name: &str) -> Option<&str> { + self.triples_maps + .values() + .find(|tm| tm.table_name() == Some(table_name)) + .and_then(|tm| tm.sql_query()) + } + + /// Whether any map is `rr:sqlQuery`-backed. + pub fn has_sql_queries(&self) -> bool { + self.triples_maps + .values() + .any(|tm| tm.sql_query().is_some()) + } + pub fn table_names(&self) -> Vec<&str> { self.table_to_maps .keys() diff --git a/fluree-db-r2rml/src/mapping/triples_map.rs b/fluree-db-r2rml/src/mapping/triples_map.rs index 15a855f6f9..ad54c08ede 100644 --- a/fluree-db-r2rml/src/mapping/triples_map.rs +++ b/fluree-db-r2rml/src/mapping/triples_map.rs @@ -69,11 +69,18 @@ impl TriplesMap { self } - /// Get the table name if this is a table-based logical table + /// The logical table's name: the `rr:tableName`, or for an `rr:sqlQuery` + /// its deterministic alias — so every consumer keyed on table names (the + /// scan operator, provider caches, `find_maps_for_table`) treats a query + /// exactly like a table. A provider that can run SQL resolves the alias + /// back to the query text through [`Self::sql_query`]. pub fn table_name(&self) -> Option<&str> { - match &self.logical_table { - LogicalTable::TableName(name) => Some(name), - } + self.logical_table.name() + } + + /// The `rr:sqlQuery` text, when this map is query-backed. + pub fn sql_query(&self) -> Option<&str> { + self.logical_table.sql_query_text() } /// Get all columns referenced by this TriplesMap @@ -259,26 +266,70 @@ impl TriplesMap { /// Logical table source /// /// Defines where the tabular data comes from. -/// For Iceberg graph sources, only table names are supported (not SQL queries). +/// Iceberg graph sources accept only table names; SQL graph sources also +/// accept `rr:sqlQuery`, which is scanned as a derived table. #[derive(Debug, Clone, Serialize, Deserialize)] pub enum LogicalTable { /// `rr:tableName` - direct table reference /// /// Table names are normalized to dot notation: "namespace.table" TableName(String), - // Note: rr:sqlQuery is explicitly NOT supported for Iceberg graph sources + /// `rr:sqlQuery` - a SQL SELECT used as the logical table. `alias` is a + /// deterministic name derived from the query text, used wherever a table + /// name is expected. + SqlQuery { sql: String, alias: String }, } +/// Prefix of every `rr:sqlQuery` alias, so a provider without SQL support can +/// recognize and refuse one. +pub const SQL_QUERY_ALIAS_PREFIX: &str = "sqlQuery:"; + impl LogicalTable { /// Create a table name logical table pub fn table(name: impl Into) -> Self { LogicalTable::TableName(name.into()) } - /// Get the table name if this is a table-based logical table + /// Create a query-backed logical table. + pub fn sql_query(sql: impl Into) -> Self { + let sql = sql.into(); + let alias = Self::alias_for_query(&sql); + LogicalTable::SqlQuery { sql, alias } + } + + fn alias_for_query(sql: &str) -> String { + use std::hash::{Hash, Hasher}; + let mut h = std::collections::hash_map::DefaultHasher::new(); + sql.trim().hash(&mut h); + format!("{SQL_QUERY_ALIAS_PREFIX}{:016x}", h.finish()) + } + + /// Whether `name` is an `rr:sqlQuery` alias rather than a real table. + pub fn is_sql_query_alias(name: &str) -> bool { + name.starts_with(SQL_QUERY_ALIAS_PREFIX) + } + + /// The table name or query alias. + pub fn name(&self) -> Option<&str> { + match self { + LogicalTable::TableName(name) => Some(name), + LogicalTable::SqlQuery { alias, .. } => Some(alias), + } + } + + /// The query text for a query-backed logical table. + pub fn sql_query_text(&self) -> Option<&str> { + match self { + LogicalTable::TableName(_) => None, + LogicalTable::SqlQuery { sql, .. } => Some(sql), + } + } + + /// The `rr:tableName`, or `None` for a query-backed logical table. pub fn as_table_name(&self) -> Option<&str> { match self { LogicalTable::TableName(name) => Some(name), + LogicalTable::SqlQuery { .. } => None, } } diff --git a/fluree-db-server/Cargo.toml b/fluree-db-server/Cargo.toml index 636fc623ca..a94b7878b7 100644 --- a/fluree-db-server/Cargo.toml +++ b/fluree-db-server/Cargo.toml @@ -23,7 +23,7 @@ name = "fluree_db_server" path = "src/lib.rs" [features] -default = ["native", "credential", "iceberg", "shacl", "bolt", "graphql"] +default = ["native", "credential", "iceberg", "sql", "shacl", "bolt", "graphql"] native = ["fluree-db-api/native"] # AWS S3 storage + DynamoDB nameservice (via connection JSON-LD config) aws = ["fluree-db-api/aws"] @@ -38,6 +38,8 @@ graphql = ["fluree-db-api/graphql"] oidc = ["fluree-db-credential/oidc", "dep:jsonwebtoken"] # Iceberg / R2RML graph source support iceberg = ["fluree-db-api/iceberg"] +# SQL graph sources (R2RML over a Trino-protocol endpoint) +sql = ["iceberg", "fluree-db-api/sql"] # Use mimalloc as the global allocator. Better multicore allocation throughput # for the allocation-heavy query/materialization paths (e.g. R2RML/Iceberg # scans). Opt-in: enable in release packaging after a soak. diff --git a/fluree-db-server/src/routes/mod.rs b/fluree-db-server/src/routes/mod.rs index b01b80d3fa..0fd9cda5b7 100644 --- a/fluree-db-server/src/routes/mod.rs +++ b/fluree-db-server/src/routes/mod.rs @@ -22,6 +22,8 @@ mod push; pub(crate) mod query; pub(crate) mod serving; mod show; +#[cfg(feature = "sql")] +mod sql; mod storage_proxy; mod stream_query; mod stubs; @@ -120,6 +122,9 @@ pub fn build_router(state: Arc) -> Router { .route("/iceberg/track", post(iceberg::iceberg_track)) .route("/iceberg/untrack", post(iceberg::iceberg_untrack)); + #[cfg(feature = "sql")] + let v1_admin_protected_writes = v1_admin_protected_writes.route("/sql/map", post(sql::sql_map)); + // Admin auth runs BEFORE leader-forward. Axum runs the // last-applied layer outermost, so `require_admin_token` // (applied after `apply_leader_forward`) is the outer layer and diff --git a/fluree-db-server/src/routes/sql.rs b/fluree-db-server/src/routes/sql.rs new file mode 100644 index 0000000000..98fd9714fe --- /dev/null +++ b/fluree-db-server/src/routes/sql.rs @@ -0,0 +1,192 @@ +//! SQL graph source endpoints: POST /v1/fluree/sql/map + +use crate::config::ServerRole; +use crate::error::{Result, ServerError}; +use crate::extract::FlureeHeaders; +use crate::state::AppState; +use crate::telemetry::{create_request_span, extract_request_id, extract_trace_id}; +use axum::extract::{Request, State}; +use axum::http::StatusCode; +use axum::response::{IntoResponse, Response}; +use axum::Json; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::sync::Arc; +use tracing::Instrument; + +use super::ledger::forward_write_request; + +/// Request body for `POST /v1/fluree/sql/map` +#[derive(Deserialize)] +pub struct SqlMapRequest { + /// Graph source name + pub name: String, + /// Statement endpoint base URL (`https://trino.example.com`, or a sidecar) + pub endpoint: String, + /// R2RML mapping content (Turtle by default) + pub r2rml: String, + /// R2RML mapping media type + pub r2rml_type: Option, + /// Branch name + pub branch: Option, + /// Rendering dialect: `trino` (default), `postgres`, `mysql`, `sqlite` + pub dialect: Option, + /// Header family: `trino` (default) or `presto` + pub protocol: Option, + /// Default catalog for unqualified table names + pub catalog: Option, + /// Default schema for unqualified table names + pub schema: Option, + /// `X-Trino-User` (defaults to `fluree`) + pub user: Option, + /// Static bearer token + pub auth_bearer: Option, + /// OAuth2 client-credentials token URL + pub oauth2_token_url: Option, + pub oauth2_client_id: Option, + pub oauth2_client_secret: Option, + pub oauth2_scope: Option, + pub oauth2_audience: Option, + /// Session properties (`X-Trino-Session`) + #[serde(default)] + pub session: BTreeMap, +} + +/// Response for `POST /v1/fluree/sql/map` +#[derive(Serialize)] +pub struct SqlMapResponse { + pub graph_source_id: String, + pub endpoint: String, + pub connection_tested: bool, + pub mapping_source: String, + pub triples_map_count: usize, + pub table_count: usize, + pub table_names: Vec, + pub mapping_validated: bool, +} + +/// Map a SQL endpoint as a graph source +/// +/// POST /v1/fluree/sql/map +pub async fn sql_map(State(state): State>, request: Request) -> Response { + if state.config.server_role == ServerRole::Peer { + return forward_write_request(&state, request).await; + } + sql_map_local(state, request).await.into_response() +} + +async fn sql_map_local(state: Arc, request: Request) -> Result { + let headers = FlureeHeaders::from_headers(request.headers())?; + let body_bytes = axum::body::to_bytes(request.into_body(), 50 * 1024 * 1024) + .await + .map_err(|e| ServerError::bad_request(format!("Failed to read body: {e}")))?; + let req: SqlMapRequest = serde_json::from_slice(&body_bytes) + .map_err(|e| ServerError::bad_request(format!("Invalid JSON: {e}")))?; + + let request_id = extract_request_id(&headers.raw, &state.telemetry_config); + let trace_id = extract_trace_id(&headers.raw); + let span = create_request_span( + "sql:map", + request_id.as_deref(), + trace_id.as_deref(), + Some(&req.name), + None, + None, + ); + async move { + tracing::info!(status = "start", name = %req.name, "sql map requested"); + + // The endpoint reaches an outbound HTTP client: refuse the + // link-local/metadata range before anything connects. (Loopback and + // private hosts are legitimate — a sidecar is the common deployment.) + fluree_db_api::validate_sql_endpoint(&req.endpoint) + .map_err(|e| ServerError::bad_request(e.to_string()))?; + if let Some(url) = &req.oauth2_token_url { + super::iceberg_ssrf::guard_connection_urls(None, Some(url), None)?; + } + + let config = build_sql_config(&req)?; + let result = state + .fluree + .create_sql_graph_source(config) + .await + .map_err(ServerError::Api)?; + + tracing::info!( + status = "success", + graph_source_id = %result.graph_source_id, + "sql graph source mapped" + ); + Ok(( + StatusCode::CREATED, + Json(SqlMapResponse { + graph_source_id: result.graph_source_id, + endpoint: result.endpoint, + connection_tested: result.connection_tested, + mapping_source: result.mapping_source, + triples_map_count: result.triples_map_count, + table_count: result.table_count, + table_names: result.table_names, + mapping_validated: result.mapping_validated, + }), + )) + } + .instrument(span) + .await +} + +fn build_sql_config(req: &SqlMapRequest) -> Result { + use fluree_db_api::{SqlAuthConfig, SqlDialect, WireProtocol}; + + let mut config = fluree_db_api::SqlCreateConfig::new(&req.name, &req.endpoint, &req.r2rml); + config.branch = req.branch.clone(); + config.mapping_media_type = req.r2rml_type.clone(); + config.catalog = req.catalog.clone(); + config.schema = req.schema.clone(); + config.user = req.user.clone(); + config.session = req.session.clone(); + + if let Some(d) = &req.dialect { + config.dialect = match d.to_lowercase().as_str() { + "trino" => SqlDialect::Trino, + "postgres" | "postgresql" => SqlDialect::Postgres, + "mysql" => SqlDialect::Mysql, + "sqlite" => SqlDialect::Sqlite, + other => { + return Err(ServerError::bad_request(format!( + "unknown dialect '{other}'. Use trino, postgres, mysql or sqlite." + ))) + } + }; + } + if let Some(p) = &req.protocol { + config.protocol = match p.to_lowercase().as_str() { + "trino" => WireProtocol::Trino, + "presto" => WireProtocol::Presto, + other => { + return Err(ServerError::bad_request(format!( + "unknown protocol '{other}'. Use trino or presto." + ))) + } + }; + } + + if let (Some(url), Some(secret)) = (&req.oauth2_token_url, &req.oauth2_client_secret) { + config.auth = SqlAuthConfig::OAuth2ClientCredentials { + token_url: url.clone(), + client_id: fluree_db_sql_config_value(req.oauth2_client_id.as_deref().unwrap_or("")), + client_secret: fluree_db_sql_config_value(secret), + scope: req.oauth2_scope.clone(), + audience: req.oauth2_audience.clone(), + }; + } else if let Some(token) = &req.auth_bearer { + config.auth = SqlAuthConfig::Bearer { + token: fluree_db_sql_config_value(token), + }; + } + Ok(config) +} + +fn fluree_db_sql_config_value(literal: &str) -> fluree_db_api::SqlConfigValue { + fluree_db_api::SqlConfigValue::Literal(literal.to_string()) +} diff --git a/fluree-db-sql/Cargo.toml b/fluree-db-sql/Cargo.toml new file mode 100644 index 0000000000..1bf51861ca --- /dev/null +++ b/fluree-db-sql/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "fluree-db-sql" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "SQL-over-HTTP graph source support for Fluree DB (Trino wire protocol)" + +[dependencies] +fluree-db-tabular = { path = "../fluree-db-tabular" } +# `ConfigValue` / `SecretResolver` / `AuthConfig` / `MappingSource` are shared with +# the Iceberg graph source. The base crate (no `aws` feature) is HTTP + serde only. +fluree-db-iceberg = { path = "../fluree-db-iceberg", default-features = false } + +async-trait.workspace = true +tokio = { workspace = true, features = ["sync", "time"] } +futures.workspace = true +async-stream = "0.3" + +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] } + +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +thiserror.workspace = true +chrono.workspace = true +base64 = "0.22" +tracing.workspace = true + +[dev-dependencies] +wiremock = { workspace = true } +tokio = { workspace = true, features = ["full"] } + +[lints] +workspace = true diff --git a/fluree-db-sql/src/config.rs b/fluree-db-sql/src/config.rs new file mode 100644 index 0000000000..af4ee42591 --- /dev/null +++ b/fluree-db-sql/src/config.rs @@ -0,0 +1,205 @@ +//! Graph-source configuration for a SQL source. +//! +//! Stored as the opaque `config` JSON of a `f:SqlMapping` nameservice record. +//! Everything reachable over the wire — endpoint, catalog/schema defaults, +//! credentials — lives here; the R2RML mapping itself is stored in CAS and only +//! referenced (`mapping.source` is a CID), exactly as for Iceberg sources. + +use std::collections::BTreeMap; +use std::sync::Arc; + +use fluree_db_iceberg::auth::AuthConfig; +use fluree_db_iceberg::config::MappingSource; +use fluree_db_iceberg::SecretResolver; +use serde::{Deserialize, Serialize}; + +use crate::dialect::SqlDialect; +use crate::error::{Result, SqlError}; + +/// Which header family the endpoint speaks. Trino renamed its headers from +/// `X-Presto-*` to `X-Trino-*` in release 351; PrestoDB still uses the old +/// names. Everything else about the protocol is identical. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum WireProtocol { + #[default] + Trino, + Presto, +} + +impl WireProtocol { + pub(crate) fn header(self, suffix: &str) -> String { + match self { + WireProtocol::Trino => format!("X-Trino-{suffix}"), + WireProtocol::Presto => format!("X-Presto-{suffix}"), + } + } +} + +fn default_request_timeout() -> u64 { + 120 +} + +fn default_user() -> String { + "fluree".to_string() +} + +/// Persisted configuration of one SQL graph source. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SqlGsConfig { + /// Base URL of the statement endpoint, e.g. `https://trino.example.com:8443` + /// or `http://localhost:8080` for a sidecar. `/v1/statement` is appended. + pub endpoint: String, + + /// How identifiers and literals are rendered. Defaults to Trino; a + /// `fluree-sql-bridge` sidecar in front of another engine reports its own. + #[serde(default)] + pub dialect: SqlDialect, + + /// Header family (`X-Trino-*` vs `X-Presto-*`). + #[serde(default)] + pub protocol: WireProtocol, + + /// Default catalog for unqualified table names (`X-Trino-Catalog`). + #[serde(default)] + pub catalog: Option, + + /// Default schema for unqualified table names (`X-Trino-Schema`). + #[serde(default)] + pub schema: Option, + + /// The `X-Trino-User` value. Required by the protocol even when a bearer + /// token identifies the caller; defaults to `fluree`. + #[serde(default = "default_user")] + pub user: String, + + /// Endpoint authentication. Shares the Iceberg REST catalog's shape so the + /// same `ConfigValue` indirection (`env_var`, `secret_ref`) applies. + #[serde(default)] + pub auth: AuthConfig, + + /// Session properties sent as `X-Trino-Session: k=v,k=v`. + #[serde(default)] + pub session: BTreeMap, + + /// Per-request HTTP timeout (each page fetch is one request). + #[serde(default = "default_request_timeout")] + pub request_timeout_secs: u64, + + /// The R2RML mapping (CAS CID + media type). Absent only transiently. + #[serde(default)] + pub mapping: Option, +} + +impl SqlGsConfig { + pub fn new(endpoint: impl Into) -> Self { + Self { + endpoint: endpoint.into(), + dialect: SqlDialect::default(), + protocol: WireProtocol::default(), + catalog: None, + schema: None, + user: default_user(), + auth: AuthConfig::default(), + session: BTreeMap::new(), + request_timeout_secs: default_request_timeout(), + mapping: None, + } + } + + pub fn from_json(json: &str) -> Result { + serde_json::from_str(json) + .map_err(|e| SqlError::Config(format!("invalid config JSON: {e}"))) + } + + pub fn to_json(&self) -> Result { + serde_json::to_string(self).map_err(|e| SqlError::Config(format!("serialize config: {e}"))) + } + + /// Structural validation: endpoint scheme/host and non-empty user. + pub fn validate(&self) -> Result<()> { + crate::net::validate_endpoint(&self.endpoint)?; + if self.user.trim().is_empty() { + return Err(SqlError::Config("user must not be empty".to_string())); + } + if self.request_timeout_secs == 0 { + return Err(SqlError::Config( + "request_timeout_secs must be positive".to_string(), + )); + } + for key in self.session.keys() { + if key.contains(',') || key.contains('=') { + return Err(SqlError::Config(format!( + "session property name '{key}' may not contain ',' or '='" + ))); + } + } + Ok(()) + } + + /// Resolve every `secret_ref` in the auth block. Fields carrying no secret + /// reference clone through untouched. + pub async fn hydrate(&self, resolver: Option<&Arc>) -> Result { + let auth = self.auth.hydrate(resolver).await?; + Ok(Self { + auth, + ..self.clone() + }) + } + + /// The endpoint with any trailing slash removed. + pub fn endpoint_base(&self) -> &str { + self.endpoint.trim_end_matches('/') + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn minimal_config_round_trips_with_defaults() { + let cfg = SqlGsConfig::from_json(r#"{"endpoint":"http://localhost:8080"}"#).unwrap(); + assert_eq!(cfg.user, "fluree"); + assert_eq!(cfg.dialect, SqlDialect::Trino); + assert_eq!(cfg.protocol, WireProtocol::Trino); + assert_eq!(cfg.request_timeout_secs, 120); + assert!(cfg.mapping.is_none()); + cfg.validate().unwrap(); + let back = SqlGsConfig::from_json(&cfg.to_json().unwrap()).unwrap(); + assert_eq!(back.endpoint, "http://localhost:8080"); + } + + #[test] + fn full_config_parses() { + let cfg = SqlGsConfig::from_json( + r#"{ + "endpoint": "https://trino.example.com/", + "dialect": "postgres", + "protocol": "presto", + "catalog": "pg", + "schema": "public", + "user": "svc", + "auth": {"type": "bearer", "token": {"env_var": "TRINO_TOKEN"}}, + "session": {"query_max_run_time": "5m"}, + "mapping": {"source": "bafy...", "media_type": "text/turtle"} + }"#, + ) + .unwrap(); + assert_eq!(cfg.dialect, SqlDialect::Postgres); + assert_eq!(cfg.protocol, WireProtocol::Presto); + assert_eq!(cfg.endpoint_base(), "https://trino.example.com"); + assert!(matches!(cfg.auth, AuthConfig::Bearer { .. })); + assert_eq!(cfg.session["query_max_run_time"], "5m"); + cfg.validate().unwrap(); + } + + #[test] + fn validate_rejects_bad_scheme_and_empty_user() { + let mut cfg = SqlGsConfig::new("ftp://x"); + assert!(cfg.validate().is_err()); + cfg = SqlGsConfig::new("http://x"); + cfg.user = " ".into(); + assert!(cfg.validate().is_err()); + } +} diff --git a/fluree-db-sql/src/dialect.rs b/fluree-db-sql/src/dialect.rs new file mode 100644 index 0000000000..028a3a3901 --- /dev/null +++ b/fluree-db-sql/src/dialect.rs @@ -0,0 +1,677 @@ +//! SQL rendering of a single-table scan. +//! +//! The query engine never sends SPARQL here. The R2RML operator asks a +//! provider for one table at a time — a projection, conjunctive filters, and +//! optionally a single-column `ORDER BY … LIMIT` — and does joins, OPTIONAL, +//! UNION and aggregation itself over the returned column batches. So what gets +//! rendered is exactly one `SELECT … FROM … WHERE …`. +//! +//! Filters are pushed **typed**: every predicate is rendered against the +//! column's known type (from a cached `LIMIT 0` probe), and a predicate whose +//! literal cannot be rendered safely for that type is dropped rather than +//! guessed — a mistyped comparison would fail the whole statement in Trino +//! ("Cannot apply operator: bigint = varchar"), and the in-engine FILTER stays +//! the authority either way, so a dropped push only costs I/O. +//! +//! The same valve carries the escaping rule: string literals are rendered with +//! standard-SQL quote doubling, and a value that a dialect might read as +//! carrying escapes is declined rather than escaped for a server mode we cannot +//! observe. See [`sql_string`]. + +use fluree_db_tabular::{BatchSchema, FieldType}; +use serde::{Deserialize, Serialize}; + +use crate::error::{Result, SqlError}; + +/// Identifier quoting and literal syntax family. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum SqlDialect { + #[default] + Trino, + Postgres, + Mysql, + Sqlite, +} + +impl SqlDialect { + fn quote_char(self) -> char { + match self { + SqlDialect::Mysql => '`', + _ => '"', + } + } + + /// Quote one identifier part, doubling any embedded quote character. + pub fn quote_ident(self, ident: &str) -> String { + let q = self.quote_char(); + let mut out = String::with_capacity(ident.len() + 2); + out.push(q); + for c in ident.chars() { + if c == q { + out.push(q); + } + out.push(c); + } + out.push(q); + out + } + + /// Quote a dotted table name part by part (`ns.table` → `"ns"."table"`). + pub fn quote_table(self, table: &str) -> String { + table + .split('.') + .map(|p| self.quote_ident(p)) + .collect::>() + .join(".") + } + + /// Whether typed literal prefixes (`DATE '…'`, `TIMESTAMP '…'`) are valid. + fn typed_literals(self) -> bool { + !matches!(self, SqlDialect::Sqlite) + } + + /// Whether the server may treat `\\` as live inside a string literal. + /// + /// Trino, SQLite and Postgres (with `standard_conforming_strings`, on by + /// default since 9.1) leave backslash inert, so doubling `'` is the whole + /// escaping rule. MySQL under its default `sql_mode` does not: there a + /// trailing `\\` escapes the closing quote and the rest of the value parses + /// as SQL. + fn backslash_may_escape(self) -> bool { + matches!(self, SqlDialect::Mysql) + } +} + +/// Where the rows come from: a table, or an `rr:sqlQuery` used as a derived table. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LogicalSource { + /// Dotted table name; each part is quoted. + Table(String), + /// Verbatim SQL from the mapping, wrapped as `(…) AS "__fluree_q"`. The + /// mapping author is trusted (a mapping is root-equivalent by design). + Query(String), +} + +impl LogicalSource { + pub fn render(&self, dialect: SqlDialect) -> String { + match self { + LogicalSource::Table(t) => dialect.quote_table(t), + LogicalSource::Query(q) => { + format!( + "({}) AS {}", + q.trim().trim_end_matches(';'), + dialect.quote_ident("__fluree_q") + ) + } + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CmpOp { + Eq, + NotEq, + Lt, + LtEq, + Gt, + GtEq, + In, +} + +impl CmpOp { + fn sql(self) -> &'static str { + match self { + CmpOp::Eq => "=", + CmpOp::NotEq => "<>", + CmpOp::Lt => "<", + CmpOp::LtEq => "<=", + CmpOp::Gt => ">", + CmpOp::GtEq => ">=", + CmpOp::In => "IN", + } + } +} + +/// A filter literal. Mirrors the engine's `ScanValue` without depending on the +/// query crate. +#[derive(Debug, Clone, PartialEq)] +pub enum Literal { + Bool(bool), + Int(i64), + Str(String), + /// Days since 1970-01-01. + Date(i32), + Double(f64), + Decimal { + unscaled: i128, + scale: i8, + }, + /// Micros since the epoch; `tz` = the source literal carried an offset. + Timestamp { + micros: i64, + tz: bool, + }, + /// A raw column value recovered by reversing a subject template. Its type + /// is whatever the column's type is; rendered only for int/string columns. + TemplateKey(String), + Set(Vec), +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Predicate { + pub column: String, + pub op: CmpOp, + pub value: Literal, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct ScanRequest { + pub source: LogicalSource, + /// Empty = every column. + pub projection: Vec, + pub predicates: Vec, +} + +/// One rendered scan plus what was dropped, so callers can log declined pushes. +#[derive(Debug, Clone)] +pub struct RenderedScan { + pub sql: String, + pub declined_predicates: Vec, +} + +/// `SELECT * FROM LIMIT 0` — the schema probe. +pub fn render_probe(source: &LogicalSource, dialect: SqlDialect) -> String { + format!("SELECT * FROM {} LIMIT 0", source.render(dialect)) +} + +/// `SELECT COUNT(*) FROM WHERE c1 IS NOT NULL AND …` +pub fn render_count( + source: &LogicalSource, + non_null_cols: &[String], + dialect: SqlDialect, +) -> String { + let mut sql = format!("SELECT COUNT(*) FROM {}", source.render(dialect)); + if !non_null_cols.is_empty() { + let conds: Vec = non_null_cols + .iter() + .map(|c| format!("{} IS NOT NULL", dialect.quote_ident(c))) + .collect(); + sql.push_str(" WHERE "); + sql.push_str(&conds.join(" AND ")); + } + sql +} + +/// Render the scan against the probed schema. Unknown projected columns are +/// an error (the mapping names a column the table does not have); predicates +/// on unknown columns or with unrenderable literals are declined, not errors. +pub fn render_scan( + req: &ScanRequest, + schema: &BatchSchema, + dialect: SqlDialect, +) -> Result { + let select_list = if req.projection.is_empty() { + schema + .fields + .iter() + .map(|f| render_projected_column(&f.name, f.field_type, dialect)) + .collect::>() + } else { + let mut cols = Vec::with_capacity(req.projection.len()); + for name in &req.projection { + let field = schema.field_by_name(name).ok_or_else(|| { + SqlError::Config(format!( + "projected column '{name}' does not exist in {}; available: {:?}", + describe(&req.source), + schema + .fields + .iter() + .map(|f| f.name.as_str()) + .collect::>() + )) + })?; + cols.push(render_projected_column(name, field.field_type, dialect)); + } + cols + }; + + let mut sql = format!( + "SELECT {} FROM {}", + select_list.join(", "), + req.source.render(dialect) + ); + + let mut conds = Vec::new(); + let mut declined = Vec::new(); + for pred in &req.predicates { + match schema.field_by_name(&pred.column) { + Some(field) => match render_predicate(pred, field.field_type, dialect) { + Some(c) => conds.push(c), + None => declined.push(pred.clone()), + }, + None => declined.push(pred.clone()), + } + } + if !conds.is_empty() { + sql.push_str(" WHERE "); + sql.push_str(&conds.join(" AND ")); + } + + Ok(RenderedScan { + sql, + declined_predicates: declined, + }) +} + +fn describe(source: &LogicalSource) -> String { + match source { + LogicalSource::Table(t) => format!("table '{t}'"), + LogicalSource::Query(_) => "the rr:sqlQuery".to_string(), + } +} + +/// A `timestamp with time zone` column is re-rendered in UTC so the wire form +/// is decodable without a zone database (Trino otherwise prints the value's +/// own zone, which may be a named region). +fn render_projected_column(name: &str, ty: FieldType, dialect: SqlDialect) -> String { + let q = dialect.quote_ident(name); + match (ty, dialect) { + (FieldType::TimestampTz, SqlDialect::Trino) => format!("{q} AT TIME ZONE 'UTC' AS {q}"), + _ => q, + } +} + +fn render_predicate(pred: &Predicate, ty: FieldType, dialect: SqlDialect) -> Option { + let col = dialect.quote_ident(&pred.column); + match (&pred.value, pred.op) { + (Literal::Set(members), CmpOp::In) => { + if members.is_empty() { + return None; + } + let rendered: Option> = members + .iter() + .map(|m| render_literal(m, ty, dialect)) + .collect(); + rendered.map(|r| format!("{col} IN ({})", r.join(", "))) + } + (Literal::Set(_), _) | (_, CmpOp::In) => None, + (lit, op) => render_literal(lit, ty, dialect).map(|l| format!("{col} {} {l}", op.sql())), + } +} + +/// Render `s` as a string literal, or decline when no rendering is safe. +/// +/// Doubling `'` is the standard-SQL rule and is sufficient wherever backslash +/// is inert. On MySQL it is not (see [`SqlDialect::backslash_may_escape`]), and +/// escaping instead of declining would be wrong in both directions: the engine +/// cannot observe the endpoint's `sql_mode`, and `dialect` names the database +/// *behind* an endpoint that need not be a bridge we configured. So a value +/// carrying a backslash is declined, which costs a pushdown and nothing else — +/// the in-engine FILTER enforces the predicate either way. +fn sql_string(s: &str, dialect: SqlDialect) -> Option { + if dialect.backslash_may_escape() && s.contains('\\') { + return None; + } + let mut out = String::with_capacity(s.len() + 2); + out.push('\''); + for c in s.chars() { + if c == '\'' { + out.push('\''); + } + out.push(c); + } + out.push('\''); + Some(out) +} + +fn is_numeric(ty: FieldType) -> bool { + matches!( + ty, + FieldType::Int32 + | FieldType::Int64 + | FieldType::Float32 + | FieldType::Float64 + | FieldType::Decimal { .. } + ) +} + +/// Render a literal for comparison against a column of type `ty`, or `None` +/// when no rendering is safe for that pairing. +fn render_literal(lit: &Literal, ty: FieldType, dialect: SqlDialect) -> Option { + match lit { + Literal::Bool(b) => { + matches!(ty, FieldType::Boolean).then(|| if *b { "TRUE" } else { "FALSE" }.to_string()) + } + Literal::Int(i) => is_numeric(ty).then(|| i.to_string()), + Literal::Str(s) => matches!(ty, FieldType::String).then(|| sql_string(s, dialect))?, + Literal::Date(days) => { + if !matches!(ty, FieldType::Date) { + return None; + } + let date = chrono::DateTime::from_timestamp(i64::from(*days) * 86_400, 0)?.date_naive(); + let text = date.format("%Y-%m-%d").to_string(); + if dialect.typed_literals() { + Some(format!("DATE '{text}'")) + } else { + sql_string(&text, dialect) + } + } + Literal::Double(d) => { + if !d.is_finite() || !is_numeric(ty) { + return None; + } + // `{:E}` gives `1.5E0`, a valid double literal in every dialect here + // and unambiguous (a bare `1.5` is a DECIMAL literal in Trino). + Some(format!("{d:E}")) + } + Literal::Decimal { unscaled, scale } => { + is_numeric(ty).then(|| render_decimal(*unscaled, *scale)) + } + Literal::Timestamp { micros, tz } => { + let matches_col = match ty { + FieldType::Timestamp => !*tz, + FieldType::TimestampTz => *tz, + _ => false, + }; + if !matches_col { + return None; + } + let dt = chrono::DateTime::from_timestamp_micros(*micros)?; + let text = dt.format("%Y-%m-%d %H:%M:%S%.6f").to_string(); + match (dialect.typed_literals(), *tz) { + (true, true) => Some(format!("TIMESTAMP '{text} UTC'")), + (true, false) => Some(format!("TIMESTAMP '{text}'")), + (false, _) => sql_string(&text, dialect), + } + } + Literal::TemplateKey(raw) => match ty { + FieldType::String => sql_string(raw, dialect), + FieldType::Int32 | FieldType::Int64 => raw.parse::().ok().map(|i| i.to_string()), + _ => None, + }, + Literal::Set(_) => None, + } +} + +fn render_decimal(unscaled: i128, scale: i8) -> String { + if scale <= 0 { + let mut s = unscaled.to_string(); + s.extend(std::iter::repeat_n('0', (-scale) as usize)); + return s; + } + let scale = scale as usize; + let negative = unscaled < 0; + let digits = unscaled.unsigned_abs().to_string(); + let padded = if digits.len() <= scale { + format!("{}{}", "0".repeat(scale + 1 - digits.len()), digits) + } else { + digits + }; + let (int_part, frac_part) = padded.split_at(padded.len() - scale); + format!("{}{int_part}.{frac_part}", if negative { "-" } else { "" }) +} + +#[cfg(test)] +mod tests { + use super::*; + use fluree_db_tabular::FieldInfo; + + fn schema() -> BatchSchema { + let f = |name: &str, ty: FieldType, id: i32| FieldInfo { + name: name.to_string(), + field_type: ty, + nullable: true, + field_id: id, + }; + BatchSchema::new(vec![ + f("id", FieldType::Int64, 1), + f("name", FieldType::String, 2), + f("born", FieldType::Date, 3), + f("score", FieldType::Float64, 4), + f( + "price", + FieldType::Decimal { + precision: 10, + scale: 2, + }, + 5, + ), + f("at", FieldType::TimestampTz, 6), + f("local_at", FieldType::Timestamp, 7), + f("ok", FieldType::Boolean, 8), + ]) + } + + fn pred(column: &str, op: CmpOp, value: Literal) -> Predicate { + Predicate { + column: column.into(), + op, + value, + } + } + + #[test] + fn quoting_doubles_embedded_quotes_and_splits_dotted_names() { + assert_eq!( + SqlDialect::Trino.quote_table("hive.sales.orders"), + r#""hive"."sales"."orders""# + ); + assert_eq!(SqlDialect::Trino.quote_ident(r#"we"ird"#), r#""we""ird""#); + assert_eq!(SqlDialect::Mysql.quote_table("db.t"), "`db`.`t`"); + assert_eq!( + sql_string("O'Brien", SqlDialect::Trino).unwrap(), + "'O''Brien'" + ); + } + + #[test] + fn probe_and_count_render() { + let src = LogicalSource::Table("s.t".into()); + assert_eq!( + render_probe(&src, SqlDialect::Trino), + r#"SELECT * FROM "s"."t" LIMIT 0"# + ); + assert_eq!( + render_count(&src, &["id".into(), "name".into()], SqlDialect::Trino), + r#"SELECT COUNT(*) FROM "s"."t" WHERE "id" IS NOT NULL AND "name" IS NOT NULL"# + ); + let q = LogicalSource::Query("select 1 as id;".into()); + assert_eq!( + render_count(&q, &[], SqlDialect::Trino), + r#"SELECT COUNT(*) FROM (select 1 as id) AS "__fluree_q""# + ); + } + + #[test] + fn typed_predicates_render_and_mismatches_decline() { + let req = ScanRequest { + source: LogicalSource::Table("t".into()), + projection: vec!["id".into(), "name".into(), "at".into()], + predicates: vec![ + pred("id", CmpOp::Eq, Literal::Int(7)), + pred("name", CmpOp::Eq, Literal::Str("O'Brien".into())), + pred("born", CmpOp::GtEq, Literal::Date(19_723)), + pred("score", CmpOp::Gt, Literal::Double(1.5)), + pred( + "price", + CmpOp::Lt, + Literal::Decimal { + unscaled: -1234, + scale: 2, + }, + ), + pred( + "at", + CmpOp::Lt, + Literal::Timestamp { + micros: 1_700_000_000_000_000, + tz: true, + }, + ), + pred( + "local_at", + CmpOp::Lt, + Literal::Timestamp { + micros: 0, + tz: false, + }, + ), + pred("ok", CmpOp::Eq, Literal::Bool(true)), + pred( + "id", + CmpOp::In, + Literal::Set(vec![Literal::Int(1), Literal::Int(2)]), + ), + // Declined: string against an int column, tz mismatch, unknown column, + // NaN, template key that is not an integer. + pred("id", CmpOp::Eq, Literal::Str("x".into())), + pred( + "at", + CmpOp::Eq, + Literal::Timestamp { + micros: 0, + tz: false, + }, + ), + pred("nope", CmpOp::Eq, Literal::Int(1)), + pred("score", CmpOp::Eq, Literal::Double(f64::NAN)), + pred("id", CmpOp::Eq, Literal::TemplateKey("abc".into())), + ], + }; + let r = render_scan(&req, &schema(), SqlDialect::Trino).unwrap(); + assert_eq!( + r.sql, + concat!( + r#"SELECT "id", "name", "at" AT TIME ZONE 'UTC' AS "at" FROM "t" WHERE "#, + r#""id" = 7 AND "name" = 'O''Brien' AND "born" >= DATE '2024-01-01' AND "score" > 1.5E0 "#, + r#"AND "price" < -12.34 AND "at" < TIMESTAMP '2023-11-14 22:13:20.000000 UTC' "#, + r#"AND "local_at" < TIMESTAMP '1970-01-01 00:00:00.000000' AND "ok" = TRUE AND "id" IN (1, 2)"# + ) + ); + assert_eq!(r.declined_predicates.len(), 5); + } + + #[test] + fn template_key_is_typed_by_the_column() { + let req = ScanRequest { + source: LogicalSource::Table("t".into()), + projection: vec![], + predicates: vec![ + pred("id", CmpOp::Eq, Literal::TemplateKey("42".into())), + pred("name", CmpOp::Eq, Literal::TemplateKey("42".into())), + pred("born", CmpOp::Eq, Literal::TemplateKey("2024-01-01".into())), + ], + }; + let r = render_scan(&req, &schema(), SqlDialect::Trino).unwrap(); + assert!( + r.sql.contains(r#""id" = 42 AND "name" = '42'"#), + "{}", + r.sql + ); + assert_eq!(r.declined_predicates.len(), 1); + assert!(r + .sql + .starts_with(r#"SELECT "id", "name", "born", "score", "price", "at" AT TIME ZONE"#)); + } + + /// A backslash is inert on Trino/Postgres/SQLite and live on MySQL under + /// its default `sql_mode`, where quote-doubling alone would let the value + /// close its own literal. Rendering must decline there, and only there. + #[test] + fn mysql_declines_string_literals_carrying_a_backslash() { + // `a\' UNION SELECT …` — the shape that escapes its closing quote when + // the server reads `\'` as an escaped quote rather than as two chars. + let hostile = r"a\' UNION SELECT price FROM other -- "; + let req = |lit: Literal| ScanRequest { + source: LogicalSource::Table("t".into()), + projection: vec!["name".into()], + predicates: vec![pred("name", CmpOp::Eq, lit)], + }; + + for lit in [ + Literal::Str(hostile.into()), + Literal::TemplateKey(hostile.into()), + // A lone trailing backslash is the minimal case, and ordinary data. + Literal::Str(r"c:\".into()), + ] { + let r = render_scan(&req(lit.clone()), &schema(), SqlDialect::Mysql).unwrap(); + assert!( + !r.sql.contains("WHERE"), + "MySQL must decline `{lit:?}`, rendered: {}", + r.sql + ); + assert_eq!(r.declined_predicates.len(), 1, "{lit:?}"); + + // Every other dialect leaves backslash inert, so the push stands. + for dialect in [SqlDialect::Trino, SqlDialect::Postgres, SqlDialect::Sqlite] { + let r = render_scan(&req(lit.clone()), &schema(), dialect).unwrap(); + assert!( + r.declined_predicates.is_empty(), + "{dialect:?} should push `{lit:?}`" + ); + } + } + + // The escaping that is applied stays standard: `'` doubles, `\` is + // passed through as the single character it is. + let r = render_scan( + &req(Literal::Str(hostile.into())), + &schema(), + SqlDialect::Trino, + ) + .unwrap(); + assert_eq!( + r.sql, + r#"SELECT "name" FROM "t" WHERE "name" = 'a\'' UNION SELECT price FROM other -- '"# + ); + } + + /// Values with no backslash are unaffected on MySQL: quote doubling and + /// backtick identifier quoting still apply. + #[test] + fn mysql_still_pushes_ordinary_string_literals() { + let req = ScanRequest { + source: LogicalSource::Table("t".into()), + projection: vec!["name".into()], + predicates: vec![pred("name", CmpOp::Eq, Literal::Str("O'Brien".into()))], + }; + let r = render_scan(&req, &schema(), SqlDialect::Mysql).unwrap(); + assert_eq!(r.sql, "SELECT `name` FROM `t` WHERE `name` = 'O''Brien'"); + assert!(r.declined_predicates.is_empty()); + } + + #[test] + fn unknown_projection_is_an_error() { + let req = ScanRequest { + source: LogicalSource::Table("t".into()), + projection: vec!["missing".into()], + predicates: vec![], + }; + let err = render_scan(&req, &schema(), SqlDialect::Trino).unwrap_err(); + assert!(err.to_string().contains("missing")); + } + + #[test] + fn sqlite_uses_plain_string_literals() { + let req = ScanRequest { + source: LogicalSource::Table("t".into()), + projection: vec!["born".into()], + predicates: vec![pred("born", CmpOp::Eq, Literal::Date(0))], + }; + let r = render_scan(&req, &schema(), SqlDialect::Sqlite).unwrap(); + assert_eq!( + r.sql, + r#"SELECT "born" FROM "t" WHERE "born" = '1970-01-01'"# + ); + } + + #[test] + fn decimal_rendering() { + assert_eq!(render_decimal(1234, 2), "12.34"); + assert_eq!(render_decimal(-5, 2), "-0.05"); + assert_eq!(render_decimal(5, 0), "5"); + assert_eq!(render_decimal(5, -2), "500"); + assert_eq!(render_decimal(0, 3), "0.000"); + } +} diff --git a/fluree-db-sql/src/error.rs b/fluree-db-sql/src/error.rs new file mode 100644 index 0000000000..04cefefcd6 --- /dev/null +++ b/fluree-db-sql/src/error.rs @@ -0,0 +1,42 @@ +//! Error type for the SQL graph source. + +/// Errors from configuring, rendering, or executing a SQL graph-source scan. +#[derive(Debug, thiserror::Error)] +pub enum SqlError { + /// The graph-source config is malformed or internally inconsistent. + #[error("SQL graph source configuration error: {0}")] + Config(String), + + /// Credential material could not be resolved or the endpoint refused it. + #[error("SQL graph source authentication error: {0}")] + Auth(String), + + /// Transport-level failure talking to the SQL endpoint. + #[error("SQL endpoint HTTP error: {0}")] + Http(String), + + /// The endpoint accepted the statement and then reported a failure. + #[error("SQL statement failed: {0}")] + Query(String), + + /// A value or type on the wire could not be turned into a column. + #[error("SQL result decode error: {0}")] + Decode(String), + + /// Something the SQL graph source deliberately does not do. + #[error("unsupported by SQL graph sources: {0}")] + Unsupported(String), +} + +impl From for SqlError { + fn from(e: fluree_db_iceberg::IcebergError) -> Self { + // Only the shared config/auth machinery is reachable through this + // conversion; anything else from that crate would be a wiring mistake. + match e { + fluree_db_iceberg::IcebergError::Config(m) => SqlError::Config(m), + other => SqlError::Auth(other.to_string()), + } + } +} + +pub type Result = std::result::Result; diff --git a/fluree-db-sql/src/lib.rs b/fluree-db-sql/src/lib.rs new file mode 100644 index 0000000000..9d70466832 --- /dev/null +++ b/fluree-db-sql/src/lib.rs @@ -0,0 +1,32 @@ +//! SQL graph sources for Fluree DB. +//! +//! An R2RML mapping over tables served by any engine that speaks the Trino +//! client protocol over HTTP: Trino / Starburst / PrestoDB directly, or a +//! `fluree-sql-bridge` sidecar in front of Postgres, MySQL or SQLite. The +//! query engine pushes one single-table scan at a time (projection + typed +//! filters), rendered here as SQL; joins and everything else stay in-engine. +//! +//! - [`config::SqlGsConfig`] — the persisted graph-source record. +//! - [`dialect`] — SQL rendering of a scan against a probed schema. +//! - [`trino::TrinoClient`] — the statement/page protocol, streaming batches. +//! - [`types`] — Trino type names and JSON page values → column batches. + +pub mod config; +pub mod dialect; +pub mod error; +pub mod net; +pub mod trino; +pub mod types; + +pub use config::{SqlGsConfig, WireProtocol}; +pub use dialect::{ + CmpOp, Literal, LogicalSource, Predicate, RenderedScan, ScanRequest, SqlDialect, +}; +pub use error::{Result, SqlError}; +pub use net::validate_endpoint as validate_sql_endpoint; +pub use trino::{SqlBatchStream, TrinoClient}; + +// Re-exported so callers wire auth/secret resolution with one import. +pub use fluree_db_iceberg::auth::{AuthConfig, SendCatalogAuth}; +pub use fluree_db_iceberg::config::MappingSource; +pub use fluree_db_iceberg::{ConfigValue, SecretResolver}; diff --git a/fluree-db-sql/src/net.rs b/fluree-db-sql/src/net.rs new file mode 100644 index 0000000000..26df71460d --- /dev/null +++ b/fluree-db-sql/src/net.rs @@ -0,0 +1,118 @@ +//! Outbound HTTP hardening for the SQL endpoint. +//! +//! A SQL endpoint legitimately lives on loopback or a private network — a +//! `fluree-sql-bridge` or Trino sidecar next to the server is the primary +//! deployment shape — so the Iceberg catalog's "public addresses only" posture +//! would block the main use case. This mirrors the narrower S3 `endpoint` +//! policy instead: redirects are never followed, and the link-local / +//! cloud-metadata range (`169.254/16`, `fe80::/10`) is refused both up front +//! (literal IPs) and at connect time (names that resolve there). + +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; +use std::sync::Arc; +use std::time::Duration; + +use reqwest::dns::{Addrs, Name, Resolve, Resolving}; + +use crate::error::{Result, SqlError}; + +fn ipv4_is_link_local_or_invalid(v4: Ipv4Addr) -> bool { + v4.is_link_local() || v4.is_unspecified() || v4.is_broadcast() +} + +fn ipv6_is_link_local(v6: Ipv6Addr) -> bool { + (v6.segments()[0] & 0xffc0) == 0xfe80 +} + +/// Whether an IP is in the range no SQL endpoint may ever be: link-local +/// (which contains the cloud-metadata address) or unspecified/broadcast. +pub fn ip_is_blocked(ip: IpAddr) -> bool { + match ip { + IpAddr::V4(v4) => ipv4_is_link_local_or_invalid(v4), + IpAddr::V6(v6) => { + if let Some(v4) = v6.to_ipv4_mapped() { + return ipv4_is_link_local_or_invalid(v4); + } + v6.is_unspecified() || ipv6_is_link_local(v6) + } + } +} + +#[derive(Debug, Default)] +struct LinkLocalGuardResolver; + +impl Resolve for LinkLocalGuardResolver { + fn resolve(&self, name: Name) -> Resolving { + Box::pin(async move { + let host = name.as_str().to_owned(); + let resolved = match tokio::net::lookup_host((host.as_str(), 0)).await { + Ok(it) => it, + Err(e) => return Err(Box::new(e) as Box), + }; + let allowed: Vec = resolved.filter(|sa| !ip_is_blocked(sa.ip())).collect(); + if allowed.is_empty() { + return Err(format!( + "SSRF guard: host '{host}' resolves only to link-local/metadata addresses" + ) + .into()); + } + Ok(Box::new(allowed.into_iter()) as Addrs) + }) + } +} + +/// A client that follows no redirects and refuses link-local targets. +pub fn build_client(request_timeout: Duration) -> Result { + reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .dns_resolver(Arc::new(LinkLocalGuardResolver)) + .connect_timeout(Duration::from_secs(30)) + .timeout(request_timeout) + .build() + .map_err(|e| SqlError::Http(format!("build HTTP client: {e}"))) +} + +/// Up-front validation of a configured endpoint: `http`/`https` only, a host +/// present, and not a literal link-local IP (the resolver never sees literals). +pub fn validate_endpoint(raw: &str) -> Result<()> { + let url = reqwest::Url::parse(raw) + .map_err(|e| SqlError::Config(format!("endpoint '{raw}' is not a valid URL: {e}")))?; + match url.scheme() { + "http" | "https" => {} + other => { + return Err(SqlError::Config(format!( + "endpoint scheme '{other}' is not allowed (use https or http)" + ))) + } + } + let host = url + .host_str() + .ok_or_else(|| SqlError::Config(format!("endpoint '{raw}' has no host")))?; + let literal = host.trim_start_matches('[').trim_end_matches(']'); + if let Ok(ip) = literal.parse::() { + if ip_is_blocked(ip) { + return Err(SqlError::Config(format!( + "SSRF guard: endpoint host '{host}' is a blocked (link-local/metadata) address" + ))); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn loopback_and_private_are_allowed_but_metadata_is_not() { + validate_endpoint("http://localhost:8080").unwrap(); + validate_endpoint("http://127.0.0.1:8080").unwrap(); + validate_endpoint("http://10.1.2.3:8080/").unwrap(); + validate_endpoint("https://trino.example.com").unwrap(); + assert!(validate_endpoint("http://169.254.169.254/latest").is_err()); + assert!(validate_endpoint("http://[fe80::1]:8080").is_err()); + assert!(validate_endpoint("http://0.0.0.0:8080").is_err()); + assert!(validate_endpoint("file:///etc/passwd").is_err()); + assert!(validate_endpoint("not a url").is_err()); + } +} diff --git a/fluree-db-sql/src/trino.rs b/fluree-db-sql/src/trino.rs new file mode 100644 index 0000000000..818b6f9e86 --- /dev/null +++ b/fluree-db-sql/src/trino.rs @@ -0,0 +1,416 @@ +//! The Trino client protocol: `POST /v1/statement`, then `GET nextUri` until +//! it disappears. Stateless from our side — every page is one plain HTTP +//! request carrying its own auth — which is what makes this usable from a +//! Lambda, and what lets a small sidecar in front of Postgres/MySQL/SQLite +//! speak the same protocol and need no driver code in this binary. + +use std::collections::HashMap; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use std::task::{Context, Poll}; +use std::time::{Duration, Instant}; + +use fluree_db_iceberg::auth::SendCatalogAuth; +use fluree_db_tabular::{BatchSchema, ColumnBatch}; +use futures::Stream; +use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; +use serde::Deserialize; +use serde_json::Value; +use tokio::sync::mpsc; +use tracing::{debug, warn}; + +use crate::config::SqlGsConfig; +use crate::dialect::{render_count, render_probe, LogicalSource, SqlDialect}; +use crate::error::{Result, SqlError}; +use crate::types::{decode_rows, schema_from_columns}; + +const SCHEMA_CACHE_TTL: Duration = Duration::from_secs(300); +const MAX_503_RETRIES: u32 = 6; +const STREAM_CHANNEL_DEPTH: usize = 4; + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct StatementResponse { + #[serde(default)] + id: Option, + #[serde(default)] + next_uri: Option, + #[serde(default)] + columns: Option>, + #[serde(default)] + data: Option>>, + #[serde(default)] + error: Option, +} + +#[derive(Debug, Deserialize)] +struct TrinoColumn { + name: String, + #[serde(rename = "type")] + type_name: String, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct TrinoError { + #[serde(default)] + message: Option, + #[serde(default)] + error_name: Option, + #[serde(default)] + error_code: Option, +} + +impl TrinoError { + fn render(&self) -> String { + let mut s = self + .message + .clone() + .unwrap_or_else(|| "unknown error".to_string()); + if let Some(name) = &self.error_name { + s.push_str(&format!(" [{name}")); + if let Some(code) = self.error_code { + s.push_str(&format!(" {code}")); + } + s.push(']'); + } + s + } +} + +/// A stream of batches fed by a background driver task. `Sync` because it +/// holds only the channel receiver — the request futures live in the task. +pub struct BatchStream { + rx: mpsc::Receiver>, +} + +impl Stream for BatchStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.rx.poll_recv(cx) + } +} + +pub type SqlBatchStream = Pin> + Send + Sync>>; + +/// A client bound to one endpoint + credential. +#[derive(Clone)] +pub struct TrinoClient { + inner: Arc, +} + +struct Inner { + http: reqwest::Client, + statement_url: String, + base_headers: HeaderMap, + auth: Arc, + dialect: SqlDialect, + schema_cache: Mutex)>>, +} + +impl std::fmt::Debug for TrinoClient { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TrinoClient") + .field("statement_url", &self.inner.statement_url) + .field("dialect", &self.inner.dialect) + .finish_non_exhaustive() + } +} + +impl TrinoClient { + /// `config` must already be hydrated (no `secret_ref` left in `auth`). + pub fn new(config: &SqlGsConfig, auth: Arc) -> Result { + config.validate()?; + let http = crate::net::build_client(Duration::from_secs(config.request_timeout_secs))?; + + let mut base_headers = HeaderMap::new(); + let h = |suffix: &str| HeaderName::from_bytes(config.protocol.header(suffix).as_bytes()); + let put = |headers: &mut HeaderMap, name: HeaderName, value: &str| -> Result<()> { + let v = HeaderValue::from_str(value) + .map_err(|_| SqlError::Config(format!("header {name} has a non-ASCII value")))?; + headers.insert(name, v); + Ok(()) + }; + put(&mut base_headers, h("User").unwrap(), &config.user)?; + put(&mut base_headers, h("Source").unwrap(), "fluree")?; + put(&mut base_headers, h("Time-Zone").unwrap(), "UTC")?; + if let Some(c) = &config.catalog { + put(&mut base_headers, h("Catalog").unwrap(), c)?; + } + if let Some(s) = &config.schema { + put(&mut base_headers, h("Schema").unwrap(), s)?; + } + if !config.session.is_empty() { + let joined = config + .session + .iter() + .map(|(k, v)| format!("{k}={v}")) + .collect::>() + .join(","); + put(&mut base_headers, h("Session").unwrap(), &joined)?; + } + + Ok(Self { + inner: Arc::new(Inner { + http, + statement_url: format!("{}/v1/statement", config.endpoint_base()), + base_headers, + auth, + dialect: config.dialect, + schema_cache: Mutex::new(HashMap::new()), + }), + }) + } + + pub fn dialect(&self) -> SqlDialect { + self.inner.dialect + } + + /// Run `sql`, streaming each protocol page as one batch. Dropping the + /// stream cancels the statement on the server. + pub fn execute(&self, sql: String) -> SqlBatchStream { + let (tx, rx) = mpsc::channel(STREAM_CHANNEL_DEPTH); + let inner = Arc::clone(&self.inner); + tokio::spawn(async move { + let mut cancel_uri: Option = None; + let outcome = inner + .drive(&sql, &mut cancel_uri, |batch| { + let tx = tx.clone(); + async move { tx.send(Ok(batch)).await.is_ok() } + }) + .await; + match outcome { + Ok(_) => {} + Err(SqlError::Http(m)) if m == CONSUMER_GONE => { + if let Some(uri) = cancel_uri { + inner.cancel(&uri).await; + } + } + Err(e) => { + let _ = tx.send(Err(e)).await; + } + } + }); + Box::pin(BatchStream { rx }) + } + + /// Run `sql` to completion. Returns the schema (present even for zero rows + /// once the statement planned) and every batch. + pub async fn execute_collect( + &self, + sql: &str, + ) -> Result<(Option>, Vec)> { + let batches = Mutex::new(Vec::new()); + let mut cancel_uri = None; + let schema = self + .inner + .drive(sql, &mut cancel_uri, |batch| { + batches.lock().unwrap().push(batch); + async { true } + }) + .await?; + Ok((schema, batches.into_inner().unwrap())) + } + + /// The source's column schema from a cached `LIMIT 0` probe. + pub async fn schema(&self, source: &LogicalSource) -> Result> { + let key = source.render(self.inner.dialect); + if let Some((at, schema)) = self.inner.schema_cache.lock().unwrap().get(&key) { + if at.elapsed() < SCHEMA_CACHE_TTL { + return Ok(Arc::clone(schema)); + } + } + let sql = render_probe(source, self.inner.dialect); + let (schema, _) = self.execute_collect(&sql).await?; + let schema = schema.ok_or_else(|| { + SqlError::Query(format!("schema probe returned no column metadata: {sql}")) + })?; + self.inner + .schema_cache + .lock() + .unwrap() + .insert(key, (Instant::now(), Arc::clone(&schema))); + Ok(schema) + } + + /// Exact `COUNT(*)` with the given columns required non-null. + pub async fn count(&self, source: &LogicalSource, non_null_cols: &[String]) -> Result { + let sql = render_count(source, non_null_cols, self.inner.dialect); + let (_, batches) = self.execute_collect(&sql).await?; + let batch = batches + .into_iter() + .find(|b| b.num_rows > 0) + .ok_or_else(|| SqlError::Query(format!("COUNT(*) returned no rows: {sql}")))?; + let col = batch + .column(0) + .ok_or_else(|| SqlError::Decode("COUNT(*) returned no column".to_string()))?; + let n = col + .get_i64(0) + .or_else(|| col.get_i32(0).map(i64::from)) + .or_else(|| col.get_f64(0).map(|f| f as i64)) + .ok_or_else(|| { + SqlError::Decode(format!("COUNT(*) value is not an integer: {col:?}")) + })?; + u64::try_from(n).map_err(|_| SqlError::Decode(format!("negative COUNT(*): {n}"))) + } +} + +const CONSUMER_GONE: &str = "consumer dropped the result stream"; + +impl Inner { + /// Post the statement and walk every page, handing each decoded batch to + /// `sink`; a `false` from the sink means the consumer went away. + async fn drive( + &self, + sql: &str, + cancel_uri: &mut Option, + mut sink: F, + ) -> Result>> + where + F: FnMut(ColumnBatch) -> Fut, + Fut: std::future::Future, + { + debug!(sql = %sql, "SQL statement"); + let mut resp = self.post_statement(sql).await?; + let mut schema: Option> = None; + loop { + if let Some(err) = &resp.error { + return Err(SqlError::Query(err.render())); + } + if schema.is_none() { + if let Some(cols) = &resp.columns { + let pairs: Vec<(String, String)> = cols + .iter() + .map(|c| (c.name.clone(), c.type_name.clone())) + .collect(); + schema = Some(schema_from_columns(&pairs)); + } + } + if let Some(rows) = resp.data.take() { + if !rows.is_empty() { + let s = schema.as_ref().ok_or_else(|| { + SqlError::Decode("page carried data before any column metadata".to_string()) + })?; + let batch = decode_rows(s, rows)?; + if !sink(batch).await { + return Err(SqlError::Http(CONSUMER_GONE.to_string())); + } + } + } + match resp.next_uri.take() { + Some(uri) => { + *cancel_uri = Some(uri.clone()); + resp = self.get_page(&uri).await?; + } + None => { + *cancel_uri = None; + return Ok(schema); + } + } + } + } + + async fn auth_headers(&self) -> Result { + let mut headers = self.base_headers.clone(); + if let Some(value) = self + .auth + .authorization_header() + .await + .map_err(|e| SqlError::Auth(e.to_string()))? + { + headers.insert( + reqwest::header::AUTHORIZATION, + HeaderValue::from_str(&value) + .map_err(|_| SqlError::Auth("invalid authorization header".into()))?, + ); + } + Ok(headers) + } + + async fn post_statement(&self, sql: &str) -> Result { + let mut attempt = 0; + loop { + let headers = self.auth_headers().await?; + let resp = self + .http + .post(&self.statement_url) + .headers(headers) + .header(reqwest::header::CONTENT_TYPE, "text/plain") + .body(sql.to_string()) + .send() + .await + .map_err(|e| SqlError::Http(format!("POST {}: {e}", self.statement_url)))?; + match self.classify(resp, attempt).await? { + Some(parsed) => return Ok(parsed), + None => attempt += 1, + } + } + } + + async fn get_page(&self, uri: &str) -> Result { + let mut attempt = 0; + loop { + let headers = self.auth_headers().await?; + let resp = self + .http + .get(uri) + .headers(headers) + .send() + .await + .map_err(|e| SqlError::Http(format!("GET {uri}: {e}")))?; + match self.classify(resp, attempt).await? { + Some(parsed) => return Ok(parsed), + None => attempt += 1, + } + } + } + + /// `Ok(Some)` = a page; `Ok(None)` = retry (503 with budget left). + async fn classify( + &self, + resp: reqwest::Response, + attempt: u32, + ) -> Result> { + let status = resp.status(); + if status == reqwest::StatusCode::SERVICE_UNAVAILABLE { + if attempt >= MAX_503_RETRIES { + return Err(SqlError::Http(format!( + "endpoint kept answering 503 after {MAX_503_RETRIES} retries" + ))); + } + let backoff = Duration::from_millis(100 * (1u64 << attempt.min(5))); + tokio::time::sleep(backoff).await; + return Ok(None); + } + if status == reqwest::StatusCode::UNAUTHORIZED || status == reqwest::StatusCode::FORBIDDEN { + return Err(SqlError::Auth(format!("endpoint returned {status}"))); + } + if !status.is_success() { + let body = resp.text().await.unwrap_or_default(); + let body = body.chars().take(500).collect::(); + return Err(SqlError::Http(format!( + "endpoint returned {status}: {body}" + ))); + } + let parsed: StatementResponse = resp + .json() + .await + .map_err(|e| SqlError::Decode(format!("statement response is not valid JSON: {e}")))?; + if let Some(id) = &parsed.id { + debug!(query_id = %id, has_next = parsed.next_uri.is_some(), "SQL page"); + } + Ok(Some(parsed)) + } + + async fn cancel(&self, uri: &str) { + match self.auth_headers().await { + Ok(headers) => { + if let Err(e) = self.http.delete(uri).headers(headers).send().await { + warn!(error = %e, "failed to cancel abandoned SQL statement"); + } + } + Err(e) => warn!(error = %e, "failed to cancel abandoned SQL statement"), + } + } +} diff --git a/fluree-db-sql/src/types.rs b/fluree-db-sql/src/types.rs new file mode 100644 index 0000000000..3c87a0ed5f --- /dev/null +++ b/fluree-db-sql/src/types.rs @@ -0,0 +1,491 @@ +//! Trino type names → `FieldType`, and JSON page values → `Column`. +//! +//! Trino's protocol renders every value as JSON: numbers for integers and +//! doubles (with `"NaN"`/`"Infinity"` strings for non-finite doubles), strings +//! for everything else — dates as `2024-01-01`, timestamps as +//! `2024-01-01 12:34:56.123`, zoned timestamps with a trailing zone, decimals as +//! their exact lexical form, varbinary as base64. + +use std::sync::Arc; + +use base64::Engine; +use chrono::{NaiveDate, NaiveDateTime, NaiveTime}; +use fluree_db_tabular::{BatchSchema, Column, ColumnBatch, FieldInfo, FieldType}; +use serde_json::Value; + +use crate::error::{Result, SqlError}; + +/// Map a Trino type signature to a column type. Unknown or structural types +/// (`row`, `array`, `map`, …) land as `String`, carrying Trino's own rendering. +pub fn field_type_from_trino(type_name: &str) -> FieldType { + let lower = type_name.trim().to_ascii_lowercase(); + // Precision sits between the base name and the zone suffix + // (`timestamp(6) with time zone`), so settle temporals before splitting. + if lower.starts_with("timestamp") { + return if lower.ends_with("with time zone") { + FieldType::TimestampTz + } else { + FieldType::Timestamp + }; + } + let (base, args) = match lower.find('(') { + Some(i) => ( + lower[..i].trim_end(), + Some(&lower[i + 1..lower.len().saturating_sub(1)]), + ), + None => (lower.as_str(), None), + }; + match base { + "boolean" => FieldType::Boolean, + "tinyint" | "smallint" | "integer" | "int" => FieldType::Int32, + "bigint" => FieldType::Int64, + "real" | "float" => FieldType::Float32, + "double" | "double precision" => FieldType::Float64, + "varbinary" | "binary" | "bytea" => FieldType::Bytes, + "date" => FieldType::Date, + "decimal" | "numeric" => { + let (p, s) = args + .and_then(|a| { + let mut it = a.split(',').map(|x| x.trim().parse::().ok()); + let p = it.next().flatten()?; + let s = it.next().flatten().unwrap_or(0); + Some((p, s)) + }) + .unwrap_or((38, 0)); + FieldType::Decimal { + precision: p.clamp(1, 76) as u8, + scale: s.clamp(-128, 127) as i8, + } + } + _ => FieldType::String, + } +} + +/// Build a batch schema from the protocol's column list. Field ids are +/// positional (1-based); the R2RML layer looks columns up by name. +pub fn schema_from_columns(columns: &[(String, String)]) -> Arc { + let fields = columns + .iter() + .enumerate() + .map(|(i, (name, ty))| FieldInfo { + name: name.clone(), + field_type: field_type_from_trino(ty), + nullable: true, + field_id: i as i32 + 1, + }) + .collect(); + Arc::new(BatchSchema::new(fields)) +} + +/// Decode one page of rows into a batch. +pub fn decode_rows(schema: &Arc, rows: Vec>) -> Result { + let n = rows.len(); + let mut columns: Vec = schema + .fields + .iter() + .map(|f| Column::with_capacity(f.field_type, n)) + .collect(); + + for (row_idx, row) in rows.into_iter().enumerate() { + if row.len() != columns.len() { + return Err(SqlError::Decode(format!( + "row {row_idx} has {} values but the schema has {} columns", + row.len(), + columns.len() + ))); + } + for (col_idx, value) in row.into_iter().enumerate() { + let field = &schema.fields[col_idx]; + push_value(&mut columns[col_idx], &field.name, field.field_type, value)?; + } + } + + ColumnBatch::new(Arc::clone(schema), columns).map_err(|e| SqlError::Decode(e.to_string())) +} + +fn push_value(column: &mut Column, name: &str, ty: FieldType, value: Value) -> Result<()> { + let bad = |what: &str, v: &Value| { + SqlError::Decode(format!( + "column '{name}' ({ty:?}): expected {what}, got {v}" + )) + }; + match column { + Column::Boolean(v) => v.push(match value { + Value::Null => None, + Value::Bool(b) => Some(b), + other => return Err(bad("boolean", &other)), + }), + Column::Int32(v) => v.push(match value { + Value::Null => None, + Value::Number(ref n) => Some( + n.as_i64() + .and_then(|i| i32::try_from(i).ok()) + .ok_or_else(|| bad("32-bit integer", &value))?, + ), + other => return Err(bad("integer", &other)), + }), + Column::Int64(v) => v.push(match value { + Value::Null => None, + Value::Number(ref n) => Some(n.as_i64().ok_or_else(|| bad("64-bit integer", &value))?), + other => return Err(bad("integer", &other)), + }), + Column::Float32(v) => v.push( + parse_double(&value) + .map_err(|()| bad("real", &value))? + .map(|d| d as f32), + ), + Column::Float64(v) => v.push(parse_double(&value).map_err(|()| bad("double", &value))?), + Column::String(v) => v.push(match value { + Value::Null => None, + Value::String(s) => Some(s), + // Structural / unknown types arrive as JSON; keep their rendering. + other => Some(other.to_string()), + }), + Column::Bytes(v) => v.push(match value { + Value::Null => None, + Value::String(ref s) => Some( + base64::engine::general_purpose::STANDARD + .decode(s) + .map_err(|_| bad("base64 varbinary", &value))?, + ), + other => return Err(bad("base64 varbinary", &other)), + }), + Column::Date(v) => v.push(match value { + Value::Null => None, + Value::String(ref s) => Some(parse_date_days(s).ok_or_else(|| bad("date", &value))?), + other => return Err(bad("date", &other)), + }), + Column::Timestamp(v) => v.push(match value { + Value::Null => None, + Value::String(ref s) => { + Some(parse_timestamp_micros(s).ok_or_else(|| bad("timestamp", &value))?) + } + other => return Err(bad("timestamp", &other)), + }), + Column::TimestampTz(v) => v.push(match value { + Value::Null => None, + Value::String(ref s) => Some(parse_timestamp_micros(s).ok_or_else(|| { + SqlError::Decode(format!( + "column '{name}' (timestamp with time zone): cannot decode '{s}' — \ + only numeric offsets and UTC/GMT/Z zones are supported; select the \ + column `AT TIME ZONE 'UTC'` or use the Trino dialect, which does so" + )) + })?), + other => return Err(bad("timestamp with time zone", &other)), + }), + Column::Decimal { values, scale, .. } => values.push(match value { + Value::Null => None, + Value::String(ref s) => { + Some(parse_decimal_unscaled(s, *scale).ok_or_else(|| bad("decimal", &value))?) + } + Value::Number(ref n) => Some( + parse_decimal_unscaled(&n.to_string(), *scale) + .ok_or_else(|| bad("decimal", &value))?, + ), + other => return Err(bad("decimal", &other)), + }), + } + Ok(()) +} + +fn parse_double(value: &Value) -> std::result::Result, ()> { + match value { + Value::Null => Ok(None), + Value::Number(n) => n.as_f64().map(Some).ok_or(()), + Value::String(s) => match s.as_str() { + "NaN" => Ok(Some(f64::NAN)), + "Infinity" | "+Infinity" => Ok(Some(f64::INFINITY)), + "-Infinity" => Ok(Some(f64::NEG_INFINITY)), + other => other.parse::().map(Some).map_err(|_| ()), + }, + _ => Err(()), + } +} + +/// `YYYY-MM-DD` → days since the epoch. +pub fn parse_date_days(s: &str) -> Option { + let d = NaiveDate::parse_from_str(s.trim(), "%Y-%m-%d").ok()?; + let epoch = NaiveDate::from_ymd_opt(1970, 1, 1)?; + i32::try_from((d - epoch).num_days()).ok() +} + +/// `YYYY-MM-DD HH:MM:SS[.f{1,12}][ ZONE]` → micros since the epoch (UTC frame +/// once the zone is applied). Sub-microsecond digits are truncated. Zones: +/// `UTC`, `GMT`, `Z`, or `±HH:MM`. Named regions return `None`. +pub fn parse_timestamp_micros(s: &str) -> Option { + let s = s.trim(); + let (date_part, rest) = s.split_once(' ')?; + let (time_part, zone) = match rest.split_once(' ') { + Some((t, z)) => (t, Some(z.trim())), + None => (rest, None), + }; + + let date = NaiveDate::parse_from_str(date_part, "%Y-%m-%d").ok()?; + let (hms, frac) = match time_part.split_once('.') { + Some((h, f)) => (h, Some(f)), + None => (time_part, None), + }; + let time = NaiveTime::parse_from_str(hms, "%H:%M:%S").ok()?; + let micros_frac: i64 = match frac { + Some(f) => { + if f.is_empty() || !f.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + let mut padded: String = f.chars().take(6).collect(); + while padded.len() < 6 { + padded.push('0'); + } + padded.parse().ok()? + } + None => 0, + }; + + let naive = NaiveDateTime::new(date, time); + let base = naive + .and_utc() + .timestamp_micros() + .checked_add(micros_frac)?; + let offset_secs = match zone { + None => 0, + Some(z) => parse_zone_offset_secs(z)?, + }; + base.checked_sub(i64::from(offset_secs) * 1_000_000) +} + +fn parse_zone_offset_secs(z: &str) -> Option { + match z { + "UTC" | "GMT" | "Z" | "+00:00" | "-00:00" => Some(0), + _ => { + let sign = match z.as_bytes().first()? { + b'+' => 1, + b'-' => -1, + _ => return None, + }; + let (h, m) = z[1..].split_once(':')?; + let h: i32 = h.parse().ok()?; + let m: i32 = m.parse().ok()?; + if h > 23 || m > 59 { + return None; + } + Some(sign * (h * 3600 + m * 60)) + } + } +} + +/// Exact lexical decimal → unscaled integer at the column's scale. Extra +/// fractional digits beyond `scale` are rejected (the engine would silently +/// misreport the value otherwise). +pub fn parse_decimal_unscaled(s: &str, scale: i8) -> Option { + let s = s.trim(); + let (negative, body) = match s.strip_prefix('-') { + Some(rest) => (true, rest), + None => (false, s.strip_prefix('+').unwrap_or(s)), + }; + let (int_part, frac_part) = body.split_once('.').unwrap_or((body, "")); + if int_part.is_empty() && frac_part.is_empty() { + return None; + } + if !int_part.bytes().all(|b| b.is_ascii_digit()) + || !frac_part.bytes().all(|b| b.is_ascii_digit()) + { + return None; + } + let scale = usize::try_from(scale).ok()?; + if frac_part.len() > scale && frac_part[scale..].bytes().any(|b| b != b'0') { + return None; + } + let mut digits = String::with_capacity(int_part.len() + scale); + digits.push_str(int_part); + digits.push_str(&frac_part[..frac_part.len().min(scale)]); + for _ in frac_part.len().min(scale)..scale { + digits.push('0'); + } + let mut v: i128 = if digits.is_empty() { + 0 + } else { + digits.parse().ok()? + }; + if negative { + v = -v; + } + Some(v) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn trino_type_names_map() { + assert_eq!(field_type_from_trino("bigint"), FieldType::Int64); + assert_eq!(field_type_from_trino("integer"), FieldType::Int32); + assert_eq!(field_type_from_trino("varchar(20)"), FieldType::String); + assert_eq!(field_type_from_trino("varchar"), FieldType::String); + assert_eq!(field_type_from_trino("double"), FieldType::Float64); + assert_eq!(field_type_from_trino("real"), FieldType::Float32); + assert_eq!(field_type_from_trino("date"), FieldType::Date); + assert_eq!(field_type_from_trino("timestamp(3)"), FieldType::Timestamp); + assert_eq!( + field_type_from_trino("timestamp(6) with time zone"), + FieldType::TimestampTz + ); + assert_eq!( + field_type_from_trino("timestamp with time zone"), + FieldType::TimestampTz + ); + assert_eq!( + field_type_from_trino("decimal(10,2)"), + FieldType::Decimal { + precision: 10, + scale: 2 + } + ); + assert_eq!( + field_type_from_trino("decimal(38, 0)"), + FieldType::Decimal { + precision: 38, + scale: 0 + } + ); + assert_eq!(field_type_from_trino("varbinary"), FieldType::Bytes); + assert_eq!(field_type_from_trino("array(varchar)"), FieldType::String); + assert_eq!(field_type_from_trino("row(a bigint)"), FieldType::String); + } + + #[test] + fn timestamps_parse_with_offsets_and_truncate_nanos() { + assert_eq!(parse_timestamp_micros("1970-01-01 00:00:00"), Some(0)); + assert_eq!( + parse_timestamp_micros("1970-01-01 00:00:00.123"), + Some(123_000) + ); + assert_eq!( + parse_timestamp_micros("1970-01-01 00:00:00.123456789"), + Some(123_456) + ); + assert_eq!( + parse_timestamp_micros("1970-01-01 01:00:00 +01:00"), + Some(0) + ); + assert_eq!( + parse_timestamp_micros("1970-01-01 00:00:00.5 UTC"), + Some(500_000) + ); + assert_eq!( + parse_timestamp_micros("1969-12-31 19:00:00 -05:00"), + Some(0) + ); + assert_eq!( + parse_timestamp_micros("1970-01-01 00:00:00 America/New_York"), + None + ); + assert_eq!(parse_timestamp_micros("garbage"), None); + assert_eq!(parse_date_days("1970-01-02"), Some(1)); + assert_eq!(parse_date_days("2024-01-01"), Some(19_723)); + } + + #[test] + fn decimals_parse_exactly() { + assert_eq!(parse_decimal_unscaled("12.34", 2), Some(1234)); + assert_eq!(parse_decimal_unscaled("-0.05", 2), Some(-5)); + assert_eq!(parse_decimal_unscaled("5", 2), Some(500)); + assert_eq!(parse_decimal_unscaled("5.1", 2), Some(510)); + assert_eq!(parse_decimal_unscaled("5.100", 2), Some(510)); + assert_eq!(parse_decimal_unscaled("5.101", 2), None); + assert_eq!(parse_decimal_unscaled("abc", 2), None); + assert_eq!(parse_decimal_unscaled(".5", 1), Some(5)); + } + + #[test] + fn decode_a_page() { + let schema = schema_from_columns(&[ + ("id".into(), "bigint".into()), + ("n".into(), "integer".into()), + ("name".into(), "varchar".into()), + ("d".into(), "double".into()), + ("born".into(), "date".into()), + ("at".into(), "timestamp(3) with time zone".into()), + ("price".into(), "decimal(10,2)".into()), + ("raw".into(), "varbinary".into()), + ("ok".into(), "boolean".into()), + ("tags".into(), "array(varchar)".into()), + ]); + let rows = vec![ + vec![ + json!(1), + json!(2), + json!("a"), + json!(1.5), + json!("2024-01-01"), + json!("2023-11-14 22:13:20.000 UTC"), + json!("12.34"), + json!("aGk="), + json!(true), + json!(["x", "y"]), + ], + vec![ + json!(2), + Value::Null, + Value::Null, + json!("NaN"), + Value::Null, + json!("2023-11-14 23:13:20.000 +01:00"), + Value::Null, + Value::Null, + Value::Null, + Value::Null, + ], + ]; + let batch = decode_rows(&schema, rows).unwrap(); + assert_eq!(batch.num_rows, 2); + let id = batch.column_by_name("id").unwrap(); + assert_eq!(id.get_i64(0), Some(1)); + assert_eq!(batch.column_by_name("n").unwrap().get_i32(0), Some(2)); + assert_eq!(batch.column_by_name("name").unwrap().get_string(1), None); + assert!(batch + .column_by_name("d") + .unwrap() + .get_f64(1) + .unwrap() + .is_nan()); + assert_eq!( + batch.column_by_name("born").unwrap().get_date(0), + Some(19_723) + ); + let at = batch.column_by_name("at").unwrap(); + assert_eq!(at.get_timestamp(0), Some(1_700_000_000_000_000)); + assert_eq!(at.get_timestamp(1), Some(1_700_000_000_000_000)); + match batch.column_by_name("price").unwrap() { + Column::Decimal { values, scale, .. } => { + assert_eq!(*scale, 2); + assert_eq!(values[0], Some(1234)); + assert_eq!(values[1], None); + } + other => panic!("{other:?}"), + } + assert_eq!( + batch.column_by_name("raw").unwrap().get_bytes(0), + Some(&b"hi"[..]) + ); + assert_eq!(batch.column_by_name("ok").unwrap().get_bool(0), Some(true)); + assert_eq!( + batch.column_by_name("tags").unwrap().get_string(0), + Some(r#"["x","y"]"#) + ); + } + + #[test] + fn decode_rejects_shape_and_type_errors() { + let schema = schema_from_columns(&[("id".into(), "bigint".into())]); + assert!(decode_rows(&schema, vec![vec![json!(1), json!(2)]]).is_err()); + assert!(decode_rows(&schema, vec![vec![json!("x")]]).is_err()); + let schema = schema_from_columns(&[("at".into(), "timestamp with time zone".into())]); + let err = decode_rows( + &schema, + vec![vec![json!("2024-01-01 00:00:00 Europe/Oslo")]], + ) + .unwrap_err(); + assert!(err.to_string().contains("AT TIME ZONE")); + } +} diff --git a/fluree-db-sql/tests/protocol.rs b/fluree-db-sql/tests/protocol.rs new file mode 100644 index 0000000000..0c78e95968 --- /dev/null +++ b/fluree-db-sql/tests/protocol.rs @@ -0,0 +1,294 @@ +//! The statement/page protocol against a fake endpoint. + +use std::sync::Arc; + +use fluree_db_iceberg::auth::NoAuth; +use fluree_db_sql::{LogicalSource, SqlGsConfig, TrinoClient}; +use futures::StreamExt; +use serde_json::json; +use wiremock::matchers::{body_string, header, method, path}; +use wiremock::{Mock, MockServer, Request, ResponseTemplate}; + +fn client(server: &MockServer) -> TrinoClient { + let mut cfg = SqlGsConfig::new(server.uri()); + cfg.catalog = Some("hive".into()); + cfg.schema = Some("sales".into()); + cfg.session.insert("query_max_run_time".into(), "5m".into()); + TrinoClient::new(&cfg, Arc::new(NoAuth)).unwrap() +} + +fn page( + id: &str, + next: Option, + columns: bool, + data: serde_json::Value, +) -> serde_json::Value { + let mut p = json!({ "id": id, "stats": { "state": "RUNNING" } }); + if let Some(n) = next { + p["nextUri"] = json!(n); + } + if columns { + p["columns"] = json!([ + { "name": "id", "type": "bigint" }, + { "name": "name", "type": "varchar" } + ]); + } + p["data"] = data; + p +} + +#[tokio::test] +async fn statement_pages_stream_as_batches_with_protocol_headers() { + let server = MockServer::start().await; + let base = server.uri(); + + Mock::given(method("POST")) + .and(path("/v1/statement")) + .and(header("X-Trino-User", "fluree")) + .and(header("X-Trino-Catalog", "hive")) + .and(header("X-Trino-Schema", "sales")) + .and(header("X-Trino-Session", "query_max_run_time=5m")) + .and(header("X-Trino-Time-Zone", "UTC")) + .and(body_string("SELECT 1")) + .respond_with(ResponseTemplate::new(200).set_body_json(page( + "q1", + Some(format!("{base}/v1/statement/q1/1")), + false, + json!(null), + ))) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/v1/statement/q1/1")) + .respond_with(ResponseTemplate::new(200).set_body_json(page( + "q1", + Some(format!("{base}/v1/statement/q1/2")), + true, + json!([[1, "a"], [2, null]]), + ))) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/v1/statement/q1/2")) + .respond_with(ResponseTemplate::new(200).set_body_json(page( + "q1", + Some(format!("{base}/v1/statement/q1/3")), + false, + json!([[3, "c"]]), + ))) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/v1/statement/q1/3")) + .respond_with(ResponseTemplate::new(200).set_body_json(page( + "q1", + None, + false, + json!(null), + ))) + .mount(&server) + .await; + + let c = client(&server); + let batches: Vec<_> = c.execute("SELECT 1".into()).collect().await; + let batches: Vec<_> = batches.into_iter().map(|b| b.unwrap()).collect(); + assert_eq!(batches.len(), 2); + assert_eq!(batches[0].num_rows, 2); + assert_eq!(batches[1].num_rows, 1); + assert_eq!(batches[0].column_by_name("id").unwrap().get_i64(1), Some(2)); + assert_eq!( + batches[0].column_by_name("name").unwrap().get_string(1), + None + ); + assert_eq!( + batches[1].column_by_name("name").unwrap().get_string(0), + Some("c") + ); + + let (schema, all) = c.execute_collect("SELECT 1").await.unwrap(); + assert_eq!(schema.unwrap().num_fields(), 2); + assert_eq!(all.iter().map(|b| b.num_rows).sum::(), 3); +} + +#[tokio::test] +async fn a_503_is_retried_and_an_error_page_fails_the_statement() { + let server = MockServer::start().await; + let base = server.uri(); + + Mock::given(method("POST")) + .and(path("/v1/statement")) + .respond_with(ResponseTemplate::new(503)) + .up_to_n_times(2) + .expect(2) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/v1/statement")) + .respond_with(ResponseTemplate::new(200).set_body_json(page( + "q2", + Some(format!("{base}/v1/statement/q2/1")), + true, + json!([[1, "a"]]), + ))) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/v1/statement/q2/1")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "q2", + "error": { "message": "line 1:8: Table 'x' does not exist", "errorName": "TABLE_NOT_FOUND", "errorCode": 43 }, + "stats": { "state": "FAILED" } + }))) + .mount(&server) + .await; + + let c = client(&server); + let items: Vec<_> = c.execute("SELECT * FROM x".into()).collect().await; + assert_eq!(items.len(), 2, "one batch then the error"); + assert!(items[0].is_ok()); + let err = items[1].as_ref().unwrap_err().to_string(); + assert!( + err.contains("does not exist") && err.contains("TABLE_NOT_FOUND 43"), + "{err}" + ); +} + +#[tokio::test] +async fn unauthorized_and_bad_json_surface_as_typed_errors() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/statement")) + .respond_with(ResponseTemplate::new(401)) + .mount(&server) + .await; + let err = client(&server) + .execute_collect("SELECT 1") + .await + .unwrap_err(); + assert!(matches!(err, fluree_db_sql::SqlError::Auth(_)), "{err}"); + + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/statement")) + .respond_with(ResponseTemplate::new(200).set_body_string("not trino")) + .mount(&server) + .await; + let err = client(&server) + .execute_collect("SELECT 1") + .await + .unwrap_err(); + assert!(matches!(err, fluree_db_sql::SqlError::Decode(_)), "{err}"); +} + +#[tokio::test] +async fn schema_probe_is_cached_and_count_reads_the_scalar() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/statement")) + .and(body_string(r#"SELECT * FROM "sales"."orders" LIMIT 0"#)) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "p", "columns": [{"name": "id", "type": "bigint"}, {"name": "total", "type": "decimal(10,2)"}], + "data": [], "stats": {"state": "FINISHED"} + }))) + .expect(1) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/v1/statement")) + .and(body_string(r#"SELECT COUNT(*) FROM "sales"."orders" WHERE "id" IS NOT NULL"#)) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "c", "columns": [{"name": "_col0", "type": "bigint"}], "data": [[42]], "stats": {"state": "FINISHED"} + }))) + .mount(&server) + .await; + + let c = client(&server); + let src = LogicalSource::Table("sales.orders".into()); + let s1 = c.schema(&src).await.unwrap(); + let s2 = c.schema(&src).await.unwrap(); + assert!(Arc::ptr_eq(&s1, &s2)); + assert_eq!( + s1.field_by_name("total").unwrap().field_type, + fluree_db_tabular::FieldType::Decimal { + precision: 10, + scale: 2 + } + ); + assert_eq!(c.count(&src, &["id".into()]).await.unwrap(), 42); +} + +#[tokio::test] +async fn dropping_the_stream_cancels_the_statement() { + let server = MockServer::start().await; + let base = server.uri(); + Mock::given(method("POST")) + .and(path("/v1/statement")) + .respond_with(ResponseTemplate::new(200).set_body_json(page( + "q3", + Some(format!("{base}/v1/statement/q3/1")), + true, + json!([[1, "a"]]), + ))) + .mount(&server) + .await; + // Every GET answers another page forever, so only a cancel ends this. + Mock::given(method("GET")) + .and(path("/v1/statement/q3/1")) + .respond_with(ResponseTemplate::new(200).set_body_json(page( + "q3", + Some(format!("{base}/v1/statement/q3/1")), + false, + json!([[2, "b"]]), + ))) + .mount(&server) + .await; + Mock::given(method("DELETE")) + .and(path("/v1/statement/q3/1")) + .respond_with(ResponseTemplate::new(204)) + .expect(1) + .mount(&server) + .await; + + let c = client(&server); + let mut stream = c.execute("SELECT 1".into()); + let first = stream.next().await.unwrap().unwrap(); + assert_eq!(first.num_rows, 1); + drop(stream); + + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5); + loop { + let cancelled = server + .received_requests() + .await + .unwrap_or_default() + .iter() + .any(|r: &Request| r.method == "DELETE"); + if cancelled { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "no DELETE observed after drop" + ); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } +} + +#[tokio::test] +async fn presto_header_family_is_selectable() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/statement")) + .and(header("X-Presto-User", "svc")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "p", "columns": [{"name": "x", "type": "integer"}], "data": [[1]], "stats": {"state": "FINISHED"} + }))) + .mount(&server) + .await; + let mut cfg = SqlGsConfig::new(server.uri()); + cfg.protocol = fluree_db_sql::WireProtocol::Presto; + cfg.user = "svc".into(); + let c = TrinoClient::new(&cfg, Arc::new(NoAuth)).unwrap(); + let (_, b) = c.execute_collect("SELECT 1").await.unwrap(); + assert_eq!(b[0].column(0).unwrap().get_i32(0), Some(1)); +} diff --git a/fluree-sql-bridge/Cargo.lock b/fluree-sql-bridge/Cargo.lock new file mode 100644 index 0000000000..219904f256 --- /dev/null +++ b/fluree-sql-bridge/Cargo.lock @@ -0,0 +1,2560 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bigdecimal" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +dependencies = [ + "serde_core", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "clap" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + +[[package]] +name = "crossbeam-queue" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" +dependencies = [ + "serde", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" + +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + +[[package]] +name = "fluree-sql-bridge" +version = "4.1.6" +dependencies = [ + "async-trait", + "axum", + "base64", + "bigdecimal", + "chrono", + "clap", + "futures", + "serde", + "serde_json", + "sqlx", + "tempfile", + "tokio", + "tower", + "tracing", + "tracing-subscriber", + "uuid", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "bytes", + "http", + "http-body", + "hyper", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" + +[[package]] +name = "icu_properties" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" + +[[package]] +name = "icu_provider" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07aa2048142242915a31d35844fb311e0e53fcca590c3a0a40dcf1b841fa09eb" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libredox" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7955dfc218a8afb29dfeffd540e3a6e96baeb94fe7138228dd7cc6937fbbf96" +dependencies = [ + "bitflags", + "libc", + "plain", + "redox_syscall 0.9.3", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_syscall" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d678d17679829e73d371e96880897e98fee2ded7acc0a50bdf8af2affa4b2fe5" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core", + "signature", + "spki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +dependencies = [ + "serde", +] + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlx" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" +dependencies = [ + "base64", + "bigdecimal", + "bytes", + "chrono", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.15.5", + "hashlink", + "indexmap", + "log", + "memchr", + "once_cell", + "percent-encoding", + "rustls", + "serde", + "serde_json", + "sha2", + "smallvec", + "thiserror", + "tokio", + "tokio-stream", + "tracing", + "url", + "uuid", + "webpki-roots 0.26.11", +] + +[[package]] +name = "sqlx-macros" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 2.0.119", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" +dependencies = [ + "dotenvy", + "either", + "heck", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 2.0.119", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" +dependencies = [ + "atoi", + "base64", + "bigdecimal", + "bitflags", + "byteorder", + "bytes", + "chrono", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" +dependencies = [ + "atoi", + "base64", + "bigdecimal", + "bitflags", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "num-bigint", + "once_cell", + "rand", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" +dependencies = [ + "atoi", + "chrono", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "serde_urlencoded", + "sqlx-core", + "thiserror", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.9", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "whoami" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" +dependencies = [ + "libredox", + "wasite", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/fluree-sql-bridge/Cargo.toml b/fluree-sql-bridge/Cargo.toml new file mode 100644 index 0000000000..3a9628e058 --- /dev/null +++ b/fluree-sql-bridge/Cargo.toml @@ -0,0 +1,78 @@ +[package] +name = "fluree-sql-bridge" +version = "4.1.6" +edition = "2021" +license = "BUSL-1.1" +repository = "https://github.com/fluree/db" +description = "Trino-protocol HTTP front for a single Postgres, MySQL or SQLite database — a sidecar for Fluree SQL graph sources" +publish = false + +# Standalone package, deliberately outside the repo workspace: it links sqlx +# and three database drivers, none of which belong in the `fluree` binary. It +# has its own Cargo.lock. +[workspace] + +# Mirrors `[workspace.lints.clippy]` in the repo root, which this package cannot +# inherit across the workspace boundary. The root is the source of truth; keep +# the two in step. +[workspace.lints.clippy] +bool_to_int_with_if = "deny" +comparison_chain = "deny" +elidable_lifetime_names = "deny" +explicit_into_iter_loop = "deny" +explicit_iter_loop = "deny" +ignored_unit_patterns = "deny" +inconsistent_struct_constructor = "deny" +manual_assert = "deny" +manual_is_variant_and = "deny" +manual_midpoint = "deny" +manual_string_new = "deny" +needless_raw_string_hashes = "deny" +option_as_ref_cloned = "deny" +range_plus_one = "deny" +redundant_closure_for_method_calls = "deny" +redundant_else = "deny" +ref_binding_to_reference = "deny" +semicolon_if_nothing_returned = "deny" +unnecessary_literal_bound = "deny" +unnecessary_semicolon = "deny" +unnested_or_patterns = "deny" +unreadable_literal = "deny" +uninlined_format_args = "deny" + +[dependencies] +axum = "0.7" +tokio = { version = "1", features = ["full"] } +clap = { version = "4", features = ["derive", "env"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +futures = "0.3" +async-trait = "0.1" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +base64 = "0.22" +uuid = { version = "1", features = ["v4"] } +sqlx = { version = "0.8", default-features = false, features = [ + "runtime-tokio", + "tls-rustls", + "postgres", + "mysql", + "sqlite", + "chrono", + "bigdecimal", + "json", + "uuid", +] } +bigdecimal = "0.4" +chrono = "0.4" + +[dev-dependencies] +tempfile = "3" +tower = { version = "0.5", features = ["util"] } + +[profile.release] +strip = true +lto = "thin" + +[lints] +workspace = true diff --git a/fluree-sql-bridge/src/backend.rs b/fluree-sql-bridge/src/backend.rs new file mode 100644 index 0000000000..d07ca64b35 --- /dev/null +++ b/fluree-sql-bridge/src/backend.rs @@ -0,0 +1,45 @@ +//! One trait over the three drivers: describe a statement's columns, then +//! stream its rows as protocol-ready JSON. + +use serde_json::Value; +use tokio::sync::mpsc; + +#[derive(Debug, Clone)] +pub struct ColumnMeta { + pub name: String, + /// Trino type name (`bigint`, `decimal(38,6)`, …). + pub trino_type: String, +} + +/// Chunks of rows, or the error that ended the statement. +pub type RowChunk = Result>, String>; + +pub const CHUNK_ROWS: usize = 500; + +#[derive(Debug, Clone)] +pub struct Session { + /// `X-Trino-Schema`: Postgres `search_path` / MySQL default database. + pub schema: Option, +} + +#[async_trait::async_trait] +pub trait Backend: Send + Sync { + fn dialect(&self) -> &'static str; + + /// Prepare the statement, report its columns, and start streaming rows + /// into `tx` from a spawned task. + async fn start( + &self, + sql: String, + session: Session, + tx: mpsc::Sender, + ) -> Result, String>; +} + +/// Drain a chunk buffer into the channel; `false` when the consumer is gone. +pub async fn flush(tx: &mpsc::Sender, buf: &mut Vec>) -> bool { + if buf.is_empty() { + return true; + } + tx.send(Ok(std::mem::take(buf))).await.is_ok() +} diff --git a/fluree-sql-bridge/src/lib.rs b/fluree-sql-bridge/src/lib.rs new file mode 100644 index 0000000000..e873dd08ff --- /dev/null +++ b/fluree-sql-bridge/src/lib.rs @@ -0,0 +1,323 @@ +//! `fluree-sql-bridge` — a Trino-protocol HTTP front for one Postgres, MySQL +//! or SQLite database. +//! +//! Fluree's SQL graph sources talk to "anything that speaks the Trino client +//! protocol". Trino itself is the general answer; this sidecar is the small +//! one for a single database when running a JVM is not wanted. It holds the +//! connection pool; the Fluree process holds nothing. +//! +//! Protocol subset served: +//! +//! - `POST /v1/statement` (body = SQL) → `{id, columns, nextUri}` +//! - `GET /v1/statement/{id}/{page}` → `{id, columns, data, nextUri?}` +//! - `DELETE /v1/statement/{id}/{page}` → cancel +//! - `GET /v1/info` → health +//! +//! Rows are streamed from the driver through a bounded channel and served a +//! page at a time, so a large result never sits in memory. + +pub mod backend; +pub mod mysql; +pub mod postgres; +pub mod render; +pub mod sqlite; + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use axum::extract::{Path, State}; +use axum::http::{HeaderMap, StatusCode}; +use axum::response::{IntoResponse, Response}; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use serde_json::{json, Value}; +use tokio::sync::{mpsc, Mutex}; +use tracing::{info, warn}; + +use backend::{Backend, ColumnMeta, RowChunk, Session}; + +/// Open the backend named by a database URL. +pub async fn connect_backend( + url: &str, + max_connections: u32, + decimal_scale: i64, +) -> Result, String> { + Ok(match url.split(':').next().unwrap_or("") { + "postgres" | "postgresql" => { + Box::new(postgres::Postgres::connect(url, max_connections, decimal_scale).await?) + } + "mysql" | "mariadb" => { + Box::new(mysql::MySql::connect(url, max_connections, decimal_scale).await?) + } + "sqlite" => Box::new(sqlite::Sqlite::connect(url, max_connections).await?), + other => { + return Err(format!( + "unsupported database URL scheme '{other}' (postgres://, mysql://, sqlite://)" + )) + } + }) +} + +pub struct Statement { + columns: Vec, + rx: mpsc::Receiver, + /// Rows already pulled from the channel but not yet served. + pending: Vec>, + next_page: u64, + last_touch: Instant, +} + +pub struct App { + backend: Box, + statements: Mutex>, + token: Option, + page_rows: usize, + idle: Duration, +} + +impl App { + pub fn new( + backend: Box, + token: Option, + page_rows: usize, + idle: Duration, + ) -> Arc { + Arc::new(Self { + backend, + statements: Mutex::new(HashMap::new()), + token, + page_rows: page_rows.max(1), + idle: idle.max(Duration::from_secs(1)), + }) + } + + /// Drop statements nobody has fetched from within the idle window. + pub fn spawn_reaper(self: &Arc) { + let app = Arc::clone(self); + tokio::spawn(async move { + loop { + tokio::time::sleep(app.idle / 2).await; + let mut st = app.statements.lock().await; + let before = st.len(); + st.retain(|_, s| s.last_touch.elapsed() < app.idle); + if st.len() != before { + info!(dropped = before - st.len(), "dropped idle statements"); + } + } + }); + } + + pub fn router(self: Arc) -> Router { + Router::new() + .route("/v1/statement", post(post_statement)) + .route("/v1/statement/:id/:page", get(get_page).delete(cancel)) + .route("/v1/info", get(info_route)) + .with_state(self) + } +} + +fn authorized(app: &App, headers: &HeaderMap) -> bool { + match &app.token { + None => true, + Some(t) => headers + .get(axum::http::header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.strip_prefix("Bearer ")) + .is_some_and(|got| got == t), + } +} + +fn header(headers: &HeaderMap, suffix: &str) -> Option { + for family in ["X-Trino-", "X-Presto-"] { + if let Some(v) = headers.get(format!("{family}{suffix}")) { + if let Ok(s) = v.to_str() { + if !s.is_empty() { + return Some(s.to_string()); + } + } + } + } + None +} + +fn column_json(columns: &[ColumnMeta]) -> Value { + Value::Array( + columns + .iter() + .map(|c| { + let raw = c + .trino_type + .split('(') + .next() + .unwrap_or(&c.trino_type) + .trim(); + json!({ + "name": c.name, + "type": c.trino_type, + "typeSignature": { "rawType": raw, "arguments": [] } + }) + }) + .collect(), + ) +} + +fn page_uri(headers: &HeaderMap, id: &str, page: u64) -> String { + let host = headers + .get(axum::http::header::HOST) + .and_then(|v| v.to_str().ok()) + .unwrap_or("localhost"); + let scheme = headers + .get("x-forwarded-proto") + .and_then(|v| v.to_str().ok()) + .unwrap_or("http"); + format!("{scheme}://{host}/v1/statement/{id}/{page}") +} + +fn error_response(id: &str, message: String, name: &str) -> Value { + json!({ + "id": id, + "error": { "message": message, "errorName": name, "errorCode": 65536 }, + "stats": { "state": "FAILED" } + }) +} + +async fn info_route(State(app): State>) -> Json { + Json(json!({ "starting": false, "dialect": app.backend.dialect(), "coordinator": true })) +} + +async fn post_statement(State(app): State>, headers: HeaderMap, body: String) -> Response { + if !authorized(&app, &headers) { + return StatusCode::UNAUTHORIZED.into_response(); + } + let sql = body.trim().trim_end_matches(';').to_string(); + if sql.is_empty() { + return (StatusCode::BAD_REQUEST, "empty statement").into_response(); + } + let id = format!( + "{}_{}", + chrono::Utc::now().format("%Y%m%d_%H%M%S"), + uuid::Uuid::new_v4().simple() + ); + let session = Session { + schema: header(&headers, "Schema"), + }; + info!(id, sql = %sql, "statement"); + + let (tx, rx) = mpsc::channel(8); + let columns = match app.backend.start(sql, session, tx).await { + Ok(c) => c, + Err(e) => { + warn!(id, error = %e, "statement failed to start"); + return Json(error_response(&id, e, "SYNTAX_ERROR")).into_response(); + } + }; + let cols = column_json(&columns); + app.statements.lock().await.insert( + id.clone(), + Statement { + columns, + rx, + pending: Vec::new(), + next_page: 1, + last_touch: Instant::now(), + }, + ); + Json(json!({ + "id": id, + "infoUri": page_uri(&headers, &id, 0), + "nextUri": page_uri(&headers, &id, 1), + "columns": cols, + "stats": { "state": "RUNNING" } + })) + .into_response() +} + +async fn get_page( + State(app): State>, + headers: HeaderMap, + Path((id, page)): Path<(String, u64)>, +) -> Response { + if !authorized(&app, &headers) { + return StatusCode::UNAUTHORIZED.into_response(); + } + let Some(mut st) = app.statements.lock().await.remove(&id) else { + return (StatusCode::GONE, "unknown or finished statement").into_response(); + }; + if page != st.next_page { + let expected = st.next_page; + app.statements.lock().await.insert(id.clone(), st); + return ( + StatusCode::GONE, + format!( + "page {page} is not the next page ({expected}); pages are served once, in order" + ), + ) + .into_response(); + } + + let mut rows = std::mem::take(&mut st.pending); + let mut finished = false; + let mut error: Option = None; + while rows.len() < app.page_rows { + let chunk = if rows.is_empty() { + st.rx.recv().await + } else { + match st.rx.try_recv() { + Ok(c) => Some(c), + Err(mpsc::error::TryRecvError::Empty) => break, + Err(mpsc::error::TryRecvError::Disconnected) => None, + } + }; + match chunk { + Some(Ok(mut c)) => rows.append(&mut c), + Some(Err(e)) => { + error = Some(e); + finished = true; + break; + } + None => { + finished = true; + break; + } + } + } + if rows.len() > app.page_rows { + st.pending = rows.split_off(app.page_rows); + } + + if let Some(e) = error { + warn!(id, error = %e, "statement failed"); + return Json(error_response(&id, e, "GENERIC_INTERNAL_ERROR")).into_response(); + } + + let cols = column_json(&st.columns); + let mut body = json!({ + "id": id, + "columns": cols, + "data": rows, + "stats": { "state": if finished { "FINISHED" } else { "RUNNING" } } + }); + if !finished { + st.next_page += 1; + st.last_touch = Instant::now(); + body["nextUri"] = json!(page_uri(&headers, &id, st.next_page)); + app.statements.lock().await.insert(id, st); + } + Json(body).into_response() +} + +async fn cancel( + State(app): State>, + headers: HeaderMap, + Path((id, _page)): Path<(String, u64)>, +) -> Response { + if !authorized(&app, &headers) { + return StatusCode::UNAUTHORIZED.into_response(); + } + // Dropping the statement drops its receiver; the driver task's next send + // fails and it stops reading. + let removed = app.statements.lock().await.remove(&id).is_some(); + info!(id, removed, "statement cancelled"); + StatusCode::NO_CONTENT.into_response() +} diff --git a/fluree-sql-bridge/src/main.rs b/fluree-sql-bridge/src/main.rs new file mode 100644 index 0000000000..279ce679ce --- /dev/null +++ b/fluree-sql-bridge/src/main.rs @@ -0,0 +1,72 @@ +use std::net::SocketAddr; +use std::time::Duration; + +use clap::Parser; +use tracing::info; + +#[derive(Parser, Debug)] +#[command(name = "fluree-sql-bridge", version, about)] +struct Args { + /// Address to listen on. + #[arg(long, default_value = "127.0.0.1:8080", env = "BRIDGE_LISTEN")] + listen: SocketAddr, + + /// Database URL: postgres://…, mysql://…, or sqlite://path.db + #[arg(long, env = "DATABASE_URL")] + database: String, + + /// Require this bearer token on every request. + #[arg(long, env = "BRIDGE_TOKEN")] + token: Option, + + /// Connection pool size. + #[arg(long, default_value_t = 8)] + max_connections: u32, + + /// Rows per protocol page. + #[arg(long, default_value_t = 5000)] + page_rows: usize, + + /// Scale reported for NUMERIC/DECIMAL columns (`decimal(38, N)`); values + /// with more fractional digits are rounded half-even. + #[arg(long, default_value_t = 6)] + decimal_scale: i64, + + /// Abandoned statements are dropped after this many seconds without a fetch. + #[arg(long, default_value_t = 300)] + idle_secs: u64, +} + +#[tokio::main] +async fn main() -> Result<(), String> { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| "info,sqlx=warn".into()), + ) + .init(); + let args = Args::parse(); + + let backend = fluree_sql_bridge::connect_backend( + &args.database, + args.max_connections, + args.decimal_scale, + ) + .await?; + info!(dialect = backend.dialect(), listen = %args.listen, "fluree-sql-bridge ready"); + + let app = fluree_sql_bridge::App::new( + backend, + args.token, + args.page_rows, + Duration::from_secs(args.idle_secs), + ); + app.spawn_reaper(); + + let listener = tokio::net::TcpListener::bind(args.listen) + .await + .map_err(|e| format!("bind {}: {e}", args.listen))?; + axum::serve(listener, app.router()) + .await + .map_err(|e| e.to_string()) +} diff --git a/fluree-sql-bridge/src/mysql.rs b/fluree-sql-bridge/src/mysql.rs new file mode 100644 index 0000000000..ff02d516d5 --- /dev/null +++ b/fluree-sql-bridge/src/mysql.rs @@ -0,0 +1,211 @@ +use futures::TryStreamExt; +use serde_json::Value; +use sqlx::mysql::{MySqlPool, MySqlPoolOptions, MySqlRow}; +use sqlx::{Column, Executor, Row, TypeInfo, ValueRef}; +use tokio::sync::mpsc; + +use crate::backend::{flush, Backend, ColumnMeta, RowChunk, Session, CHUNK_ROWS}; +use crate::render::{self, trino}; + +/// Make MySQL read string literals by the standard-SQL rule, where `''` is an +/// escaped quote and a backslash is an ordinary character. +/// +/// Clients render literals with quote doubling alone, which is the whole rule +/// on Trino, Postgres and SQLite. MySQL's default `sql_mode` additionally +/// treats `\\` as live inside a literal, so a value ending in a backslash would +/// escape its own closing quote and the remainder of the statement would parse +/// as SQL. `NO_BACKSLASH_ESCAPES` removes that difference for every session +/// this pool hands out. +/// +/// The `IF` guards an empty `sql_mode`, which `CONCAT` would turn into a +/// leading comma that MySQL rejects. +const ENFORCE_STANDARD_STRING_LITERALS: &str = concat!( + "SET SESSION sql_mode = IF(@@sql_mode = '', 'NO_BACKSLASH_ESCAPES', ", + "CONCAT(@@sql_mode, ',NO_BACKSLASH_ESCAPES'))" +); + +pub struct MySql { + pool: MySqlPool, + decimal_scale: i64, +} + +impl MySql { + pub async fn connect( + url: &str, + max_connections: u32, + decimal_scale: i64, + ) -> Result { + let pool = MySqlPoolOptions::new() + .max_connections(max_connections) + .after_connect(|conn, _meta| { + Box::pin(async move { + conn.execute(ENFORCE_STANDARD_STRING_LITERALS).await?; + Ok(()) + }) + }) + .connect(url) + .await + .map_err(|e| format!("connect mysql: {e}"))?; + Ok(Self { + pool, + decimal_scale, + }) + } +} + +fn base_type(t: &str) -> (&str, bool) { + match t.strip_suffix(" UNSIGNED") { + Some(b) => (b, true), + None => (t, false), + } +} + +fn trino_type(t: &str, scale: i64) -> Result { + let (base, unsigned) = base_type(t); + Ok(match base { + "BOOLEAN" => trino::BOOLEAN.into(), + "TINYINT" | "SMALLINT" | "MEDIUMINT" | "YEAR" => trino::INTEGER.into(), + "INT" => { + if unsigned { + trino::BIGINT.into() + } else { + trino::INTEGER.into() + } + } + "BIGINT" | "BIT" => trino::BIGINT.into(), + "FLOAT" => trino::REAL.into(), + "DOUBLE" => trino::DOUBLE.into(), + "DECIMAL" => trino::decimal(scale), + "CHAR" | "VARCHAR" | "TEXT" | "TINYTEXT" | "MEDIUMTEXT" | "LONGTEXT" | "ENUM" | "SET" + | "JSON" => trino::VARCHAR.into(), + "BINARY" | "VARBINARY" | "BLOB" | "TINYBLOB" | "MEDIUMBLOB" | "LONGBLOB" => { + trino::VARBINARY.into() + } + "DATE" => trino::DATE.into(), + "DATETIME" => trino::TIMESTAMP.into(), + "TIMESTAMP" => trino::TIMESTAMP_TZ.into(), + "TIME" => trino::TIME.into(), + other => { + return Err(format!( + "unsupported MySQL column type {other}; CAST it in the rr:sqlQuery" + )) + } + }) +} + +fn cell(row: &MySqlRow, i: usize, t: &str, scale: i64) -> Result { + let raw = row.try_get_raw(i).map_err(|e| e.to_string())?; + if raw.is_null() { + return Ok(Value::Null); + } + macro_rules! get { + ($ty:ty) => { + row.try_get::<$ty, _>(i) + .map_err(|e| format!("column {i} ({t}): {e}"))? + }; + } + let (base, unsigned) = base_type(t); + Ok(match (base, unsigned) { + ("BOOLEAN", _) => render::bool(get!(bool)), + ("TINYINT", false) => render::int(i64::from(get!(i8))), + ("TINYINT", true) => render::int(i64::from(get!(u8))), + ("SMALLINT" | "YEAR", false) => render::int(i64::from(get!(i16))), + ("SMALLINT" | "YEAR", true) => render::int(i64::from(get!(u16))), + ("MEDIUMINT" | "INT", false) => render::int(i64::from(get!(i32))), + ("MEDIUMINT" | "INT", true) => render::int(i64::from(get!(u32))), + ("BIGINT", false) => render::int(get!(i64)), + ("BIGINT", true) => render::uint(get!(u64)), + ("BIT", _) => render::uint(get!(u64)), + ("FLOAT", _) => render::double(f64::from(get!(f32))), + ("DOUBLE", _) => render::double(get!(f64)), + ("DECIMAL", _) => render::decimal(&get!(sqlx::types::BigDecimal), scale), + ( + "CHAR" | "VARCHAR" | "TEXT" | "TINYTEXT" | "MEDIUMTEXT" | "LONGTEXT" | "ENUM" | "SET", + _, + ) => render::string(get!(String)), + ("JSON", _) => render::jsonish(&get!(serde_json::Value)), + ("BINARY" | "VARBINARY" | "BLOB" | "TINYBLOB" | "MEDIUMBLOB" | "LONGBLOB", _) => { + render::bytes(&get!(Vec)) + } + ("DATE", _) => render::date(get!(chrono::NaiveDate)), + ("DATETIME", _) => render::timestamp(get!(chrono::NaiveDateTime)), + ("TIMESTAMP", _) => render::timestamp_tz(get!(chrono::DateTime)), + ("TIME", _) => render::time(get!(chrono::NaiveTime)), + (other, _) => return Err(format!("unsupported MySQL column type {other}")), + }) +} + +#[async_trait::async_trait] +impl Backend for MySql { + fn dialect(&self) -> &'static str { + "mysql" + } + + async fn start( + &self, + sql: String, + session: Session, + tx: mpsc::Sender, + ) -> Result, String> { + let mut conn = self + .pool + .acquire() + .await + .map_err(|e| format!("acquire: {e}"))?; + if let Some(schema) = &session.schema { + let stmt = format!("USE `{}`", schema.replace('`', "``")); + sqlx::query(&stmt) + .execute(&mut *conn) + .await + .map_err(|e| format!("use database: {e}"))?; + } + let described = conn.describe(&sql).await.map_err(|e| e.to_string())?; + let types: Vec = described + .columns() + .iter() + .map(|c| c.type_info().name().to_string()) + .collect(); + let scale = self.decimal_scale; + let mut columns = Vec::with_capacity(types.len()); + for (c, t) in described.columns().iter().zip(&types) { + columns.push(ColumnMeta { + name: c.name().to_string(), + trino_type: trino_type(t, scale)?, + }); + } + + tokio::spawn(async move { + let mut stream = sqlx::query(&sql).fetch(&mut *conn); + let mut buf: Vec> = Vec::with_capacity(CHUNK_ROWS); + loop { + match stream.try_next().await { + Ok(Some(row)) => { + let mut out = Vec::with_capacity(types.len()); + for (i, t) in types.iter().enumerate() { + match cell(&row, i, t, scale) { + Ok(v) => out.push(v), + Err(e) => { + let _ = tx.send(Err(e)).await; + return; + } + } + } + buf.push(out); + if buf.len() >= CHUNK_ROWS && !flush(&tx, &mut buf).await { + return; + } + } + Ok(None) => { + flush(&tx, &mut buf).await; + return; + } + Err(e) => { + let _ = tx.send(Err(e.to_string())).await; + return; + } + } + } + }); + Ok(columns) + } +} diff --git a/fluree-sql-bridge/src/postgres.rs b/fluree-sql-bridge/src/postgres.rs new file mode 100644 index 0000000000..9112c457e5 --- /dev/null +++ b/fluree-sql-bridge/src/postgres.rs @@ -0,0 +1,198 @@ +use futures::TryStreamExt; +use serde_json::Value; +use sqlx::postgres::{PgPool, PgPoolOptions, PgRow}; +use sqlx::{Column, Executor, Row, TypeInfo, ValueRef}; +use tokio::sync::mpsc; + +use crate::backend::{flush, Backend, ColumnMeta, RowChunk, Session, CHUNK_ROWS}; +use crate::render::{self, trino}; + +/// Keep Postgres on the standard-SQL string rule this bridge's clients assume. +/// +/// `standard_conforming_strings` has defaulted to `on` since 9.1, which is why +/// Postgres is not exposed the way MySQL is (see `mysql.rs`). But it is still a +/// settable GUC: a server, database or role with it `off` would process +/// backslash escapes inside ordinary literals, and a value ending in one could +/// escape its own closing quote. Setting it per session costs nothing and +/// removes the dependency on how the server happens to be configured. +const ENFORCE_STANDARD_STRING_LITERALS: &str = "SET standard_conforming_strings = on"; + +pub struct Postgres { + pool: PgPool, + decimal_scale: i64, +} + +impl Postgres { + pub async fn connect( + url: &str, + max_connections: u32, + decimal_scale: i64, + ) -> Result { + let pool = PgPoolOptions::new() + .max_connections(max_connections) + .after_connect(|conn, _meta| { + Box::pin(async move { + conn.execute(ENFORCE_STANDARD_STRING_LITERALS).await?; + Ok(()) + }) + }) + .connect(url) + .await + .map_err(|e| format!("connect postgres: {e}"))?; + Ok(Self { + pool, + decimal_scale, + }) + } + + fn trino_type(&self, pg: &str) -> Result { + Ok(match pg { + "BOOL" => trino::BOOLEAN.into(), + "INT2" | "INT4" => trino::INTEGER.into(), + "INT8" | "OID" => trino::BIGINT.into(), + "FLOAT4" => trino::REAL.into(), + "FLOAT8" => trino::DOUBLE.into(), + "NUMERIC" | "MONEY" => trino::decimal(self.decimal_scale), + "TEXT" | "VARCHAR" | "BPCHAR" | "CHAR" | "NAME" | "UUID" | "JSON" | "JSONB" + | "INTERVAL" | "CITEXT" => trino::VARCHAR.into(), + "BYTEA" => trino::VARBINARY.into(), + "DATE" => trino::DATE.into(), + "TIMESTAMP" => trino::TIMESTAMP.into(), + "TIMESTAMPTZ" => trino::TIMESTAMP_TZ.into(), + "TIME" => trino::TIME.into(), + other if other.ends_with("[]") => trino::VARCHAR.into(), + other => { + return Err(format!( + "unsupported Postgres column type {other}; CAST it in the rr:sqlQuery" + )) + } + }) + } +} + +fn cell(row: &PgRow, i: usize, pg: &str, decimal_scale: i64) -> Result { + let raw = row.try_get_raw(i).map_err(|e| e.to_string())?; + if raw.is_null() { + return Ok(Value::Null); + } + macro_rules! get { + ($t:ty) => { + row.try_get::<$t, _>(i) + .map_err(|e| format!("column {i} ({pg}): {e}"))? + }; + } + Ok(match pg { + "BOOL" => render::bool(get!(bool)), + "INT2" => render::int(i64::from(get!(i16))), + "INT4" => render::int(i64::from(get!(i32))), + "INT8" => render::int(get!(i64)), + "OID" => render::uint(u64::from(get!(sqlx::postgres::types::Oid).0)), + "FLOAT4" => render::double(f64::from(get!(f32))), + "FLOAT8" => render::double(get!(f64)), + "NUMERIC" => render::decimal(&get!(sqlx::types::BigDecimal), decimal_scale), + "MONEY" => { + let m = get!(sqlx::postgres::types::PgMoney); + render::decimal(&m.to_bigdecimal(2), decimal_scale) + } + "TEXT" | "VARCHAR" | "BPCHAR" | "CHAR" | "NAME" | "CITEXT" => render::string(get!(String)), + "UUID" => render::string(get!(sqlx::types::Uuid).to_string()), + "JSON" | "JSONB" => render::jsonish(&get!(serde_json::Value)), + "INTERVAL" => { + let iv = get!(sqlx::postgres::types::PgInterval); + render::string(format!( + "{} months {} days {} microseconds", + iv.months, iv.days, iv.microseconds + )) + } + "BYTEA" => render::bytes(&get!(Vec)), + "DATE" => render::date(get!(chrono::NaiveDate)), + "TIMESTAMP" => render::timestamp(get!(chrono::NaiveDateTime)), + "TIMESTAMPTZ" => render::timestamp_tz(get!(chrono::DateTime)), + "TIME" => render::time(get!(chrono::NaiveTime)), + "TEXT[]" | "VARCHAR[]" => Value::String(serde_json::to_string(&get!(Vec)).unwrap()), + "INT4[]" => Value::String(serde_json::to_string(&get!(Vec)).unwrap()), + "INT8[]" => Value::String(serde_json::to_string(&get!(Vec)).unwrap()), + "FLOAT8[]" => Value::String(serde_json::to_string(&get!(Vec)).unwrap()), + "BOOL[]" => Value::String(serde_json::to_string(&get!(Vec)).unwrap()), + other => return Err(format!("unsupported Postgres column type {other}")), + }) +} + +#[async_trait::async_trait] +impl Backend for Postgres { + fn dialect(&self) -> &'static str { + "postgres" + } + + async fn start( + &self, + sql: String, + session: Session, + tx: mpsc::Sender, + ) -> Result, String> { + let mut conn = self + .pool + .acquire() + .await + .map_err(|e| format!("acquire: {e}"))?; + if let Some(schema) = &session.schema { + let stmt = format!("SET search_path TO {}", quote_ident(schema)); + sqlx::query(&stmt) + .execute(&mut *conn) + .await + .map_err(|e| format!("set search_path: {e}"))?; + } + let described = conn.describe(&sql).await.map_err(|e| e.to_string())?; + let pg_types: Vec = described + .columns() + .iter() + .map(|c| c.type_info().name().to_string()) + .collect(); + let mut columns = Vec::with_capacity(pg_types.len()); + for (c, t) in described.columns().iter().zip(&pg_types) { + columns.push(ColumnMeta { + name: c.name().to_string(), + trino_type: self.trino_type(t)?, + }); + } + + let scale = self.decimal_scale; + tokio::spawn(async move { + let mut stream = sqlx::query(&sql).fetch(&mut *conn); + let mut buf: Vec> = Vec::with_capacity(CHUNK_ROWS); + loop { + match stream.try_next().await { + Ok(Some(row)) => { + let mut out = Vec::with_capacity(pg_types.len()); + for (i, t) in pg_types.iter().enumerate() { + match cell(&row, i, t, scale) { + Ok(v) => out.push(v), + Err(e) => { + let _ = tx.send(Err(e)).await; + return; + } + } + } + buf.push(out); + if buf.len() >= CHUNK_ROWS && !flush(&tx, &mut buf).await { + return; + } + } + Ok(None) => { + flush(&tx, &mut buf).await; + return; + } + Err(e) => { + let _ = tx.send(Err(e.to_string())).await; + return; + } + } + } + }); + Ok(columns) + } +} + +pub fn quote_ident(s: &str) -> String { + format!("\"{}\"", s.replace('"', "\"\"")) +} diff --git a/fluree-sql-bridge/src/render.rs b/fluree-sql-bridge/src/render.rs new file mode 100644 index 0000000000..51f63e0732 --- /dev/null +++ b/fluree-sql-bridge/src/render.rs @@ -0,0 +1,83 @@ +//! Values → the JSON the Trino client protocol uses, and driver type names → +//! Trino type names. Kept driver-agnostic so the three backends share it. + +use base64::Engine; +use bigdecimal::BigDecimal; +use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc}; +use serde_json::{json, Value}; + +pub fn bool(b: bool) -> Value { + Value::Bool(b) +} + +pub fn int(i: i64) -> Value { + json!(i) +} + +pub fn uint(u: u64) -> Value { + json!(u) +} + +pub fn double(d: f64) -> Value { + if d.is_nan() { + json!("NaN") + } else if d.is_infinite() { + json!(if d > 0.0 { "Infinity" } else { "-Infinity" }) + } else { + json!(d) + } +} + +/// Exact decimal at the bridge's fixed scale (the column type is reported as +/// `decimal(38, scale)`); rounds half-even when the value carries more digits. +pub fn decimal(d: &BigDecimal, scale: i64) -> Value { + let fixed = d.with_scale_round(scale, bigdecimal::RoundingMode::HalfEven); + json!(fixed.to_plain_string()) +} + +pub fn string(s: impl Into) -> Value { + Value::String(s.into()) +} + +pub fn bytes(b: &[u8]) -> Value { + json!(base64::engine::general_purpose::STANDARD.encode(b)) +} + +pub fn date(d: NaiveDate) -> Value { + json!(d.format("%Y-%m-%d").to_string()) +} + +pub fn timestamp(t: NaiveDateTime) -> Value { + json!(t.format("%Y-%m-%d %H:%M:%S%.6f").to_string()) +} + +pub fn timestamp_tz(t: DateTime) -> Value { + json!(t.format("%Y-%m-%d %H:%M:%S%.6f UTC").to_string()) +} + +pub fn time(t: NaiveTime) -> Value { + json!(t.format("%H:%M:%S%.3f").to_string()) +} + +pub fn jsonish(v: &Value) -> Value { + Value::String(v.to_string()) +} + +/// Trino's spelling of the types this bridge produces. +pub mod trino { + pub const BOOLEAN: &str = "boolean"; + pub const INTEGER: &str = "integer"; + pub const BIGINT: &str = "bigint"; + pub const REAL: &str = "real"; + pub const DOUBLE: &str = "double"; + pub const VARCHAR: &str = "varchar"; + pub const VARBINARY: &str = "varbinary"; + pub const DATE: &str = "date"; + pub const TIMESTAMP: &str = "timestamp(6)"; + pub const TIMESTAMP_TZ: &str = "timestamp(6) with time zone"; + pub const TIME: &str = "time(3)"; + + pub fn decimal(scale: i64) -> String { + format!("decimal(38,{scale})") + } +} diff --git a/fluree-sql-bridge/src/sqlite.rs b/fluree-sql-bridge/src/sqlite.rs new file mode 100644 index 0000000000..609177eedd --- /dev/null +++ b/fluree-sql-bridge/src/sqlite.rs @@ -0,0 +1,123 @@ +use futures::TryStreamExt; +use serde_json::Value; +use sqlx::sqlite::{SqlitePool, SqlitePoolOptions, SqliteRow}; +use sqlx::{Column, Executor, Row, TypeInfo, ValueRef}; +use tokio::sync::mpsc; + +use crate::backend::{flush, Backend, ColumnMeta, RowChunk, Session, CHUNK_ROWS}; +use crate::render::{self, trino}; + +pub struct Sqlite { + pool: SqlitePool, +} + +impl Sqlite { + pub async fn connect(url: &str, max_connections: u32) -> Result { + let pool = SqlitePoolOptions::new() + .max_connections(max_connections) + .connect(url) + .await + .map_err(|e| format!("connect sqlite: {e}"))?; + Ok(Self { pool }) + } +} + +/// SQLite is dynamically typed; the declared type only hints. Everything +/// unknown is read as text, which is what SQLite itself would hand back. +fn trino_type(t: &str) -> &'static str { + match t.to_ascii_uppercase().as_str() { + "BOOLEAN" => trino::BOOLEAN, + "INTEGER" | "INT" | "BIGINT" | "SMALLINT" | "TINYINT" => trino::BIGINT, + "REAL" | "FLOAT" | "DOUBLE" | "NUMERIC" | "DECIMAL" => trino::DOUBLE, + "BLOB" => trino::VARBINARY, + "DATE" => trino::DATE, + "DATETIME" | "TIMESTAMP" => trino::TIMESTAMP, + _ => trino::VARCHAR, + } +} + +fn cell(row: &SqliteRow, i: usize, trino: &str) -> Result { + let raw = row.try_get_raw(i).map_err(|e| e.to_string())?; + if raw.is_null() { + return Ok(Value::Null); + } + macro_rules! get { + ($ty:ty) => { + row.try_get::<$ty, _>(i) + .map_err(|e| format!("column {i}: {e}"))? + }; + } + Ok(match trino { + t if t == trino::BOOLEAN => render::bool(get!(bool)), + t if t == trino::BIGINT => render::int(get!(i64)), + t if t == trino::DOUBLE => render::double(get!(f64)), + t if t == trino::VARBINARY => render::bytes(&get!(Vec)), + t if t == trino::DATE => render::date(get!(chrono::NaiveDate)), + t if t == trino::TIMESTAMP => render::timestamp(get!(chrono::NaiveDateTime)), + _ => render::string(get!(String)), + }) +} + +#[async_trait::async_trait] +impl Backend for Sqlite { + fn dialect(&self) -> &'static str { + "sqlite" + } + + async fn start( + &self, + sql: String, + _session: Session, + tx: mpsc::Sender, + ) -> Result, String> { + let mut conn = self + .pool + .acquire() + .await + .map_err(|e| format!("acquire: {e}"))?; + let described = conn.describe(&sql).await.map_err(|e| e.to_string())?; + let columns: Vec = described + .columns() + .iter() + .map(|c| ColumnMeta { + name: c.name().to_string(), + trino_type: trino_type(c.type_info().name()).to_string(), + }) + .collect(); + let types: Vec = columns.iter().map(|c| c.trino_type.clone()).collect(); + + tokio::spawn(async move { + let mut stream = sqlx::query(&sql).fetch(&mut *conn); + let mut buf: Vec> = Vec::with_capacity(CHUNK_ROWS); + loop { + match stream.try_next().await { + Ok(Some(row)) => { + let mut out = Vec::with_capacity(types.len()); + for (i, t) in types.iter().enumerate() { + match cell(&row, i, t) { + Ok(v) => out.push(v), + Err(e) => { + let _ = tx.send(Err(e)).await; + return; + } + } + } + buf.push(out); + if buf.len() >= CHUNK_ROWS && !flush(&tx, &mut buf).await { + return; + } + } + Ok(None) => { + flush(&tx, &mut buf).await; + return; + } + Err(e) => { + let _ = tx.send(Err(e.to_string())).await; + return; + } + } + } + }); + Ok(columns) + } +} diff --git a/fluree-sql-bridge/tests/server_backends.rs b/fluree-sql-bridge/tests/server_backends.rs new file mode 100644 index 0000000000..3640efbd2c --- /dev/null +++ b/fluree-sql-bridge/tests/server_backends.rs @@ -0,0 +1,379 @@ +//! Protocol tests that need a real MySQL or Postgres server. +//! +//! SQLite covers the protocol shape (`sqlite_protocol.rs`), but it is the one +//! backend whose string literals cannot misbehave, so the escaping rule these +//! tests pin is invisible there. +//! +//! Gated on `FLUREE_BRIDGE_MYSQL_URL` / `FLUREE_BRIDGE_POSTGRES_URL`. CI's +//! `sql-bridge` job supplies both from service containers, and +//! `server_backends_are_configured_in_ci` fails if it ever stops — a skipped +//! test must not read as a passing one. + +use std::sync::Arc; +use std::time::Duration; + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use serde_json::Value; +use sqlx::Executor; +use tower::ServiceExt; + +/// The value a hostile query author supplies. Under MySQL's default `sql_mode` +/// the rendered form `'a\'' UNION …' ` closes its literal after `a'`, leaving +/// `UNION SELECT token FROM bridge_secrets --` to parse as SQL. +const PAYLOAD: &str = r"a\' UNION SELECT token FROM bridge_secrets -- "; + +/// A Windows-ish path: the minimal trailing-backslash case, and ordinary data. +const TRAILING_BACKSLASH: &str = r"c:\"; + +/// What `bridge_secrets` holds — a mapping-scoped query must never see it. +const SECRET: &str = "topsecret"; + +/// What a `LIMIT 0` probe over `bridge_types` reports, on either backend. +/// +/// Both produce the same vector: MySQL stores `BOOLEAN` as `TINYINT(1)`, but +/// sqlx reports the declared type, so it maps to Trino `boolean` like +/// Postgres's. That equivalence is what lets a mapping move between the two +/// backends unchanged, so it is shared here rather than written out twice. +const PROBE_TYPES: [&str; 8] = [ + "integer", + "bigint", + "double", + "varchar", + "decimal(38,6)", + "date", + "timestamp(6)", + "boolean", +]; + +fn backend_url(var: &str) -> Option { + match std::env::var(var) { + Ok(url) if !url.is_empty() => Some(url), + _ => { + eprintln!("SKIPPED: {var} is unset"); + None + } + } +} + +/// Render a string the way the engine's `sql_string` does: wrap in `'…'` and +/// double any embedded `'`. Nothing else. Reproduced here rather than imported +/// because the point is to pin the *bridge's* behaviour against exactly this +/// rendering, independent of what the engine chooses to push. +fn sql_string(s: &str) -> String { + format!("'{}'", s.replace('\'', "''")) +} + +async fn router_for(url: &str) -> axum::Router { + let backend = fluree_sql_bridge::connect_backend(url, 2, 6) + .await + .unwrap_or_else(|e| panic!("connect {url}: {e}")); + let app = fluree_sql_bridge::App::new(backend, None, 100, Duration::from_secs(60)); + Arc::clone(&app).router() +} + +async fn call(router: &axum::Router, req: Request) -> (StatusCode, Value) { + let resp = router.clone().oneshot(req).await.unwrap(); + let status = resp.status(); + let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + ( + status, + serde_json::from_slice(&bytes).unwrap_or(Value::Null), + ) +} + +/// Run one statement through the protocol and drain every page. +/// +/// Returns `Err` with the protocol error message when the statement fails, so a +/// test can distinguish "the server rejected it" from "the server ran it". +async fn rows(router: &axum::Router, sql: &str) -> Result, String> { + let req = Request::post("/v1/statement") + .header("host", "bridge:8080") + .header("X-Trino-User", "fluree") + .body(Body::from(sql.to_string())) + .unwrap(); + let (status, first) = call(router, req).await; + assert_eq!(status, StatusCode::OK, "{first}"); + if let Some(msg) = first["error"]["message"].as_str() { + return Err(msg.to_string()); + } + + let mut out = Vec::new(); + let mut next = first["nextUri"].as_str().unwrap().to_string(); + loop { + let path = next.trim_start_matches("http://bridge:8080").to_string(); + let req = Request::get(&path) + .header("host", "bridge:8080") + .body(Body::empty()) + .unwrap(); + let (status, page) = call(router, req).await; + assert_eq!(status, StatusCode::OK, "{page}"); + if let Some(msg) = page["error"]["message"].as_str() { + return Err(msg.to_string()); + } + out.extend(page["data"].as_array().cloned().unwrap_or_default()); + match page["nextUri"].as_str() { + Some(n) => next = n.to_string(), + None => break, + } + } + Ok(out) +} + +/// Assert that a literal reaches the database as data, not as code. +/// +/// `quote` renders an identifier for the dialect under test. Seeding goes +/// through bound parameters, so the row really does hold `PAYLOAD` byte for +/// byte and a correct server returns exactly it. +async fn assert_literals_are_data(router: &axum::Router, quote: fn(&str) -> String) { + let name = quote("name"); + let people = quote("bridge_people"); + + let found = rows( + router, + &format!( + "SELECT {name} FROM {people} WHERE {name} = {}", + sql_string(PAYLOAD) + ), + ) + .await + .expect("the payload is a well-formed literal and the statement must run"); + + let values: Vec<&str> = found.iter().filter_map(|r| r[0].as_str()).collect(); + assert!( + !values.contains(&SECRET), + "injected UNION reached bridge_secrets: {values:?}" + ); + assert_eq!( + values, + [PAYLOAD], + "the payload must match itself and nothing else" + ); + + let found = rows( + router, + &format!( + "SELECT {name} FROM {people} WHERE {name} = {}", + sql_string(TRAILING_BACKSLASH) + ), + ) + .await + .expect("a trailing backslash is a well-formed literal"); + let values: Vec<&str> = found.iter().filter_map(|r| r[0].as_str()).collect(); + assert_eq!(values, [TRAILING_BACKSLASH]); + + // Quote doubling itself still works, and is not disturbed by the mode change. + let found = rows( + router, + &format!( + "SELECT {name} FROM {people} WHERE {name} = {}", + sql_string("O'Brien") + ), + ) + .await + .unwrap(); + let values: Vec<&str> = found.iter().filter_map(|r| r[0].as_str()).collect(); + assert_eq!(values, ["O'Brien"]); +} + +fn mysql_quote(ident: &str) -> String { + format!("`{}`", ident.replace('`', "``")) +} + +fn pg_quote(ident: &str) -> String { + format!("\"{}\"", ident.replace('"', "\"\"")) +} + +#[tokio::test] +async fn mysql_string_literals_are_data_not_code() { + let Some(url) = backend_url("FLUREE_BRIDGE_MYSQL_URL") else { + return; + }; + let pool = sqlx::mysql::MySqlPool::connect(&url).await.unwrap(); + for stmt in [ + "DROP TABLE IF EXISTS bridge_people", + "DROP TABLE IF EXISTS bridge_secrets", + "CREATE TABLE bridge_people (id INT PRIMARY KEY, name VARCHAR(255))", + "CREATE TABLE bridge_secrets (token VARCHAR(255))", + ] { + pool.execute(stmt).await.unwrap(); + } + for (id, name) in [(1, PAYLOAD), (2, TRAILING_BACKSLASH), (3, "O'Brien")] { + sqlx::query("INSERT INTO bridge_people (id, name) VALUES (?, ?)") + .bind(id) + .bind(name) + .execute(&pool) + .await + .unwrap(); + } + sqlx::query("INSERT INTO bridge_secrets (token) VALUES (?)") + .bind(SECRET) + .execute(&pool) + .await + .unwrap(); + drop(pool); + + assert_literals_are_data(&router_for(&url).await, mysql_quote).await; +} + +#[tokio::test] +async fn postgres_string_literals_are_data_not_code() { + let Some(url) = backend_url("FLUREE_BRIDGE_POSTGRES_URL") else { + return; + }; + let pool = sqlx::postgres::PgPool::connect(&url).await.unwrap(); + for stmt in [ + "DROP TABLE IF EXISTS bridge_people", + "DROP TABLE IF EXISTS bridge_secrets", + "CREATE TABLE bridge_people (id INT PRIMARY KEY, name TEXT)", + "CREATE TABLE bridge_secrets (token TEXT)", + ] { + pool.execute(stmt).await.unwrap(); + } + for (id, name) in [(1, PAYLOAD), (2, TRAILING_BACKSLASH), (3, "O'Brien")] { + sqlx::query("INSERT INTO bridge_people (id, name) VALUES ($1, $2)") + .bind(id) + .bind(name) + .execute(&pool) + .await + .unwrap(); + } + sqlx::query("INSERT INTO bridge_secrets (token) VALUES ($1)") + .bind(SECRET) + .execute(&pool) + .await + .unwrap(); + drop(pool); + + assert_literals_are_data(&router_for(&url).await, pg_quote).await; +} + +/// The probe the engine's schema cache depends on: `SELECT * … LIMIT 0` must +/// name every column with a Trino type, and values must round-trip. +/// +/// See [`PROBE_TYPES`] for why both backends report the same vector. +#[tokio::test] +async fn mysql_probe_names_trino_types_and_values_round_trip() { + let Some(url) = backend_url("FLUREE_BRIDGE_MYSQL_URL") else { + return; + }; + let pool = sqlx::mysql::MySqlPool::connect(&url).await.unwrap(); + for stmt in [ + "DROP TABLE IF EXISTS bridge_types", + "CREATE TABLE bridge_types (i INT, b BIGINT, d DOUBLE, s VARCHAR(8), \ + n DECIMAL(10,2), dt DATE, ts DATETIME, ok BOOLEAN)", + "INSERT INTO bridge_types VALUES (1, 2, 1.5, 'x', 3.25, '2024-01-02', \ + '2024-01-02 03:04:05', 1)", + ] { + pool.execute(stmt).await.unwrap(); + } + drop(pool); + let router = router_for(&url).await; + + assert_eq!(probe_types(&router).await, PROBE_TYPES); + + let found = rows(&router, "SELECT i, b, d, s, dt, ok FROM bridge_types") + .await + .unwrap(); + assert_eq!(found.len(), 1); + assert_eq!( + found[0], + serde_json::json!([1, 2, 1.5, "x", "2024-01-02", true]) + ); +} + +#[tokio::test] +async fn postgres_probe_names_trino_types_and_values_round_trip() { + let Some(url) = backend_url("FLUREE_BRIDGE_POSTGRES_URL") else { + return; + }; + let pool = sqlx::postgres::PgPool::connect(&url).await.unwrap(); + for stmt in [ + "DROP TABLE IF EXISTS bridge_types", + "CREATE TABLE bridge_types (i INT, b BIGINT, d DOUBLE PRECISION, s VARCHAR(8), \ + n DECIMAL(10,2), dt DATE, ts TIMESTAMP, ok BOOLEAN)", + "INSERT INTO bridge_types VALUES (1, 2, 1.5, 'x', 3.25, '2024-01-02', \ + '2024-01-02 03:04:05', true)", + ] { + pool.execute(stmt).await.unwrap(); + } + drop(pool); + let router = router_for(&url).await; + + assert_eq!(probe_types(&router).await, PROBE_TYPES); + + let found = rows(&router, "SELECT i, b, d, s, dt, ok FROM bridge_types") + .await + .unwrap(); + assert_eq!(found.len(), 1); + assert_eq!( + found[0], + serde_json::json!([1, 2, 1.5, "x", "2024-01-02", true]) + ); +} + +/// The column types a `LIMIT 0` probe reports. +async fn probe_types(router: &axum::Router) -> Vec { + let req = Request::post("/v1/statement") + .header("host", "bridge:8080") + .header("X-Trino-User", "fluree") + .body(Body::from("SELECT * FROM bridge_types LIMIT 0")) + .unwrap(); + let (status, first) = call(router, req).await; + assert_eq!(status, StatusCode::OK, "{first}"); + first["columns"] + .as_array() + .unwrap_or_else(|| panic!("no columns: {first}")) + .iter() + .map(|c| c["type"].as_str().unwrap().to_string()) + .collect() +} + +/// The session settings the escaping rule depends on, asserted directly. +/// +/// For MySQL this is the whole fix: a default server does *not* set +/// `NO_BACKSLASH_ESCAPES`, so dropping the pool's `after_connect` makes this +/// fail. For Postgres it is weaker — `standard_conforming_strings` is already +/// `on` by default, so this passes with or without the session `SET`, and only +/// a server configured with it `off` would tell the two apart. It is asserted +/// anyway so the setting cannot be dropped without a visible reason. +#[tokio::test] +async fn sessions_use_standard_sql_string_literals() { + if let Some(url) = backend_url("FLUREE_BRIDGE_MYSQL_URL") { + let found = rows(&router_for(&url).await, "SELECT @@sql_mode") + .await + .unwrap(); + let mode = found[0][0].as_str().unwrap(); + assert!( + mode.contains("NO_BACKSLASH_ESCAPES"), + "MySQL sessions must disable backslash escapes, got: {mode}" + ); + } + if let Some(url) = backend_url("FLUREE_BRIDGE_POSTGRES_URL") { + let found = rows( + &router_for(&url).await, + "SELECT current_setting('standard_conforming_strings')", + ) + .await + .unwrap(); + assert_eq!(found[0][0].as_str(), Some("on")); + } +} + +/// A skipped test is not a passing one. CI must supply both backends. +#[test] +fn server_backends_are_configured_in_ci() { + if std::env::var("CI").is_err() { + eprintln!("SKIPPED: not CI"); + return; + } + for var in ["FLUREE_BRIDGE_MYSQL_URL", "FLUREE_BRIDGE_POSTGRES_URL"] { + assert!( + std::env::var(var).is_ok_and(|v| !v.is_empty()), + "{var} must be set in CI, or the server-backed tests above pass by \ + doing nothing" + ); + } +} diff --git a/fluree-sql-bridge/tests/sqlite_protocol.rs b/fluree-sql-bridge/tests/sqlite_protocol.rs new file mode 100644 index 0000000000..053c7627cf --- /dev/null +++ b/fluree-sql-bridge/tests/sqlite_protocol.rs @@ -0,0 +1,184 @@ +//! The protocol over a real SQLite file, in-process. + +use std::sync::Arc; +use std::time::Duration; + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use serde_json::{json, Value}; +use tower::ServiceExt; + +async fn app(token: Option<&str>, page_rows: usize) -> (axum::Router, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("t.db"); + let url = format!("sqlite://{}?mode=rwc", path.display()); + let pool = sqlx::sqlite::SqlitePool::connect(&url).await.unwrap(); + sqlx::query( + "CREATE TABLE people (id INTEGER PRIMARY KEY, name TEXT, score REAL, born DATE, ok BOOLEAN, raw BLOB)", + ) + .execute(&pool) + .await + .unwrap(); + for i in 1..=12 { + sqlx::query( + "INSERT INTO people (id, name, score, born, ok, raw) VALUES (?, ?, ?, ?, ?, ?)", + ) + .bind(i) + .bind(if i == 3 { None } else { Some(format!("p{i}")) }) + .bind(f64::from(i) * 1.5) + .bind("2024-01-02") + .bind(i % 2 == 0) + .bind(vec![1u8, 2, 3]) + .execute(&pool) + .await + .unwrap(); + } + drop(pool); + let backend = fluree_sql_bridge::connect_backend(&url, 2, 6) + .await + .unwrap(); + let app = fluree_sql_bridge::App::new( + backend, + token.map(String::from), + page_rows, + Duration::from_secs(60), + ); + (Arc::clone(&app).router(), dir) +} + +async fn call(router: &axum::Router, req: Request) -> (StatusCode, Value) { + let resp = router.clone().oneshot(req).await.unwrap(); + let status = resp.status(); + let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let v = serde_json::from_slice(&bytes).unwrap_or(Value::Null); + (status, v) +} + +fn post(sql: &str, token: Option<&str>) -> Request { + let mut b = Request::post("/v1/statement") + .header("host", "bridge:8080") + .header("X-Trino-User", "fluree"); + if let Some(t) = token { + b = b.header("authorization", format!("Bearer {t}")); + } + b.body(Body::from(sql.to_string())).unwrap() +} + +fn get(uri: &str, token: Option<&str>) -> Request { + let path = uri.trim_start_matches("http://bridge:8080"); + let mut b = Request::get(path).header("host", "bridge:8080"); + if let Some(t) = token { + b = b.header("authorization", format!("Bearer {t}")); + } + b.body(Body::empty()).unwrap() +} + +#[tokio::test] +async fn pages_stream_in_order_with_trino_typed_columns() { + let (router, _dir) = app(None, 5).await; + let (status, first) = call( + &router, + post( + "SELECT id, name, score, born, ok, raw FROM people ORDER BY id", + None, + ), + ) + .await; + assert_eq!(status, StatusCode::OK, "{first}"); + let types: Vec<&str> = first["columns"] + .as_array() + .unwrap() + .iter() + .map(|c| c["type"].as_str().unwrap()) + .collect(); + assert_eq!( + types, + [ + "bigint", + "varchar", + "double", + "date", + "boolean", + "varbinary" + ] + ); + assert!(first["data"].is_null()); + + let mut next = first["nextUri"].as_str().unwrap().to_string(); + let mut rows: Vec = Vec::new(); + let mut pages = 0; + loop { + let (status, page) = call(&router, get(&next, None)).await; + assert_eq!(status, StatusCode::OK, "{page}"); + assert!(page["error"].is_null(), "{page}"); + pages += 1; + rows.extend(page["data"].as_array().cloned().unwrap_or_default()); + match page["nextUri"].as_str() { + Some(n) => next = n.to_string(), + None => break, + } + } + assert_eq!(rows.len(), 12); + assert!(pages >= 3, "5 rows per page: {pages}"); + assert_eq!(rows[0], json!([1, "p1", 1.5, "2024-01-02", false, "AQID"])); + assert_eq!(rows[2][1], Value::Null, "NULL name"); + + // A finished statement is gone. + let (status, _) = call(&router, get(&next, None)).await; + assert_eq!(status, StatusCode::GONE); +} + +#[tokio::test] +async fn probe_with_no_rows_still_reports_columns_and_count_is_a_scalar() { + let (router, _dir) = app(None, 100).await; + let (_, first) = call(&router, post("SELECT * FROM people LIMIT 0", None)).await; + assert_eq!(first["columns"].as_array().unwrap().len(), 6); + let (_, page) = call(&router, get(first["nextUri"].as_str().unwrap(), None)).await; + assert_eq!(page["data"], json!([])); + assert!(page["nextUri"].is_null()); + + let (_, first) = call( + &router, + post("SELECT COUNT(*) FROM people WHERE name IS NOT NULL", None), + ) + .await; + let (_, page) = call(&router, get(first["nextUri"].as_str().unwrap(), None)).await; + assert_eq!(page["data"], json!([[11]])); +} + +#[tokio::test] +async fn sql_errors_come_back_as_protocol_errors() { + let (router, _dir) = app(None, 100).await; + let (status, resp) = call(&router, post("SELECT nope FROM missing", None)).await; + assert_eq!(status, StatusCode::OK); + assert!( + resp["error"]["message"] + .as_str() + .unwrap() + .contains("missing"), + "{resp}" + ); + assert_eq!(resp["stats"]["state"], "FAILED"); +} + +#[tokio::test] +async fn bearer_token_is_enforced_and_cancel_drops_the_statement() { + let (router, _dir) = app(Some("s3cret"), 2).await; + let (status, _) = call(&router, post("SELECT 1", None)).await; + assert_eq!(status, StatusCode::UNAUTHORIZED); + + let (status, first) = call(&router, post("SELECT id FROM people", Some("s3cret"))).await; + assert_eq!(status, StatusCode::OK); + let next = first["nextUri"].as_str().unwrap().to_string(); + let path = next.trim_start_matches("http://bridge:8080").to_string(); + let del = Request::delete(&path) + .header("authorization", "Bearer s3cret") + .body(Body::empty()) + .unwrap(); + let (status, _) = call(&router, del).await; + assert_eq!(status, StatusCode::NO_CONTENT); + let (status, _) = call(&router, get(&next, Some("s3cret"))).await; + assert_eq!(status, StatusCode::GONE); +} diff --git a/fluree-vocab/src/lib.rs b/fluree-vocab/src/lib.rs index 3b581e10a0..32d1ca5e4c 100644 --- a/fluree-vocab/src/lib.rs +++ b/fluree-vocab/src/lib.rs @@ -2116,6 +2116,9 @@ pub mod ns_types { /// `https://ns.flur.ee/db#R2rmlMapping` - R2RML relational mapping pub const R2RML_MAPPING: &str = "https://ns.flur.ee/db#R2rmlMapping"; + + /// `https://ns.flur.ee/db#SqlMapping` - R2RML mapping over a SQL endpoint + pub const SQL_MAPPING: &str = "https://ns.flur.ee/db#SqlMapping"; } /// Graph source nameservice field local names (under `https://ns.flur.ee/db#`)