From ab7c3844deb737b098214310f868e4e601bda3a2 Mon Sep 17 00:00:00 2001 From: bplatz Date: Sat, 29 Aug 2026 15:54:49 -0400 Subject: [PATCH 01/13] =?UTF-8?q?feat(sql):=20fluree-db-sql=20crate=20?= =?UTF-8?q?=E2=80=94=20Trino-protocol=20SQL=20scans=20for=20R2RML=20graph?= =?UTF-8?q?=20sources?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A SQL graph source scans tables through any endpoint speaking the Trino client protocol (POST /v1/statement + nextUri pages): Trino, Starburst, PrestoDB, or a sidecar in front of another engine. Every page is one plain HTTP request, so nothing is stateful on our side and no database driver is linked into the binary. The engine's R2RML operator asks a provider for one table at a time — projection, conjunctive filters, optional top-k — so this crate renders exactly one SELECT per scan. Filters are pushed typed against a cached LIMIT 0 schema probe; a literal that cannot be rendered safely for the column's type is declined rather than guessed, since a mistyped comparison fails the whole statement in Trino and the in-engine FILTER stays the authority regardless. Top-k is not pushed: a null in a key or required column would consume LIMIT slots and break the superset contract. Type decoding covers Trino's JSON renderings (dates, precision timestamps, numeric-offset zones, exact decimals, base64 varbinary, NaN/Infinity). timestamp-with-time-zone columns are selected AT TIME ZONE 'UTC' so a named-region zone never reaches the decoder. The endpoint guard blocks only the link-local/metadata range and follows no redirects: loopback/private hosts are the sidecar deployment shape. --- Cargo.lock | 20 ++ Cargo.toml | 1 + fluree-db-sql/Cargo.toml | 31 ++ fluree-db-sql/src/config.rs | 205 +++++++++++ fluree-db-sql/src/dialect.rs | 580 ++++++++++++++++++++++++++++++++ fluree-db-sql/src/error.rs | 42 +++ fluree-db-sql/src/lib.rs | 31 ++ fluree-db-sql/src/net.rs | 118 +++++++ fluree-db-sql/src/trino.rs | 416 +++++++++++++++++++++++ fluree-db-sql/src/types.rs | 491 +++++++++++++++++++++++++++ fluree-db-sql/tests/protocol.rs | 294 ++++++++++++++++ 11 files changed, 2229 insertions(+) create mode 100644 fluree-db-sql/Cargo.toml create mode 100644 fluree-db-sql/src/config.rs create mode 100644 fluree-db-sql/src/dialect.rs create mode 100644 fluree-db-sql/src/error.rs create mode 100644 fluree-db-sql/src/lib.rs create mode 100644 fluree-db-sql/src/net.rs create mode 100644 fluree-db-sql/src/trino.rs create mode 100644 fluree-db-sql/src/types.rs create mode 100644 fluree-db-sql/tests/protocol.rs diff --git a/Cargo.lock b/Cargo.lock index a44a7aa593..32e5f2ec9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3407,6 +3407,26 @@ dependencies = [ "zstd", ] +[[package]] +name = "fluree-db-sql" +version = "4.1.6" +dependencies = [ + "async-stream", + "async-trait", + "base64 0.22.1", + "chrono", + "fluree-db-iceberg", + "fluree-db-tabular", + "futures", + "reqwest", + "serde", + "serde_json", + "thiserror 1.0.69", + "tokio", + "tracing", + "wiremock", +] + [[package]] name = "fluree-db-storage-aws" version = "4.1.6" diff --git a/Cargo.toml b/Cargo.toml index cabd4e57a7..416b094a82 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,7 @@ members = [ "fluree-db-iceberg", "fluree-db-tabular", "fluree-db-r2rml", + "fluree-db-sql", "fluree-db-server", "fluree-db-bolt", "fluree-db-peer", diff --git a/fluree-db-sql/Cargo.toml b/fluree-db-sql/Cargo.toml new file mode 100644 index 0000000000..54a02f31b8 --- /dev/null +++ b/fluree-db-sql/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "fluree-db-sql" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "SQL-over-HTTP graph source support for Fluree DB (Trino wire protocol)" + +[dependencies] +fluree-db-tabular = { path = "../fluree-db-tabular" } +# `ConfigValue` / `SecretResolver` / `AuthConfig` / `MappingSource` are shared with +# the Iceberg graph source. The base crate (no `aws` feature) is HTTP + serde only. +fluree-db-iceberg = { path = "../fluree-db-iceberg", default-features = false } + +async-trait.workspace = true +tokio = { workspace = true, features = ["sync", "time"] } +futures.workspace = true +async-stream = "0.3" + +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] } + +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +thiserror.workspace = true +chrono.workspace = true +base64 = "0.22" +tracing.workspace = true + +[dev-dependencies] +wiremock = { workspace = true } +tokio = { workspace = true, features = ["full"] } diff --git a/fluree-db-sql/src/config.rs b/fluree-db-sql/src/config.rs new file mode 100644 index 0000000000..af4ee42591 --- /dev/null +++ b/fluree-db-sql/src/config.rs @@ -0,0 +1,205 @@ +//! Graph-source configuration for a SQL source. +//! +//! Stored as the opaque `config` JSON of a `f:SqlMapping` nameservice record. +//! Everything reachable over the wire — endpoint, catalog/schema defaults, +//! credentials — lives here; the R2RML mapping itself is stored in CAS and only +//! referenced (`mapping.source` is a CID), exactly as for Iceberg sources. + +use std::collections::BTreeMap; +use std::sync::Arc; + +use fluree_db_iceberg::auth::AuthConfig; +use fluree_db_iceberg::config::MappingSource; +use fluree_db_iceberg::SecretResolver; +use serde::{Deserialize, Serialize}; + +use crate::dialect::SqlDialect; +use crate::error::{Result, SqlError}; + +/// Which header family the endpoint speaks. Trino renamed its headers from +/// `X-Presto-*` to `X-Trino-*` in release 351; PrestoDB still uses the old +/// names. Everything else about the protocol is identical. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum WireProtocol { + #[default] + Trino, + Presto, +} + +impl WireProtocol { + pub(crate) fn header(self, suffix: &str) -> String { + match self { + WireProtocol::Trino => format!("X-Trino-{suffix}"), + WireProtocol::Presto => format!("X-Presto-{suffix}"), + } + } +} + +fn default_request_timeout() -> u64 { + 120 +} + +fn default_user() -> String { + "fluree".to_string() +} + +/// Persisted configuration of one SQL graph source. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SqlGsConfig { + /// Base URL of the statement endpoint, e.g. `https://trino.example.com:8443` + /// or `http://localhost:8080` for a sidecar. `/v1/statement` is appended. + pub endpoint: String, + + /// How identifiers and literals are rendered. Defaults to Trino; a + /// `fluree-sql-bridge` sidecar in front of another engine reports its own. + #[serde(default)] + pub dialect: SqlDialect, + + /// Header family (`X-Trino-*` vs `X-Presto-*`). + #[serde(default)] + pub protocol: WireProtocol, + + /// Default catalog for unqualified table names (`X-Trino-Catalog`). + #[serde(default)] + pub catalog: Option, + + /// Default schema for unqualified table names (`X-Trino-Schema`). + #[serde(default)] + pub schema: Option, + + /// The `X-Trino-User` value. Required by the protocol even when a bearer + /// token identifies the caller; defaults to `fluree`. + #[serde(default = "default_user")] + pub user: String, + + /// Endpoint authentication. Shares the Iceberg REST catalog's shape so the + /// same `ConfigValue` indirection (`env_var`, `secret_ref`) applies. + #[serde(default)] + pub auth: AuthConfig, + + /// Session properties sent as `X-Trino-Session: k=v,k=v`. + #[serde(default)] + pub session: BTreeMap, + + /// Per-request HTTP timeout (each page fetch is one request). + #[serde(default = "default_request_timeout")] + pub request_timeout_secs: u64, + + /// The R2RML mapping (CAS CID + media type). Absent only transiently. + #[serde(default)] + pub mapping: Option, +} + +impl SqlGsConfig { + pub fn new(endpoint: impl Into) -> Self { + Self { + endpoint: endpoint.into(), + dialect: SqlDialect::default(), + protocol: WireProtocol::default(), + catalog: None, + schema: None, + user: default_user(), + auth: AuthConfig::default(), + session: BTreeMap::new(), + request_timeout_secs: default_request_timeout(), + mapping: None, + } + } + + pub fn from_json(json: &str) -> Result { + serde_json::from_str(json) + .map_err(|e| SqlError::Config(format!("invalid config JSON: {e}"))) + } + + pub fn to_json(&self) -> Result { + serde_json::to_string(self).map_err(|e| SqlError::Config(format!("serialize config: {e}"))) + } + + /// Structural validation: endpoint scheme/host and non-empty user. + pub fn validate(&self) -> Result<()> { + crate::net::validate_endpoint(&self.endpoint)?; + if self.user.trim().is_empty() { + return Err(SqlError::Config("user must not be empty".to_string())); + } + if self.request_timeout_secs == 0 { + return Err(SqlError::Config( + "request_timeout_secs must be positive".to_string(), + )); + } + for key in self.session.keys() { + if key.contains(',') || key.contains('=') { + return Err(SqlError::Config(format!( + "session property name '{key}' may not contain ',' or '='" + ))); + } + } + Ok(()) + } + + /// Resolve every `secret_ref` in the auth block. Fields carrying no secret + /// reference clone through untouched. + pub async fn hydrate(&self, resolver: Option<&Arc>) -> Result { + let auth = self.auth.hydrate(resolver).await?; + Ok(Self { + auth, + ..self.clone() + }) + } + + /// The endpoint with any trailing slash removed. + pub fn endpoint_base(&self) -> &str { + self.endpoint.trim_end_matches('/') + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn minimal_config_round_trips_with_defaults() { + let cfg = SqlGsConfig::from_json(r#"{"endpoint":"http://localhost:8080"}"#).unwrap(); + assert_eq!(cfg.user, "fluree"); + assert_eq!(cfg.dialect, SqlDialect::Trino); + assert_eq!(cfg.protocol, WireProtocol::Trino); + assert_eq!(cfg.request_timeout_secs, 120); + assert!(cfg.mapping.is_none()); + cfg.validate().unwrap(); + let back = SqlGsConfig::from_json(&cfg.to_json().unwrap()).unwrap(); + assert_eq!(back.endpoint, "http://localhost:8080"); + } + + #[test] + fn full_config_parses() { + let cfg = SqlGsConfig::from_json( + r#"{ + "endpoint": "https://trino.example.com/", + "dialect": "postgres", + "protocol": "presto", + "catalog": "pg", + "schema": "public", + "user": "svc", + "auth": {"type": "bearer", "token": {"env_var": "TRINO_TOKEN"}}, + "session": {"query_max_run_time": "5m"}, + "mapping": {"source": "bafy...", "media_type": "text/turtle"} + }"#, + ) + .unwrap(); + assert_eq!(cfg.dialect, SqlDialect::Postgres); + assert_eq!(cfg.protocol, WireProtocol::Presto); + assert_eq!(cfg.endpoint_base(), "https://trino.example.com"); + assert!(matches!(cfg.auth, AuthConfig::Bearer { .. })); + assert_eq!(cfg.session["query_max_run_time"], "5m"); + cfg.validate().unwrap(); + } + + #[test] + fn validate_rejects_bad_scheme_and_empty_user() { + let mut cfg = SqlGsConfig::new("ftp://x"); + assert!(cfg.validate().is_err()); + cfg = SqlGsConfig::new("http://x"); + cfg.user = " ".into(); + assert!(cfg.validate().is_err()); + } +} diff --git a/fluree-db-sql/src/dialect.rs b/fluree-db-sql/src/dialect.rs new file mode 100644 index 0000000000..aa8955e0be --- /dev/null +++ b/fluree-db-sql/src/dialect.rs @@ -0,0 +1,580 @@ +//! SQL rendering of a single-table scan. +//! +//! The query engine never sends SPARQL here. The R2RML operator asks a +//! provider for one table at a time — a projection, conjunctive filters, and +//! optionally a single-column `ORDER BY … LIMIT` — and does joins, OPTIONAL, +//! UNION and aggregation itself over the returned column batches. So what gets +//! rendered is exactly one `SELECT … FROM … WHERE …`. +//! +//! Filters are pushed **typed**: every predicate is rendered against the +//! column's known type (from a cached `LIMIT 0` probe), and a predicate whose +//! literal cannot be rendered safely for that type is dropped rather than +//! guessed — a mistyped comparison would fail the whole statement in Trino +//! ("Cannot apply operator: bigint = varchar"), and the in-engine FILTER stays +//! the authority either way, so a dropped push only costs I/O. + +use fluree_db_tabular::{BatchSchema, FieldType}; +use serde::{Deserialize, Serialize}; + +use crate::error::{Result, SqlError}; + +/// Identifier quoting and literal syntax family. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum SqlDialect { + #[default] + Trino, + Postgres, + Mysql, + Sqlite, +} + +impl SqlDialect { + fn quote_char(self) -> char { + match self { + SqlDialect::Mysql => '`', + _ => '"', + } + } + + /// Quote one identifier part, doubling any embedded quote character. + pub fn quote_ident(self, ident: &str) -> String { + let q = self.quote_char(); + let mut out = String::with_capacity(ident.len() + 2); + out.push(q); + for c in ident.chars() { + if c == q { + out.push(q); + } + out.push(c); + } + out.push(q); + out + } + + /// Quote a dotted table name part by part (`ns.table` → `"ns"."table"`). + pub fn quote_table(self, table: &str) -> String { + table + .split('.') + .map(|p| self.quote_ident(p)) + .collect::>() + .join(".") + } + + /// Whether typed literal prefixes (`DATE '…'`, `TIMESTAMP '…'`) are valid. + fn typed_literals(self) -> bool { + !matches!(self, SqlDialect::Sqlite) + } +} + +/// Where the rows come from: a table, or an `rr:sqlQuery` used as a derived table. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LogicalSource { + /// Dotted table name; each part is quoted. + Table(String), + /// Verbatim SQL from the mapping, wrapped as `(…) AS "__fluree_q"`. The + /// mapping author is trusted (a mapping is root-equivalent by design). + Query(String), +} + +impl LogicalSource { + pub fn render(&self, dialect: SqlDialect) -> String { + match self { + LogicalSource::Table(t) => dialect.quote_table(t), + LogicalSource::Query(q) => { + format!( + "({}) AS {}", + q.trim().trim_end_matches(';'), + dialect.quote_ident("__fluree_q") + ) + } + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CmpOp { + Eq, + NotEq, + Lt, + LtEq, + Gt, + GtEq, + In, +} + +impl CmpOp { + fn sql(self) -> &'static str { + match self { + CmpOp::Eq => "=", + CmpOp::NotEq => "<>", + CmpOp::Lt => "<", + CmpOp::LtEq => "<=", + CmpOp::Gt => ">", + CmpOp::GtEq => ">=", + CmpOp::In => "IN", + } + } +} + +/// A filter literal. Mirrors the engine's `ScanValue` without depending on the +/// query crate. +#[derive(Debug, Clone, PartialEq)] +pub enum Literal { + Bool(bool), + Int(i64), + Str(String), + /// Days since 1970-01-01. + Date(i32), + Double(f64), + Decimal { + unscaled: i128, + scale: i8, + }, + /// Micros since the epoch; `tz` = the source literal carried an offset. + Timestamp { + micros: i64, + tz: bool, + }, + /// A raw column value recovered by reversing a subject template. Its type + /// is whatever the column's type is; rendered only for int/string columns. + TemplateKey(String), + Set(Vec), +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Predicate { + pub column: String, + pub op: CmpOp, + pub value: Literal, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct ScanRequest { + pub source: LogicalSource, + /// Empty = every column. + pub projection: Vec, + pub predicates: Vec, +} + +/// One rendered scan plus what was dropped, so callers can log declined pushes. +#[derive(Debug, Clone)] +pub struct RenderedScan { + pub sql: String, + pub declined_predicates: Vec, +} + +/// `SELECT * FROM LIMIT 0` — the schema probe. +pub fn render_probe(source: &LogicalSource, dialect: SqlDialect) -> String { + format!("SELECT * FROM {} LIMIT 0", source.render(dialect)) +} + +/// `SELECT COUNT(*) FROM WHERE c1 IS NOT NULL AND …` +pub fn render_count( + source: &LogicalSource, + non_null_cols: &[String], + dialect: SqlDialect, +) -> String { + let mut sql = format!("SELECT COUNT(*) FROM {}", source.render(dialect)); + if !non_null_cols.is_empty() { + let conds: Vec = non_null_cols + .iter() + .map(|c| format!("{} IS NOT NULL", dialect.quote_ident(c))) + .collect(); + sql.push_str(" WHERE "); + sql.push_str(&conds.join(" AND ")); + } + sql +} + +/// Render the scan against the probed schema. Unknown projected columns are +/// an error (the mapping names a column the table does not have); predicates +/// on unknown columns or with unrenderable literals are declined, not errors. +pub fn render_scan( + req: &ScanRequest, + schema: &BatchSchema, + dialect: SqlDialect, +) -> Result { + let select_list = if req.projection.is_empty() { + schema + .fields + .iter() + .map(|f| render_projected_column(&f.name, f.field_type, dialect)) + .collect::>() + } else { + let mut cols = Vec::with_capacity(req.projection.len()); + for name in &req.projection { + let field = schema.field_by_name(name).ok_or_else(|| { + SqlError::Config(format!( + "projected column '{name}' does not exist in {}; available: {:?}", + describe(&req.source), + schema + .fields + .iter() + .map(|f| f.name.as_str()) + .collect::>() + )) + })?; + cols.push(render_projected_column(name, field.field_type, dialect)); + } + cols + }; + + let mut sql = format!( + "SELECT {} FROM {}", + select_list.join(", "), + req.source.render(dialect) + ); + + let mut conds = Vec::new(); + let mut declined = Vec::new(); + for pred in &req.predicates { + match schema.field_by_name(&pred.column) { + Some(field) => match render_predicate(pred, field.field_type, dialect) { + Some(c) => conds.push(c), + None => declined.push(pred.clone()), + }, + None => declined.push(pred.clone()), + } + } + if !conds.is_empty() { + sql.push_str(" WHERE "); + sql.push_str(&conds.join(" AND ")); + } + + Ok(RenderedScan { + sql, + declined_predicates: declined, + }) +} + +fn describe(source: &LogicalSource) -> String { + match source { + LogicalSource::Table(t) => format!("table '{t}'"), + LogicalSource::Query(_) => "the rr:sqlQuery".to_string(), + } +} + +/// A `timestamp with time zone` column is re-rendered in UTC so the wire form +/// is decodable without a zone database (Trino otherwise prints the value's +/// own zone, which may be a named region). +fn render_projected_column(name: &str, ty: FieldType, dialect: SqlDialect) -> String { + let q = dialect.quote_ident(name); + match (ty, dialect) { + (FieldType::TimestampTz, SqlDialect::Trino) => format!("{q} AT TIME ZONE 'UTC' AS {q}"), + _ => q, + } +} + +fn render_predicate(pred: &Predicate, ty: FieldType, dialect: SqlDialect) -> Option { + let col = dialect.quote_ident(&pred.column); + match (&pred.value, pred.op) { + (Literal::Set(members), CmpOp::In) => { + if members.is_empty() { + return None; + } + let rendered: Option> = members + .iter() + .map(|m| render_literal(m, ty, dialect)) + .collect(); + rendered.map(|r| format!("{col} IN ({})", r.join(", "))) + } + (Literal::Set(_), _) | (_, CmpOp::In) => None, + (lit, op) => render_literal(lit, ty, dialect).map(|l| format!("{col} {} {l}", op.sql())), + } +} + +fn sql_string(s: &str) -> String { + let mut out = String::with_capacity(s.len() + 2); + out.push('\''); + for c in s.chars() { + if c == '\'' { + out.push('\''); + } + out.push(c); + } + out.push('\''); + out +} + +fn is_numeric(ty: FieldType) -> bool { + matches!( + ty, + FieldType::Int32 + | FieldType::Int64 + | FieldType::Float32 + | FieldType::Float64 + | FieldType::Decimal { .. } + ) +} + +/// Render a literal for comparison against a column of type `ty`, or `None` +/// when no rendering is safe for that pairing. +fn render_literal(lit: &Literal, ty: FieldType, dialect: SqlDialect) -> Option { + match lit { + Literal::Bool(b) => { + matches!(ty, FieldType::Boolean).then(|| if *b { "TRUE" } else { "FALSE" }.to_string()) + } + Literal::Int(i) => is_numeric(ty).then(|| i.to_string()), + Literal::Str(s) => matches!(ty, FieldType::String).then(|| sql_string(s)), + Literal::Date(days) => { + if !matches!(ty, FieldType::Date) { + return None; + } + let date = chrono::DateTime::from_timestamp(i64::from(*days) * 86_400, 0)?.date_naive(); + let text = date.format("%Y-%m-%d").to_string(); + Some(if dialect.typed_literals() { + format!("DATE '{text}'") + } else { + sql_string(&text) + }) + } + Literal::Double(d) => { + if !d.is_finite() || !is_numeric(ty) { + return None; + } + // `{:E}` gives `1.5E0`, a valid double literal in every dialect here + // and unambiguous (a bare `1.5` is a DECIMAL literal in Trino). + Some(format!("{d:E}")) + } + Literal::Decimal { unscaled, scale } => { + is_numeric(ty).then(|| render_decimal(*unscaled, *scale)) + } + Literal::Timestamp { micros, tz } => { + let matches_col = match ty { + FieldType::Timestamp => !*tz, + FieldType::TimestampTz => *tz, + _ => false, + }; + if !matches_col { + return None; + } + let dt = chrono::DateTime::from_timestamp_micros(*micros)?; + let text = dt.format("%Y-%m-%d %H:%M:%S%.6f").to_string(); + Some(match (dialect.typed_literals(), *tz) { + (true, true) => format!("TIMESTAMP '{text} UTC'"), + (true, false) => format!("TIMESTAMP '{text}'"), + (false, _) => sql_string(&text), + }) + } + Literal::TemplateKey(raw) => match ty { + FieldType::String => Some(sql_string(raw)), + FieldType::Int32 | FieldType::Int64 => raw.parse::().ok().map(|i| i.to_string()), + _ => None, + }, + Literal::Set(_) => None, + } +} + +fn render_decimal(unscaled: i128, scale: i8) -> String { + if scale <= 0 { + let mut s = unscaled.to_string(); + s.extend(std::iter::repeat_n('0', (-scale) as usize)); + return s; + } + let scale = scale as usize; + let negative = unscaled < 0; + let digits = unscaled.unsigned_abs().to_string(); + let padded = if digits.len() <= scale { + format!("{}{}", "0".repeat(scale + 1 - digits.len()), digits) + } else { + digits + }; + let (int_part, frac_part) = padded.split_at(padded.len() - scale); + format!("{}{int_part}.{frac_part}", if negative { "-" } else { "" }) +} + +#[cfg(test)] +mod tests { + use super::*; + use fluree_db_tabular::FieldInfo; + + fn schema() -> BatchSchema { + let f = |name: &str, ty: FieldType, id: i32| FieldInfo { + name: name.to_string(), + field_type: ty, + nullable: true, + field_id: id, + }; + BatchSchema::new(vec![ + f("id", FieldType::Int64, 1), + f("name", FieldType::String, 2), + f("born", FieldType::Date, 3), + f("score", FieldType::Float64, 4), + f( + "price", + FieldType::Decimal { + precision: 10, + scale: 2, + }, + 5, + ), + f("at", FieldType::TimestampTz, 6), + f("local_at", FieldType::Timestamp, 7), + f("ok", FieldType::Boolean, 8), + ]) + } + + fn pred(column: &str, op: CmpOp, value: Literal) -> Predicate { + Predicate { + column: column.into(), + op, + value, + } + } + + #[test] + fn quoting_doubles_embedded_quotes_and_splits_dotted_names() { + assert_eq!( + SqlDialect::Trino.quote_table("hive.sales.orders"), + r#""hive"."sales"."orders""# + ); + assert_eq!(SqlDialect::Trino.quote_ident(r#"we"ird"#), r#""we""ird""#); + assert_eq!(SqlDialect::Mysql.quote_table("db.t"), "`db`.`t`"); + assert_eq!(sql_string("O'Brien"), "'O''Brien'"); + } + + #[test] + fn probe_and_count_render() { + let src = LogicalSource::Table("s.t".into()); + assert_eq!( + render_probe(&src, SqlDialect::Trino), + r#"SELECT * FROM "s"."t" LIMIT 0"# + ); + assert_eq!( + render_count(&src, &["id".into(), "name".into()], SqlDialect::Trino), + r#"SELECT COUNT(*) FROM "s"."t" WHERE "id" IS NOT NULL AND "name" IS NOT NULL"# + ); + let q = LogicalSource::Query("select 1 as id;".into()); + assert_eq!( + render_count(&q, &[], SqlDialect::Trino), + r#"SELECT COUNT(*) FROM (select 1 as id) AS "__fluree_q""# + ); + } + + #[test] + fn typed_predicates_render_and_mismatches_decline() { + let req = ScanRequest { + source: LogicalSource::Table("t".into()), + projection: vec!["id".into(), "name".into(), "at".into()], + predicates: vec![ + pred("id", CmpOp::Eq, Literal::Int(7)), + pred("name", CmpOp::Eq, Literal::Str("O'Brien".into())), + pred("born", CmpOp::GtEq, Literal::Date(19_723)), + pred("score", CmpOp::Gt, Literal::Double(1.5)), + pred( + "price", + CmpOp::Lt, + Literal::Decimal { + unscaled: -1234, + scale: 2, + }, + ), + pred( + "at", + CmpOp::Lt, + Literal::Timestamp { + micros: 1_700_000_000_000_000, + tz: true, + }, + ), + pred( + "local_at", + CmpOp::Lt, + Literal::Timestamp { + micros: 0, + tz: false, + }, + ), + pred("ok", CmpOp::Eq, Literal::Bool(true)), + pred( + "id", + CmpOp::In, + Literal::Set(vec![Literal::Int(1), Literal::Int(2)]), + ), + // Declined: string against an int column, tz mismatch, unknown column, + // NaN, template key that is not an integer. + pred("id", CmpOp::Eq, Literal::Str("x".into())), + pred( + "at", + CmpOp::Eq, + Literal::Timestamp { + micros: 0, + tz: false, + }, + ), + pred("nope", CmpOp::Eq, Literal::Int(1)), + pred("score", CmpOp::Eq, Literal::Double(f64::NAN)), + pred("id", CmpOp::Eq, Literal::TemplateKey("abc".into())), + ], + }; + let r = render_scan(&req, &schema(), SqlDialect::Trino).unwrap(); + assert_eq!( + r.sql, + concat!( + r#"SELECT "id", "name", "at" AT TIME ZONE 'UTC' AS "at" FROM "t" WHERE "#, + r#""id" = 7 AND "name" = 'O''Brien' AND "born" >= DATE '2024-01-01' AND "score" > 1.5E0 "#, + r#"AND "price" < -12.34 AND "at" < TIMESTAMP '2023-11-14 22:13:20.000000 UTC' "#, + r#"AND "local_at" < TIMESTAMP '1970-01-01 00:00:00.000000' AND "ok" = TRUE AND "id" IN (1, 2)"# + ) + ); + assert_eq!(r.declined_predicates.len(), 5); + } + + #[test] + fn template_key_is_typed_by_the_column() { + let req = ScanRequest { + source: LogicalSource::Table("t".into()), + projection: vec![], + predicates: vec![ + pred("id", CmpOp::Eq, Literal::TemplateKey("42".into())), + pred("name", CmpOp::Eq, Literal::TemplateKey("42".into())), + pred("born", CmpOp::Eq, Literal::TemplateKey("2024-01-01".into())), + ], + }; + let r = render_scan(&req, &schema(), SqlDialect::Trino).unwrap(); + assert!( + r.sql.contains(r#""id" = 42 AND "name" = '42'"#), + "{}", + r.sql + ); + assert_eq!(r.declined_predicates.len(), 1); + assert!(r + .sql + .starts_with(r#"SELECT "id", "name", "born", "score", "price", "at" AT TIME ZONE"#)); + } + + #[test] + fn unknown_projection_is_an_error() { + let req = ScanRequest { + source: LogicalSource::Table("t".into()), + projection: vec!["missing".into()], + predicates: vec![], + }; + let err = render_scan(&req, &schema(), SqlDialect::Trino).unwrap_err(); + assert!(err.to_string().contains("missing")); + } + + #[test] + fn sqlite_uses_plain_string_literals() { + let req = ScanRequest { + source: LogicalSource::Table("t".into()), + projection: vec!["born".into()], + predicates: vec![pred("born", CmpOp::Eq, Literal::Date(0))], + }; + let r = render_scan(&req, &schema(), SqlDialect::Sqlite).unwrap(); + assert_eq!( + r.sql, + r#"SELECT "born" FROM "t" WHERE "born" = '1970-01-01'"# + ); + } + + #[test] + fn decimal_rendering() { + assert_eq!(render_decimal(1234, 2), "12.34"); + assert_eq!(render_decimal(-5, 2), "-0.05"); + assert_eq!(render_decimal(5, 0), "5"); + assert_eq!(render_decimal(5, -2), "500"); + assert_eq!(render_decimal(0, 3), "0.000"); + } +} diff --git a/fluree-db-sql/src/error.rs b/fluree-db-sql/src/error.rs new file mode 100644 index 0000000000..04cefefcd6 --- /dev/null +++ b/fluree-db-sql/src/error.rs @@ -0,0 +1,42 @@ +//! Error type for the SQL graph source. + +/// Errors from configuring, rendering, or executing a SQL graph-source scan. +#[derive(Debug, thiserror::Error)] +pub enum SqlError { + /// The graph-source config is malformed or internally inconsistent. + #[error("SQL graph source configuration error: {0}")] + Config(String), + + /// Credential material could not be resolved or the endpoint refused it. + #[error("SQL graph source authentication error: {0}")] + Auth(String), + + /// Transport-level failure talking to the SQL endpoint. + #[error("SQL endpoint HTTP error: {0}")] + Http(String), + + /// The endpoint accepted the statement and then reported a failure. + #[error("SQL statement failed: {0}")] + Query(String), + + /// A value or type on the wire could not be turned into a column. + #[error("SQL result decode error: {0}")] + Decode(String), + + /// Something the SQL graph source deliberately does not do. + #[error("unsupported by SQL graph sources: {0}")] + Unsupported(String), +} + +impl From for SqlError { + fn from(e: fluree_db_iceberg::IcebergError) -> Self { + // Only the shared config/auth machinery is reachable through this + // conversion; anything else from that crate would be a wiring mistake. + match e { + fluree_db_iceberg::IcebergError::Config(m) => SqlError::Config(m), + other => SqlError::Auth(other.to_string()), + } + } +} + +pub type Result = std::result::Result; diff --git a/fluree-db-sql/src/lib.rs b/fluree-db-sql/src/lib.rs new file mode 100644 index 0000000000..5605bf510c --- /dev/null +++ b/fluree-db-sql/src/lib.rs @@ -0,0 +1,31 @@ +//! SQL graph sources for Fluree DB. +//! +//! An R2RML mapping over tables served by any engine that speaks the Trino +//! client protocol over HTTP: Trino / Starburst / PrestoDB directly, or a +//! `fluree-sql-bridge` sidecar in front of Postgres, MySQL or SQLite. The +//! query engine pushes one single-table scan at a time (projection + typed +//! filters), rendered here as SQL; joins and everything else stay in-engine. +//! +//! - [`config::SqlGsConfig`] — the persisted graph-source record. +//! - [`dialect`] — SQL rendering of a scan against a probed schema. +//! - [`trino::TrinoClient`] — the statement/page protocol, streaming batches. +//! - [`types`] — Trino type names and JSON page values → column batches. + +pub mod config; +pub mod dialect; +pub mod error; +pub mod net; +pub mod trino; +pub mod types; + +pub use config::{SqlGsConfig, WireProtocol}; +pub use dialect::{ + CmpOp, Literal, LogicalSource, Predicate, RenderedScan, ScanRequest, SqlDialect, +}; +pub use error::{Result, SqlError}; +pub use trino::{SqlBatchStream, TrinoClient}; + +// Re-exported so callers wire auth/secret resolution with one import. +pub use fluree_db_iceberg::auth::{AuthConfig, SendCatalogAuth}; +pub use fluree_db_iceberg::config::MappingSource; +pub use fluree_db_iceberg::{ConfigValue, SecretResolver}; diff --git a/fluree-db-sql/src/net.rs b/fluree-db-sql/src/net.rs new file mode 100644 index 0000000000..26df71460d --- /dev/null +++ b/fluree-db-sql/src/net.rs @@ -0,0 +1,118 @@ +//! Outbound HTTP hardening for the SQL endpoint. +//! +//! A SQL endpoint legitimately lives on loopback or a private network — a +//! `fluree-sql-bridge` or Trino sidecar next to the server is the primary +//! deployment shape — so the Iceberg catalog's "public addresses only" posture +//! would block the main use case. This mirrors the narrower S3 `endpoint` +//! policy instead: redirects are never followed, and the link-local / +//! cloud-metadata range (`169.254/16`, `fe80::/10`) is refused both up front +//! (literal IPs) and at connect time (names that resolve there). + +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; +use std::sync::Arc; +use std::time::Duration; + +use reqwest::dns::{Addrs, Name, Resolve, Resolving}; + +use crate::error::{Result, SqlError}; + +fn ipv4_is_link_local_or_invalid(v4: Ipv4Addr) -> bool { + v4.is_link_local() || v4.is_unspecified() || v4.is_broadcast() +} + +fn ipv6_is_link_local(v6: Ipv6Addr) -> bool { + (v6.segments()[0] & 0xffc0) == 0xfe80 +} + +/// Whether an IP is in the range no SQL endpoint may ever be: link-local +/// (which contains the cloud-metadata address) or unspecified/broadcast. +pub fn ip_is_blocked(ip: IpAddr) -> bool { + match ip { + IpAddr::V4(v4) => ipv4_is_link_local_or_invalid(v4), + IpAddr::V6(v6) => { + if let Some(v4) = v6.to_ipv4_mapped() { + return ipv4_is_link_local_or_invalid(v4); + } + v6.is_unspecified() || ipv6_is_link_local(v6) + } + } +} + +#[derive(Debug, Default)] +struct LinkLocalGuardResolver; + +impl Resolve for LinkLocalGuardResolver { + fn resolve(&self, name: Name) -> Resolving { + Box::pin(async move { + let host = name.as_str().to_owned(); + let resolved = match tokio::net::lookup_host((host.as_str(), 0)).await { + Ok(it) => it, + Err(e) => return Err(Box::new(e) as Box), + }; + let allowed: Vec = resolved.filter(|sa| !ip_is_blocked(sa.ip())).collect(); + if allowed.is_empty() { + return Err(format!( + "SSRF guard: host '{host}' resolves only to link-local/metadata addresses" + ) + .into()); + } + Ok(Box::new(allowed.into_iter()) as Addrs) + }) + } +} + +/// A client that follows no redirects and refuses link-local targets. +pub fn build_client(request_timeout: Duration) -> Result { + reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .dns_resolver(Arc::new(LinkLocalGuardResolver)) + .connect_timeout(Duration::from_secs(30)) + .timeout(request_timeout) + .build() + .map_err(|e| SqlError::Http(format!("build HTTP client: {e}"))) +} + +/// Up-front validation of a configured endpoint: `http`/`https` only, a host +/// present, and not a literal link-local IP (the resolver never sees literals). +pub fn validate_endpoint(raw: &str) -> Result<()> { + let url = reqwest::Url::parse(raw) + .map_err(|e| SqlError::Config(format!("endpoint '{raw}' is not a valid URL: {e}")))?; + match url.scheme() { + "http" | "https" => {} + other => { + return Err(SqlError::Config(format!( + "endpoint scheme '{other}' is not allowed (use https or http)" + ))) + } + } + let host = url + .host_str() + .ok_or_else(|| SqlError::Config(format!("endpoint '{raw}' has no host")))?; + let literal = host.trim_start_matches('[').trim_end_matches(']'); + if let Ok(ip) = literal.parse::() { + if ip_is_blocked(ip) { + return Err(SqlError::Config(format!( + "SSRF guard: endpoint host '{host}' is a blocked (link-local/metadata) address" + ))); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn loopback_and_private_are_allowed_but_metadata_is_not() { + validate_endpoint("http://localhost:8080").unwrap(); + validate_endpoint("http://127.0.0.1:8080").unwrap(); + validate_endpoint("http://10.1.2.3:8080/").unwrap(); + validate_endpoint("https://trino.example.com").unwrap(); + assert!(validate_endpoint("http://169.254.169.254/latest").is_err()); + assert!(validate_endpoint("http://[fe80::1]:8080").is_err()); + assert!(validate_endpoint("http://0.0.0.0:8080").is_err()); + assert!(validate_endpoint("file:///etc/passwd").is_err()); + assert!(validate_endpoint("not a url").is_err()); + } +} diff --git a/fluree-db-sql/src/trino.rs b/fluree-db-sql/src/trino.rs new file mode 100644 index 0000000000..818b6f9e86 --- /dev/null +++ b/fluree-db-sql/src/trino.rs @@ -0,0 +1,416 @@ +//! The Trino client protocol: `POST /v1/statement`, then `GET nextUri` until +//! it disappears. Stateless from our side — every page is one plain HTTP +//! request carrying its own auth — which is what makes this usable from a +//! Lambda, and what lets a small sidecar in front of Postgres/MySQL/SQLite +//! speak the same protocol and need no driver code in this binary. + +use std::collections::HashMap; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use std::task::{Context, Poll}; +use std::time::{Duration, Instant}; + +use fluree_db_iceberg::auth::SendCatalogAuth; +use fluree_db_tabular::{BatchSchema, ColumnBatch}; +use futures::Stream; +use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; +use serde::Deserialize; +use serde_json::Value; +use tokio::sync::mpsc; +use tracing::{debug, warn}; + +use crate::config::SqlGsConfig; +use crate::dialect::{render_count, render_probe, LogicalSource, SqlDialect}; +use crate::error::{Result, SqlError}; +use crate::types::{decode_rows, schema_from_columns}; + +const SCHEMA_CACHE_TTL: Duration = Duration::from_secs(300); +const MAX_503_RETRIES: u32 = 6; +const STREAM_CHANNEL_DEPTH: usize = 4; + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct StatementResponse { + #[serde(default)] + id: Option, + #[serde(default)] + next_uri: Option, + #[serde(default)] + columns: Option>, + #[serde(default)] + data: Option>>, + #[serde(default)] + error: Option, +} + +#[derive(Debug, Deserialize)] +struct TrinoColumn { + name: String, + #[serde(rename = "type")] + type_name: String, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct TrinoError { + #[serde(default)] + message: Option, + #[serde(default)] + error_name: Option, + #[serde(default)] + error_code: Option, +} + +impl TrinoError { + fn render(&self) -> String { + let mut s = self + .message + .clone() + .unwrap_or_else(|| "unknown error".to_string()); + if let Some(name) = &self.error_name { + s.push_str(&format!(" [{name}")); + if let Some(code) = self.error_code { + s.push_str(&format!(" {code}")); + } + s.push(']'); + } + s + } +} + +/// A stream of batches fed by a background driver task. `Sync` because it +/// holds only the channel receiver — the request futures live in the task. +pub struct BatchStream { + rx: mpsc::Receiver>, +} + +impl Stream for BatchStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.rx.poll_recv(cx) + } +} + +pub type SqlBatchStream = Pin> + Send + Sync>>; + +/// A client bound to one endpoint + credential. +#[derive(Clone)] +pub struct TrinoClient { + inner: Arc, +} + +struct Inner { + http: reqwest::Client, + statement_url: String, + base_headers: HeaderMap, + auth: Arc, + dialect: SqlDialect, + schema_cache: Mutex)>>, +} + +impl std::fmt::Debug for TrinoClient { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TrinoClient") + .field("statement_url", &self.inner.statement_url) + .field("dialect", &self.inner.dialect) + .finish_non_exhaustive() + } +} + +impl TrinoClient { + /// `config` must already be hydrated (no `secret_ref` left in `auth`). + pub fn new(config: &SqlGsConfig, auth: Arc) -> Result { + config.validate()?; + let http = crate::net::build_client(Duration::from_secs(config.request_timeout_secs))?; + + let mut base_headers = HeaderMap::new(); + let h = |suffix: &str| HeaderName::from_bytes(config.protocol.header(suffix).as_bytes()); + let put = |headers: &mut HeaderMap, name: HeaderName, value: &str| -> Result<()> { + let v = HeaderValue::from_str(value) + .map_err(|_| SqlError::Config(format!("header {name} has a non-ASCII value")))?; + headers.insert(name, v); + Ok(()) + }; + put(&mut base_headers, h("User").unwrap(), &config.user)?; + put(&mut base_headers, h("Source").unwrap(), "fluree")?; + put(&mut base_headers, h("Time-Zone").unwrap(), "UTC")?; + if let Some(c) = &config.catalog { + put(&mut base_headers, h("Catalog").unwrap(), c)?; + } + if let Some(s) = &config.schema { + put(&mut base_headers, h("Schema").unwrap(), s)?; + } + if !config.session.is_empty() { + let joined = config + .session + .iter() + .map(|(k, v)| format!("{k}={v}")) + .collect::>() + .join(","); + put(&mut base_headers, h("Session").unwrap(), &joined)?; + } + + Ok(Self { + inner: Arc::new(Inner { + http, + statement_url: format!("{}/v1/statement", config.endpoint_base()), + base_headers, + auth, + dialect: config.dialect, + schema_cache: Mutex::new(HashMap::new()), + }), + }) + } + + pub fn dialect(&self) -> SqlDialect { + self.inner.dialect + } + + /// Run `sql`, streaming each protocol page as one batch. Dropping the + /// stream cancels the statement on the server. + pub fn execute(&self, sql: String) -> SqlBatchStream { + let (tx, rx) = mpsc::channel(STREAM_CHANNEL_DEPTH); + let inner = Arc::clone(&self.inner); + tokio::spawn(async move { + let mut cancel_uri: Option = None; + let outcome = inner + .drive(&sql, &mut cancel_uri, |batch| { + let tx = tx.clone(); + async move { tx.send(Ok(batch)).await.is_ok() } + }) + .await; + match outcome { + Ok(_) => {} + Err(SqlError::Http(m)) if m == CONSUMER_GONE => { + if let Some(uri) = cancel_uri { + inner.cancel(&uri).await; + } + } + Err(e) => { + let _ = tx.send(Err(e)).await; + } + } + }); + Box::pin(BatchStream { rx }) + } + + /// Run `sql` to completion. Returns the schema (present even for zero rows + /// once the statement planned) and every batch. + pub async fn execute_collect( + &self, + sql: &str, + ) -> Result<(Option>, Vec)> { + let batches = Mutex::new(Vec::new()); + let mut cancel_uri = None; + let schema = self + .inner + .drive(sql, &mut cancel_uri, |batch| { + batches.lock().unwrap().push(batch); + async { true } + }) + .await?; + Ok((schema, batches.into_inner().unwrap())) + } + + /// The source's column schema from a cached `LIMIT 0` probe. + pub async fn schema(&self, source: &LogicalSource) -> Result> { + let key = source.render(self.inner.dialect); + if let Some((at, schema)) = self.inner.schema_cache.lock().unwrap().get(&key) { + if at.elapsed() < SCHEMA_CACHE_TTL { + return Ok(Arc::clone(schema)); + } + } + let sql = render_probe(source, self.inner.dialect); + let (schema, _) = self.execute_collect(&sql).await?; + let schema = schema.ok_or_else(|| { + SqlError::Query(format!("schema probe returned no column metadata: {sql}")) + })?; + self.inner + .schema_cache + .lock() + .unwrap() + .insert(key, (Instant::now(), Arc::clone(&schema))); + Ok(schema) + } + + /// Exact `COUNT(*)` with the given columns required non-null. + pub async fn count(&self, source: &LogicalSource, non_null_cols: &[String]) -> Result { + let sql = render_count(source, non_null_cols, self.inner.dialect); + let (_, batches) = self.execute_collect(&sql).await?; + let batch = batches + .into_iter() + .find(|b| b.num_rows > 0) + .ok_or_else(|| SqlError::Query(format!("COUNT(*) returned no rows: {sql}")))?; + let col = batch + .column(0) + .ok_or_else(|| SqlError::Decode("COUNT(*) returned no column".to_string()))?; + let n = col + .get_i64(0) + .or_else(|| col.get_i32(0).map(i64::from)) + .or_else(|| col.get_f64(0).map(|f| f as i64)) + .ok_or_else(|| { + SqlError::Decode(format!("COUNT(*) value is not an integer: {col:?}")) + })?; + u64::try_from(n).map_err(|_| SqlError::Decode(format!("negative COUNT(*): {n}"))) + } +} + +const CONSUMER_GONE: &str = "consumer dropped the result stream"; + +impl Inner { + /// Post the statement and walk every page, handing each decoded batch to + /// `sink`; a `false` from the sink means the consumer went away. + async fn drive( + &self, + sql: &str, + cancel_uri: &mut Option, + mut sink: F, + ) -> Result>> + where + F: FnMut(ColumnBatch) -> Fut, + Fut: std::future::Future, + { + debug!(sql = %sql, "SQL statement"); + let mut resp = self.post_statement(sql).await?; + let mut schema: Option> = None; + loop { + if let Some(err) = &resp.error { + return Err(SqlError::Query(err.render())); + } + if schema.is_none() { + if let Some(cols) = &resp.columns { + let pairs: Vec<(String, String)> = cols + .iter() + .map(|c| (c.name.clone(), c.type_name.clone())) + .collect(); + schema = Some(schema_from_columns(&pairs)); + } + } + if let Some(rows) = resp.data.take() { + if !rows.is_empty() { + let s = schema.as_ref().ok_or_else(|| { + SqlError::Decode("page carried data before any column metadata".to_string()) + })?; + let batch = decode_rows(s, rows)?; + if !sink(batch).await { + return Err(SqlError::Http(CONSUMER_GONE.to_string())); + } + } + } + match resp.next_uri.take() { + Some(uri) => { + *cancel_uri = Some(uri.clone()); + resp = self.get_page(&uri).await?; + } + None => { + *cancel_uri = None; + return Ok(schema); + } + } + } + } + + async fn auth_headers(&self) -> Result { + let mut headers = self.base_headers.clone(); + if let Some(value) = self + .auth + .authorization_header() + .await + .map_err(|e| SqlError::Auth(e.to_string()))? + { + headers.insert( + reqwest::header::AUTHORIZATION, + HeaderValue::from_str(&value) + .map_err(|_| SqlError::Auth("invalid authorization header".into()))?, + ); + } + Ok(headers) + } + + async fn post_statement(&self, sql: &str) -> Result { + let mut attempt = 0; + loop { + let headers = self.auth_headers().await?; + let resp = self + .http + .post(&self.statement_url) + .headers(headers) + .header(reqwest::header::CONTENT_TYPE, "text/plain") + .body(sql.to_string()) + .send() + .await + .map_err(|e| SqlError::Http(format!("POST {}: {e}", self.statement_url)))?; + match self.classify(resp, attempt).await? { + Some(parsed) => return Ok(parsed), + None => attempt += 1, + } + } + } + + async fn get_page(&self, uri: &str) -> Result { + let mut attempt = 0; + loop { + let headers = self.auth_headers().await?; + let resp = self + .http + .get(uri) + .headers(headers) + .send() + .await + .map_err(|e| SqlError::Http(format!("GET {uri}: {e}")))?; + match self.classify(resp, attempt).await? { + Some(parsed) => return Ok(parsed), + None => attempt += 1, + } + } + } + + /// `Ok(Some)` = a page; `Ok(None)` = retry (503 with budget left). + async fn classify( + &self, + resp: reqwest::Response, + attempt: u32, + ) -> Result> { + let status = resp.status(); + if status == reqwest::StatusCode::SERVICE_UNAVAILABLE { + if attempt >= MAX_503_RETRIES { + return Err(SqlError::Http(format!( + "endpoint kept answering 503 after {MAX_503_RETRIES} retries" + ))); + } + let backoff = Duration::from_millis(100 * (1u64 << attempt.min(5))); + tokio::time::sleep(backoff).await; + return Ok(None); + } + if status == reqwest::StatusCode::UNAUTHORIZED || status == reqwest::StatusCode::FORBIDDEN { + return Err(SqlError::Auth(format!("endpoint returned {status}"))); + } + if !status.is_success() { + let body = resp.text().await.unwrap_or_default(); + let body = body.chars().take(500).collect::(); + return Err(SqlError::Http(format!( + "endpoint returned {status}: {body}" + ))); + } + let parsed: StatementResponse = resp + .json() + .await + .map_err(|e| SqlError::Decode(format!("statement response is not valid JSON: {e}")))?; + if let Some(id) = &parsed.id { + debug!(query_id = %id, has_next = parsed.next_uri.is_some(), "SQL page"); + } + Ok(Some(parsed)) + } + + async fn cancel(&self, uri: &str) { + match self.auth_headers().await { + Ok(headers) => { + if let Err(e) = self.http.delete(uri).headers(headers).send().await { + warn!(error = %e, "failed to cancel abandoned SQL statement"); + } + } + Err(e) => warn!(error = %e, "failed to cancel abandoned SQL statement"), + } + } +} diff --git a/fluree-db-sql/src/types.rs b/fluree-db-sql/src/types.rs new file mode 100644 index 0000000000..f2923b480a --- /dev/null +++ b/fluree-db-sql/src/types.rs @@ -0,0 +1,491 @@ +//! Trino type names → `FieldType`, and JSON page values → `Column`. +//! +//! Trino's protocol renders every value as JSON: numbers for integers and +//! doubles (with `"NaN"`/`"Infinity"` strings for non-finite doubles), strings +//! for everything else — dates as `2024-01-01`, timestamps as +//! `2024-01-01 12:34:56.123`, zoned timestamps with a trailing zone, decimals as +//! their exact lexical form, varbinary as base64. + +use std::sync::Arc; + +use base64::Engine; +use chrono::{NaiveDate, NaiveDateTime, NaiveTime}; +use fluree_db_tabular::{BatchSchema, Column, ColumnBatch, FieldInfo, FieldType}; +use serde_json::Value; + +use crate::error::{Result, SqlError}; + +/// Map a Trino type signature to a column type. Unknown or structural types +/// (`row`, `array`, `map`, …) land as `String`, carrying Trino's own rendering. +pub fn field_type_from_trino(type_name: &str) -> FieldType { + let lower = type_name.trim().to_ascii_lowercase(); + // Precision sits between the base name and the zone suffix + // (`timestamp(6) with time zone`), so settle temporals before splitting. + if lower.starts_with("timestamp") { + return if lower.ends_with("with time zone") { + FieldType::TimestampTz + } else { + FieldType::Timestamp + }; + } + let (base, args) = match lower.find('(') { + Some(i) => ( + lower[..i].trim_end(), + Some(&lower[i + 1..lower.len().saturating_sub(1)]), + ), + None => (lower.as_str(), None), + }; + match base { + "boolean" => FieldType::Boolean, + "tinyint" | "smallint" | "integer" | "int" => FieldType::Int32, + "bigint" => FieldType::Int64, + "real" | "float" => FieldType::Float32, + "double" | "double precision" => FieldType::Float64, + "varbinary" | "binary" | "bytea" => FieldType::Bytes, + "date" => FieldType::Date, + "decimal" | "numeric" => { + let (p, s) = args + .and_then(|a| { + let mut it = a.split(',').map(|x| x.trim().parse::().ok()); + let p = it.next().flatten()?; + let s = it.next().flatten().unwrap_or(0); + Some((p, s)) + }) + .unwrap_or((38, 0)); + FieldType::Decimal { + precision: p.clamp(1, 76) as u8, + scale: s.clamp(-128, 127) as i8, + } + } + _ => FieldType::String, + } +} + +/// Build a batch schema from the protocol's column list. Field ids are +/// positional (1-based); the R2RML layer looks columns up by name. +pub fn schema_from_columns(columns: &[(String, String)]) -> Arc { + let fields = columns + .iter() + .enumerate() + .map(|(i, (name, ty))| FieldInfo { + name: name.clone(), + field_type: field_type_from_trino(ty), + nullable: true, + field_id: i as i32 + 1, + }) + .collect(); + Arc::new(BatchSchema::new(fields)) +} + +/// Decode one page of rows into a batch. +pub fn decode_rows(schema: &Arc, rows: Vec>) -> Result { + let n = rows.len(); + let mut columns: Vec = schema + .fields + .iter() + .map(|f| Column::with_capacity(f.field_type, n)) + .collect(); + + for (row_idx, row) in rows.into_iter().enumerate() { + if row.len() != columns.len() { + return Err(SqlError::Decode(format!( + "row {row_idx} has {} values but the schema has {} columns", + row.len(), + columns.len() + ))); + } + for (col_idx, value) in row.into_iter().enumerate() { + let field = &schema.fields[col_idx]; + push_value(&mut columns[col_idx], &field.name, field.field_type, value)?; + } + } + + ColumnBatch::new(Arc::clone(schema), columns).map_err(|e| SqlError::Decode(e.to_string())) +} + +fn push_value(column: &mut Column, name: &str, ty: FieldType, value: Value) -> Result<()> { + let bad = |what: &str, v: &Value| { + SqlError::Decode(format!( + "column '{name}' ({ty:?}): expected {what}, got {v}" + )) + }; + match column { + Column::Boolean(v) => v.push(match value { + Value::Null => None, + Value::Bool(b) => Some(b), + other => return Err(bad("boolean", &other)), + }), + Column::Int32(v) => v.push(match value { + Value::Null => None, + Value::Number(ref n) => Some( + n.as_i64() + .and_then(|i| i32::try_from(i).ok()) + .ok_or_else(|| bad("32-bit integer", &value))?, + ), + other => return Err(bad("integer", &other)), + }), + Column::Int64(v) => v.push(match value { + Value::Null => None, + Value::Number(ref n) => Some(n.as_i64().ok_or_else(|| bad("64-bit integer", &value))?), + other => return Err(bad("integer", &other)), + }), + Column::Float32(v) => v.push( + parse_double(&value) + .map_err(|_| bad("real", &value))? + .map(|d| d as f32), + ), + Column::Float64(v) => v.push(parse_double(&value).map_err(|_| bad("double", &value))?), + Column::String(v) => v.push(match value { + Value::Null => None, + Value::String(s) => Some(s), + // Structural / unknown types arrive as JSON; keep their rendering. + other => Some(other.to_string()), + }), + Column::Bytes(v) => v.push(match value { + Value::Null => None, + Value::String(ref s) => Some( + base64::engine::general_purpose::STANDARD + .decode(s) + .map_err(|_| bad("base64 varbinary", &value))?, + ), + other => return Err(bad("base64 varbinary", &other)), + }), + Column::Date(v) => v.push(match value { + Value::Null => None, + Value::String(ref s) => Some(parse_date_days(s).ok_or_else(|| bad("date", &value))?), + other => return Err(bad("date", &other)), + }), + Column::Timestamp(v) => v.push(match value { + Value::Null => None, + Value::String(ref s) => { + Some(parse_timestamp_micros(s).ok_or_else(|| bad("timestamp", &value))?) + } + other => return Err(bad("timestamp", &other)), + }), + Column::TimestampTz(v) => v.push(match value { + Value::Null => None, + Value::String(ref s) => Some(parse_timestamp_micros(s).ok_or_else(|| { + SqlError::Decode(format!( + "column '{name}' (timestamp with time zone): cannot decode '{s}' — \ + only numeric offsets and UTC/GMT/Z zones are supported; select the \ + column `AT TIME ZONE 'UTC'` or use the Trino dialect, which does so" + )) + })?), + other => return Err(bad("timestamp with time zone", &other)), + }), + Column::Decimal { values, scale, .. } => values.push(match value { + Value::Null => None, + Value::String(ref s) => { + Some(parse_decimal_unscaled(s, *scale).ok_or_else(|| bad("decimal", &value))?) + } + Value::Number(ref n) => Some( + parse_decimal_unscaled(&n.to_string(), *scale) + .ok_or_else(|| bad("decimal", &value))?, + ), + other => return Err(bad("decimal", &other)), + }), + } + Ok(()) +} + +fn parse_double(value: &Value) -> std::result::Result, ()> { + match value { + Value::Null => Ok(None), + Value::Number(n) => n.as_f64().map(Some).ok_or(()), + Value::String(s) => match s.as_str() { + "NaN" => Ok(Some(f64::NAN)), + "Infinity" | "+Infinity" => Ok(Some(f64::INFINITY)), + "-Infinity" => Ok(Some(f64::NEG_INFINITY)), + other => other.parse::().map(Some).map_err(|_| ()), + }, + _ => Err(()), + } +} + +/// `YYYY-MM-DD` → days since the epoch. +pub fn parse_date_days(s: &str) -> Option { + let d = NaiveDate::parse_from_str(s.trim(), "%Y-%m-%d").ok()?; + let epoch = NaiveDate::from_ymd_opt(1970, 1, 1)?; + i32::try_from((d - epoch).num_days()).ok() +} + +/// `YYYY-MM-DD HH:MM:SS[.f{1,12}][ ZONE]` → micros since the epoch (UTC frame +/// once the zone is applied). Sub-microsecond digits are truncated. Zones: +/// `UTC`, `GMT`, `Z`, or `±HH:MM`. Named regions return `None`. +pub fn parse_timestamp_micros(s: &str) -> Option { + let s = s.trim(); + let (date_part, rest) = s.split_once(' ')?; + let (time_part, zone) = match rest.split_once(' ') { + Some((t, z)) => (t, Some(z.trim())), + None => (rest, None), + }; + + let date = NaiveDate::parse_from_str(date_part, "%Y-%m-%d").ok()?; + let (hms, frac) = match time_part.split_once('.') { + Some((h, f)) => (h, Some(f)), + None => (time_part, None), + }; + let time = NaiveTime::parse_from_str(hms, "%H:%M:%S").ok()?; + let micros_frac: i64 = match frac { + Some(f) => { + if f.is_empty() || !f.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + let mut padded: String = f.chars().take(6).collect(); + while padded.len() < 6 { + padded.push('0'); + } + padded.parse().ok()? + } + None => 0, + }; + + let naive = NaiveDateTime::new(date, time); + let base = naive + .and_utc() + .timestamp_micros() + .checked_add(micros_frac)?; + let offset_secs = match zone { + None => 0, + Some(z) => parse_zone_offset_secs(z)?, + }; + base.checked_sub(i64::from(offset_secs) * 1_000_000) +} + +fn parse_zone_offset_secs(z: &str) -> Option { + match z { + "UTC" | "GMT" | "Z" | "+00:00" | "-00:00" => Some(0), + _ => { + let sign = match z.as_bytes().first()? { + b'+' => 1, + b'-' => -1, + _ => return None, + }; + let (h, m) = z[1..].split_once(':')?; + let h: i32 = h.parse().ok()?; + let m: i32 = m.parse().ok()?; + if h > 23 || m > 59 { + return None; + } + Some(sign * (h * 3600 + m * 60)) + } + } +} + +/// Exact lexical decimal → unscaled integer at the column's scale. Extra +/// fractional digits beyond `scale` are rejected (the engine would silently +/// misreport the value otherwise). +pub fn parse_decimal_unscaled(s: &str, scale: i8) -> Option { + let s = s.trim(); + let (negative, body) = match s.strip_prefix('-') { + Some(rest) => (true, rest), + None => (false, s.strip_prefix('+').unwrap_or(s)), + }; + let (int_part, frac_part) = body.split_once('.').unwrap_or((body, "")); + if int_part.is_empty() && frac_part.is_empty() { + return None; + } + if !int_part.bytes().all(|b| b.is_ascii_digit()) + || !frac_part.bytes().all(|b| b.is_ascii_digit()) + { + return None; + } + let scale = usize::try_from(scale).ok()?; + if frac_part.len() > scale && frac_part[scale..].bytes().any(|b| b != b'0') { + return None; + } + let mut digits = String::with_capacity(int_part.len() + scale); + digits.push_str(int_part); + digits.push_str(&frac_part[..frac_part.len().min(scale)]); + for _ in frac_part.len().min(scale)..scale { + digits.push('0'); + } + let mut v: i128 = if digits.is_empty() { + 0 + } else { + digits.parse().ok()? + }; + if negative { + v = -v; + } + Some(v) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn trino_type_names_map() { + assert_eq!(field_type_from_trino("bigint"), FieldType::Int64); + assert_eq!(field_type_from_trino("integer"), FieldType::Int32); + assert_eq!(field_type_from_trino("varchar(20)"), FieldType::String); + assert_eq!(field_type_from_trino("varchar"), FieldType::String); + assert_eq!(field_type_from_trino("double"), FieldType::Float64); + assert_eq!(field_type_from_trino("real"), FieldType::Float32); + assert_eq!(field_type_from_trino("date"), FieldType::Date); + assert_eq!(field_type_from_trino("timestamp(3)"), FieldType::Timestamp); + assert_eq!( + field_type_from_trino("timestamp(6) with time zone"), + FieldType::TimestampTz + ); + assert_eq!( + field_type_from_trino("timestamp with time zone"), + FieldType::TimestampTz + ); + assert_eq!( + field_type_from_trino("decimal(10,2)"), + FieldType::Decimal { + precision: 10, + scale: 2 + } + ); + assert_eq!( + field_type_from_trino("decimal(38, 0)"), + FieldType::Decimal { + precision: 38, + scale: 0 + } + ); + assert_eq!(field_type_from_trino("varbinary"), FieldType::Bytes); + assert_eq!(field_type_from_trino("array(varchar)"), FieldType::String); + assert_eq!(field_type_from_trino("row(a bigint)"), FieldType::String); + } + + #[test] + fn timestamps_parse_with_offsets_and_truncate_nanos() { + assert_eq!(parse_timestamp_micros("1970-01-01 00:00:00"), Some(0)); + assert_eq!( + parse_timestamp_micros("1970-01-01 00:00:00.123"), + Some(123_000) + ); + assert_eq!( + parse_timestamp_micros("1970-01-01 00:00:00.123456789"), + Some(123_456) + ); + assert_eq!( + parse_timestamp_micros("1970-01-01 01:00:00 +01:00"), + Some(0) + ); + assert_eq!( + parse_timestamp_micros("1970-01-01 00:00:00.5 UTC"), + Some(500_000) + ); + assert_eq!( + parse_timestamp_micros("1969-12-31 19:00:00 -05:00"), + Some(0) + ); + assert_eq!( + parse_timestamp_micros("1970-01-01 00:00:00 America/New_York"), + None + ); + assert_eq!(parse_timestamp_micros("garbage"), None); + assert_eq!(parse_date_days("1970-01-02"), Some(1)); + assert_eq!(parse_date_days("2024-01-01"), Some(19_723)); + } + + #[test] + fn decimals_parse_exactly() { + assert_eq!(parse_decimal_unscaled("12.34", 2), Some(1234)); + assert_eq!(parse_decimal_unscaled("-0.05", 2), Some(-5)); + assert_eq!(parse_decimal_unscaled("5", 2), Some(500)); + assert_eq!(parse_decimal_unscaled("5.1", 2), Some(510)); + assert_eq!(parse_decimal_unscaled("5.100", 2), Some(510)); + assert_eq!(parse_decimal_unscaled("5.101", 2), None); + assert_eq!(parse_decimal_unscaled("abc", 2), None); + assert_eq!(parse_decimal_unscaled(".5", 1), Some(5)); + } + + #[test] + fn decode_a_page() { + let schema = schema_from_columns(&[ + ("id".into(), "bigint".into()), + ("n".into(), "integer".into()), + ("name".into(), "varchar".into()), + ("d".into(), "double".into()), + ("born".into(), "date".into()), + ("at".into(), "timestamp(3) with time zone".into()), + ("price".into(), "decimal(10,2)".into()), + ("raw".into(), "varbinary".into()), + ("ok".into(), "boolean".into()), + ("tags".into(), "array(varchar)".into()), + ]); + let rows = vec![ + vec![ + json!(1), + json!(2), + json!("a"), + json!(1.5), + json!("2024-01-01"), + json!("2023-11-14 22:13:20.000 UTC"), + json!("12.34"), + json!("aGk="), + json!(true), + json!(["x", "y"]), + ], + vec![ + json!(2), + Value::Null, + Value::Null, + json!("NaN"), + Value::Null, + json!("2023-11-14 23:13:20.000 +01:00"), + Value::Null, + Value::Null, + Value::Null, + Value::Null, + ], + ]; + let batch = decode_rows(&schema, rows).unwrap(); + assert_eq!(batch.num_rows, 2); + let id = batch.column_by_name("id").unwrap(); + assert_eq!(id.get_i64(0), Some(1)); + assert_eq!(batch.column_by_name("n").unwrap().get_i32(0), Some(2)); + assert_eq!(batch.column_by_name("name").unwrap().get_string(1), None); + assert!(batch + .column_by_name("d") + .unwrap() + .get_f64(1) + .unwrap() + .is_nan()); + assert_eq!( + batch.column_by_name("born").unwrap().get_date(0), + Some(19_723) + ); + let at = batch.column_by_name("at").unwrap(); + assert_eq!(at.get_timestamp(0), Some(1_700_000_000_000_000)); + assert_eq!(at.get_timestamp(1), Some(1_700_000_000_000_000)); + match batch.column_by_name("price").unwrap() { + Column::Decimal { values, scale, .. } => { + assert_eq!(*scale, 2); + assert_eq!(values[0], Some(1234)); + assert_eq!(values[1], None); + } + other => panic!("{other:?}"), + } + assert_eq!( + batch.column_by_name("raw").unwrap().get_bytes(0), + Some(&b"hi"[..]) + ); + assert_eq!(batch.column_by_name("ok").unwrap().get_bool(0), Some(true)); + assert_eq!( + batch.column_by_name("tags").unwrap().get_string(0), + Some(r#"["x","y"]"#) + ); + } + + #[test] + fn decode_rejects_shape_and_type_errors() { + let schema = schema_from_columns(&[("id".into(), "bigint".into())]); + assert!(decode_rows(&schema, vec![vec![json!(1), json!(2)]]).is_err()); + assert!(decode_rows(&schema, vec![vec![json!("x")]]).is_err()); + let schema = schema_from_columns(&[("at".into(), "timestamp with time zone".into())]); + let err = decode_rows( + &schema, + vec![vec![json!("2024-01-01 00:00:00 Europe/Oslo")]], + ) + .unwrap_err(); + assert!(err.to_string().contains("AT TIME ZONE")); + } +} diff --git a/fluree-db-sql/tests/protocol.rs b/fluree-db-sql/tests/protocol.rs new file mode 100644 index 0000000000..0c78e95968 --- /dev/null +++ b/fluree-db-sql/tests/protocol.rs @@ -0,0 +1,294 @@ +//! The statement/page protocol against a fake endpoint. + +use std::sync::Arc; + +use fluree_db_iceberg::auth::NoAuth; +use fluree_db_sql::{LogicalSource, SqlGsConfig, TrinoClient}; +use futures::StreamExt; +use serde_json::json; +use wiremock::matchers::{body_string, header, method, path}; +use wiremock::{Mock, MockServer, Request, ResponseTemplate}; + +fn client(server: &MockServer) -> TrinoClient { + let mut cfg = SqlGsConfig::new(server.uri()); + cfg.catalog = Some("hive".into()); + cfg.schema = Some("sales".into()); + cfg.session.insert("query_max_run_time".into(), "5m".into()); + TrinoClient::new(&cfg, Arc::new(NoAuth)).unwrap() +} + +fn page( + id: &str, + next: Option, + columns: bool, + data: serde_json::Value, +) -> serde_json::Value { + let mut p = json!({ "id": id, "stats": { "state": "RUNNING" } }); + if let Some(n) = next { + p["nextUri"] = json!(n); + } + if columns { + p["columns"] = json!([ + { "name": "id", "type": "bigint" }, + { "name": "name", "type": "varchar" } + ]); + } + p["data"] = data; + p +} + +#[tokio::test] +async fn statement_pages_stream_as_batches_with_protocol_headers() { + let server = MockServer::start().await; + let base = server.uri(); + + Mock::given(method("POST")) + .and(path("/v1/statement")) + .and(header("X-Trino-User", "fluree")) + .and(header("X-Trino-Catalog", "hive")) + .and(header("X-Trino-Schema", "sales")) + .and(header("X-Trino-Session", "query_max_run_time=5m")) + .and(header("X-Trino-Time-Zone", "UTC")) + .and(body_string("SELECT 1")) + .respond_with(ResponseTemplate::new(200).set_body_json(page( + "q1", + Some(format!("{base}/v1/statement/q1/1")), + false, + json!(null), + ))) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/v1/statement/q1/1")) + .respond_with(ResponseTemplate::new(200).set_body_json(page( + "q1", + Some(format!("{base}/v1/statement/q1/2")), + true, + json!([[1, "a"], [2, null]]), + ))) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/v1/statement/q1/2")) + .respond_with(ResponseTemplate::new(200).set_body_json(page( + "q1", + Some(format!("{base}/v1/statement/q1/3")), + false, + json!([[3, "c"]]), + ))) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/v1/statement/q1/3")) + .respond_with(ResponseTemplate::new(200).set_body_json(page( + "q1", + None, + false, + json!(null), + ))) + .mount(&server) + .await; + + let c = client(&server); + let batches: Vec<_> = c.execute("SELECT 1".into()).collect().await; + let batches: Vec<_> = batches.into_iter().map(|b| b.unwrap()).collect(); + assert_eq!(batches.len(), 2); + assert_eq!(batches[0].num_rows, 2); + assert_eq!(batches[1].num_rows, 1); + assert_eq!(batches[0].column_by_name("id").unwrap().get_i64(1), Some(2)); + assert_eq!( + batches[0].column_by_name("name").unwrap().get_string(1), + None + ); + assert_eq!( + batches[1].column_by_name("name").unwrap().get_string(0), + Some("c") + ); + + let (schema, all) = c.execute_collect("SELECT 1").await.unwrap(); + assert_eq!(schema.unwrap().num_fields(), 2); + assert_eq!(all.iter().map(|b| b.num_rows).sum::(), 3); +} + +#[tokio::test] +async fn a_503_is_retried_and_an_error_page_fails_the_statement() { + let server = MockServer::start().await; + let base = server.uri(); + + Mock::given(method("POST")) + .and(path("/v1/statement")) + .respond_with(ResponseTemplate::new(503)) + .up_to_n_times(2) + .expect(2) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/v1/statement")) + .respond_with(ResponseTemplate::new(200).set_body_json(page( + "q2", + Some(format!("{base}/v1/statement/q2/1")), + true, + json!([[1, "a"]]), + ))) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/v1/statement/q2/1")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "q2", + "error": { "message": "line 1:8: Table 'x' does not exist", "errorName": "TABLE_NOT_FOUND", "errorCode": 43 }, + "stats": { "state": "FAILED" } + }))) + .mount(&server) + .await; + + let c = client(&server); + let items: Vec<_> = c.execute("SELECT * FROM x".into()).collect().await; + assert_eq!(items.len(), 2, "one batch then the error"); + assert!(items[0].is_ok()); + let err = items[1].as_ref().unwrap_err().to_string(); + assert!( + err.contains("does not exist") && err.contains("TABLE_NOT_FOUND 43"), + "{err}" + ); +} + +#[tokio::test] +async fn unauthorized_and_bad_json_surface_as_typed_errors() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/statement")) + .respond_with(ResponseTemplate::new(401)) + .mount(&server) + .await; + let err = client(&server) + .execute_collect("SELECT 1") + .await + .unwrap_err(); + assert!(matches!(err, fluree_db_sql::SqlError::Auth(_)), "{err}"); + + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/statement")) + .respond_with(ResponseTemplate::new(200).set_body_string("not trino")) + .mount(&server) + .await; + let err = client(&server) + .execute_collect("SELECT 1") + .await + .unwrap_err(); + assert!(matches!(err, fluree_db_sql::SqlError::Decode(_)), "{err}"); +} + +#[tokio::test] +async fn schema_probe_is_cached_and_count_reads_the_scalar() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/statement")) + .and(body_string(r#"SELECT * FROM "sales"."orders" LIMIT 0"#)) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "p", "columns": [{"name": "id", "type": "bigint"}, {"name": "total", "type": "decimal(10,2)"}], + "data": [], "stats": {"state": "FINISHED"} + }))) + .expect(1) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/v1/statement")) + .and(body_string(r#"SELECT COUNT(*) FROM "sales"."orders" WHERE "id" IS NOT NULL"#)) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "c", "columns": [{"name": "_col0", "type": "bigint"}], "data": [[42]], "stats": {"state": "FINISHED"} + }))) + .mount(&server) + .await; + + let c = client(&server); + let src = LogicalSource::Table("sales.orders".into()); + let s1 = c.schema(&src).await.unwrap(); + let s2 = c.schema(&src).await.unwrap(); + assert!(Arc::ptr_eq(&s1, &s2)); + assert_eq!( + s1.field_by_name("total").unwrap().field_type, + fluree_db_tabular::FieldType::Decimal { + precision: 10, + scale: 2 + } + ); + assert_eq!(c.count(&src, &["id".into()]).await.unwrap(), 42); +} + +#[tokio::test] +async fn dropping_the_stream_cancels_the_statement() { + let server = MockServer::start().await; + let base = server.uri(); + Mock::given(method("POST")) + .and(path("/v1/statement")) + .respond_with(ResponseTemplate::new(200).set_body_json(page( + "q3", + Some(format!("{base}/v1/statement/q3/1")), + true, + json!([[1, "a"]]), + ))) + .mount(&server) + .await; + // Every GET answers another page forever, so only a cancel ends this. + Mock::given(method("GET")) + .and(path("/v1/statement/q3/1")) + .respond_with(ResponseTemplate::new(200).set_body_json(page( + "q3", + Some(format!("{base}/v1/statement/q3/1")), + false, + json!([[2, "b"]]), + ))) + .mount(&server) + .await; + Mock::given(method("DELETE")) + .and(path("/v1/statement/q3/1")) + .respond_with(ResponseTemplate::new(204)) + .expect(1) + .mount(&server) + .await; + + let c = client(&server); + let mut stream = c.execute("SELECT 1".into()); + let first = stream.next().await.unwrap().unwrap(); + assert_eq!(first.num_rows, 1); + drop(stream); + + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5); + loop { + let cancelled = server + .received_requests() + .await + .unwrap_or_default() + .iter() + .any(|r: &Request| r.method == "DELETE"); + if cancelled { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "no DELETE observed after drop" + ); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } +} + +#[tokio::test] +async fn presto_header_family_is_selectable() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/statement")) + .and(header("X-Presto-User", "svc")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "p", "columns": [{"name": "x", "type": "integer"}], "data": [[1]], "stats": {"state": "FINISHED"} + }))) + .mount(&server) + .await; + let mut cfg = SqlGsConfig::new(server.uri()); + cfg.protocol = fluree_db_sql::WireProtocol::Presto; + cfg.user = "svc".into(); + let c = TrinoClient::new(&cfg, Arc::new(NoAuth)).unwrap(); + let (_, b) = c.execute_collect("SELECT 1").await.unwrap(); + assert_eq!(b[0].column(0).unwrap().get_i32(0), Some(1)); +} From c6db3aae102ca1663093331a59248772504462c6 Mon Sep 17 00:00:00 2001 From: bplatz Date: Sat, 29 Aug 2026 16:05:04 -0400 Subject: [PATCH 02/13] feat(sql): SQL graph sources through the R2RML provider Adds GraphSourceType::Sql (f:SqlMapping) and routes it through FlureeR2rmlProvider: has_r2rml_mapping / compiled_mapping read the mapping reference per source family, and scan_table / table_row_count dispatch to a Trino-protocol client when the record is SQL-backed. The COUNT shortcut is exact for SQL (SELECT COUNT(*) WHERE key IS NOT NULL), where Iceberg can only answer it from manifest stats. Fluree::create_sql_graph_source mirrors the R2RML registration path: the mapping is compiled and stored in CAS, the endpoint is probed with SELECT 1 (a failure is logged, not fatal), and the record is published. Clients are cached process-wide keyed by the raw-config fingerprint, so a rotated secret behind an env var does not rebuild the client every query. A SQL source has no snapshot to pin. Its build watermark records endpoint, table and first-touch time, and the loadTable-cache precondition in verify_build_snapshot_integrity is skipped for it. The end-to-end test drives registration, a plain scan, a pushed typed equality, date decoding and the count shortcut against a fake endpoint and asserts the SQL actually sent. --- Cargo.lock | 2 + fluree-db-api/Cargo.toml | 15 +- fluree-db-api/src/graph_source/cache.rs | 26 + .../src/graph_source/catalog_session.rs | 15 + fluree-db-api/src/graph_source/mod.rs | 6 + fluree-db-api/src/graph_source/r2rml.rs | 97 ++-- fluree-db-api/src/graph_source/sql.rs | 483 ++++++++++++++++++ fluree-db-api/src/ledger_info.rs | 1 + fluree-db-api/src/lib.rs | 7 + fluree-db-api/tests/it_sql_graph_source.rs | 283 ++++++++++ fluree-db-nameservice/src/lib.rs | 9 +- fluree-db-sql/src/lib.rs | 1 + fluree-vocab/src/lib.rs | 3 + 13 files changed, 915 insertions(+), 33 deletions(-) create mode 100644 fluree-db-api/src/graph_source/sql.rs create mode 100644 fluree-db-api/tests/it_sql_graph_source.rs diff --git a/Cargo.lock b/Cargo.lock index 32e5f2ec9a..beed27a227 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2650,6 +2650,7 @@ dependencies = [ "fluree-db-shacl", "fluree-db-sparql", "fluree-db-spatial", + "fluree-db-sql", "fluree-db-storage-aws", "fluree-db-storage-ipfs", "fluree-db-tabular", @@ -2685,6 +2686,7 @@ dependencies = [ "tokio", "tracing", "tracing-subscriber", + "wiremock", "xxhash-rust", "zstd", ] diff --git a/fluree-db-api/Cargo.toml b/fluree-db-api/Cargo.toml index e377a33420..388af7fc7e 100644 --- a/fluree-db-api/Cargo.toml +++ b/fluree-db-api/Cargo.toml @@ -17,6 +17,10 @@ aws = ["fluree-db-connection/aws", "dep:fluree-db-storage-aws", "dep:aws-sdk-sts # Arrow columnar reader is the single graph-source read path (native predicate # pushdown: row-group skipping + exact row filtering). iceberg = ["dep:fluree-db-iceberg", "fluree-db-iceberg/aws"] +# SQL graph sources (R2RML over a Trino-protocol endpoint). Stacks on `iceberg` +# because the graph-source provider dispatch lives there; the crate itself adds +# only an HTTP client, no database drivers. +sql = ["iceberg", "dep:fluree-db-sql"] # Opt-in LocalStack-backed S3/DynamoDB tests (auto-start via testcontainers) aws-testcontainers = [ "dep:fluree-db-storage-aws", @@ -42,7 +46,7 @@ vector = ["fluree-db-query/vector"] # IPFS-backed storage (via Kubo HTTP RPC) ipfs = ["dep:fluree-db-storage-ipfs"] # Convenience bundle (excludes vector, aws, and test-only features) -full = ["native", "credential", "iceberg", "shacl", "ipfs", "graphql"] +full = ["native", "credential", "iceberg", "sql", "shacl", "ipfs", "graphql"] [dependencies] fluree-db-core = { path = "../fluree-db-core" } @@ -64,6 +68,7 @@ fluree-db-graphql = { path = "../fluree-db-graphql", optional = true } fluree-db-spatial = { path = "../fluree-db-spatial" } fluree-db-iceberg = { path = "../fluree-db-iceberg", default-features = false, optional = true, features = ["aws"] } fluree-db-r2rml = { path = "../fluree-db-r2rml", features = ["turtle"] } +fluree-db-sql = { path = "../fluree-db-sql", optional = true } fluree-db-tabular = { path = "../fluree-db-tabular" } fluree-db-storage-aws = { path = "../fluree-db-storage-aws", optional = true } fluree-db-storage-ipfs = { path = "../fluree-db-storage-ipfs", optional = true } @@ -144,6 +149,7 @@ fluree-bench-support = { path = "../fluree-bench-support" } fluree-bench-alloc = { path = "../fluree-bench-alloc" } rand = { workspace = true } fluree-db-nameservice-sync = { path = "../fluree-db-nameservice-sync" } +wiremock = { workspace = true } [[test]] name = "grp_graphsource" @@ -176,6 +182,13 @@ name = "it_iceberg_warehouse_root" path = "tests/it_iceberg_warehouse_root.rs" required-features = ["iceberg", "native"] +# End-to-end over a SQL graph source against a fake Trino-protocol endpoint +# (wiremock), through registration, the R2RML query path and pushdown. +[[test]] +name = "it_sql_graph_source" +path = "tests/it_sql_graph_source.rs" +required-features = ["sql", "native"] + [[test]] name = "grp_index" path = "tests/grp_index.rs" diff --git a/fluree-db-api/src/graph_source/cache.rs b/fluree-db-api/src/graph_source/cache.rs index ae246da31c..37ef49f3f4 100644 --- a/fluree-db-api/src/graph_source/cache.rs +++ b/fluree-db-api/src/graph_source/cache.rs @@ -16,6 +16,8 @@ use super::catalog_session::CachedLoadTable; use fluree_db_iceberg::catalog::RestCatalogClient; #[cfg(feature = "iceberg")] use fluree_db_iceberg::{io::parquet::ParquetFooterCache, metadata::TableMetadata, DataFile}; +#[cfg(feature = "sql")] +use fluree_db_sql::TrinoClient; #[cfg(feature = "iceberg")] use std::time::Duration; @@ -132,6 +134,13 @@ pub struct R2rmlCache { /// skip the ~1.3–3s catalog GET. #[cfg(feature = "iceberg")] rest_load_tables: moka::sync::Cache>, + + /// Process-wide SQL endpoint clients keyed like `rest_clients` (id + raw + /// config fingerprint), sharing its TTL rationale. Each client also holds + /// the per-table schema probes, so reuse across queries skips the + /// `LIMIT 0` round trip. + #[cfg(feature = "sql")] + sql_clients: moka::sync::Cache>, } // moka::sync::Cache is Send+Sync but doesn't implement Debug @@ -183,6 +192,11 @@ impl R2rmlCache { .max_capacity(metadata_cap) .time_to_live(Duration::from_secs(rest_loadtable_ttl_secs())) .build(), + #[cfg(feature = "sql")] + sql_clients: moka::sync::Cache::builder() + .max_capacity(64) + .time_to_live(Duration::from_secs(rest_client_ttl_secs())) + .build(), } } @@ -289,6 +303,16 @@ impl R2rmlCache { self.rest_clients.insert(fingerprint, client); } + #[cfg(feature = "sql")] + pub(crate) fn sql_client(&self, key: &str) -> Option> { + self.sql_clients.get(key) + } + + #[cfg(feature = "sql")] + pub(crate) fn put_sql_client(&self, key: String, client: Arc) { + self.sql_clients.insert(key, client); + } + /// Get a cross-query `loadTable` response if cached, within TTL, and its /// vended credentials are not near expiry; otherwise `None` (an expired /// entry is invalidated). Returns `None` when caching or the cross-query @@ -329,6 +353,8 @@ impl R2rmlCache { self.rest_clients.invalidate_all(); self.rest_load_tables.invalidate_all(); } + #[cfg(feature = "sql")] + self.sql_clients.invalidate_all(); } /// Get cache statistics. diff --git a/fluree-db-api/src/graph_source/catalog_session.rs b/fluree-db-api/src/graph_source/catalog_session.rs index fc730921fa..d70e20b159 100644 --- a/fluree-db-api/src/graph_source/catalog_session.rs +++ b/fluree-db-api/src/graph_source/catalog_session.rs @@ -130,9 +130,24 @@ pub(crate) struct IcebergCatalogSession { /// build (not once per table). Always cached (independent of the loadTable /// cache toggle) — the listing is stable for the build. warehouse_listings: Mutex>>>, + /// Graph sources this session has scanned that are SQL-backed. A SQL source + /// has no snapshot to pin, so the loadTable-cache precondition in + /// `verify_build_snapshot_integrity` does not apply to it. + sql_sources: Mutex>, } impl IcebergCatalogSession { + pub(crate) fn mark_sql_source(&self, graph_source_id: &str) { + self.sql_sources + .lock() + .unwrap() + .insert(graph_source_id.to_string()); + } + + pub(crate) fn is_sql_source(&self, graph_source_id: &str) -> bool { + self.sql_sources.lock().unwrap().contains(graph_source_id) + } + /// Cache key for a `loadTable` response: source id + fully-qualified table. pub(crate) fn load_table_key(graph_source_id: &str, namespace: &str, table: &str) -> String { format!("{graph_source_id}\u{1f}{namespace}.{table}") diff --git a/fluree-db-api/src/graph_source/mod.rs b/fluree-db-api/src/graph_source/mod.rs index 9318ccb575..a0f9909d71 100644 --- a/fluree-db-api/src/graph_source/mod.rs +++ b/fluree-db-api/src/graph_source/mod.rs @@ -141,6 +141,12 @@ mod ephemeral; #[cfg(feature = "iceberg")] mod r2rml_materialize; +#[cfg(feature = "sql")] +mod sql; + +#[cfg(feature = "sql")] +pub use sql::{SqlCreateConfig, SqlCreateResult}; + // Re-export configuration types pub use config::Bm25CreateConfig; diff --git a/fluree-db-api/src/graph_source/r2rml.rs b/fluree-db-api/src/graph_source/r2rml.rs index 24461d9486..9f34d0ee66 100644 --- a/fluree-db-api/src/graph_source/r2rml.rs +++ b/fluree-db-api/src/graph_source/r2rml.rs @@ -93,6 +93,24 @@ fn iceberg_scan_concurrency(num_files: usize) -> usize { /// reference, so rotating the underlying secret leaves the fingerprint unchanged /// — the client cache's TTL (see `cache::DEFAULT_REST_CLIENT_TTL_SECS`), not this /// fingerprint, is what bounds staleness in that case. +/// The R2RML mapping reference carried by a mapped graph-source record, per +/// source family. `None` for a non-mapped type, an unparseable config, or a +/// record registered without a mapping. +fn mapping_source_of( + record: &fluree_db_nameservice::GraphSourceRecord, +) -> Option { + match record.source_type { + GraphSourceType::R2rml | GraphSourceType::Iceberg => { + IcebergGsConfig::from_json(&record.config) + .ok() + .and_then(|c| c.mapping) + } + #[cfg(feature = "sql")] + GraphSourceType::Sql => super::sql::mapping_source(record), + _ => None, + } +} + fn config_fingerprint(config: &str) -> u64 { use std::hash::{Hash, Hasher}; let mut h = std::collections::hash_map::DefaultHasher::new(); @@ -684,7 +702,7 @@ impl crate::Fluree { /// `media_type` is given. Format selection goes through the shared /// [`fluree_db_r2rml::loader::MappingFormat`] resolver (default Turtle) so /// registration and query time can never disagree (issue #1397). - fn compile_r2rml_content( + pub(crate) fn compile_r2rml_content( content: &str, media_type: Option<&str>, source: &str, @@ -734,7 +752,7 @@ impl crate::Fluree { /// Collect the distinct logical table names referenced by a compiled /// mapping, sorted for deterministic reporting. - fn sorted_table_names(compiled: &CompiledR2rmlMapping) -> Vec { + pub(crate) fn sorted_table_names(compiled: &CompiledR2rmlMapping) -> Vec { let mut names: Vec = compiled .table_names() .into_iter() @@ -782,6 +800,26 @@ impl<'a> FlureeR2rmlProvider<'a> { } } + /// The SQL source behind `graph_source_id`, or `None` when it is Iceberg-backed. + #[cfg(feature = "sql")] + async fn sql_source( + &self, + graph_source_id: &str, + ) -> QueryResult> { + let record = self + .fluree + .nameservice() + .lookup_graph_source(graph_source_id) + .await + .map_err(|e| QueryError::Internal(format!("Nameservice error: {e}")))?; + match record { + Some(r) if r.source_type == GraphSourceType::Sql => { + Ok(Some(super::sql::SqlSource::open(self.fluree, &r).await?)) + } + _ => Ok(None), + } + } + /// Resolve a graph source's storage backend, parsed table metadata, and /// metadata-location — the shared setup behind both full and incremental /// scans (REST/Direct × GCS/S3 × credentials × caching). @@ -1402,21 +1440,7 @@ impl R2rmlProvider for FlureeR2rmlProvider<'_> { .lookup_graph_source(graph_source_id) .await { - Ok(Some(record)) => { - // First check if this is an R2RML or Iceberg graph source type - if !matches!( - record.source_type, - GraphSourceType::R2rml | GraphSourceType::Iceberg - ) { - return false; - } - - // Parse into typed config to stay aligned with real config schema - match IcebergGsConfig::from_json(&record.config) { - Ok(config) => config.mapping.is_some(), - Err(_) => false, - } - } + Ok(Some(record)) => mapping_source_of(&record).is_some(), Ok(None) => false, Err(_) => false, } @@ -1441,29 +1465,23 @@ impl R2rmlProvider for FlureeR2rmlProvider<'_> { QueryError::InvalidQuery(format!("Graph source '{graph_source_id}' not found")) })?; - // Verify it's an R2RML or Iceberg graph source - if !matches!( - record.source_type, - GraphSourceType::R2rml | GraphSourceType::Iceberg - ) { + if !record + .source_type + .kind() + .eq(&fluree_db_nameservice::GraphSourceKind::Mapped) + { return Err(QueryError::InvalidQuery(format!( "Graph source '{}' is not an R2RML graph source (type: {:?})", graph_source_id, record.source_type ))); } - // Parse into typed config - let iceberg_config = IcebergGsConfig::from_json(&record.config).map_err(|e| { - QueryError::Internal(format!( - "Failed to parse graph source config for '{graph_source_id}': {e}" - )) - })?; - - let mapping_config = iceberg_config.mapping.as_ref().ok_or_else(|| { + let mapping_config = mapping_source_of(&record).ok_or_else(|| { QueryError::InvalidQuery(format!( "Graph source '{graph_source_id}' is missing 'mapping' in config" )) })?; + let mapping_config = &mapping_config; let mapping_source = &mapping_config.source; let media_type = mapping_config.media_type.as_deref(); @@ -1587,7 +1605,12 @@ impl R2rmlProvider for FlureeR2rmlProvider<'_> { graph_source_id: &str, ) -> std::result::Result<(), fluree_db_r2rml::R2rmlError> { use fluree_db_r2rml::R2rmlError; - if !super::catalog_session::cache_enabled() { + // A SQL source pins nothing (its watermark is endpoint+table+time), so + // the loadTable-cache precondition is meaningless for it. Known only + // once a scan has run, which is fine: the up-front check at build start + // still applies to a mixed session's Iceberg sources. + if !super::catalog_session::cache_enabled() && !self.session.is_sql_source(graph_source_id) + { return Err(R2rmlError::BuildSnapshotIntegrity( "the loadTable metadata cache is disabled (FLUREE_ICEBERG_LOADTABLE_CACHE=0), so \ Iceberg snapshot pinning is a no-op and the twin's stamped watermark cannot be \ @@ -1966,6 +1989,12 @@ impl FlureeR2rmlProvider<'_> { non_null_cols: &[String], _as_of_t: Option, ) -> QueryResult> { + #[cfg(feature = "sql")] + if let Some(sql) = self.sql_source(graph_source_id).await? { + return sql + .row_count(&self.session, table_name, non_null_cols) + .await; + } // Same pinned context as the scan: one Iceberg snapshot per query (the // shared `self.session` pin), so a count and a scan cannot disagree. // GREP: r2rml-as-of-t — `as_of_t` is ignored here exactly as the scan path @@ -2688,6 +2717,12 @@ impl FlureeR2rmlProvider<'_> { // `_as_of_t` is deliberately ignored. If as-of semantics ever land here, // `table_row_count_inner` MUST honor them identically (matching breadcrumb // there): a COUNT and a scan in one query must read the same snapshot. + #[cfg(feature = "sql")] + if let Some(sql) = self.sql_source(graph_source_id).await? { + return sql + .scan(&self.session, table_name, projection, filters) + .await; + } info!( graph_source_id = %graph_source_id, table_name = %table_name, diff --git a/fluree-db-api/src/graph_source/sql.rs b/fluree-db-api/src/graph_source/sql.rs new file mode 100644 index 0000000000..7206ac2e38 --- /dev/null +++ b/fluree-db-api/src/graph_source/sql.rs @@ -0,0 +1,483 @@ +//! SQL graph sources: an R2RML mapping over tables reached through a +//! Trino-protocol HTTP endpoint. +//! +//! Registration mirrors the Iceberg/R2RML path (mapping compiled and stored in +//! CAS, record published under `f:SqlMapping`), and scans are served through +//! the same [`super::FlureeR2rmlProvider`], which dispatches here when the +//! record's type is `Sql`. A SQL source has no snapshot to pin, so its build +//! watermark records the endpoint, table and first-touch time. + +use std::collections::BTreeMap; +use std::sync::Arc; + +use fluree_db_nameservice::{GraphSourceRecord, GraphSourceType}; +use fluree_db_query::error::{QueryError, Result as QueryResult}; +use fluree_db_query::r2rml::{ColumnBatchStream, ScanFilter, ScanValue, TableWatermark}; +use fluree_db_sql::{ + AuthConfig, CmpOp, Literal, LogicalSource, MappingSource, Predicate, ScanRequest, SqlDialect, + SqlError, SqlGsConfig, TrinoClient, WireProtocol, +}; +use futures::StreamExt; +use tracing::{debug, info, warn}; + +use super::config::R2rmlMappingInput; +use crate::graph_source::catalog_session::IcebergCatalogSession; + +/// Everything needed to register a SQL graph source. +#[derive(Debug, Clone)] +pub struct SqlCreateConfig { + /// Graph source name (e.g. `"warehouse-sql"`). + pub name: String, + /// Branch (defaults to `"main"`). + pub branch: Option, + /// Statement endpoint base URL. + pub endpoint: String, + pub dialect: SqlDialect, + pub protocol: WireProtocol, + pub catalog: Option, + pub schema: Option, + /// `X-Trino-User`; defaults to `fluree`. + pub user: Option, + pub auth: AuthConfig, + pub session: BTreeMap, + /// The R2RML mapping — inline content or a pre-existing address. + pub mapping: R2rmlMappingInput, + pub mapping_media_type: Option, +} + +impl SqlCreateConfig { + pub fn new( + name: impl Into, + endpoint: impl Into, + mapping_content: impl Into, + ) -> Self { + Self { + name: name.into(), + branch: None, + endpoint: endpoint.into(), + dialect: SqlDialect::default(), + protocol: WireProtocol::default(), + catalog: None, + schema: None, + user: None, + auth: AuthConfig::default(), + session: BTreeMap::new(), + mapping: R2rmlMappingInput::Content(mapping_content.into()), + mapping_media_type: None, + } + } + + pub fn effective_branch(&self) -> &str { + self.branch.as_deref().unwrap_or("main") + } + + pub fn graph_source_id(&self) -> String { + format!("{}:{}", self.name, self.effective_branch()) + } + + /// The persisted config, with the mapping's CAS address filled in. + pub fn to_gs_config(&self, mapping_address: &str) -> SqlGsConfig { + let mut cfg = SqlGsConfig::new(self.endpoint.clone()); + cfg.dialect = self.dialect; + cfg.protocol = self.protocol; + cfg.catalog = self.catalog.clone(); + cfg.schema = self.schema.clone(); + if let Some(u) = &self.user { + cfg.user = u.clone(); + } + cfg.auth = self.auth.clone(); + cfg.session = self.session.clone(); + let media_type = self.mapping_media_type.clone().unwrap_or_else(|| { + fluree_db_r2rml::loader::MappingFormat::resolve(None, mapping_address) + .media_type() + .to_string() + }); + cfg.mapping = Some(MappingSource { + source: mapping_address.to_string(), + media_type: Some(media_type), + }); + cfg + } + + pub fn validate(&self) -> crate::Result<()> { + if self.name.trim().is_empty() { + return Err(crate::ApiError::Config( + "graph source name must not be empty".to_string(), + )); + } + if self.name.contains(':') { + return Err(crate::ApiError::Config(format!( + "graph source name '{}' may not contain ':'", + self.name + ))); + } + self.to_gs_config("") + .validate() + .map_err(|e| crate::ApiError::Config(e.to_string())) + } +} + +/// What `create_sql_graph_source` reports back. +#[derive(Debug, Clone, serde::Serialize)] +pub struct SqlCreateResult { + pub graph_source_id: String, + pub endpoint: String, + pub mapping_source: String, + pub triples_map_count: usize, + pub table_count: usize, + pub table_names: Vec, + /// Whether `SELECT 1` succeeded against the endpoint. A failure is logged, + /// not fatal: the record is still created (credentials may arrive later). + pub connection_tested: bool, + pub mapping_validated: bool, +} + +impl crate::Fluree { + /// Register a SQL graph source. Compiles the mapping, stores it in CAS, + /// probes the endpoint, and publishes the record. + pub async fn create_sql_graph_source( + &self, + config: SqlCreateConfig, + ) -> crate::Result { + let graph_source_id = config.graph_source_id(); + info!(graph_source_id = %graph_source_id, "Creating SQL graph source"); + config.validate()?; + + let (mapping_address, triples_map_count, table_names, mapping_validated) = match &config + .mapping + { + R2rmlMappingInput::Content(content) => { + let compiled = + Self::compile_r2rml_content(content, config.mapping_media_type.as_deref(), "")?; + let count = compiled.len(); + let tables = Self::sorted_table_names(&compiled); + let cid = self + .content_store(&graph_source_id) + .put( + fluree_db_core::ContentKind::GraphSourceMapping, + content.as_bytes(), + ) + .await + .map_err(|e| { + crate::ApiError::Config(format!("Failed to store R2RML mapping: {e}")) + })?; + (cid.to_string(), count, tables, true) + } + R2rmlMappingInput::Address(address) => { + let storage = self.admin_storage().ok_or_else(|| { + crate::ApiError::Config( + "address-based mappings are not supported on this backend".to_string(), + ) + })?; + let (count, tables, validated) = match storage.read_bytes(address).await { + Ok(bytes) => match String::from_utf8(bytes) + .map_err(|e| crate::ApiError::Config(e.to_string())) + .and_then(|content| { + Self::compile_r2rml_content( + &content, + config.mapping_media_type.as_deref(), + address, + ) + }) { + Ok(compiled) => (compiled.len(), Self::sorted_table_names(&compiled), true), + Err(e) => { + warn!(graph_source_id = %graph_source_id, error = %e, "Could not validate R2RML mapping from address"); + (0, Vec::new(), false) + } + }, + Err(e) => { + warn!(graph_source_id = %graph_source_id, error = %e, "Could not read R2RML mapping from address"); + (0, Vec::new(), false) + } + }; + (address.clone(), count, tables, validated) + } + }; + + let gs_config = config.to_gs_config(&mapping_address); + let connection_tested = match self.test_sql_connection(&gs_config).await { + Ok(()) => true, + Err(e) => { + warn!(graph_source_id = %graph_source_id, error = %e, "SQL endpoint connection test failed; registering anyway"); + false + } + }; + + let config_json = gs_config + .to_json() + .map_err(|e| crate::ApiError::Config(format!("Failed to serialize config: {e}")))?; + self.publisher()? + .publish_graph_source( + &config.name, + config.effective_branch(), + GraphSourceType::Sql, + &config_json, + &[], + ) + .await?; + + info!(graph_source_id = %graph_source_id, mapping_address = %mapping_address, "Created SQL graph source"); + Ok(SqlCreateResult { + graph_source_id, + endpoint: gs_config.endpoint, + mapping_source: mapping_address, + triples_map_count, + table_count: table_names.len(), + table_names, + connection_tested, + mapping_validated, + }) + } + + /// `SELECT 1` against the endpoint with the configured credentials. + pub async fn test_sql_connection(&self, config: &SqlGsConfig) -> crate::Result<()> { + let client = build_sql_client(config, self.secret_resolver()) + .await + .map_err(|e| crate::ApiError::Config(e.to_string()))?; + client + .execute_collect("SELECT 1") + .await + .map(|_| ()) + .map_err(|e| { + crate::ApiError::Config(format!("SQL endpoint connection test failed: {e}")) + }) + } +} + +/// Hydrate secrets, build the auth provider, and construct the client. +async fn build_sql_client( + config: &SqlGsConfig, + resolver: Option<&Arc>, +) -> Result { + let hydrated = config.hydrate(resolver).await?; + let auth = hydrated.auth.create_provider_arc()?; + TrinoClient::new(&hydrated, auth) +} + +/// One SQL source resolved from its nameservice record. +pub(crate) struct SqlSource { + pub(crate) graph_source_id: String, + pub(crate) config: SqlGsConfig, + pub(crate) client: Arc, +} + +impl SqlSource { + /// Resolve the record's config and the (process-cached) client. The cache + /// key is a fingerprint of the RAW config so a secret rotation behind an + /// env var / secret ref does not rebuild the client every query. + pub(crate) async fn open( + fluree: &crate::Fluree, + record: &GraphSourceRecord, + ) -> QueryResult { + let config = SqlGsConfig::from_json(&record.config).map_err(|e| { + QueryError::Internal(format!( + "Failed to parse SQL graph source config for '{}': {e}", + record.graph_source_id + )) + })?; + let cache = fluree.r2rml_cache(); + let key = super::r2rml::rest_client_cache_key(&record.graph_source_id, &record.config); + let client = match cache.sql_client(&key) { + Some(c) => c, + None => { + let c = Arc::new( + build_sql_client(&config, fluree.secret_resolver()) + .await + .map_err(|e| { + QueryError::Internal(format!( + "SQL graph source '{}': {e}", + record.graph_source_id + )) + })?, + ); + cache.put_sql_client(key, Arc::clone(&c)); + c + } + }; + Ok(Self { + graph_source_id: record.graph_source_id.clone(), + config, + client, + }) + } + + fn source(&self, table_name: &str) -> LogicalSource { + LogicalSource::Table(table_name.to_string()) + } + + /// Stamp this table into the build watermark on first touch. + fn record_watermark(&self, session: &IcebergCatalogSession, table_name: &str) { + session.mark_sql_source(&self.graph_source_id); + session.record_snapshot( + IcebergCatalogSession::snapshot_key(&self.graph_source_id, table_name), + TableWatermark { + metadata_location: format!( + "sql://{}/{}@{}", + self.config + .endpoint_base() + .trim_start_matches("https://") + .trim_start_matches("http://"), + table_name, + chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true) + ), + snapshot_id: None, + sequence_number: None, + }, + ); + } + + pub(crate) async fn scan( + &self, + session: &IcebergCatalogSession, + table_name: &str, + projection: &[String], + filters: &[ScanFilter], + ) -> QueryResult { + let source = self.source(table_name); + let schema = self + .client + .schema(&source) + .await + .map_err(|e| sql_query_error(&self.graph_source_id, table_name, e))?; + self.record_watermark(session, table_name); + + let request = ScanRequest { + source, + projection: projection.to_vec(), + predicates: filters.iter().map(to_predicate).collect(), + }; + let rendered = + fluree_db_sql::dialect::render_scan(&request, &schema, self.client.dialect()) + .map_err(|e| sql_query_error(&self.graph_source_id, table_name, e))?; + if !rendered.declined_predicates.is_empty() { + debug!( + graph_source_id = %self.graph_source_id, + table_name, + declined = ?rendered.declined_predicates, + "SQL pushdown declined some predicates (in-engine FILTER enforces them)" + ); + } + info!( + graph_source_id = %self.graph_source_id, + table_name, + sql = %rendered.sql, + "SQL table scan" + ); + + let gs = self.graph_source_id.clone(); + let table = table_name.to_string(); + let stream = self + .client + .execute(rendered.sql) + .map(move |item| item.map_err(|e| sql_query_error(&gs, &table, e))); + Ok(Box::pin(stream)) + } + + pub(crate) async fn row_count( + &self, + session: &IcebergCatalogSession, + table_name: &str, + non_null_cols: &[String], + ) -> QueryResult> { + let source = self.source(table_name); + self.record_watermark(session, table_name); + let n = self + .client + .count(&source, non_null_cols) + .await + .map_err(|e| sql_query_error(&self.graph_source_id, table_name, e))?; + Ok(Some(n)) + } +} + +fn sql_query_error(graph_source_id: &str, table_name: &str, e: SqlError) -> QueryError { + let msg = format!("SQL graph source '{graph_source_id}', table '{table_name}': {e}"); + match e { + SqlError::Config(_) | SqlError::Unsupported(_) => QueryError::InvalidQuery(msg), + _ => QueryError::Internal(msg), + } +} + +fn to_predicate(f: &ScanFilter) -> Predicate { + use fluree_db_query::r2rml::ScanCmpOp; + Predicate { + column: f.column.clone(), + op: match f.op { + ScanCmpOp::Eq => CmpOp::Eq, + ScanCmpOp::NotEq => CmpOp::NotEq, + ScanCmpOp::Lt => CmpOp::Lt, + ScanCmpOp::LtEq => CmpOp::LtEq, + ScanCmpOp::Gt => CmpOp::Gt, + ScanCmpOp::GtEq => CmpOp::GtEq, + ScanCmpOp::In => CmpOp::In, + }, + value: to_literal(&f.value), + } +} + +fn to_literal(v: &ScanValue) -> Literal { + match v { + ScanValue::Bool(b) => Literal::Bool(*b), + ScanValue::Int(i) => Literal::Int(*i), + ScanValue::Date(d) => Literal::Date(*d), + ScanValue::Str(s) => Literal::Str(s.clone()), + ScanValue::Double(d) => Literal::Double(*d), + ScanValue::Decimal { + unscaled, scale, .. + } => Literal::Decimal { + unscaled: *unscaled, + scale: *scale, + }, + ScanValue::TemplateKey(k) => Literal::TemplateKey(k.clone()), + ScanValue::Set(members) => Literal::Set(members.iter().map(to_literal).collect()), + ScanValue::Timestamp { micros, tz } => Literal::Timestamp { + micros: *micros, + tz: *tz, + }, + } +} + +/// The mapping reference of a SQL record, if it has one. +pub(crate) fn mapping_source(record: &GraphSourceRecord) -> Option { + SqlGsConfig::from_json(&record.config) + .ok() + .and_then(|c| c.mapping) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn create_config_round_trips_into_gs_config() { + let mut c = SqlCreateConfig::new("wh", "http://localhost:8080/", "@prefix rr: ."); + c.catalog = Some("pg".into()); + c.user = Some("svc".into()); + assert_eq!(c.graph_source_id(), "wh:main"); + let gs = c.to_gs_config("bafy123"); + assert_eq!(gs.endpoint_base(), "http://localhost:8080"); + assert_eq!(gs.catalog.as_deref(), Some("pg")); + assert_eq!(gs.user, "svc"); + let m = gs.mapping.unwrap(); + assert_eq!(m.source, "bafy123"); + assert_eq!(m.media_type.as_deref(), Some("text/turtle")); + c.validate().unwrap(); + c.name = "a:b".into(); + assert!(c.validate().is_err()); + } + + #[test] + fn scan_filters_convert() { + let f = ScanFilter { + column: "id".into(), + op: fluree_db_query::r2rml::ScanCmpOp::In, + value: ScanValue::Set(vec![ScanValue::Int(1), ScanValue::TemplateKey("2".into())]), + }; + let p = to_predicate(&f); + assert_eq!(p.op, CmpOp::In); + assert_eq!( + p.value, + Literal::Set(vec![Literal::Int(1), Literal::TemplateKey("2".into())]) + ); + } +} diff --git a/fluree-db-api/src/ledger_info.rs b/fluree-db-api/src/ledger_info.rs index 45095062ab..ff1fdf49a2 100644 --- a/fluree-db-api/src/ledger_info.rs +++ b/fluree-db-api/src/ledger_info.rs @@ -1184,6 +1184,7 @@ pub fn graph_source_type_label(source_type: &GraphSourceType) -> String { GraphSourceType::Geo => "Geo".to_string(), GraphSourceType::R2rml => "R2RML".to_string(), GraphSourceType::Iceberg => "Iceberg".to_string(), + GraphSourceType::Sql => "SQL".to_string(), GraphSourceType::Unknown(s) => format!("Unknown({s})"), } } diff --git a/fluree-db-api/src/lib.rs b/fluree-db-api/src/lib.rs index 9d3cf0f67e..d12bdc3d5b 100644 --- a/fluree-db-api/src/lib.rs +++ b/fluree-db-api/src/lib.rs @@ -220,6 +220,13 @@ pub use graph_source::{ ValidateR2rmlResponse, }; +#[cfg(feature = "sql")] +pub use fluree_db_sql::{ + validate_sql_endpoint, AuthConfig as SqlAuthConfig, SqlDialect, SqlGsConfig, WireProtocol, +}; +#[cfg(feature = "sql")] +pub use graph_source::{SqlCreateConfig, SqlCreateResult}; + /// Secret-resolution injection point for `ConfigValue::SecretRef` in Iceberg /// graph-source auth. The host constructs a [`SecretResolver`] with the tenant /// captured and injects it via [`Fluree::with_secret_resolver`]; db stays diff --git a/fluree-db-api/tests/it_sql_graph_source.rs b/fluree-db-api/tests/it_sql_graph_source.rs new file mode 100644 index 0000000000..d2fbbb0f81 --- /dev/null +++ b/fluree-db-api/tests/it_sql_graph_source.rs @@ -0,0 +1,283 @@ +//! End-to-end over a SQL graph source: registration, the R2RML query path, +//! typed filter pushdown and the exact COUNT shortcut — against a fake +//! Trino-protocol endpoint, so the SQL the engine actually sends is asserted. + +#![cfg(all(feature = "sql", feature = "native"))] + +use fluree_db_api::{FlureeBuilder, SqlCreateConfig}; +use serde_json::{json, Value}; +use wiremock::matchers::{body_string_contains, method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +const PEOPLE_R2RML: &str = r#" + @prefix rr: . + @prefix ex: . + + + a rr:TriplesMap ; + rr:logicalTable [ rr:tableName "sales.people" ] ; + rr:subjectMap [ + rr:template "http://example.org/person/{id}" ; + rr:class ex:Person + ] ; + rr:predicateObjectMap [ + rr:predicate ex:name ; + rr:objectMap [ rr:column "name" ] + ] ; + rr:predicateObjectMap [ + rr:predicate ex:score ; + rr:objectMap [ rr:column "score" ] + ] ; + rr:predicateObjectMap [ + rr:predicate ex:born ; + rr:objectMap [ rr:column "born" ] + ] . +"#; + +fn columns() -> Value { + json!([ + {"name": "id", "type": "bigint"}, + {"name": "name", "type": "varchar"}, + {"name": "score", "type": "double"}, + {"name": "born", "type": "date"} + ]) +} + +fn finished(data: Value) -> ResponseTemplate { + ResponseTemplate::new(200).set_body_json(json!({ + "id": "q", + "columns": columns(), + "data": data, + "stats": {"state": "FINISHED"} + })) +} + +/// The fake endpoint. Mocks are tried in priority order (lower first). +async fn fake_trino() -> MockServer { + let server = MockServer::start().await; + // SELECT 1 — the registration-time connection test. + Mock::given(method("POST")) + .and(path("/v1/statement")) + .and(body_string_contains("SELECT 1")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "t", "columns": [{"name": "_col0", "type": "integer"}], "data": [[1]], "stats": {"state": "FINISHED"} + }))) + .with_priority(1) + .mount(&server) + .await; + // Schema probe. + Mock::given(method("POST")) + .and(path("/v1/statement")) + .and(body_string_contains("LIMIT 0")) + .respond_with(finished(json!([]))) + .with_priority(2) + .mount(&server) + .await; + // Exact count. + Mock::given(method("POST")) + .and(path("/v1/statement")) + .and(body_string_contains("COUNT(*)")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "c", "columns": [{"name": "_col0", "type": "bigint"}], "data": [[3]], "stats": {"state": "FINISHED"} + }))) + .with_priority(3) + .mount(&server) + .await; + // A pushed equality on name. + Mock::given(method("POST")) + .and(path("/v1/statement")) + .and(body_string_contains(r#""name" = 'bob'"#)) + .respond_with(finished(json!([[2, "bob", 7.5, "1990-05-04"]]))) + .with_priority(4) + .mount(&server) + .await; + // Any other scan of the table: every row. + Mock::given(method("POST")) + .and(path("/v1/statement")) + .and(body_string_contains(r#"FROM "sales"."people""#)) + .respond_with(finished(json!([ + [1, "alice", 9.25, "1985-01-02"], + [2, "bob", 7.5, "1990-05-04"], + [3, null, null, null] + ]))) + .with_priority(5) + .mount(&server) + .await; + server +} + +/// SPARQL JSON results → the binding rows. +fn bindings(v: &Value) -> Vec { + v.pointer("/results/bindings") + .and_then(Value::as_array) + .cloned() + .unwrap_or_else(|| panic!("not SPARQL JSON results: {v}")) +} + +async fn statements(server: &MockServer) -> Vec { + server + .received_requests() + .await + .unwrap_or_default() + .iter() + .filter(|r| r.method == "POST") + .map(|r| String::from_utf8_lossy(&r.body).to_string()) + .collect() +} + +#[tokio::test] +async fn sql_graph_source_end_to_end() { + let server = fake_trino().await; + let fluree = FlureeBuilder::memory().build_memory(); + + // 1. Register. + let mut config = SqlCreateConfig::new("people-sql", server.uri(), PEOPLE_R2RML); + config.catalog = Some("pg".into()); + let created = fluree + .create_sql_graph_source(config) + .await + .expect("create sql graph source"); + assert_eq!(created.graph_source_id, "people-sql:main"); + assert!(created.connection_tested, "SELECT 1 probe succeeded"); + assert!(created.mapping_validated); + assert_eq!(created.table_names, vec!["sales.people".to_string()]); + assert_eq!(created.triples_map_count, 1); + + let info = fluree + .nameservice() + .lookup_graph_source("people-sql:main") + .await + .expect("lookup") + .expect("record"); + assert_eq!( + info.source_type, + fluree_db_nameservice::GraphSourceType::Sql + ); + + // 2. A plain scan. + let query = json!({ + "@context": {"ex": "http://example.org/"}, + "from": "people-sql:main", + "select": ["?name"], + "where": {"@id": "?s", "ex:name": "?name"}, + }); + let rows = fluree + .query_from() + .jsonld(&query) + .execute_formatted() + .await + .expect("query sql source"); + let names: Vec = rows + .as_array() + .expect("array") + .iter() + .map(std::string::ToString::to_string) + .collect(); + assert_eq!( + names.len(), + 2, + "the null-name row yields no ex:name triple: {names:?}" + ); + assert!(names.iter().any(|n| n.contains("alice")) && names.iter().any(|n| n.contains("bob"))); + + let sent = statements(&server).await; + let probe = sent + .iter() + .find(|s| s.contains("LIMIT 0")) + .expect("schema probe was issued"); + assert_eq!(probe, r#"SELECT * FROM "sales"."people" LIMIT 0"#); + let scan = sent + .iter() + .find(|s| s.starts_with("SELECT \"") && !s.contains("WHERE")) + .expect("scan statement"); + assert!( + scan.contains(r#""id""#) && scan.contains(r#""name""#), + "{scan}" + ); + assert!( + !scan.contains(r#""score""#), + "only mapped+needed columns are projected: {scan}" + ); + + // 3. A constant object is pushed as a typed WHERE. + let sparql = r#" + PREFIX ex: + SELECT ?s ?score FROM + WHERE { ?s ex:name "bob" ; ex:score ?score } + "#; + let rows = fluree + .query_from() + .sparql(sparql) + .execute_formatted() + .await + .expect("filtered query"); + let rows = bindings(&rows); + assert_eq!(rows.len(), 1, "{rows:?}"); + assert!(rows[0].to_string().contains("person/2"), "{rows:?}"); + let sent = statements(&server).await; + assert!( + sent.iter().any(|s| s.contains(r#"WHERE "name" = 'bob'"#)), + "equality pushed to SQL: {sent:?}" + ); + + // 4. Typed decoding: a date column round-trips as xsd:date. + let sparql = " + PREFIX ex: + SELECT ?born FROM + WHERE { ex:born ?born } + "; + let rows = fluree + .query_from() + .sparql(sparql) + .execute_formatted() + .await + .expect("date query"); + assert!(rows.to_string().contains("1985-01-02"), "{rows}"); + + // 5. COUNT over the class answers 3 whether the exact shortcut fired or + // the scan counted (the fake is consistent); record which. + let sparql = " + PREFIX ex: + SELECT (COUNT(?s) AS ?n) FROM + WHERE { ?s a ex:Person } + "; + let rows = fluree + .query_from() + .sparql(sparql) + .execute_formatted() + .await + .expect("count query"); + assert!(rows.to_string().contains('3'), "{rows}"); + let sent = statements(&server).await; + eprintln!( + "COUNT(*) shortcut fired: {}", + sent.iter().any(|s| s.contains("COUNT(*)")) + ); +} + +#[tokio::test] +async fn registration_survives_an_unreachable_endpoint() { + let fluree = FlureeBuilder::memory().build_memory(); + let config = SqlCreateConfig::new("dead-sql", "http://127.0.0.1:9", PEOPLE_R2RML); + let created = fluree + .create_sql_graph_source(config) + .await + .expect("registration does not require a live endpoint"); + assert!(!created.connection_tested); + assert!(created.mapping_validated); + + // Querying it surfaces the transport error rather than empty results. + let query = json!({ + "@context": {"ex": "http://example.org/"}, + "from": "dead-sql:main", + "select": ["?name"], + "where": {"@id": "?s", "ex:name": "?name"}, + }); + let err = fluree + .query_from() + .jsonld(&query) + .execute_formatted() + .await + .expect_err("unreachable endpoint fails the query"); + assert!(err.to_string().contains("dead-sql:main"), "{err}"); +} diff --git a/fluree-db-nameservice/src/lib.rs b/fluree-db-nameservice/src/lib.rs index 1c7ffd548f..93081b8ccd 100644 --- a/fluree-db-nameservice/src/lib.rs +++ b/fluree-db-nameservice/src/lib.rs @@ -268,6 +268,8 @@ pub enum GraphSourceType { R2rml, /// Apache Iceberg table Iceberg, + /// R2RML mapping over tables reached through a SQL endpoint + Sql, /// Unknown/custom graph source type Unknown(String), } @@ -279,7 +281,9 @@ impl GraphSourceType { GraphSourceType::Bm25 | GraphSourceType::Vector | GraphSourceType::Geo => { GraphSourceKind::Index } - GraphSourceType::R2rml | GraphSourceType::Iceberg => GraphSourceKind::Mapped, + GraphSourceType::R2rml | GraphSourceType::Iceberg | GraphSourceType::Sql => { + GraphSourceKind::Mapped + } GraphSourceType::Unknown(_) => GraphSourceKind::Index, // default assumption } } @@ -295,6 +299,7 @@ impl GraphSourceType { GraphSourceType::Geo => "f:GeoIndex".to_string(), GraphSourceType::R2rml => "f:R2rmlMapping".to_string(), GraphSourceType::Iceberg => "f:IcebergMapping".to_string(), + GraphSourceType::Sql => "f:SqlMapping".to_string(), GraphSourceType::Unknown(s) => s.clone(), } } @@ -311,12 +316,14 @@ impl GraphSourceType { "f:GeoIndex" => GraphSourceType::Geo, "f:R2rmlMapping" => GraphSourceType::R2rml, "f:IcebergMapping" => GraphSourceType::Iceberg, + "f:SqlMapping" => GraphSourceType::Sql, // Full IRI forms ns_types::BM25_INDEX => GraphSourceType::Bm25, ns_types::HNSW_INDEX => GraphSourceType::Vector, ns_types::GEO_INDEX => GraphSourceType::Geo, ns_types::R2RML_MAPPING => GraphSourceType::R2rml, ns_types::ICEBERG_MAPPING => GraphSourceType::Iceberg, + ns_types::SQL_MAPPING => GraphSourceType::Sql, _ => GraphSourceType::Unknown(s.to_string()), } } diff --git a/fluree-db-sql/src/lib.rs b/fluree-db-sql/src/lib.rs index 5605bf510c..9d70466832 100644 --- a/fluree-db-sql/src/lib.rs +++ b/fluree-db-sql/src/lib.rs @@ -23,6 +23,7 @@ pub use dialect::{ CmpOp, Literal, LogicalSource, Predicate, RenderedScan, ScanRequest, SqlDialect, }; pub use error::{Result, SqlError}; +pub use net::validate_endpoint as validate_sql_endpoint; pub use trino::{SqlBatchStream, TrinoClient}; // Re-exported so callers wire auth/secret resolution with one import. diff --git a/fluree-vocab/src/lib.rs b/fluree-vocab/src/lib.rs index 3b581e10a0..32d1ca5e4c 100644 --- a/fluree-vocab/src/lib.rs +++ b/fluree-vocab/src/lib.rs @@ -2116,6 +2116,9 @@ pub mod ns_types { /// `https://ns.flur.ee/db#R2rmlMapping` - R2RML relational mapping pub const R2RML_MAPPING: &str = "https://ns.flur.ee/db#R2rmlMapping"; + + /// `https://ns.flur.ee/db#SqlMapping` - R2RML mapping over a SQL endpoint + pub const SQL_MAPPING: &str = "https://ns.flur.ee/db#SqlMapping"; } /// Graph source nameservice field local names (under `https://ns.flur.ee/db#`) From cd0900ad90027ce8e13baebf18288014fbb6568d Mon Sep 17 00:00:00 2001 From: bplatz Date: Sat, 29 Aug 2026 16:16:52 -0400 Subject: [PATCH 03/13] feat(sql): rr:sqlQuery logical tables, POST /sql/map, and fluree sql CLI rr:sqlQuery compiles to a deterministic alias (sqlQuery:) that stands in wherever a table name is expected, so the scan operator, caches and find_maps_for_table need no changes; a SQL source resolves the alias back to the query and scans it as a derived table. Iceberg-backed sources refuse such a mapping at registration instead of at first query. The server route mirrors /iceberg/map with the SQL config surface (dialect, protocol, catalog/schema, user, bearer or OAuth2, session properties) and guards the endpoint against the link-local/metadata range. The CLI gains fluree sql map|list|info|drop; list/info/drop share the mapped-source implementations with fluree iceberg, whose family predicate now includes SQL. --- fluree-db-api/src/graph_source/r2rml.rs | 26 ++- fluree-db-api/src/graph_source/sql.rs | 15 +- fluree-db-api/src/lib.rs | 3 +- fluree-db-api/tests/it_sql_graph_source.rs | 114 ++++++++++ fluree-db-cli/Cargo.toml | 4 +- fluree-db-cli/src/cli.rs | 127 +++++++++++ fluree-db-cli/src/commands/iceberg.rs | 9 +- fluree-db-cli/src/commands/mod.rs | 1 + fluree-db-cli/src/commands/sql.rs | 249 +++++++++++++++++++++ fluree-db-cli/src/lib.rs | 36 +++ fluree-db-cli/src/remote_client.rs | 17 ++ fluree-db-r2rml/src/loader/extractor.rs | 20 +- fluree-db-r2rml/src/loader/mod.rs | 75 +++++++ fluree-db-r2rml/src/mapping/compiled.rs | 16 ++ fluree-db-r2rml/src/mapping/triples_map.rs | 65 +++++- fluree-db-server/Cargo.toml | 4 +- fluree-db-server/src/routes/mod.rs | 5 + fluree-db-server/src/routes/sql.rs | 192 ++++++++++++++++ 18 files changed, 951 insertions(+), 27 deletions(-) create mode 100644 fluree-db-cli/src/commands/sql.rs create mode 100644 fluree-db-server/src/routes/sql.rs diff --git a/fluree-db-api/src/graph_source/r2rml.rs b/fluree-db-api/src/graph_source/r2rml.rs index 9f34d0ee66..e096946cf9 100644 --- a/fluree-db-api/src/graph_source/r2rml.rs +++ b/fluree-db-api/src/graph_source/r2rml.rs @@ -111,6 +111,19 @@ fn mapping_source_of( } } +/// An Iceberg-backed source scans tables, never queries: refuse a mapping with +/// `rr:sqlQuery` at registration rather than at first query. +fn reject_sql_queries(compiled: &CompiledR2rmlMapping) -> Result<()> { + if compiled.has_sql_queries() { + return Err(crate::ApiError::Config( + "rr:sqlQuery logical tables are only supported by SQL graph sources; \ + use rr:tableName for Iceberg-backed mappings" + .to_string(), + )); + } + Ok(()) +} + fn config_fingerprint(config: &str) -> u64 { use std::hash::{Hash, Hasher}; let mut h = std::collections::hash_map::DefaultHasher::new(); @@ -569,6 +582,7 @@ impl crate::Fluree { // CID address, which is also extensionless). let compiled = Self::compile_r2rml_content(content, config.mapping_media_type.as_deref(), "")?; + reject_sql_queries(&compiled)?; let count = compiled.len(); let tables = Self::sorted_table_names(&compiled); let gs_id = config.graph_source_id(); @@ -1991,8 +2005,9 @@ impl FlureeR2rmlProvider<'_> { ) -> QueryResult> { #[cfg(feature = "sql")] if let Some(sql) = self.sql_source(graph_source_id).await? { + let mapping = self.compiled_mapping(graph_source_id, None).await?; return sql - .row_count(&self.session, table_name, non_null_cols) + .row_count(&self.session, &mapping, table_name, non_null_cols) .await; } // Same pinned context as the scan: one Iceberg snapshot per query (the @@ -2209,6 +2224,12 @@ impl FlureeR2rmlProvider<'_> { graph_source_id: &str, table_name: &str, ) -> QueryResult<(Arc>, Arc, String)> { + if fluree_db_r2rml::mapping::LogicalTable::is_sql_query_alias(table_name) { + return Err(QueryError::InvalidQuery(format!( + "Graph source '{graph_source_id}': rr:sqlQuery logical tables are only \ + supported by SQL graph sources" + ))); + } // Look up the graph source record to get Iceberg connection info let record = self .fluree @@ -2719,8 +2740,9 @@ impl FlureeR2rmlProvider<'_> { // there): a COUNT and a scan in one query must read the same snapshot. #[cfg(feature = "sql")] if let Some(sql) = self.sql_source(graph_source_id).await? { + let mapping = self.compiled_mapping(graph_source_id, None).await?; return sql - .scan(&self.session, table_name, projection, filters) + .scan(&self.session, &mapping, table_name, projection, filters) .await; } info!( diff --git a/fluree-db-api/src/graph_source/sql.rs b/fluree-db-api/src/graph_source/sql.rs index 7206ac2e38..263364a60e 100644 --- a/fluree-db-api/src/graph_source/sql.rs +++ b/fluree-db-api/src/graph_source/sql.rs @@ -13,6 +13,7 @@ use std::sync::Arc; use fluree_db_nameservice::{GraphSourceRecord, GraphSourceType}; use fluree_db_query::error::{QueryError, Result as QueryResult}; use fluree_db_query::r2rml::{ColumnBatchStream, ScanFilter, ScanValue, TableWatermark}; +use fluree_db_r2rml::mapping::CompiledR2rmlMapping; use fluree_db_sql::{ AuthConfig, CmpOp, Literal, LogicalSource, MappingSource, Predicate, ScanRequest, SqlDialect, SqlError, SqlGsConfig, TrinoClient, WireProtocol, @@ -301,8 +302,12 @@ impl SqlSource { }) } - fn source(&self, table_name: &str) -> LogicalSource { - LogicalSource::Table(table_name.to_string()) + /// A table name, or the `rr:sqlQuery` text behind a query alias. + fn source(&self, mapping: &CompiledR2rmlMapping, table_name: &str) -> LogicalSource { + match mapping.sql_query_for_table(table_name) { + Some(sql) => LogicalSource::Query(sql.to_string()), + None => LogicalSource::Table(table_name.to_string()), + } } /// Stamp this table into the build watermark on first touch. @@ -329,11 +334,12 @@ impl SqlSource { pub(crate) async fn scan( &self, session: &IcebergCatalogSession, + mapping: &CompiledR2rmlMapping, table_name: &str, projection: &[String], filters: &[ScanFilter], ) -> QueryResult { - let source = self.source(table_name); + let source = self.source(mapping, table_name); let schema = self .client .schema(&source) @@ -376,10 +382,11 @@ impl SqlSource { pub(crate) async fn row_count( &self, session: &IcebergCatalogSession, + mapping: &CompiledR2rmlMapping, table_name: &str, non_null_cols: &[String], ) -> QueryResult> { - let source = self.source(table_name); + let source = self.source(mapping, table_name); self.record_watermark(session, table_name); let n = self .client diff --git a/fluree-db-api/src/lib.rs b/fluree-db-api/src/lib.rs index d12bdc3d5b..053e7dd0ac 100644 --- a/fluree-db-api/src/lib.rs +++ b/fluree-db-api/src/lib.rs @@ -222,7 +222,8 @@ pub use graph_source::{ #[cfg(feature = "sql")] pub use fluree_db_sql::{ - validate_sql_endpoint, AuthConfig as SqlAuthConfig, SqlDialect, SqlGsConfig, WireProtocol, + validate_sql_endpoint, AuthConfig as SqlAuthConfig, ConfigValue as SqlConfigValue, SqlDialect, + SqlGsConfig, WireProtocol, }; #[cfg(feature = "sql")] pub use graph_source::{SqlCreateConfig, SqlCreateResult}; diff --git a/fluree-db-api/tests/it_sql_graph_source.rs b/fluree-db-api/tests/it_sql_graph_source.rs index d2fbbb0f81..6c5cdcb08d 100644 --- a/fluree-db-api/tests/it_sql_graph_source.rs +++ b/fluree-db-api/tests/it_sql_graph_source.rs @@ -255,6 +255,120 @@ async fn sql_graph_source_end_to_end() { ); } +const ORDERS_SQLQUERY_R2RML: &str = r#" + @prefix rr: . + @prefix ex: . + + + a rr:TriplesMap ; + rr:logicalTable [ rr:sqlQuery "SELECT id, total FROM sales.orders WHERE status = 'open'" ] ; + rr:subjectMap [ + rr:template "http://example.org/order/{id}" ; + rr:class ex:Order + ] ; + rr:predicateObjectMap [ + rr:predicate ex:total ; + rr:objectMap [ rr:column "total" ] + ] . +"#; + +/// An `rr:sqlQuery` logical table is scanned as a derived table, with the +/// projection and pushed filters applied on top of the mapping's query. +#[tokio::test] +async fn sql_query_logical_table_is_scanned_as_a_derived_table() { + let server = MockServer::start().await; + let orders_columns = + json!([{"name": "id", "type": "bigint"}, {"name": "total", "type": "decimal(10,2)"}]); + Mock::given(method("POST")) + .and(path("/v1/statement")) + .and(body_string_contains("SELECT 1")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "t", "columns": [{"name": "_col0", "type": "integer"}], "data": [[1]], "stats": {"state": "FINISHED"} + }))) + .with_priority(1) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/v1/statement")) + .and(body_string_contains("LIMIT 0")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "p", "columns": orders_columns, "data": [], "stats": {"state": "FINISHED"} + }))) + .with_priority(2) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/v1/statement")) + .and(body_string_contains(r#"AS "__fluree_q""#)) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "s", "columns": orders_columns, "data": [[10, "99.50"], [11, "5.00"]], "stats": {"state": "FINISHED"} + }))) + .with_priority(3) + .mount(&server) + .await; + + let fluree = FlureeBuilder::memory().build_memory(); + let created = fluree + .create_sql_graph_source(SqlCreateConfig::new( + "orders-sql", + server.uri(), + ORDERS_SQLQUERY_R2RML, + )) + .await + .expect("create"); + assert_eq!(created.table_count, 1); + assert!( + created.table_names[0].starts_with("sqlQuery:"), + "{:?}", + created.table_names + ); + + let sparql = " + PREFIX ex: + SELECT ?o ?total FROM + WHERE { ?o ex:total ?total } ORDER BY ?o + "; + let rows = fluree + .query_from() + .sparql(sparql) + .execute_formatted() + .await + .expect("query over rr:sqlQuery"); + let rows = bindings(&rows); + assert_eq!(rows.len(), 2, "{rows:?}"); + assert!( + rows[0].to_string().contains("order/10") && rows[0].to_string().contains("99.50"), + "{rows:?}" + ); + + let sent = statements(&server).await; + let scan = sent + .iter() + .find(|s| s.contains(r#"AS "__fluree_q""#) && !s.contains("LIMIT 0")) + .expect("derived-table scan"); + assert_eq!( + scan, + r#"SELECT "id", "total" FROM (SELECT id, total FROM sales.orders WHERE status = 'open') AS "__fluree_q""# + ); +} + +/// The Iceberg-backed registration path refuses `rr:sqlQuery` up front. +#[tokio::test] +async fn iceberg_sources_refuse_sql_query_mappings() { + let fluree = FlureeBuilder::memory().build_memory(); + let config = fluree_db_api::R2rmlCreateConfig::new( + "ice", + "https://polaris.example.invalid", + "default.default", + ORDERS_SQLQUERY_R2RML, + ); + let err = fluree + .create_r2rml_graph_source(config) + .await + .expect_err("rr:sqlQuery is not for Iceberg"); + assert!(err.to_string().contains("rr:sqlQuery"), "{err}"); +} + #[tokio::test] async fn registration_survives_an_unreachable_endpoint() { let fluree = FlureeBuilder::memory().build_memory(); diff --git a/fluree-db-cli/Cargo.toml b/fluree-db-cli/Cargo.toml index 12778522c1..80edc92995 100644 --- a/fluree-db-cli/Cargo.toml +++ b/fluree-db-cli/Cargo.toml @@ -20,9 +20,11 @@ path = "src/lib.rs" # reuses the dist binary) can use S3 storage + DynamoDB nameservice via # connection config. Dormant unless configured; ~+2 MB over the AWS SDK that # `iceberg` already pulls in. -default = ["server", "iceberg", "shacl", "aws", "graphql"] +default = ["server", "iceberg", "sql", "shacl", "aws", "graphql"] server = ["dep:fluree-db-server"] iceberg = ["fluree-db-api/iceberg"] +# SQL graph sources (R2RML over a Trino-protocol endpoint) +sql = ["iceberg", "fluree-db-api/sql", "fluree-db-server?/sql"] aws = ["fluree-db-server/aws", "fluree-db-nameservice-sync/aws"] # SHACL constraint validation at transaction time shacl = ["fluree-db-api/shacl"] diff --git a/fluree-db-cli/src/cli.rs b/fluree-db-cli/src/cli.rs index 2fcdfa90a6..7bf7ba386b 100644 --- a/fluree-db-cli/src/cli.rs +++ b/fluree-db-cli/src/cli.rs @@ -1348,6 +1348,12 @@ pub enum Commands { action: IcebergAction, }, + /// Manage SQL graph sources (R2RML over a Trino-protocol endpoint) + Sql { + #[command(subcommand)] + action: SqlAction, + }, + /// Materialize a native twin ledger from a virtual (R2RML-over-Iceberg) /// graph source: bulk-build every triple, verify it against the source, and /// write it as a native ledger or a .flpack pack (DEC-003 Deliverable 1). @@ -3033,6 +3039,127 @@ pub enum IcebergAction { }, } +#[derive(Debug, Clone, Subcommand)] +pub enum SqlAction { + /// Map tables behind a SQL endpoint as an R2RML graph source + /// + /// The endpoint speaks the Trino client protocol: Trino, Starburst, + /// PrestoDB, or a `fluree-sql-bridge` sidecar in front of Postgres, + /// MySQL or SQLite. + /// + /// Examples: + /// fluree sql map warehouse --endpoint https://trino.example.com:8443 --r2rml mappings/orders.ttl --auth-bearer $TOKEN + /// fluree sql map crm --endpoint http://localhost:8080 --catalog pg --schema public --r2rml crm.ttl + Map(Box), + + /// List mapped graph sources (SQL, Iceberg and R2RML) + List { + /// List graph sources on a remote server (by remote name, e.g., "origin") + #[arg(long)] + remote: Option, + }, + + /// Show details for a mapped graph source + Info { + /// Graph source name + name: String, + + /// Query a remote server (by remote name, e.g., "origin") + #[arg(long)] + remote: Option, + }, + + /// Drop a mapped graph source + Drop { + /// Graph source name + name: String, + + /// Required flag to confirm deletion + #[arg(long)] + force: bool, + + /// Execute against a remote server (by remote name, e.g., "origin") + #[arg(long)] + remote: Option, + }, +} + +/// Arguments for mapping a SQL endpoint as a graph source. +#[derive(Debug, Clone, clap::Args)] +pub struct SqlMapArgs { + /// Graph source name (e.g., "warehouse") + pub name: String, + + /// Execute against a remote server (by remote name, e.g., "origin") + #[arg(long)] + pub remote: Option, + + /// Statement endpoint base URL (e.g., "https://trino.example.com:8443") + #[arg(long)] + pub endpoint: String, + + /// R2RML mapping file. Each rr:tableName names a table reachable through + /// the endpoint; rr:sqlQuery is also accepted. + #[arg(long)] + pub r2rml: PathBuf, + + /// R2RML mapping media type (e.g., "text/turtle"); inferred from extension if omitted + #[arg(long)] + pub r2rml_type: Option, + + /// Branch name (defaults to "main") + #[arg(long)] + pub branch: Option, + + /// SQL rendering dialect: trino (default), postgres, mysql, sqlite + #[arg(long)] + pub dialect: Option, + + /// Header family: trino (default) or presto + #[arg(long)] + pub protocol: Option, + + /// Default catalog for unqualified table names + #[arg(long)] + pub catalog: Option, + + /// Default schema for unqualified table names + #[arg(long)] + pub schema: Option, + + /// Protocol user (X-Trino-User); defaults to "fluree" + #[arg(long)] + pub user: Option, + + /// Bearer token for endpoint authentication + #[arg(long)] + pub auth_bearer: Option, + + /// OAuth2 token URL for client credentials auth + #[arg(long)] + pub oauth2_token_url: Option, + + /// OAuth2 client ID + #[arg(long)] + pub oauth2_client_id: Option, + + /// OAuth2 client secret + #[arg(long)] + pub oauth2_client_secret: Option, + + /// OAuth2 scope + #[arg(long)] + pub oauth2_scope: Option, + + /// OAuth2 audience + #[arg(long)] + pub oauth2_audience: Option, + + /// Session property (repeatable): --session query_max_run_time=5m + #[arg(long = "session", value_name = "KEY=VALUE")] + pub session: Vec, +} + /// Arguments for mapping an Iceberg table as a graph source. #[derive(Debug, Clone, clap::Args)] pub struct IcebergMapArgs { diff --git a/fluree-db-cli/src/commands/iceberg.rs b/fluree-db-cli/src/commands/iceberg.rs index d90fc8d34e..f7db363e65 100644 --- a/fluree-db-cli/src/commands/iceberg.rs +++ b/fluree-db-cli/src/commands/iceberg.rs @@ -343,7 +343,7 @@ async fn run_iceberg_map_remote( /// `text/turtle` (the resolver's default), case-insensitively. Returns `None` /// only when the path has no extension at all, leaving the server to apply the /// same default. An explicit `--r2rml-type` still overrides this at the call site. -fn infer_mapping_media_type(path: &std::path::Path) -> Option { +pub(crate) fn infer_mapping_media_type(path: &std::path::Path) -> Option { use fluree_db_r2rml::loader::MappingFormat; // No extension means no signal to infer from — defer to the server default. path.extension()?; @@ -712,7 +712,7 @@ fn build_iceberg_config(args: &IcebergMapArgs) -> CliResult String { +pub(crate) fn format_table_summary(count: usize, names: &[String]) -> String { if names.is_empty() { count.to_string() } else { @@ -720,16 +720,19 @@ fn format_table_summary(count: usize, names: &[String]) -> String { } } +/// Mapped (R2RML-backed) graph sources: Iceberg, R2RML and SQL. `fluree sql` +/// and `fluree iceberg` share list/info/drop over this family. fn is_iceberg_family_source_type(st: &fluree_db_nameservice::GraphSourceType) -> bool { matches!( st, fluree_db_nameservice::GraphSourceType::Iceberg | fluree_db_nameservice::GraphSourceType::R2rml + | fluree_db_nameservice::GraphSourceType::Sql ) } fn is_iceberg_family_type_str(s: &str) -> bool { - matches!(s, "Iceberg" | "R2RML") + matches!(s, "Iceberg" | "R2RML" | "SQL") } #[cfg(test)] diff --git a/fluree-db-cli/src/commands/mod.rs b/fluree-db-cli/src/commands/mod.rs index 22689101c9..6905a6c132 100644 --- a/fluree-db-cli/src/commands/mod.rs +++ b/fluree-db-cli/src/commands/mod.rs @@ -39,6 +39,7 @@ pub mod remote; #[cfg(feature = "server")] pub mod server; pub mod show; +pub mod sql; pub mod sweep; pub mod sync; pub mod token; diff --git a/fluree-db-cli/src/commands/sql.rs b/fluree-db-cli/src/commands/sql.rs new file mode 100644 index 0000000000..282093a136 --- /dev/null +++ b/fluree-db-cli/src/commands/sql.rs @@ -0,0 +1,249 @@ +//! `fluree sql map` — register a SQL graph source. +//! +//! `fluree sql list|info|drop` share the mapped-source implementations in +//! [`super::iceberg`]. + +use crate::cli::SqlMapArgs; +use crate::error::{CliError, CliResult}; +use fluree_db_api::server_defaults::FlureeDir; + +pub async fn run_sql_map(args: SqlMapArgs, dirs: &FlureeDir, direct: bool) -> CliResult<()> { + if let Some(remote_name) = args.remote.as_deref() { + let client = crate::context::build_remote_client(remote_name, dirs).await?; + let result = run_sql_map_remote(&client, &args).await.map_err(|e| { + CliError::Remote(format!( + "failed to map SQL graph source on '{remote_name}': {e}" + )) + }); + crate::context::persist_refreshed_tokens(&client, remote_name, dirs).await; + return result; + } + + if !direct { + if let Some(client) = crate::context::try_server_route_client(dirs) { + return run_sql_map_remote(&client, &args) + .await + .map_err(|e| CliError::Remote(format!("failed to map SQL graph source: {e}"))); + } + } + + run_sql_map_local(args, dirs).await +} + +fn read_mapping(args: &SqlMapArgs) -> CliResult { + std::fs::read_to_string(&args.r2rml).map_err(|e| { + CliError::Input(format!( + "Failed to read R2RML mapping file '{}': {e}", + args.r2rml.display() + )) + }) +} + +fn mapping_media_type(args: &SqlMapArgs) -> Option { + args.r2rml_type + .clone() + .or_else(|| super::iceberg::infer_mapping_media_type(&args.r2rml)) +} + +fn session_pairs(args: &SqlMapArgs) -> CliResult> { + args.session + .iter() + .map(|kv| { + kv.split_once('=') + .map(|(k, v)| (k.trim().to_string(), v.trim().to_string())) + .filter(|(k, _)| !k.is_empty()) + .ok_or_else(|| CliError::Usage(format!("--session expects KEY=VALUE, got '{kv}'"))) + }) + .collect() +} + +fn args_to_json(args: &SqlMapArgs) -> CliResult { + let mut body = serde_json::json!({ + "name": args.name, + "endpoint": args.endpoint, + "r2rml": read_mapping(args)?, + }); + let obj = body.as_object_mut().unwrap(); + if let Some(v) = mapping_media_type(args) { + obj.insert("r2rml_type".into(), v.into()); + } + for (key, value) in [ + ("branch", &args.branch), + ("dialect", &args.dialect), + ("protocol", &args.protocol), + ("catalog", &args.catalog), + ("schema", &args.schema), + ("user", &args.user), + ("auth_bearer", &args.auth_bearer), + ("oauth2_token_url", &args.oauth2_token_url), + ("oauth2_client_id", &args.oauth2_client_id), + ("oauth2_client_secret", &args.oauth2_client_secret), + ("oauth2_scope", &args.oauth2_scope), + ("oauth2_audience", &args.oauth2_audience), + ] { + if let Some(v) = value { + obj.insert(key.into(), v.clone().into()); + } + } + let session = session_pairs(args)?; + if !session.is_empty() { + obj.insert("session".into(), serde_json::to_value(session).unwrap()); + } + Ok(body) +} + +async fn run_sql_map_remote( + client: &crate::remote_client::RemoteLedgerClient, + args: &SqlMapArgs, +) -> CliResult<()> { + let body = args_to_json(args)?; + let result = client.sql_map(&body).await?; + let get = |k: &str| { + result + .get(k) + .and_then(serde_json::Value::as_str) + .unwrap_or("-") + .to_string() + }; + let n = |k: &str| { + result + .get(k) + .and_then(serde_json::Value::as_u64) + .unwrap_or(0) + }; + let flag = |k: &str| { + result + .get(k) + .and_then(serde_json::Value::as_bool) + .unwrap_or(false) + }; + let tables: Vec = result + .get("table_names") + .and_then(|v| v.as_array()) + .map(|a| { + a.iter() + .filter_map(|t| t.as_str().map(String::from)) + .collect() + }) + .unwrap_or_default(); + print_created( + &get("graph_source_id"), + &get("endpoint"), + &get("mapping_source"), + n("triples_map_count") as usize, + n("table_count") as usize, + &tables, + flag("connection_tested"), + flag("mapping_validated"), + ); + Ok(()) +} + +#[cfg(feature = "sql")] +async fn run_sql_map_local(args: SqlMapArgs, dirs: &FlureeDir) -> CliResult<()> { + use fluree_db_api::{SqlAuthConfig, SqlConfigValue, SqlDialect, WireProtocol}; + + let fluree = crate::context::build_fluree(dirs)?; + let mapping = read_mapping(&args)?; + let mut config = fluree_db_api::SqlCreateConfig::new(&args.name, &args.endpoint, mapping); + config.branch = args.branch.clone(); + config.mapping_media_type = mapping_media_type(&args); + config.catalog = args.catalog.clone(); + config.schema = args.schema.clone(); + config.user = args.user.clone(); + config.session = session_pairs(&args)?; + if let Some(d) = &args.dialect { + config.dialect = match d.to_lowercase().as_str() { + "trino" => SqlDialect::Trino, + "postgres" | "postgresql" => SqlDialect::Postgres, + "mysql" => SqlDialect::Mysql, + "sqlite" => SqlDialect::Sqlite, + other => { + return Err(CliError::Usage(format!( + "unknown --dialect '{other}' (trino, postgres, mysql, sqlite)" + ))) + } + }; + } + if let Some(p) = &args.protocol { + config.protocol = match p.to_lowercase().as_str() { + "trino" => WireProtocol::Trino, + "presto" => WireProtocol::Presto, + other => { + return Err(CliError::Usage(format!( + "unknown --protocol '{other}' (trino, presto)" + ))) + } + }; + } + if let (Some(url), Some(secret)) = (&args.oauth2_token_url, &args.oauth2_client_secret) { + config.auth = SqlAuthConfig::OAuth2ClientCredentials { + token_url: url.clone(), + client_id: SqlConfigValue::Literal(args.oauth2_client_id.clone().unwrap_or_default()), + client_secret: SqlConfigValue::Literal(secret.clone()), + scope: args.oauth2_scope.clone(), + audience: args.oauth2_audience.clone(), + }; + } else if let Some(token) = &args.auth_bearer { + config.auth = SqlAuthConfig::Bearer { + token: SqlConfigValue::Literal(token.clone()), + }; + } + + let result = fluree.create_sql_graph_source(config).await?; + print_created( + &result.graph_source_id, + &result.endpoint, + &result.mapping_source, + result.triples_map_count, + result.table_count, + &result.table_names, + result.connection_tested, + result.mapping_validated, + ); + Ok(()) +} + +#[cfg(not(feature = "sql"))] +async fn run_sql_map_local(_args: SqlMapArgs, _dirs: &FlureeDir) -> CliResult<()> { + Err(CliError::Usage( + "SQL graph source support not compiled. Rebuild with `--features sql`.".into(), + )) +} + +#[allow(clippy::too_many_arguments)] +fn print_created( + graph_source_id: &str, + endpoint: &str, + mapping_source: &str, + triples_map_count: usize, + table_count: usize, + table_names: &[String], + connection_tested: bool, + mapping_validated: bool, +) { + println!("Mapped SQL endpoint as graph source '{graph_source_id}'"); + println!(" Endpoint: {endpoint}"); + println!(" R2RML: {mapping_source}"); + println!(" TriplesMaps: {triples_map_count}"); + println!( + " Tables: {}", + super::iceberg::format_table_summary(table_count, table_names) + ); + println!( + " Connection: {}", + if connection_tested { + "verified" + } else { + "not tested (endpoint unreachable or credentials rejected)" + } + ); + println!( + " Mapping: {}", + if mapping_validated { + "validated" + } else { + "not validated (check mapping source)" + } + ); +} diff --git a/fluree-db-cli/src/lib.rs b/fluree-db-cli/src/lib.rs index 896b0fc0e9..ac65a305f6 100644 --- a/fluree-db-cli/src/lib.rs +++ b/fluree-db-cli/src/lib.rs @@ -776,6 +776,42 @@ pub async fn run(cli: Cli) -> error::CliResult<()> { commands::memory::run(action, &fluree_dir).await } + Commands::Sql { action } => { + let fluree_dir = config::require_fluree_dir(config_path)?; + match action { + cli::SqlAction::Map(args) => { + commands::sql::run_sql_map(*args, &fluree_dir, direct).await + } + cli::SqlAction::List { remote } => { + commands::iceberg::run_iceberg_list(&fluree_dir, remote.as_deref(), direct) + .await + } + cli::SqlAction::Info { name, remote } => { + commands::iceberg::run_iceberg_info( + &name, + &fluree_dir, + remote.as_deref(), + direct, + ) + .await + } + cli::SqlAction::Drop { + name, + force, + remote, + } => { + commands::iceberg::run_iceberg_drop( + &name, + force, + &fluree_dir, + remote.as_deref(), + direct, + ) + .await + } + } + } + Commands::Iceberg { action } => { let fluree_dir = config::require_fluree_dir(config_path)?; match action { diff --git a/fluree-db-cli/src/remote_client.rs b/fluree-db-cli/src/remote_client.rs index 58870315a0..2c125b8968 100644 --- a/fluree-db-cli/src/remote_client.rs +++ b/fluree-db-cli/src/remote_client.rs @@ -2763,6 +2763,23 @@ impl RemoteLedgerClient { // Iceberg graph source operations // ========================================================================= + /// Map a SQL endpoint as a graph source on the remote server. + /// + /// Calls `POST {base_url}/sql/map`. + pub async fn sql_map( + &self, + body: &serde_json::Value, + ) -> Result { + let url = self.op_url_root("sql/map"); + self.send_json( + reqwest::Method::POST, + &url, + "application/json", + Some(RequestBody::Json(body)), + ) + .await + } + /// Map an Iceberg table as a graph source on the remote server. /// /// Calls `POST {base_url}/iceberg/map`. diff --git a/fluree-db-r2rml/src/loader/extractor.rs b/fluree-db-r2rml/src/loader/extractor.rs index 866d536fa7..52b667be32 100644 --- a/fluree-db-r2rml/src/loader/extractor.rs +++ b/fluree-db-r2rml/src/loader/extractor.rs @@ -140,14 +140,18 @@ impl<'a> MappingExtractor<'a> { } } - // Check for rr:sqlQuery (not supported) - if self - .find_object_optional(&table_triples, R2RML::SQL_QUERY) - .is_some() - { - return Err(R2rmlError::Unsupported( - "rr:sqlQuery is not supported for Iceberg graph sources".to_string(), - )); + // rr:sqlQuery — scanned as a derived table by SQL graph sources; + // Iceberg-backed sources refuse the alias at registration. + if let Some(query) = self.find_object_optional(&table_triples, R2RML::SQL_QUERY) { + if let Some(sql) = self.term_to_string(&query) { + if sql.trim().is_empty() { + return Err(R2rmlError::InvalidValue { + property: "rr:sqlQuery".to_string(), + message: "query text is empty".to_string(), + }); + } + return Ok(LogicalTable::sql_query(sql)); + } } Err(R2rmlError::MissingProperty( diff --git a/fluree-db-r2rml/src/loader/mod.rs b/fluree-db-r2rml/src/loader/mod.rs index e0b1da6640..88a7c6d338 100644 --- a/fluree-db-r2rml/src/loader/mod.rs +++ b/fluree-db-r2rml/src/loader/mod.rs @@ -259,3 +259,78 @@ mod tests { .is_some()); } } + +#[cfg(all(test, feature = "turtle"))] +mod sql_query_tests { + use super::R2rmlLoader; + use crate::mapping::LogicalTable; + + const MAPPING: &str = r#" + @prefix rr: . + @prefix ex: . + + a rr:TriplesMap ; + rr:logicalTable [ rr:sqlQuery """SELECT id, total FROM sales.orders WHERE status = 'open'""" ] ; + rr:subjectMap [ rr:template "http://example.org/order/{id}" ; rr:class ex:Order ] ; + rr:predicateObjectMap [ rr:predicate ex:total ; rr:objectMap [ rr:column "total" ] ] . + + a rr:TriplesMap ; + rr:logicalTable [ rr:tableName "sales.customers" ] ; + rr:subjectMap [ rr:template "http://example.org/customer/{id}" ] ; + rr:predicateObjectMap [ rr:predicate ex:name ; rr:objectMap [ rr:column "name" ] ] . + "#; + + #[test] + fn sql_query_compiles_to_a_stable_alias_that_resolves_back_to_the_query() { + let compiled = R2rmlLoader::from_turtle(MAPPING) + .unwrap() + .compile() + .unwrap(); + assert!(compiled.has_sql_queries()); + + let orders = compiled + .get("http://example.org/m#Orders") + .expect("orders map"); + let alias = orders + .table_name() + .expect("alias stands in for the table name"); + assert!(LogicalTable::is_sql_query_alias(alias), "{alias}"); + assert_eq!( + compiled.sql_query_for_table(alias), + Some("SELECT id, total FROM sales.orders WHERE status = 'open'") + ); + assert_eq!(compiled.sql_query_for_table("sales.customers"), None); + + // Same query text → same alias, so caches keyed on the name are stable. + let again = R2rmlLoader::from_turtle(MAPPING) + .unwrap() + .compile() + .unwrap(); + let alias_again = again + .triples_maps + .values() + .find(|tm| tm.iri.ends_with("#Orders")) + .and_then(|tm| tm.table_name()) + .unwrap(); + assert_eq!(alias, alias_again); + + // Both maps are reachable by table name. + assert_eq!(compiled.find_maps_for_table(alias).len(), 1); + assert_eq!(compiled.find_maps_for_table("sales.customers").len(), 1); + } + + #[test] + fn empty_sql_query_is_rejected() { + let mapping = r#" + @prefix rr: . + a rr:TriplesMap ; + rr:logicalTable [ rr:sqlQuery " " ] ; + rr:subjectMap [ rr:template "http://example.org/{id}" ] . + "#; + let err = R2rmlLoader::from_turtle(mapping) + .unwrap() + .compile() + .unwrap_err(); + assert!(err.to_string().contains("rr:sqlQuery"), "{err}"); + } +} diff --git a/fluree-db-r2rml/src/mapping/compiled.rs b/fluree-db-r2rml/src/mapping/compiled.rs index f4051139c3..5606281045 100644 --- a/fluree-db-r2rml/src/mapping/compiled.rs +++ b/fluree-db-r2rml/src/mapping/compiled.rs @@ -171,6 +171,22 @@ impl CompiledR2rmlMapping { } /// Get all unique table names referenced by the mapping + /// The `rr:sqlQuery` text behind a query alias returned by + /// [`TriplesMap::table_name`], if `table_name` is one. + pub fn sql_query_for_table(&self, table_name: &str) -> Option<&str> { + self.triples_maps + .values() + .find(|tm| tm.table_name() == Some(table_name)) + .and_then(|tm| tm.sql_query()) + } + + /// Whether any map is `rr:sqlQuery`-backed. + pub fn has_sql_queries(&self) -> bool { + self.triples_maps + .values() + .any(|tm| tm.sql_query().is_some()) + } + pub fn table_names(&self) -> Vec<&str> { self.table_to_maps .keys() diff --git a/fluree-db-r2rml/src/mapping/triples_map.rs b/fluree-db-r2rml/src/mapping/triples_map.rs index 15a855f6f9..ad54c08ede 100644 --- a/fluree-db-r2rml/src/mapping/triples_map.rs +++ b/fluree-db-r2rml/src/mapping/triples_map.rs @@ -69,11 +69,18 @@ impl TriplesMap { self } - /// Get the table name if this is a table-based logical table + /// The logical table's name: the `rr:tableName`, or for an `rr:sqlQuery` + /// its deterministic alias — so every consumer keyed on table names (the + /// scan operator, provider caches, `find_maps_for_table`) treats a query + /// exactly like a table. A provider that can run SQL resolves the alias + /// back to the query text through [`Self::sql_query`]. pub fn table_name(&self) -> Option<&str> { - match &self.logical_table { - LogicalTable::TableName(name) => Some(name), - } + self.logical_table.name() + } + + /// The `rr:sqlQuery` text, when this map is query-backed. + pub fn sql_query(&self) -> Option<&str> { + self.logical_table.sql_query_text() } /// Get all columns referenced by this TriplesMap @@ -259,26 +266,70 @@ impl TriplesMap { /// Logical table source /// /// Defines where the tabular data comes from. -/// For Iceberg graph sources, only table names are supported (not SQL queries). +/// Iceberg graph sources accept only table names; SQL graph sources also +/// accept `rr:sqlQuery`, which is scanned as a derived table. #[derive(Debug, Clone, Serialize, Deserialize)] pub enum LogicalTable { /// `rr:tableName` - direct table reference /// /// Table names are normalized to dot notation: "namespace.table" TableName(String), - // Note: rr:sqlQuery is explicitly NOT supported for Iceberg graph sources + /// `rr:sqlQuery` - a SQL SELECT used as the logical table. `alias` is a + /// deterministic name derived from the query text, used wherever a table + /// name is expected. + SqlQuery { sql: String, alias: String }, } +/// Prefix of every `rr:sqlQuery` alias, so a provider without SQL support can +/// recognize and refuse one. +pub const SQL_QUERY_ALIAS_PREFIX: &str = "sqlQuery:"; + impl LogicalTable { /// Create a table name logical table pub fn table(name: impl Into) -> Self { LogicalTable::TableName(name.into()) } - /// Get the table name if this is a table-based logical table + /// Create a query-backed logical table. + pub fn sql_query(sql: impl Into) -> Self { + let sql = sql.into(); + let alias = Self::alias_for_query(&sql); + LogicalTable::SqlQuery { sql, alias } + } + + fn alias_for_query(sql: &str) -> String { + use std::hash::{Hash, Hasher}; + let mut h = std::collections::hash_map::DefaultHasher::new(); + sql.trim().hash(&mut h); + format!("{SQL_QUERY_ALIAS_PREFIX}{:016x}", h.finish()) + } + + /// Whether `name` is an `rr:sqlQuery` alias rather than a real table. + pub fn is_sql_query_alias(name: &str) -> bool { + name.starts_with(SQL_QUERY_ALIAS_PREFIX) + } + + /// The table name or query alias. + pub fn name(&self) -> Option<&str> { + match self { + LogicalTable::TableName(name) => Some(name), + LogicalTable::SqlQuery { alias, .. } => Some(alias), + } + } + + /// The query text for a query-backed logical table. + pub fn sql_query_text(&self) -> Option<&str> { + match self { + LogicalTable::TableName(_) => None, + LogicalTable::SqlQuery { sql, .. } => Some(sql), + } + } + + /// The `rr:tableName`, or `None` for a query-backed logical table. pub fn as_table_name(&self) -> Option<&str> { match self { LogicalTable::TableName(name) => Some(name), + LogicalTable::SqlQuery { .. } => None, } } diff --git a/fluree-db-server/Cargo.toml b/fluree-db-server/Cargo.toml index a57f52b8f3..6716d8aef8 100644 --- a/fluree-db-server/Cargo.toml +++ b/fluree-db-server/Cargo.toml @@ -22,7 +22,7 @@ name = "fluree_db_server" path = "src/lib.rs" [features] -default = ["native", "credential", "iceberg", "shacl", "bolt", "graphql"] +default = ["native", "credential", "iceberg", "sql", "shacl", "bolt", "graphql"] native = ["fluree-db-api/native"] # AWS S3 storage + DynamoDB nameservice (via connection JSON-LD config) aws = ["fluree-db-api/aws"] @@ -37,6 +37,8 @@ graphql = ["fluree-db-api/graphql"] oidc = ["fluree-db-credential/oidc", "dep:jsonwebtoken"] # Iceberg / R2RML graph source support iceberg = ["fluree-db-api/iceberg"] +# SQL graph sources (R2RML over a Trino-protocol endpoint) +sql = ["iceberg", "fluree-db-api/sql"] # Use mimalloc as the global allocator. Better multicore allocation throughput # for the allocation-heavy query/materialization paths (e.g. R2RML/Iceberg # scans). Opt-in: enable in release packaging after a soak. diff --git a/fluree-db-server/src/routes/mod.rs b/fluree-db-server/src/routes/mod.rs index b01b80d3fa..0fd9cda5b7 100644 --- a/fluree-db-server/src/routes/mod.rs +++ b/fluree-db-server/src/routes/mod.rs @@ -22,6 +22,8 @@ mod push; pub(crate) mod query; pub(crate) mod serving; mod show; +#[cfg(feature = "sql")] +mod sql; mod storage_proxy; mod stream_query; mod stubs; @@ -120,6 +122,9 @@ pub fn build_router(state: Arc) -> Router { .route("/iceberg/track", post(iceberg::iceberg_track)) .route("/iceberg/untrack", post(iceberg::iceberg_untrack)); + #[cfg(feature = "sql")] + let v1_admin_protected_writes = v1_admin_protected_writes.route("/sql/map", post(sql::sql_map)); + // Admin auth runs BEFORE leader-forward. Axum runs the // last-applied layer outermost, so `require_admin_token` // (applied after `apply_leader_forward`) is the outer layer and diff --git a/fluree-db-server/src/routes/sql.rs b/fluree-db-server/src/routes/sql.rs new file mode 100644 index 0000000000..98fd9714fe --- /dev/null +++ b/fluree-db-server/src/routes/sql.rs @@ -0,0 +1,192 @@ +//! SQL graph source endpoints: POST /v1/fluree/sql/map + +use crate::config::ServerRole; +use crate::error::{Result, ServerError}; +use crate::extract::FlureeHeaders; +use crate::state::AppState; +use crate::telemetry::{create_request_span, extract_request_id, extract_trace_id}; +use axum::extract::{Request, State}; +use axum::http::StatusCode; +use axum::response::{IntoResponse, Response}; +use axum::Json; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::sync::Arc; +use tracing::Instrument; + +use super::ledger::forward_write_request; + +/// Request body for `POST /v1/fluree/sql/map` +#[derive(Deserialize)] +pub struct SqlMapRequest { + /// Graph source name + pub name: String, + /// Statement endpoint base URL (`https://trino.example.com`, or a sidecar) + pub endpoint: String, + /// R2RML mapping content (Turtle by default) + pub r2rml: String, + /// R2RML mapping media type + pub r2rml_type: Option, + /// Branch name + pub branch: Option, + /// Rendering dialect: `trino` (default), `postgres`, `mysql`, `sqlite` + pub dialect: Option, + /// Header family: `trino` (default) or `presto` + pub protocol: Option, + /// Default catalog for unqualified table names + pub catalog: Option, + /// Default schema for unqualified table names + pub schema: Option, + /// `X-Trino-User` (defaults to `fluree`) + pub user: Option, + /// Static bearer token + pub auth_bearer: Option, + /// OAuth2 client-credentials token URL + pub oauth2_token_url: Option, + pub oauth2_client_id: Option, + pub oauth2_client_secret: Option, + pub oauth2_scope: Option, + pub oauth2_audience: Option, + /// Session properties (`X-Trino-Session`) + #[serde(default)] + pub session: BTreeMap, +} + +/// Response for `POST /v1/fluree/sql/map` +#[derive(Serialize)] +pub struct SqlMapResponse { + pub graph_source_id: String, + pub endpoint: String, + pub connection_tested: bool, + pub mapping_source: String, + pub triples_map_count: usize, + pub table_count: usize, + pub table_names: Vec, + pub mapping_validated: bool, +} + +/// Map a SQL endpoint as a graph source +/// +/// POST /v1/fluree/sql/map +pub async fn sql_map(State(state): State>, request: Request) -> Response { + if state.config.server_role == ServerRole::Peer { + return forward_write_request(&state, request).await; + } + sql_map_local(state, request).await.into_response() +} + +async fn sql_map_local(state: Arc, request: Request) -> Result { + let headers = FlureeHeaders::from_headers(request.headers())?; + let body_bytes = axum::body::to_bytes(request.into_body(), 50 * 1024 * 1024) + .await + .map_err(|e| ServerError::bad_request(format!("Failed to read body: {e}")))?; + let req: SqlMapRequest = serde_json::from_slice(&body_bytes) + .map_err(|e| ServerError::bad_request(format!("Invalid JSON: {e}")))?; + + let request_id = extract_request_id(&headers.raw, &state.telemetry_config); + let trace_id = extract_trace_id(&headers.raw); + let span = create_request_span( + "sql:map", + request_id.as_deref(), + trace_id.as_deref(), + Some(&req.name), + None, + None, + ); + async move { + tracing::info!(status = "start", name = %req.name, "sql map requested"); + + // The endpoint reaches an outbound HTTP client: refuse the + // link-local/metadata range before anything connects. (Loopback and + // private hosts are legitimate — a sidecar is the common deployment.) + fluree_db_api::validate_sql_endpoint(&req.endpoint) + .map_err(|e| ServerError::bad_request(e.to_string()))?; + if let Some(url) = &req.oauth2_token_url { + super::iceberg_ssrf::guard_connection_urls(None, Some(url), None)?; + } + + let config = build_sql_config(&req)?; + let result = state + .fluree + .create_sql_graph_source(config) + .await + .map_err(ServerError::Api)?; + + tracing::info!( + status = "success", + graph_source_id = %result.graph_source_id, + "sql graph source mapped" + ); + Ok(( + StatusCode::CREATED, + Json(SqlMapResponse { + graph_source_id: result.graph_source_id, + endpoint: result.endpoint, + connection_tested: result.connection_tested, + mapping_source: result.mapping_source, + triples_map_count: result.triples_map_count, + table_count: result.table_count, + table_names: result.table_names, + mapping_validated: result.mapping_validated, + }), + )) + } + .instrument(span) + .await +} + +fn build_sql_config(req: &SqlMapRequest) -> Result { + use fluree_db_api::{SqlAuthConfig, SqlDialect, WireProtocol}; + + let mut config = fluree_db_api::SqlCreateConfig::new(&req.name, &req.endpoint, &req.r2rml); + config.branch = req.branch.clone(); + config.mapping_media_type = req.r2rml_type.clone(); + config.catalog = req.catalog.clone(); + config.schema = req.schema.clone(); + config.user = req.user.clone(); + config.session = req.session.clone(); + + if let Some(d) = &req.dialect { + config.dialect = match d.to_lowercase().as_str() { + "trino" => SqlDialect::Trino, + "postgres" | "postgresql" => SqlDialect::Postgres, + "mysql" => SqlDialect::Mysql, + "sqlite" => SqlDialect::Sqlite, + other => { + return Err(ServerError::bad_request(format!( + "unknown dialect '{other}'. Use trino, postgres, mysql or sqlite." + ))) + } + }; + } + if let Some(p) = &req.protocol { + config.protocol = match p.to_lowercase().as_str() { + "trino" => WireProtocol::Trino, + "presto" => WireProtocol::Presto, + other => { + return Err(ServerError::bad_request(format!( + "unknown protocol '{other}'. Use trino or presto." + ))) + } + }; + } + + if let (Some(url), Some(secret)) = (&req.oauth2_token_url, &req.oauth2_client_secret) { + config.auth = SqlAuthConfig::OAuth2ClientCredentials { + token_url: url.clone(), + client_id: fluree_db_sql_config_value(req.oauth2_client_id.as_deref().unwrap_or("")), + client_secret: fluree_db_sql_config_value(secret), + scope: req.oauth2_scope.clone(), + audience: req.oauth2_audience.clone(), + }; + } else if let Some(token) = &req.auth_bearer { + config.auth = SqlAuthConfig::Bearer { + token: fluree_db_sql_config_value(token), + }; + } + Ok(config) +} + +fn fluree_db_sql_config_value(literal: &str) -> fluree_db_api::SqlConfigValue { + fluree_db_api::SqlConfigValue::Literal(literal.to_string()) +} From f392c646a60a7f2343cd99455cd54da355f5fff7 Mon Sep 17 00:00:00 2001 From: bplatz Date: Sat, 29 Aug 2026 16:20:10 -0400 Subject: [PATCH 04/13] docs: SQL graph sources (Trino-protocol endpoints and the bridge sidecar) --- docs/SUMMARY.md | 2 + docs/api/endpoints.md | 67 +++++++++ docs/cli/sql.md | 113 +++++++++++++++ docs/concepts/graph-sources.md | 16 +++ docs/graph-sources/README.md | 21 +++ docs/graph-sources/overview.md | 17 +++ docs/graph-sources/sql.md | 249 +++++++++++++++++++++++++++++++++ docs/reference/crate-map.md | 17 ++- docs/reference/vocabulary.md | 1 + 9 files changed, 502 insertions(+), 1 deletion(-) create mode 100644 docs/cli/sql.md create mode 100644 docs/graph-sources/sql.md diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 917cf09d4d..fc3caa6d84 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -50,6 +50,7 @@ - [mcp](cli/mcp.md) - [docs](cli/docs.md) - [iceberg](cli/iceberg.md) + - [sql](cli/sql.md) - [bm25](cli/bm25.md) - [materialize](cli/materialize.md) - [completions](cli/completions.md) @@ -174,6 +175,7 @@ - [Overview](graph-sources/overview.md) - [Iceberg / Parquet](graph-sources/iceberg.md) - [R2RML](graph-sources/r2rml.md) + - [SQL endpoints (Trino / bridge)](graph-sources/sql.md) - [BM25 graph source](graph-sources/bm25.md) - [Fluree for AI and agents](ai/README.md) diff --git a/docs/api/endpoints.md b/docs/api/endpoints.md index 92fdbaf481..691b386a10 100644 --- a/docs/api/endpoints.md +++ b/docs/api/endpoints.md @@ -3042,6 +3042,73 @@ By default the server does not sync on commit, so an index only advances when so See also the CLI equivalent: [fluree bm25 sync](../cli/bm25.md#fluree-bm25-sync). +### POST {api_base_url}/sql/map + +Map tables behind a SQL endpoint as an R2RML graph source. The endpoint speaks the Trino client protocol (Trino, Starburst, PrestoDB, or a `fluree-sql-bridge` sidecar). Admin-protected — requires the admin Bearer token when an admin token is configured. Available only when the server is built with the `sql` feature (on by default). See [SQL graph sources](../graph-sources/sql.md). + +**URL:** +``` +POST {api_base_url}/sql/map +``` + +**Request Body:** + +```json +{ + "name": "warehouse", + "endpoint": "https://trino.example.com:8443", + "r2rml": "@prefix rr: . ...", + "r2rml_type": "text/turtle", + "branch": "main", + "dialect": "trino", + "protocol": "trino", + "catalog": "hive", + "schema": "sales", + "user": "fluree", + "auth_bearer": "…", + "session": { "query_max_run_time": "5m" } +} +``` + +| Field | Type | Description | +|-------|------|-------------| +| `name` | string | Graph source name (required) | +| `endpoint` | string | Statement endpoint base URL (required); `/v1/statement` is appended. Loopback/private hosts are allowed; the link-local/metadata range is refused. | +| `r2rml` | string | Inline R2RML mapping (required). `rr:tableName` and `rr:sqlQuery` logical tables are both accepted. | +| `r2rml_type` | string | Media type of `r2rml` (`text/turtle`, `application/ld+json`) | +| `branch` | string | Branch name (default: `main`) | +| `dialect` | string | `trino` (default), `postgres`, `mysql`, `sqlite` — the engine behind a bridge | +| `protocol` | string | `trino` (default, `X-Trino-*` headers) or `presto` | +| `catalog`, `schema` | string | Defaults for unqualified table names | +| `user` | string | Protocol user header (default `fluree`) | +| `auth_bearer` | string | Static bearer token | +| `oauth2_token_url`, `oauth2_client_id`, `oauth2_client_secret`, `oauth2_scope`, `oauth2_audience` | string | OAuth2 client-credentials flow (refreshes); `oauth2_token_url` is guarded against internal hosts | +| `session` | object | Session properties sent as `X-Trino-Session` | + +**Response:** + +```json +{ + "graph_source_id": "warehouse:main", + "endpoint": "https://trino.example.com:8443", + "connection_tested": true, + "mapping_source": "bafy…", + "triples_map_count": 3, + "table_count": 2, + "table_names": ["sales.customers", "sales.orders"], + "mapping_validated": true +} +``` + +`connection_tested` reports whether `SELECT 1` succeeded against the endpoint; a failure does not block registration. + +**Status Codes:** +- `201 Created` — graph source created +- `400 Bad Request` — invalid body, unknown `dialect`/`protocol`, endpoint refused by the SSRF guard, or an invalid mapping +- `401 Unauthorized` — admin token required + +--- + ### POST {api_base_url}/iceberg/materialize Materialize a graph source into a native ledger (so BM25 / vector / reasoning can run over it). Reads incrementally from a per-`(source, target, table)` watermark persisted in a shared `fluree_materialize_state:main` ledger, or fully with `force_full`. `target` may be a template that fans out into one ledger per partition (see the field table). Admin-protected; `iceberg` feature only. See [Materialization](../graph-sources/iceberg.md#materialization-into-a-native-ledger). diff --git a/docs/cli/sql.md b/docs/cli/sql.md new file mode 100644 index 0000000000..377f0185d1 --- /dev/null +++ b/docs/cli/sql.md @@ -0,0 +1,113 @@ +# fluree sql + +Manage SQL graph sources — R2RML mappings over tables reached through a +Trino-protocol endpoint (Trino, Starburst, PrestoDB, or a `fluree-sql-bridge` +sidecar). See [SQL graph sources](../graph-sources/sql.md). + +## Subcommands + +| Subcommand | Description | +|------------|-------------| +| `map` | Map tables behind a SQL endpoint as a graph source | +| `list` | List mapped graph sources (SQL, Iceberg and R2RML) | +| `info` | Show details for a mapped graph source | +| `drop` | Drop a mapped graph source | + +`list`, `info` and `drop` are shared with [`fluree iceberg`](iceberg.md): both +commands operate on the same family of mapped sources. + +## fluree sql map + +### Usage + +```bash +fluree sql map --endpoint --r2rml [OPTIONS] +``` + +### Arguments + +| Argument | Description | +|----------|-------------| +| `` | Graph source name (e.g., "warehouse") | + +### Options + +**Endpoint:** + +| Option | Description | +|--------|-------------| +| `--endpoint ` | Statement endpoint base URL (required), e.g. `https://trino.example.com:8443` or `http://localhost:8080` for a sidecar | +| `--dialect ` | SQL rendering dialect: `trino` (default), `postgres`, `mysql`, `sqlite`. Use the engine behind a bridge. | +| `--protocol ` | Header family: `trino` (default) or `presto` | +| `--catalog ` | Default catalog for unqualified table names | +| `--schema ` | Default schema for unqualified table names | +| `--user ` | Protocol user (`X-Trino-User`); defaults to `fluree` | +| `--session KEY=VALUE` | Session property (repeatable), e.g. `--session query_max_run_time=5m` | + +**R2RML mapping:** + +| Option | Description | +|--------|-------------| +| `--r2rml ` | Mapping file (required). Each `rr:tableName` names a table reachable through the endpoint; `rr:sqlQuery` is also accepted. | +| `--r2rml-type ` | Mapping media type (e.g., `text/turtle`); inferred from extension if omitted | + +**Authentication:** + +| Option | Description | +|--------|-------------| +| `--auth-bearer ` | Static bearer token | +| `--oauth2-token-url ` | OAuth2 client-credentials token endpoint | +| `--oauth2-client-id ` | OAuth2 client ID | +| `--oauth2-client-secret ` | OAuth2 client secret | +| `--oauth2-scope ` | OAuth2 scope | +| `--oauth2-audience ` | OAuth2 audience | + +**General:** + +| Option | Description | +|--------|-------------| +| `--branch ` | Branch name (defaults to `main`) | +| `--remote ` | Execute against a remote server | + +### Examples + +```bash +# Trino with a bearer token; tables are qualified inside hive.sales +fluree sql map warehouse \ + --endpoint https://trino.example.com:8443 \ + --catalog hive --schema sales \ + --auth-bearer "$TRINO_TOKEN" \ + --r2rml mappings/orders.ttl + +# A bridge sidecar in front of Postgres +fluree sql map crm \ + --endpoint http://localhost:8080 \ + --dialect postgres --schema public \ + --r2rml mappings/crm.ttl +``` + +### Output + +``` +Mapped SQL endpoint as graph source 'warehouse:main' + Endpoint: https://trino.example.com:8443 + R2RML: bafy… + TriplesMaps: 3 + Tables: 2 (sales.orders, sales.customers) + Connection: verified + Mapping: validated +``` + +`Connection: not tested` means the `SELECT 1` probe failed; the source is +still registered and the first query reports the underlying error. + +## fluree sql list / info / drop + +```bash +fluree sql list +fluree sql info warehouse +fluree sql drop warehouse --force +``` + +Behave exactly as the [`fluree iceberg`](iceberg.md) equivalents; SQL sources +show the type `SQL`. diff --git a/docs/concepts/graph-sources.md b/docs/concepts/graph-sources.md index e015becde8..71c24e26d6 100644 --- a/docs/concepts/graph-sources.md +++ b/docs/concepts/graph-sources.md @@ -138,6 +138,22 @@ WHERE { See the [R2RML documentation](../graph-sources/r2rml.md) for details. +### SQL Endpoints + +**Differentiator**: The R2RML mapping runs over a live relational database or warehouse through a Trino-protocol HTTP endpoint — no copy, and no database driver inside Fluree. One Trino coordinator reaches Postgres, MySQL, SQL Server, Oracle, Snowflake, BigQuery and more; a small `fluree-sql-bridge` sidecar covers a single Postgres/MySQL/SQLite database without a JVM. + +**Use Cases:** +- A virtual graph over an operational database +- Federating a ledger with warehouse tables in one query +- Serverless deployments — every scan is a stateless HTTP request + +**Key Features:** +- Typed filter pushdown and exact `COUNT` per table; joins in the engine +- `rr:sqlQuery` logical tables +- Reads the current table state (no snapshots or time travel) + +See [SQL graph sources](../graph-sources/sql.md) for details. + ## Graph Source Lifecycle ### Creation diff --git a/docs/graph-sources/README.md b/docs/graph-sources/README.md index adc0fab1ee..f98c08f29e 100644 --- a/docs/graph-sources/README.md +++ b/docs/graph-sources/README.md @@ -31,6 +31,14 @@ Relational database mapping: - Join optimization - Supported databases (PostgreSQL, MySQL, etc.) +### [SQL endpoints](sql.md) + +Relational databases and warehouses through a Trino-protocol endpoint: +- Trino / Starburst / PrestoDB, or the `fluree-sql-bridge` sidecar +- R2RML mappings with `rr:tableName` and `rr:sqlQuery` +- Typed filter pushdown and exact `COUNT` +- No database drivers in the Fluree binary + ### [BM25 Graph Source](bm25.md) Full-text search as graph source: @@ -233,6 +241,19 @@ See [Iceberg / Parquet](iceberg.md). See [R2RML](r2rml.md). +### SQL Endpoints + +**Purpose:** Query relational databases and warehouses as RDF, live + +**Backend:** Any Trino-protocol endpoint — Trino/Starburst in front of Postgres, MySQL, SQL Server, Oracle, Snowflake, BigQuery, …, or the `fluree-sql-bridge` sidecar for a single database + +**Use Cases:** +- Virtual graph over an operational database, no copy +- One SPARQL query spanning a ledger and a warehouse table +- Lambda deployments (every scan is a stateless HTTP request) + +See [SQL endpoints](sql.md). + ## Architecture ### Graph Source Registry diff --git a/docs/graph-sources/overview.md b/docs/graph-sources/overview.md index 012487b413..81791f7e8e 100644 --- a/docs/graph-sources/overview.md +++ b/docs/graph-sources/overview.md @@ -159,6 +159,23 @@ See [Iceberg / Parquet](iceberg.md) for full configuration details and examples. } ``` +### 4. SQL Endpoints + +**Backend:** Tables behind a Trino-protocol HTTP endpoint (Trino / Starburst / PrestoDB, or the `fluree-sql-bridge` sidecar), via R2RML mapping + +**Purpose:** Virtual graph over a relational database or warehouse, read live + +SQL sources use the same [R2RML mapping](r2rml.md) as Iceberg sources and additionally accept `rr:sqlQuery`. The engine pushes one typed single-table `SELECT` per triples map and performs joins itself. No database driver is linked into Fluree; the endpoint holds the connections. + +See [SQL endpoints](sql.md) for configuration, pushdown rules and the bridge. + +**Query:** +```sparql +PREFIX ex: +SELECT ?name ?total FROM +WHERE { ?o ex:customer ?c ; ex:total ?total . ?c ex:name ?name . FILTER(?total > 100) } +``` + ## Creating Graph Sources ### Via Rust API diff --git a/docs/graph-sources/sql.md b/docs/graph-sources/sql.md new file mode 100644 index 0000000000..52fbe125c5 --- /dev/null +++ b/docs/graph-sources/sql.md @@ -0,0 +1,249 @@ +# SQL Graph Sources + +Query tables in a relational database or data warehouse as RDF, through an +[R2RML mapping](r2rml.md), without loading the data into a ledger. A SQL graph +source reaches its tables over HTTP through any engine that speaks the **Trino +client protocol** — so nothing in Fluree holds a database connection, no JDBC +or native driver is compiled into the binary, and the same source works from a +long-running server and from a Lambda. + +## Where the SQL runs + +Fluree does not talk to Postgres, MySQL, Snowflake or Oracle directly. It sends +one `POST /v1/statement` per table scan to an endpoint and pages through the +result. Anything that implements that protocol works: + +| Endpoint | When to use it | +|----------|----------------| +| **Trino / Starburst** | The general answer. One Trino coordinator fronts Postgres, MySQL, SQL Server, Oracle, Snowflake, BigQuery, Redshift, Iceberg, Delta and dozens more through its connectors, and its client protocol is plain HTTP + JSON. | +| **PrestoDB** | Same protocol with the older `X-Presto-*` headers (`"protocol": "presto"`). | +| **`fluree-sql-bridge`** | A small sidecar for a single Postgres, MySQL or SQLite database when running a JVM is not wanted. It speaks the same protocol, so Fluree treats it exactly like Trino. See [Running the bridge](#running-the-bridge). | + +Every page of a result is one stateless HTTP request carrying its own +credentials; dropping a result stream cancels the statement server-side. + +## Registering a source + +=== CLI + +```bash +fluree sql map warehouse \ + --endpoint https://trino.example.com:8443 \ + --catalog hive --schema sales \ + --auth-bearer "$TRINO_TOKEN" \ + --r2rml mappings/orders.ttl +``` + +=== HTTP + +```bash +curl -X POST http://localhost:8090/v1/fluree/sql/map \ + -H "Authorization: Bearer $ADMIN_TOKEN" \ + -H "Content-Type: application/json" \ + -d @- <<'JSON' +{ + "name": "warehouse", + "endpoint": "https://trino.example.com:8443", + "catalog": "hive", + "schema": "sales", + "auth_bearer": "…", + "r2rml": "@prefix rr: . …" +} +JSON +``` + +=== Rust + +```rust +use fluree_db_api::{FlureeBuilder, SqlCreateConfig}; + +let fluree = FlureeBuilder::memory().build_memory(); +let mut config = SqlCreateConfig::new("warehouse", "https://trino.example.com:8443", MAPPING_TTL); +config.catalog = Some("hive".into()); +config.schema = Some("sales".into()); +fluree.create_sql_graph_source(config).await?; +``` + +Registration compiles the mapping, stores it in content-addressed storage, and +probes the endpoint with `SELECT 1`. A failed probe is reported +(`connection_tested: false`) but does not block registration — credentials can +be fixed later; the first query surfaces the real error. + +### Configuration + +| Field | Default | Meaning | +|-------|---------|---------| +| `endpoint` | — | Base URL; `/v1/statement` is appended | +| `dialect` | `trino` | How identifiers and literals are rendered: `trino`, `postgres`, `mysql`, `sqlite`. Use the engine *behind* a bridge. | +| `protocol` | `trino` | Header family: `trino` (`X-Trino-*`) or `presto` (`X-Presto-*`) | +| `catalog`, `schema` | — | Defaults for unqualified `rr:tableName`s | +| `user` | `fluree` | The protocol's user header; required even with a bearer token | +| `auth` | none | `bearer` (static token) or `oauth2_client_credentials`; values accept the same `env_var` / `secret_ref` indirection as Iceberg catalog auth | +| `session` | `{}` | Session properties, e.g. `{"query_max_run_time": "5m"}` | +| `request_timeout_secs` | `120` | Per page fetch | + +Table names in the mapping are dotted and quoted part by part: +`rr:tableName "sales.orders"` becomes `"sales"."orders"`; with `catalog` +set, an unqualified name resolves inside it. + +### `rr:sqlQuery` + +Unlike Iceberg sources, a SQL source accepts the R2RML `rr:sqlQuery` logical +table. The query is scanned as a derived table, with Fluree's projection and +pushed filters applied on top of it: + +```turtle +<#OpenOrders> a rr:TriplesMap ; + rr:logicalTable [ rr:sqlQuery "SELECT id, total FROM sales.orders WHERE status = 'open'" ] ; + rr:subjectMap [ rr:template "http://example.org/order/{id}" ; rr:class ex:Order ] ; + rr:predicateObjectMap [ rr:predicate ex:total ; rr:objectMap [ rr:column "total" ] ] . +``` + +```sql +-- what the engine sends for ?o ex:total ?total +SELECT "id", "total" FROM (SELECT id, total FROM sales.orders WHERE status = 'open') AS "__fluree_q" +``` + +The query text is trusted as written — a mapping author already has +root-equivalent read access to the source, exactly as with `rr:tableName`. + +## Querying + +A SQL source is queried like any other mapped source — as a `from` target, in +`FROM <…>`, or inside `GRAPH`: + +```sparql +PREFIX ex: +SELECT ?name ?total +FROM +WHERE { + ?o a ex:Order ; ex:customer ?c ; ex:total ?total . + ?c ex:name ?name . + FILTER(?total > 100) +} +``` + +### What is pushed to SQL + +The query engine asks the source for **one table at a time** — a projection, +conjunctive filters, and nothing else — and does joins, `OPTIONAL`, `UNION`, +property paths and aggregation itself over the returned rows. So each triples +map touched by a query becomes one statement of the shape: + +```sql +SELECT "id", "customer_id", "total" FROM "sales"."orders" WHERE "total" > 1E2 +``` + +Pushed as `WHERE`: + +- `FILTER` comparisons and `IN` / single-variable `VALUES` on a mapped column +- constant objects (`?o ex:status "open"`) +- a bound subject (` ex:total ?t`), reversed + through the subject template to the key column + +Every predicate is rendered **against the column's type**, learned from a +cached `SELECT * FROM … LIMIT 0` probe. A literal that cannot be compared +safely with the column — a string against a `bigint`, a naive timestamp +against a `timestamp with time zone`, a NaN — is simply not pushed. The +in-engine `FILTER` remains the authority in every case, so a declined push +costs I/O, never correctness. + +`COUNT` over a single triples map is answered by an exact +`SELECT COUNT(*) … WHERE IS NOT NULL` — exact where the Iceberg +source can only use manifest statistics. + +**Not pushed:** `ORDER BY … LIMIT`. A NULL in a key or required column would +consume `LIMIT` slots for rows the mapping drops, so the engine's own sort +runs over the full scan. Joins between triples maps on the same source are +also performed in the engine (a whole-query SQL rewrite in the Ontop style is +a possible later optimization, not a v1 requirement). + +### Types + +Trino's column types map onto Fluree's tabular types; the R2RML datatype +rules then apply as for any source. + +| SQL / Trino type | Fluree column | RDF datatype (default) | +|------------------|---------------|------------------------| +| `boolean` | Boolean | `xsd:boolean` | +| `tinyint`, `smallint`, `integer` | Int32 | `xsd:integer` | +| `bigint` | Int64 | `xsd:integer` | +| `real` | Float32 | `xsd:float` | +| `double` | Float64 | `xsd:double` | +| `decimal(p,s)` | Decimal | `xsd:decimal` (exact) | +| `varchar`, `char`, `json`, `uuid`, … | String | `xsd:string` | +| `varbinary` | Bytes | `xsd:base64Binary` | +| `date` | Date | `xsd:date` | +| `timestamp(p)` | Timestamp | `xsd:dateTime` | +| `timestamp(p) with time zone` | TimestampTz | `xsd:dateTime` (UTC) | +| `array`, `map`, `row` | String (Trino's JSON rendering) | `xsd:string` | + +Zoned timestamps are selected `AT TIME ZONE 'UTC'` on the Trino dialect, so a +value stored in a named region never has to be decoded client-side. Fractional +seconds beyond microseconds are truncated. + +## Freshness and materialization + +A SQL source has no snapshot: every query reads the tables as they are at that +moment. Consequently + +- `as-of` time travel is not available on a SQL source; +- a [materialized twin](iceberg.md#materialization) built from a SQL source is + stamped with `sql:///@
@