feat(stack-encrypt, eql): json_tree tokens derivable from the path, a self tag, client-side wildcards

.cargo/mutants.toml

@@ -113,11 +113,6 @@ exclude_re = [
   'stack-encrypt/src/json/mod\.rs:\d+:\d+: replace \| with \^ in unhex$',
   # `JsonPath::root` is `Self::default()`.
   'stack-encrypt/src/json/path\.rs:\d+:\d+: replace JsonPath::root -> Self with Default::default\(\)$',
-  # The sealed root's `None` token: with that arm gone, the root falls to
-  # the arm that wraps its parent's token, and the root has no parent, so
-  # it is `None` there too. `the_sealed_root_has_no_token_and_its_fields_are_heads`
-  # pins the value; the arm stays so the match says it.
-  'stack-encrypt/src/json/derive\.rs:\d+:\d+: delete match arm \(0, true\) in Walker<\x27_>::visit$',
   # A document always has a root, so its terms are never empty: `false` is
   # the value `is_empty` returns. The `true` replacement is still caught.
   'stack-encrypt/src/json/derive\.rs:\d+:\d+: replace JsonTerms::is_empty -> bool with false$',

.changeset/eql-json-tree-derivable-tokens.md

@@ -0,0 +1,9 @@
+---
+"@cipherstash/eql": minor
+---
+
+EQL v4 `json_tree` (unreleased): a subtree read sends the token of exactly the node it reads, and returns that node. Every token is now derivable from its node's path alone: an object member's is `H("t" ‖ t(parent) ‖ s)`, with no ordinal, and an array element's keeps its index. Every stored entry gains a self tag `x`, the first eight bytes of `H("x" ‖ iv ‖ t)`, by which the server finds the node a token names; the entry validator requires it, so a document written by the earlier build of this unreleased family is refused. A read of `$.profile.address.city` used to send `$.profile`'s token, a capability over that whole field in every row, and walk to the leaf through gates; it now sends the leaf's token and nothing that covers any node above it.
+
+**SQL signature changes.** `eql_v4.json_tree_subtree(doc, tokens text[], budget integer DEFAULT 64)` replaces `json_tree_subtree(doc, token, budget)` and `json_tree_subtree_at(doc, token, gates, budget)`, which is removed: for each token in order it returns the entry whose `x` matches at `d = 0`, then its descendants with `d` their depth below it, so `{}`, `[]`, a scalar and a missing node are four answers, as `jsonb` gives them (#1356). The budget counts rows across all the tokens. `json_tree_children(doc, token)` and `json_tree_child_count(doc, token)` lose their `gates` argument and take the token of any node; `json_tree_child_count` of a token that names no node is NULL, as `jsonb_array_length` of a missing path is (it was 0), and so is the sealed root's, whose top-level fields chain from nothing. `eql_v4_internal.json_tree_child_token` is replaced by `json_tree_member_token`, `json_tree_element_token`, `json_tree_self_tag`, `json_tree_hit_token` and `json_tree_index`, which maps every entry's `x`, and every chained entry's `p`, to its position in `sv`; `json_tree_by_parent` is removed. The chain primitives carry the `eql-inline-critical` marker, so the post-install step leaves their `search_path` unpinned and the planner inlines them: pinned, every probe of a walk was a full SQL-function call, which was most of a read's cost.
+
+A wildcard (`$.items[*].price`) is expanded by the client into one token per position below a bound, 64 by default, plus a probe at the bound; a row where the probe finds an element fails the read with the bound rather than return part of the list, walked or fallen back to the sealed root. A read of `$` is legal under the sealed root and returns the root entry, which opens to the whole document. Stored documents grow by about a tenth in characters (+9.8% on a nested document, +9.4% on a flat one). `JsonHeader::FORMAT_VERSION` and the document's `v` are unchanged. In `eql-bindings`, `JsonTreeEntry` gains `x` and `JsonTreeSubtree` opens to `Option<Value>`; in Stack Encrypt, `JsonSubtreeKey` carries the token list (`tokens()`, `probes()`, `bound()`), `KeysetCipher::json_subtree_key_with_bound` sets the bound, `Error::JsonWildcardTruncated` and `TermError::JsonWildcardTooWide` are new, and `TermError::JsonSealedRoot` is removed. What the database learns is in `docs/reference/json-tree-leakage.md`; the design and the decisions behind it are in `docs/plans/2026-10-17-json-tree-index.md`, "Design change (2026-10-24)".

biome.json

@@ -14,6 +14,7 @@
       "!packages/stack-encrypt/tests/fixtures/query_operators.json",
       "!packages/stack-encrypt/tests/fixtures/value_kinds.json",
       "!packages/stack-encrypt/tests/fixtures/stored_rows_v1.json",
+      "!packages/stack-encrypt/tests/fixtures/json_tree_vectors.json",
       "!packages/eql/tests/encryption/fixtures/ste_vec_selectors.json",
       "!packages/eql/tests/encryption/fixtures/v4_rows.json",
       "!packages/eql/tests/encryption/fixtures/stored_v4_rows_v1.json",

packages/eql/crates/eql-bindings/CHANGELOG.md

@@ -13,17 +13,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   feature). `JsonTreeDocument` is `public.eql_v4_json_tree`, `{v, i, h, sv,
   q}`: the document's entries, one per JSON node, and `q`, its containment
   projection, which the server compares with a needle by native `jsonb @>`.
-  `JsonTreeEntry` is one entry as `->` and the walks return it,
+  `JsonTreeEntry` is one entry as `->` and the subtree reads return it,
+  `{s, c, x, p?, op?}` with the extractors' grafted fields,
   `JsonTreeQuery` the containment needle and `JsonTreeLeafQuery` the
-  ordering operand at a path. `JsonTreeSubtree` collects a walk's rows; past
-  the walk's budget the server returns the sealed root instead, which
-  opens at the path the walk's key was derived from: the rows are built
+  ordering operand at a path. `JsonTreeSubtree` collects a read's rows
+  (`eql_v4.json_tree_subtree(doc, key.tokens(), budget)`) and opens to the
+  value at the path as a `Value`, to `Option<Value>` (`None` where the row
+  has no node there) or to one value per concrete path as a `Vec<Value>`;
+  past the read's budget the server returns the sealed root instead, which
+  opens to the same answer under the read's key: the rows are built
   with `JsonTreeSubtree::from_key(&key, rows)`, and there is no
   constructor without the key.
   An entry `->` returned opens through the selector that asked for it,
   `JsonTreeExtracted::from_selector(&selector, entry)`, which refuses an
   entry carrying another selector (`stack_encrypt::Error::JsonWrongEntry`),
-  and a walk's rows through the key that keyed it, which checks each row
+  and a read's rows through the key that keyed it, which checks each row
   against the path the same way; `JsonTreeEntry` does not open on its own.
   The document is an `EncryptFrom<Value>` and `DecryptInto<Value>` target,
   the extracted entry and the subtree open through `DecryptInto`, and the needle and
@@ -33,7 +37,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   seals with `keyset.run(JsonTree::new(options).encryption(), ..)`), such
   as the paths that carry an ordering term. The wire fields are typed apart
   from the v3 hex terms, and each names its text encoding:
-  `JsonTreeSelector` (`s`, and the pairs of `q`), `JsonTreeTag` (`p`),
+  `JsonTreeSelector` (`s`, and the pairs of `q`), `JsonTreeTag` (`x` and `p`),
   `JsonTreeOrderingTerm` (`op`) and `JsonTreeKeyHeader` (`h`) hold an
   `encoding::Base64Text`, padded standard base64, and `JsonTreeCiphertext`
   (`c`) an `encoding::Base85Text`, RFC 1924 base85. Each encoding newtype

packages/eql/crates/eql-bindings/src/v4/json_tree.rs

@@ -48,10 +48,13 @@ pub struct JsonTreeSelector(pub Base64Text);
 #[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, TS, JsonSchema)]
 pub struct JsonTreeCiphertext(pub Base85Text);
 
-/// An eight-byte parent tag `p`, in [`Base64Text`]: the first eight bytes
-/// of the parent-tag hash, per document, salted by the document's `iv`, so
-/// unlinkable across rows at rest. `eql_v4.json_tree_subtree_at` derives
-/// the same text with `encode(substr(…, 1, 8), 'base64')` to find a child.
+/// An eight-byte chain tag in [`Base64Text`]: an entry's self tag `x`, the
+/// first eight bytes of `H("x" ‖ iv ‖ t)` for the entry's own token `t`, or
+/// its parent tag `p`, the first eight bytes of
+/// `H("p" ‖ iv ‖ t(parent) ‖ LE32(o))`. Salted by the document's `iv`, so
+/// unlinkable across rows at rest. `eql_v4.json_tree_subtree` derives the
+/// same text with `encode(substr(…, 1, 8), 'base64')` to find the node a
+/// token names (`x`) and its children (`p`).
 #[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, TS, JsonSchema)]
 pub struct JsonTreeTag(pub Base64Text);
 
@@ -91,21 +94,23 @@ pub struct JsonTreeDocument {
 }
 
 /// `public.eql_v4_json_tree_entry`: one node. `s` is the path selector,
-/// `c` the node's own ciphertext; `p` the parent tag below the chain's
-/// heads, which the walk probes by ordinal; `op` the leaf ordering term.
+/// `c` the node's own ciphertext, `x` the self tag by which a read finds
+/// the node its token names; `p` the parent tag below the chain's heads,
+/// which a read probes by ordinal; `op` the leaf ordering term.
 /// `h`, `j`, `d`, `v` and `i` are what the extractors graft onto an entry
 /// they return, so that it can be opened.
 ///
 /// An entry a query returned is the database's word and does not open on
 /// its own: open the entry `->` returned through the selector that asked
-/// for it ([`JsonTreeExtracted::from_selector`]), and a walk's rows through
+/// for it ([`JsonTreeExtracted::from_selector`]), and a read's rows through
 /// the key that keyed it ([`JsonTreeSubtree::from_key`]). Each checks that
 /// the rows are the ones it asked for.
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)]
 #[serde(deny_unknown_fields)]
 pub struct JsonTreeEntry {
     pub s: JsonTreeSelector,
     pub c: JsonTreeCiphertext,
+    pub x: JsonTreeTag,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     #[ts(optional)]
     pub p: Option<JsonTreeTag>,
@@ -207,26 +212,31 @@ impl JsonTreeExtracted {
     }
 }
 
-/// The rows a subtree walk returned (`eql_v4.json_tree_subtree` or
-/// `json_tree_subtree_at`), collected with the key that keyed the walk:
-/// opens to the value under the walked path, or to the list of complete
-/// subtrees the rows hold.
+/// The rows a subtree read returned,
+/// `eql_v4.json_tree_subtree(doc, key.tokens(), budget)`, collected with the
+/// key that keyed the read. Every read returns each node a token names at
+/// depth `0` with everything beneath it, so the rows open to the value at
+/// the path, `{}`, `[]` and scalars included; to `None` as an
+/// `Option<Value>` when the row has no node there; or to a list, one value
+/// per concrete path a wildcard reaches. A wildcard read of an array longer
+/// than the key's bound fails with
+/// [`Error::JsonWildcardTruncated`](stack_encrypt::Error::JsonWildcardTruncated)
+/// rather than open to part of the answer.
 ///
-/// Past its budget a walk returns the sealed root alone, at depth `0`;
-/// those rows open to the whole document, and the key's path selects the
-/// same value(s) the walked rows open to. The key also says which walk the
-/// rows came from (no gates: `json_tree_subtree`, the rows below the path's
-/// node; gates: `json_tree_subtree_at`, the nodes the path reaches), so the
-/// only constructor, [`from_key`](Self::from_key), takes it, and rows
-/// cannot be opened without it.
+/// Past its budget a read returns the sealed root alone, at depth `0`;
+/// those rows open to the whole document, and the key's concrete paths and
+/// probes are evaluated over it to the same answers. The only constructor,
+/// [`from_key`](Self::from_key), takes the key, which checks every row
+/// against its path when the rows are opened, so rows cannot be opened
+/// without it.
 #[derive(Clone, Debug, PartialEq)]
 pub struct JsonTreeSubtree {
     rows: Vec<JsonTreeEntry>,
     key: JsonSubtreeKey,
 }
 
 impl JsonTreeSubtree {
-    /// The rows the walk `key` keyed returned, as the server sent them.
+    /// The rows the read `key` keyed returned, as the server sent them.
     pub fn from_key(key: &JsonSubtreeKey, rows: Vec<JsonTreeEntry>) -> Self {
         Self {
             rows,
@@ -239,7 +249,7 @@ impl JsonTreeSubtree {
         &self.rows
     }
 
-    /// The path the walk was keyed from.
+    /// The path the read was keyed from.
     pub fn path(&self) -> &JsonPath {
         self.key.path()
     }
@@ -455,8 +465,8 @@ impl DecryptInto<Value> for JsonTreeExtracted {
     }
 }
 
-/// The rows of `json_tree_subtree(doc, token)` open to the value under the
-/// walked node.
+/// The rows of a read open to the value at its path; no node there is an
+/// error, which `Option<Value>` reads as `None`.
 impl DecryptInto<Value> for JsonTreeSubtree {
     type Context = NonEmpty<Identifier>;
     fn decryption<K: 'static>(self, context: Self::Context) -> Decryption<Value, K> {
@@ -467,8 +477,20 @@ impl DecryptInto<Value> for JsonTreeSubtree {
     }
 }
 
-/// The rows of a walk open to the complete subtrees they hold, in order:
-/// one per node the gates of `json_tree_subtree_at` reached.
+/// The rows of a read open to the value at its path, or `None` where the
+/// row has no node there: the four answers `jsonb #>` gives.
+impl DecryptInto<Option<Value>> for JsonTreeSubtree {
+    type Context = NonEmpty<Identifier>;
+    fn decryption<K: 'static>(self, context: Self::Context) -> Decryption<Option<Value>, K> {
+        match self.subtree() {
+            Ok(subtree) => subtree.decryption(AeadContext::from(context)),
+            Err(error) => Decryption::failed(error),
+        }
+    }
+}
+
+/// The rows of a read open to a list: one value per concrete path the key
+/// named that the row has, in order.
 impl DecryptInto<Vec<Value>> for JsonTreeSubtree {
     type Context = NonEmpty<Identifier>;
     fn decryption<K: 'static>(self, context: Self::Context) -> Decryption<Vec<Value>, K> {
@@ -499,6 +521,11 @@ mod tests {
         Base64Text::encode([byte; 16]).as_str().to_owned()
     }
 
+    /// Eight bytes of `byte` as a self or parent tag's wire text.
+    fn b64_8(byte: u8) -> String {
+        Base64Text::encode([byte; 8]).as_str().to_owned()
+    }
+
     /// A ciphertext's wire text.
     fn b85(byte: u8) -> String {
         Base85Text::encode([byte; 28]).as_str().to_owned()
@@ -542,7 +569,7 @@ mod tests {
     fn a_document_carries_its_projection() {
         let document = serde_json::json!({
             "v": 4, "i": {"t": "users", "c": "profile"}, "h": "AA==",
-            "sv": [{"s": b64(0xaa), "c": b85(0)}],
+            "sv": [{"s": b64(0xaa), "c": b85(0), "x": b64_8(0xcc)}],
             "q": {"sv": [{"s": b64(0xaa), "vs": b64(0xbb)}], "el": []},
         });
         let stored: JsonTreeDocument = serde_json::from_value(document.clone()).unwrap();
@@ -559,9 +586,11 @@ mod tests {
     #[test]
     fn an_entry_with_a_field_in_the_wrong_encoding_is_refused() {
         for entry in [
-            serde_json::json!({"s": "aa", "c": b85(0)}),
-            serde_json::json!({"s": b64(0), "c": "z"}),
-            serde_json::json!({"s": b64(0), "c": b85(0), "p": "not base64"}),
+            serde_json::json!({"s": "aa", "c": b85(0), "x": b64_8(0)}),
+            serde_json::json!({"s": b64(0), "c": "z", "x": b64_8(0)}),
+            serde_json::json!({"s": b64(0), "c": b85(0), "x": "not base64"}),
+            serde_json::json!({"s": b64(0), "c": b85(0), "x": b64_8(0), "p": "not base64"}),
+            serde_json::json!({"s": b64(0), "c": b85(0)}),
         ] {
             assert!(
                 serde_json::from_value::<JsonTreeEntry>(entry.clone()).is_err(),
@@ -573,7 +602,7 @@ mod tests {
     #[test]
     fn an_entry_without_the_grafted_fields_does_not_open() {
         let entry: JsonTreeEntry = serde_json::from_value(serde_json::json!({
-            "s": b64(0), "c": b85(0)
+            "s": b64(0), "c": b85(0), "x": b64_8(0)
         }))
         .unwrap();
         assert!(entry.node().is_err());