From 1f13b77a29e6446de84a6b845dfd1cc0cfdb97e4 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 00:58:13 +1100 Subject: [PATCH 01/26] feat(stack-encrypt): answer a document field's JSON operators and open its reads from a data plan A data plan could store an EncryptedJson document but not query or read one: the operators were reachable only from typed Rust, and a read's rows opened only against a JsonSubtreeKey or JsonSelector a binding cannot hold. OpSpec gains the document operators (contains, contained_by, eq_at, the orderings at a path, between_at, selector, subtree, children, child_count), each taking an object of named inputs with JSON values as JSON text. query_term derives the typed chain's operands bit for bit and tags them: the term's wire form gains `encoding`, and Operand gains Text and TextList, so a selector or a token list is named, not anonymous bytes; an operator entry gains `input`. A position is refused where the operand is a selector, which reads it as the wildcard, and a wildcard where the operator names one node. json_open opens a selector or subtree read's rows against the request that prepared it: the engine derives the selector or the key again from the request, the field's resolved options and the keyset, runs the index's own row checks on each database row's group, and confines opening to that keyset. Nothing in the rows says which path they are for. The guest exports it as se_json_open, and maps a wrong entry and a truncated wildcard to STATUS_ENCODING. The Go runtime reads the new keys (QueryTerm.Encoding/Text, record.Operator.Input) and adds Cipher.JSONOpen. tests/json_tree_data_plan.rs holds every operand to the typed chain's and opens reads of `->` and of a wildcard subtree across documents. Refs #1347, #1350, #1352 Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- .changeset/eql-encrypted-json-operations.md | 5 + languages/golang/CHANGELOG.md | 8 + languages/golang/encrypt/checker.go | 19 +- languages/golang/encrypt/guest.go | 2 + languages/golang/encrypt/guest/src/abi.rs | 43 ++ languages/golang/encrypt/guest/src/ops.rs | 63 +- languages/golang/encrypt/guest/src/status.rs | 7 + languages/golang/encrypt/guest/src/targets.rs | 7 +- .../golang/encrypt/guest/tests/native_ops.rs | 34 +- languages/golang/encrypt/json.go | 93 +++ languages/golang/encrypt/json_tree_test.go | 122 ++++ languages/golang/encrypt/query.go | 57 +- languages/golang/encrypt/query_test.go | 31 +- languages/golang/encrypt/targets_test.go | 27 +- languages/golang/internal/record/record.go | 25 +- packages/eql/crates/eql-bindings/CHANGELOG.md | 6 + .../eql-bindings/src/encryption/targets.rs | 10 + .../encryption/tests/json_tree_data_plan.rs | 618 +++++++++++++++++- .../tests/encryption/tests/typed_queries.rs | 8 +- packages/stack-encrypt/CHANGELOG.md | 29 + .../stack-encrypt/src/dynamic/json_query.rs | 525 +++++++++++++++ packages/stack-encrypt/src/dynamic/mod.rs | 24 +- packages/stack-encrypt/src/dynamic/query.rs | 554 ++++++++++++++-- packages/stack-encrypt/src/dynamic/target.rs | 16 + packages/stack-encrypt/src/json/derive.rs | 34 +- packages/stack-encrypt/src/json/query.rs | 2 +- .../stack-encrypt/tests/query_operators.rs | 2 + .../tests/ui/op_at_path_not_a_leaf.stderr | 6 +- .../tests/ui/op_json_not_answered.stderr | 6 +- packages/stack-encrypt/tests/value_kinds.rs | 2 +- 30 files changed, 2256 insertions(+), 129 deletions(-) create mode 100644 .changeset/eql-encrypted-json-operations.md create mode 100644 languages/golang/encrypt/json.go create mode 100644 packages/stack-encrypt/src/dynamic/json_query.rs diff --git a/.changeset/eql-encrypted-json-operations.md b/.changeset/eql-encrypted-json-operations.md new file mode 100644 index 000000000..3edfbb491 --- /dev/null +++ b/.changeset/eql-encrypted-json-operations.md @@ -0,0 +1,5 @@ +--- +"@cipherstash/eql": minor +--- + +**`EncryptedJson` answers its JSON operators by name.** `eql-bindings`' target table row now tells Stack Encrypt the domain an ordering at a path binds as (`Target::leaf_query_sql_domain`, `eql_v4.query_json_tree_leaf`), so a data plan's `lt_at`, `lte_at`, `gt_at`, `gte_at` and `between_at` bind their operand as that domain, beside `contains`, `contained_by` and `eq_at`, which bind the needle as `eql_v4.query_json_tree`. No SQL changes. diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index c390530a7..aec2688b0 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -12,6 +12,14 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. ### Added +- **Query and read an encrypted JSON document, at the record level** + (#1350, #1352). `Cipher.QueryTerm` derives a document field's operands: + `QueryTerm.Encoding` says what the term holds, and `QueryTerm.Text` + carries a selector, a node's token or a read's tokens. + `Cipher.JSONOpen` opens the rows a selector or subtree read returned + against the request that prepared it. `record.Operator.Input` names an + operator's input. Generated methods follow with `stashgen`. + - **The EQL types for an encrypted JSON document** (#1348): `eql.EncryptedJSON` (`public.eql_v4_json_tree`), its containment needle `eql.EncryptedJSONQuery`, its ordering operand diff --git a/languages/golang/encrypt/checker.go b/languages/golang/encrypt/checker.go index 3ea4c81eb..d79f22fa8 100644 --- a/languages/golang/encrypt/checker.go +++ b/languages/golang/encrypt/checker.go @@ -218,7 +218,7 @@ func parseTarget(entry vcvalue.Object) (record.Target, error) { // parseOperators reads a list of operators, an se_targets entry's or // se_operators's: a list of objects, each with exactly the keys operator, -// fidelity, arity and index, every value one the engine writes. +// fidelity, arity, index and input, every value one the engine writes. func parseOperators(value any) ([]record.Operator, error) { list, ok := value.([]any) if !ok { @@ -256,6 +256,13 @@ func parseOperators(value any) ([]record.Operator, error) { } case "index": op.Index, err = parseOperatorIndex(s) + case "input": + switch s { + case "value", "pair", "json", "path_json", "path_pair", "path", "path_bound": + op.Input = s + default: + return nil, fmt.Errorf("operator %d: unknown input %q", i, s) + } default: return nil, fmt.Errorf("operator %d: unknown key %q", i, f.Key) } @@ -263,7 +270,7 @@ func parseOperators(value any) ([]record.Operator, error) { return nil, fmt.Errorf("operator %d: %v", i, err) } } - if len(seen) != 4 || op.Name == "" { + if len(seen) != 5 || op.Name == "" { return nil, fmt.Errorf("operator %d: not every key, or no name", i) } operators = append(operators, op) @@ -278,7 +285,9 @@ func parseOperators(value any) ([]record.Operator, error) { // readers refuse the same values. func parseOperatorName(s string) (string, error) { switch s { - case "eq", "lt", "lte", "gt", "gte", "between", "matches": + case "eq", "lt", "lte", "gt", "gte", "between", "matches", + "contains", "contained_by", "eq_at", "lt_at", "lte_at", "gt_at", "gte_at", "between_at", + "selector", "subtree", "children", "child_count": return s, nil } return "", fmt.Errorf("unknown operator %q", s) @@ -294,10 +303,10 @@ func parseFidelity(s string) (record.Fidelity, error) { } // parseOperatorIndex reads the index that answers an operator: eq, match, -// ore, ope or ore_block. +// ore, ope, ore_block or json. func parseOperatorIndex(s string) (record.Output, error) { switch o := record.Output(s); o { - case record.Equality, record.Match, record.Ore, record.Ope, record.OreBlock: + case record.Equality, record.Match, record.Ore, record.Ope, record.OreBlock, record.JSONIndex: return o, nil } return "", fmt.Errorf("unknown index %q", s) diff --git a/languages/golang/encrypt/guest.go b/languages/golang/encrypt/guest.go index 04d01730c..878ce860c 100644 --- a/languages/golang/encrypt/guest.go +++ b/languages/golang/encrypt/guest.go @@ -86,6 +86,7 @@ type instance struct { cipherInit, shutdown, keyset api.Function term api.Function queryTerm, verify, limit api.Function + jsonOpen api.Function encryptRecord, decryptRecord api.Function planCheck, targets, operators api.Function // The step loop (step.go): take a paused export's step, resume it with @@ -179,6 +180,7 @@ func newInstance(ctx context.Context, wasm []byte, t *transport, policy guest.Lo "se_keyset": &inst.keyset, "se_term": &inst.term, "se_query_term": &inst.queryTerm, + "se_json_open": &inst.jsonOpen, "se_verify": &inst.verify, "se_target_limit": &inst.limit, "se_encrypt_record": &inst.encryptRecord, diff --git a/languages/golang/encrypt/guest/src/abi.rs b/languages/golang/encrypt/guest/src/abi.rs index e1df6095c..fd539231b 100644 --- a/languages/golang/encrypt/guest/src/abi.rs +++ b/languages/golang/encrypt/guest/src/abi.rs @@ -571,6 +571,49 @@ pub unsafe extern "C" fn se_query_term( }) } +/// Open the rows a read of a document field returned ([`ops::json_open`]): +/// the codec-encoded rows (a list per database row of entries), the +/// codec-encoded plan, the field's name (UTF-8 bytes) and the codec-encoded +/// read request in, under the keyset `opts` selects (the keyset the read +/// was prepared under); a codec-encoded list per row of JSON texts out. The +/// output buffer contains **plaintext**: the host copies it out and +/// immediately `se_dealloc`s it (which wipes it). The rows and the request +/// hold no plaintext and are left untouched. +/// +/// # Safety +/// +/// As for [`se_term`]. +#[no_mangle] +pub unsafe extern "C" fn se_json_open( + rows_ptr: *const u8, + rows_len: u32, + plan_ptr: *const u8, + plan_len: u32, + field_ptr: *const u8, + field_len: u32, + req_ptr: *const u8, + req_len: u32, + opt_ptr: *const u8, + opt_len: u32, +) -> u64 { + export(|| { + // SAFETY: host-owned ranges the export was handed; the borrows end + // before it returns. + let rows = unsafe { input(rows_ptr, rows_len)? }.to_vec(); + let plan = unsafe { input(plan_ptr, plan_len)? }.to_vec(); + let field = unsafe { input(field_ptr, field_len)? }.to_vec(); + let request = unsafe { input(req_ptr, req_len)? }.to_vec(); + let opts = unsafe { input(opt_ptr, opt_len)? }; + ops::validate::json_open(&rows, &plan, &field, &request)?; + let options = parse_options(opts, Side::Mint)?; + let cipher = cipher()?; + run(Box::pin(async move { + let keyset = options.keyset.resolve(&cipher).await?; + ops::json_open(&keyset, &rows, &plan, &field, &request).await + })) + }) +} + /// Verify a page of decrypted values against a probabilistic query /// ([`ops::verify`]): the query's codec-encoded input and a codec-encoded /// list of values, the codec-encoded plan, and the field's and the diff --git a/languages/golang/encrypt/guest/src/ops.rs b/languages/golang/encrypt/guest/src/ops.rs index aaba4a393..90bcf9047 100644 --- a/languages/golang/encrypt/guest/src/ops.rs +++ b/languages/golang/encrypt/guest/src/ops.rs @@ -272,6 +272,43 @@ where encode_answer(&term) } +/// Open the rows a read of a document field returned +/// ([`dynamic::json_open`]): a codec-encoded list per database row of the +/// entries the SQL returned (`public.eql_v4_json_tree_entry` JSON, text or +/// bytes), the plan, the field's name and the codec-encoded read request +/// (`{"operation": "selector" | "subtree", "path", "wildcard_bound"?}`) in; +/// a codec-encoded list per row of JSON texts (bytes) out, which is +/// **plaintext**. The selector or the read key is derived again from the +/// request under `keyset`, which the host names as it named it to prepare +/// the read, and the rows open through that keyset only: a document sealed +/// under another is refused before any key is retrieved. One batched key +/// request for every row. +pub async fn json_open( + keyset: &KeysetCipher<'_, K>, + rows: &[u8], + plan: &[u8], + field: &[u8], + request: &[u8], +) -> Result, u32> +where + K: DataKeySource + Sync + 'static, +{ + let (plan, field) = parse_field(plan, field)?; + let opened = dynamic::json_open( + keyset, + Scope::Keyset(keyset.clone()), + &plan, + field, + parse_frame(request)?, + parse_frame(rows)?, + &resolver(), + ) + .map_err(|e| fail_dynamic(&e))? + .await + .map_err(|e| fail_error(&e))?; + encode_frame(opened) +} + /// Verify a page of decrypted values against a probabilistic query: the /// query's codec-encoded input and a codec-encoded list of values, with the /// plan, the field's name and the operator's name, in; a codec-encoded list @@ -336,6 +373,14 @@ pub fn target_limit( /// this build's resolver, the field's name UTF-8, and the operator one the /// engine knows ([`dynamic::OpSpec::parse`]); an unknown operator is /// refused naming the field, as an operator the field does not answer is. +/// A plan and one of its fields' names. +fn parse_field<'f>(plan: &[u8], field: &'f [u8]) -> Result<(dynamic::record::Plan, &'f str), u32> { + let plan = parse_plan(plan)?; + let field = std::str::from_utf8(field) + .map_err(|_| last_error::malformed("the field name is not UTF-8"))?; + Ok((plan, field)) +} + fn parse_query<'f>( plan: &[u8], field: &'f [u8], @@ -500,6 +545,21 @@ pub mod validate { .map_err(|e| fail_dynamic(&e)) } + /// A read's inputs, as [`json_open`](super::json_open) takes them: the + /// plan parses, the field stores a document, and the request and rows + /// are the read's ([`dynamic::check_json_open`]). + pub fn json_open(rows: &[u8], plan: &[u8], field: &[u8], request: &[u8]) -> Result<(), u32> { + let (plan, field) = parse_field(plan, field)?; + dynamic::check_json_open( + &plan, + field, + parse_frame(request)?, + parse_frame(rows)?, + &resolver(), + ) + .map_err(|e| fail_dynamic(&e)) + } + /// A record tree against its plan, as [`decrypt_record`] takes them: /// the plan parses, the tree decodes with well-formed leaves, every /// ciphertext-bearing field has a `"c"` node that is not a passthrough, @@ -894,7 +954,8 @@ mod tests { panic!("a query term is an object") }; assert!(matches!(&term[3].1, Tree::String(s) if s.as_bytes() == b"eql_v4.query_text_eq")); - let Tree::Bytes(query_bytes) = &term[4].1 else { + assert!(matches!(&term[4].1, Tree::String(s) if s.as_bytes() == b"eql")); + let Tree::Bytes(query_bytes) = &term[5].1 else { panic!("the term is the query value's bytes") }; let probe_json: serde_json::Value = diff --git a/languages/golang/encrypt/guest/src/status.rs b/languages/golang/encrypt/guest/src/status.rs index 07da13b1a..733725039 100644 --- a/languages/golang/encrypt/guest/src/status.rs +++ b/languages/golang/encrypt/guest/src/status.rs @@ -141,6 +141,13 @@ pub fn status_for_error(error: &stack_encrypt::Error) -> u32 { // one it was opened as: sealed as that type, not tampered with, so // the caller's input and not STATUS_AUTH. stack_encrypt::Error::OpenedType { .. } => STATUS_ENCODING, + // A JSON read's rows that are not the ones its request asked for, + // or a wildcard read of an array longer than its bound: statements + // about the rows the host handed over, decided before any key is + // retrieved, which the host acts on (asks again, raises the bound). + // The diagnostic's code tells the two apart. + stack_encrypt::Error::JsonWrongEntry { .. } + | stack_encrypt::Error::JsonWildcardTruncated { .. } => STATUS_ENCODING, _ => STATUS_INTERNAL, } } diff --git a/languages/golang/encrypt/guest/src/targets.rs b/languages/golang/encrypt/guest/src/targets.rs index d32ab609a..943a55bfc 100644 --- a/languages/golang/encrypt/guest/src/targets.rs +++ b/languages/golang/encrypt/guest/src/targets.rs @@ -110,9 +110,14 @@ fn describe(target: &eql_bindings::encryption::targets::Target) -> TargetDescrip ); // A type with a match index answers `matches` through its twin only // when the engine knows the options its filter is built under. - match target.match_options() { + let descriptor = match target.match_options() { Some(options) => descriptor.with_match_options(options), None => descriptor, + }; + // A document's orderings at a path bind as its leaf operand. + match target.leaf_query_sql_domain() { + Some(domain) => descriptor.with_leaf_query_sql_domain(domain), + None => descriptor, } } diff --git a/languages/golang/encrypt/guest/tests/native_ops.rs b/languages/golang/encrypt/guest/tests/native_ops.rs index e91aae234..5f08cdafb 100644 --- a/languages/golang/encrypt/guest/tests/native_ops.rs +++ b/languages/golang/encrypt/guest/tests/native_ops.rs @@ -1796,7 +1796,7 @@ fn the_catalog_lists_each_types_operators() { op.iter().map(|(k, v)| (k.as_str(), text(v))).collect() }) .collect(); - let only = |want: [(&str, &str); 4]| { + let only = |want: [(&str, &str); 5]| { assert_eq!(listed, [want.to_vec()], "{name}"); }; let exact = |operator: &'static str, index: &'static str| { @@ -1805,6 +1805,7 @@ fn the_catalog_lists_each_types_operators() { ("fidelity", "exact"), ("arity", "one"), ("index", index), + ("input", "value"), ] }; // OPE for Ord and OrdOpe, block ORE for OrdOre and SearchOre. @@ -1833,13 +1834,41 @@ fn the_catalog_lists_each_types_operators() { ("fidelity", "exact"), ("arity", "one"), ("index", "eq"), + ("input", "value"), ]), "TextMatch" => only([ ("operator", "matches"), ("fidelity", "probabilistic"), ("arity", "one"), ("index", "match"), + ("input", "value"), ]), + // A document's operators under the index's defaults: the + // orderings at a path only for a field that orders one. + "EncryptedJson" => { + let json = |operator: &'static str, arity: &'static str, input: &'static str| { + vec![ + ("operator", operator), + ("fidelity", "exact"), + ("arity", arity), + ("index", "json"), + ("input", input), + ] + }; + assert_eq!( + listed, + [ + json("contains", "one", "json"), + json("contained_by", "one", "json"), + json("eq_at", "one", "path_json"), + json("selector", "one", "path"), + json("subtree", "one", "path_bound"), + json("children", "one", "path"), + json("child_count", "one", "path"), + ], + "{name}" + ); + } // Each operator through its own index, whose term alone the // query carries. "TextSearch" | "TextSearchOre" => { @@ -1850,6 +1879,7 @@ fn the_catalog_lists_each_types_operators() { ("fidelity", "probabilistic"), ("arity", "one"), ("index", "match"), + ("input", "value"), ]); assert_eq!(listed, want, "{name}"); } @@ -1967,7 +1997,7 @@ fn operators_of(plan: &[u8], field: &str) -> Vec<[String; 4]> { panic!("an operator is an object"); }; let keys: Vec<&str> = entries.iter().map(|(k, _)| k.as_str()).collect(); - assert_eq!(keys, ["operator", "fidelity", "arity", "index"]); + assert_eq!(keys, ["operator", "fidelity", "arity", "index", "input"]); [0, 1, 2, 3].map(|i| text(&entries[i].1).to_owned()) }) .collect() diff --git a/languages/golang/encrypt/json.go b/languages/golang/encrypt/json.go new file mode 100644 index 000000000..988a50547 --- /dev/null +++ b/languages/golang/encrypt/json.go @@ -0,0 +1,93 @@ +package encrypt + +import ( + "context" + "fmt" + + "github.com/cipherstash/stack/languages/golang/internal/record" + "github.com/cipherstash/vitaminc/bindings/go/vcffi" + "github.com/cipherstash/vitaminc/bindings/go/vcvalue" +) + +// JSONOpen opens the rows a read of a field that stores a JSON document +// returned (the guest's se_json_open), against the request that prepared +// the read: {"operation": "selector" | "subtree", "path", "wildcard_bound"}. +// rows holds one group per database row: for selector, the entry col -> +// selector returned, or none for SQL NULL; for subtree, the rows +// eql_v4.json_tree_subtree returned. Each entry is the +// public.eql_v4_json_tree_entry JSON the database sent. +// +// The engine derives the selector or the read's key again from the request, +// the field's declaration and this cipher's keyset, and checks every row +// against it; the rows never say which path they are for. A row that is not +// one the read asked for is [ErrEncoding]; a wildcard read of an array +// longer than its bound is too. What comes back is, per group, the JSON +// text of each value the read reached, in document order: plaintext, which +// the caller owns. One ZeroKMS request for every group. For generated code. +// +// Opening does not prove the database returned every row the read asked +// for, or rows of the database row the caller meant: see the EQL json_tree +// leakage reference. +func (cph *Cipher) JSONOpen(ctx context.Context, plan *record.Plan, field string, request vcvalue.Object, rows [][][]byte) ([][][]byte, error) { + p, err := cph.queryPlan(plan, field) + if err != nil { + return nil, err + } + groups := make([]any, len(rows)) + for i, group := range rows { + entries := make([]any, len(group)) + for j, entry := range group { + entries[j] = entry + } + groups[i] = entries + } + encodedRows, err := vcffi.Marshal(groups) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrEncoding, err) + } + encodedRequest, err := vcffi.Marshal(request) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrEncoding, err) + } + encodedPlan, err := vcffi.Marshal(p.Wire()) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrEncoding, err) + } + opts, err := vcffi.Marshal(options(cph.keyset, "")) + if err != nil { + return nil, err + } + out, err := cph.client.callKMS(ctx, func(ctx context.Context, inst *instance) ([]byte, error) { + return inst.call(ctx, inst.jsonOpen, buf(encodedRows), buf(encodedPlan), buf([]byte(field)), buf(encodedRequest), buf(opts)) + }) + if err != nil { + return nil, err + } + // The output is plaintext: wiped once it is decoded into the caller's + // slices. + defer wipe(out) + decoded, err := vcffi.Unmarshal(out) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrInternal, err) + } + list, ok := decoded.([]any) + if !ok || len(list) != len(rows) { + return nil, fmt.Errorf("%w: se_json_open returned %T for %d groups", ErrInternal, decoded, len(rows)) + } + opened := make([][][]byte, len(list)) + for i, item := range list { + values, ok := item.([]any) + if !ok { + return nil, fmt.Errorf("%w: se_json_open group %d is %T", ErrInternal, i, item) + } + opened[i] = make([][]byte, len(values)) + for j, v := range values { + text, ok := v.([]byte) + if !ok { + return nil, fmt.Errorf("%w: se_json_open value %d.%d is %T", ErrInternal, i, j, v) + } + opened[i][j] = text + } + } + return opened, nil +} diff --git a/languages/golang/encrypt/json_tree_test.go b/languages/golang/encrypt/json_tree_test.go index b0a3db4c2..c5050af0e 100644 --- a/languages/golang/encrypt/json_tree_test.go +++ b/languages/golang/encrypt/json_tree_test.go @@ -11,6 +11,7 @@ import ( "github.com/cipherstash/stack/languages/golang/encrypt" "github.com/cipherstash/stack/languages/golang/encrypt/eql" "github.com/cipherstash/stack/languages/golang/internal/record" + "github.com/cipherstash/vitaminc/bindings/go/vcvalue" ) // An encrypted JSON document (the EQL type EncryptedJson) through the guest @@ -105,3 +106,124 @@ func TestJSONTextThatIsNotADocumentIsErrEncoding(t *testing.T) { } } } + +// Every operator a document field answers, through the guest: the engine +// lists them with their inputs, and each derives a term of the encoding and +// SQL type the operator binds as. +func TestJSONOperatorsDeriveTaggedTerms(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + cipher := c.DefaultKeyset() + checker, err := encrypt.NewChecker(ctx) + if err != nil { + t.Fatal(err) + } + defer func() { _ = checker.Close() }() + ops, err := checker.Operators(ctx, jsonPlan(), "profile") + if err != nil { + t.Fatal(err) + } + inputs := map[string]string{} + for _, op := range ops { + if op.Index != record.JSONIndex { + t.Fatalf("%s through %s", op.Name, op.Index) + } + inputs[op.Name] = op.Input + } + want := map[string]string{ + "contains": "json", "contained_by": "json", "eq_at": "path_json", + "lt_at": "path_json", "lte_at": "path_json", "gt_at": "path_json", "gte_at": "path_json", + "between_at": "path_pair", "selector": "path", "subtree": "path_bound", + "children": "path", "child_count": "path", + } + if !reflect.DeepEqual(inputs, want) { + t.Fatalf("operators = %v", inputs) + } + type obj = vcvalue.Object + for _, tc := range []struct { + op, encoding, domain string + input obj + parts int + }{ + {"contains", "eql", "eql_v4.query_json_tree", obj{{Key: "json", Value: []byte(`{"age":34}`)}}, 1}, + {"contained_by", "eql", "eql_v4.query_json_tree", obj{{Key: "json", Value: `{"age":34}`}}, 1}, + {"eq_at", "eql", "eql_v4.query_json_tree", obj{{Key: "path", Value: "$.name"}, {Key: "json", Value: []byte(`"Alice"`)}}, 1}, + {"lt_at", "eql", "eql_v4.query_json_tree_leaf", obj{{Key: "path", Value: "$.age"}, {Key: "json", Value: []byte(`40`)}}, 1}, + {"between_at", "eql", "eql_v4.query_json_tree_leaf", obj{{Key: "path", Value: "$.age"}, {Key: "low", Value: []byte(`18`)}, {Key: "high", Value: []byte(`40`)}}, 2}, + {"selector", "text", "text", obj{{Key: "path", Value: "$.name"}}, 1}, + {"subtree", "text_array", "text[]", obj{{Key: "path", Value: "$.visits[*]"}, {Key: "wildcard_bound", Value: uint32(3)}}, 4}, + {"children", "text", "text", obj{{Key: "path", Value: "$.visits"}}, 1}, + {"child_count", "text", "text", obj{{Key: "path", Value: "$.visits"}}, 1}, + } { + term, err := cipher.QueryTerm(ctx, jsonPlan(), "profile", tc.op, tc.input) + if err != nil { + t.Fatalf("%s: %v", tc.op, err) + } + parts := len(term.Terms) + len(term.Text) + if term.Encoding != tc.encoding || term.Domain != tc.domain || parts != tc.parts || term.Operator != tc.op { + t.Fatalf("%s: %+v", tc.op, term) + } + } + // A position where the operand is a selector is refused, naming the + // field and the reason. + _, err = cipher.QueryTerm(ctx, jsonPlan(), "profile", "selector", obj{{Key: "path", Value: "$.visits[1]"}}) + var d *encrypt.Diagnostic + if !errors.Is(err, encrypt.ErrEncoding) || !errors.As(err, &d) || d.Reason() != "json_path_position" || d.Field() != "profile" { + t.Fatalf("a positional selector: %v", err) + } +} + +// A selector read through the guest: the entry `col -> selector` returns, +// as the SQL grafts it (the document's header and the entry's ordinal), +// opens to the scalar; no entry opens to nothing; and another path's entry +// is refused, since the request names the path and the rows do not. +func TestJSONSelectorReadsOpenAgainstTheirRequest(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + cipher := c.DefaultKeyset() + sealed, err := cipher.Seal(ctx, jsonPlan(), []record.Source{{"profile": []byte(`{"name":"Alice","age":34}`)}}) + if err != nil { + t.Fatal(err) + } + selector := func(path string) string { + term, err := cipher.QueryTerm(ctx, jsonPlan(), "profile", "selector", vcvalue.Object{{Key: "path", Value: path}}) + if err != nil { + t.Fatal(err) + } + return term.Text[0] + } + // col -> selector, as eql_v4.json_tree_graft writes it. + extract := func(s string) []byte { + var doc struct { + H string `json:"h"` + SV []map[string]json.RawMessage `json:"sv"` + } + if err := json.Unmarshal(sealed[0]["profile"].EQL, &doc); err != nil { + t.Fatal(err) + } + for j, entry := range doc.SV { + var es string + _ = json.Unmarshal(entry["s"], &es) + if es == s { + entry["h"], _ = json.Marshal(doc.H) + entry["j"], _ = json.Marshal(j) + out, _ := json.Marshal(entry) + return out + } + } + t.Fatalf("no entry carries %s", s) + return nil + } + name := extract(selector("$.name")) + opened, err := cipher.JSONOpen(ctx, jsonPlan(), "profile", vcvalue.Object{{Key: "operation", Value: "selector"}, {Key: "path", Value: "$.name"}}, [][][]byte{{name}, {}}) + if err != nil { + t.Fatal(err) + } + if len(opened) != 2 || len(opened[0]) != 1 || string(opened[0][0]) != `"Alice"` || len(opened[1]) != 0 { + t.Fatalf("opened %q", opened) + } + _, err = cipher.JSONOpen(ctx, jsonPlan(), "profile", vcvalue.Object{{Key: "operation", Value: "selector"}, {Key: "path", Value: "$.age"}}, [][][]byte{{name}}) + if !errors.Is(err, encrypt.ErrEncoding) { + t.Fatalf("another path's entry: %v", err) + } +} diff --git a/languages/golang/encrypt/query.go b/languages/golang/encrypt/query.go index bcff679bc..4e4f948c0 100644 --- a/languages/golang/encrypt/query.go +++ b/languages/golang/encrypt/query.go @@ -36,8 +36,16 @@ type QueryTerm struct { Domain string // Terms is the term: one, or for between two, the low bound's then the // high's. For a field that names an EQL type it is the query value's - // JSON. + // JSON. Empty when the term is Text. Terms [][]byte + // Encoding says what the term holds: bytes (an index term), eql (an EQL + // query value's JSON), text (a JSON path's selector or one node's + // token, in Text[0]) or text_array (a read's tokens, in Text). + Encoding string + // Text is the term as text, for the text and text_array encodings: what + // a document's selector, children, child_count and subtree operators + // derive. It binds as SQL text, or text[]. + Text []string } // Bucket is one bar of a histogram of the stored match filters' set-bit @@ -184,20 +192,39 @@ func parseQueryTerm(decoded any) (QueryTerm, error) { } case "domain": t.Domain, err = targetOptionalString(f) + case "encoding": + var s string + if s, err = targetString(f); err == nil { + switch s { + case "bytes", "eql", "text", "text_array": + t.Encoding = s + default: + err = fmt.Errorf("unknown encoding %q", s) + } + } case "term": switch v := f.Value.(type) { case []byte: t.Terms = [][]byte{v} + case string: + t.Text = []string{v} case []any: - if len(v) != 2 { - return QueryTerm{}, fmt.Errorf("a range of %d terms", len(v)) - } - for _, bound := range v { - b, ok := bound.([]byte) - if !ok { - return QueryTerm{}, fmt.Errorf("a bound is %T, not bytes", bound) + for _, item := range v { + switch x := item.(type) { + case []byte: + t.Terms = append(t.Terms, x) + case string: + t.Text = append(t.Text, x) + default: + return QueryTerm{}, fmt.Errorf("a term part is %T", item) } - t.Terms = append(t.Terms, b) + } + if t.Terms != nil && (len(t.Terms) != 2 || t.Text != nil) { + return QueryTerm{}, fmt.Errorf("a range of %d terms", len(t.Terms)) + } + if t.Text == nil && t.Terms == nil { + // An empty list is a read that sends no token. + t.Text = []string{} } default: return QueryTerm{}, fmt.Errorf("term is %T", f.Value) @@ -209,8 +236,16 @@ func parseQueryTerm(decoded any) (QueryTerm, error) { return QueryTerm{}, err } } - if len(seen) != 5 { - return QueryTerm{}, fmt.Errorf("%d keys, not 5", len(seen)) + if len(seen) != 6 { + return QueryTerm{}, fmt.Errorf("%d keys, not 6", len(seen)) + } + // The encoding and the term agree: text for the text encodings, + // bytes for the others. + if text := t.Encoding == "text" || t.Encoding == "text_array"; text != (t.Text != nil) { + return QueryTerm{}, fmt.Errorf("a %s term that holds %d texts and %d byte strings", t.Encoding, len(t.Text), len(t.Terms)) + } + if t.Encoding == "text" && len(t.Text) != 1 { + return QueryTerm{}, fmt.Errorf("a text term of %d texts", len(t.Text)) } return t, nil } diff --git a/languages/golang/encrypt/query_test.go b/languages/golang/encrypt/query_test.go index c3b14cd60..9b21aa00a 100644 --- a/languages/golang/encrypt/query_test.go +++ b/languages/golang/encrypt/query_test.go @@ -419,7 +419,7 @@ func TestTheCatalogListsEachTypesOperators(t *testing.T) { orderings := func(index record.Output) []record.Operator { var ops []record.Operator for _, name := range []string{"lt", "lte", "gt", "gte"} { - ops = append(ops, record.Operator{Name: name, Fidelity: record.Exact, Index: index}) + ops = append(ops, record.Operator{Name: name, Fidelity: record.Exact, Index: index, Input: "value"}) } return ops } @@ -427,21 +427,21 @@ func TestTheCatalogListsEachTypesOperators(t *testing.T) { index, ordered := orderIndex[target.Name] switch { case strings.HasSuffix(target.Name, "Eq") && slices.Contains(families, strings.TrimSuffix(target.Name, "Eq")): - want := []record.Operator{{Name: "eq", Fidelity: record.Exact, Index: record.Equality}} + want := []record.Operator{{Name: "eq", Fidelity: record.Exact, Index: record.Equality, Input: "value"}} if len(target.Operators) != 1 || target.Operators[0] != want[0] { t.Fatalf("%s answers %+v, want %+v", target.Name, target.Operators, want) } case ordered: var want []record.Operator if strings.HasPrefix(target.Name, "Text") { - want = append(want, record.Operator{Name: "eq", Fidelity: record.Exact, Index: record.Equality}) + want = append(want, record.Operator{Name: "eq", Fidelity: record.Exact, Index: record.Equality, Input: "value"}) } want = append(want, orderings(index)...) if !slices.Equal(target.Operators, want) { t.Fatalf("%s answers %+v, want %+v", target.Name, target.Operators, want) } case target.Name == "TextMatch": - want := record.Operator{Name: "matches", Fidelity: record.Probabilistic, Index: record.Match} + want := record.Operator{Name: "matches", Fidelity: record.Probabilistic, Index: record.Match, Input: "value"} if len(target.Operators) != 1 || target.Operators[0] != want { t.Fatalf("TextMatch answers %+v, want %+v", target.Operators, want) } @@ -451,12 +451,25 @@ func TestTheCatalogListsEachTypesOperators(t *testing.T) { if target.Name == "TextSearchOre" { index = record.OreBlock } - want := []record.Operator{{Name: "eq", Fidelity: record.Exact, Index: record.Equality}} + want := []record.Operator{{Name: "eq", Fidelity: record.Exact, Index: record.Equality, Input: "value"}} want = append(want, orderings(index)...) - want = append(want, record.Operator{Name: "matches", Fidelity: record.Probabilistic, Index: record.Match}) + want = append(want, record.Operator{Name: "matches", Fidelity: record.Probabilistic, Index: record.Match, Input: "value"}) if !slices.Equal(target.Operators, want) { t.Fatalf("%s answers %+v, want %+v", target.Name, target.Operators, want) } + case target.Name == "EncryptedJson": + // A document's operators, under the index's defaults: no + // ordering without an ordered path. Each through the JSON index. + var names []string + for _, op := range target.Operators { + if op.Index != record.JSONIndex || op.Fidelity != record.Exact { + t.Fatalf("EncryptedJson answers %+v", op) + } + names = append(names, op.Name) + } + if want := []string{"contains", "contained_by", "eq_at", "selector", "subtree", "children", "child_count"}; !slices.Equal(names, want) { + t.Fatalf("EncryptedJson answers %v, want %v", names, want) + } case len(target.Operators) != 0: // Every other type is storage-only or not produced. t.Fatalf("%s (producible %v) answers %+v, want none", target.Name, target.Producible, target.Operators) @@ -481,16 +494,16 @@ func TestTheCheckerListsAFieldsOperators(t *testing.T) { } defer func() { _ = checker.Close() }() exact := func(name string, index record.Output) record.Operator { - return record.Operator{Name: name, Fidelity: record.Exact, Index: index} + return record.Operator{Name: name, Fidelity: record.Exact, Index: index, Input: "value"} } for field, want := range map[string][]record.Operator{ "email": {exact("eq", record.Equality)}, "age": { exact("eq", record.Equality), exact("lt", record.Ore), exact("lte", record.Ore), exact("gt", record.Ore), exact("gte", record.Ore), - {Name: "between", Fidelity: record.Exact, Pair: true, Index: record.Ore}, + {Name: "between", Fidelity: record.Exact, Pair: true, Index: record.Ore, Input: "pair"}, }, - "name": {{Name: "matches", Fidelity: record.Probabilistic, Index: record.Match}}, + "name": {{Name: "matches", Fidelity: record.Probabilistic, Index: record.Match, Input: "value"}}, } { got, err := checker.Operators(ctx, queryPlan, field) if err != nil { diff --git a/languages/golang/encrypt/targets_test.go b/languages/golang/encrypt/targets_test.go index 4cb8676be..6b07edb11 100644 --- a/languages/golang/encrypt/targets_test.go +++ b/languages/golang/encrypt/targets_test.go @@ -39,12 +39,12 @@ func TestParseTargetsReadsEachEntry(t *testing.T) { } want := []record.Target{ {Name: "TextEq", Family: "Text", Suffix: "Eq", Plaintext: record.String, SQLDomain: "eql_v4_text_eq", Indexes: []record.Output{record.Equality}, Query: "TextEqQuery", QuerySQLDomain: "eql_v4.query_text_eq", Producible: true, - Operators: []record.Operator{{Name: "eq", Fidelity: record.Exact, Index: record.Equality}}}, + Operators: []record.Operator{{Name: "eq", Fidelity: record.Exact, Index: record.Equality, Input: "value"}}}, {Name: "JSON", Family: "JSON", Plaintext: record.Untyped, SQLDomain: "eql_v4_json", Indexes: []record.Output{"json"}, Reason: "the JSON index is a new operation in the engine", Operators: []record.Operator{}}, {Name: "IntegerOrdOre", Plaintext: record.Int64, Indexes: []record.Output{record.Ore}, Producible: true, Operators: []record.Operator{ - {Name: "lt", Fidelity: record.Exact, Index: record.Ore}, - {Name: "between", Fidelity: record.Exact, Pair: true, Index: record.Ore}, - {Name: "matches", Fidelity: record.Probabilistic, Index: record.Match}, + {Name: "lt", Fidelity: record.Exact, Index: record.Ore, Input: "value"}, + {Name: "between", Fidelity: record.Exact, Pair: true, Index: record.Ore, Input: "pair"}, + {Name: "matches", Fidelity: record.Probabilistic, Index: record.Match, Input: "value"}, }}, } if !reflect.DeepEqual(got, want) { @@ -73,14 +73,15 @@ func TestParseTargetsReadsEachEntry(t *testing.T) { "operator not object": withOperators([]any{"eq"}), "unknown fidelity": withOperators([]any{operator("eq", "approximate", "one", "eq")}), "unknown arity": withOperators([]any{operator("eq", "exact", "three", "eq")}), - "unknown op index": withOperators([]any{operator("eq", "exact", "one", "json")}), + "unknown op index": withOperators([]any{operator("eq", "exact", "one", "equality")}), + "unknown op input": withOperators([]any{append(operator("eq", "exact", "one", "eq")[:4], vcvalue.Field{Key: "input", Value: "path_value"})}), "no op name": withOperators([]any{operator("", "exact", "one", "eq")}), "unknown operator": withOperators([]any{operator("like", "exact", "one", "eq")}), "operator case": withOperators([]any{operator("Eq", "exact", "one", "eq")}), "op key missing": withOperators([]any{vcvalue.Object{{Key: "operator", Value: "eq"}}}), "op unknown key": withOperators([]any{append(operator("eq", "exact", "one", "eq"), vcvalue.Field{Key: "case", Value: "insensitive"})}), "op key twice": withOperators([]any{append(operator("eq", "exact", "one", "eq"), vcvalue.Field{Key: "index", Value: "eq"})}), - "op value not text": withOperators([]any{vcvalue.Object{{Key: "operator", Value: int64(1)}, {Key: "fidelity", Value: "exact"}, {Key: "arity", Value: "one"}, {Key: "index", Value: "eq"}}}), + "op value not text": withOperators([]any{vcvalue.Object{{Key: "operator", Value: int64(1)}, {Key: "fidelity", Value: "exact"}, {Key: "arity", Value: "one"}, {Key: "index", Value: "eq"}, {Key: "input", Value: "value"}}}), } { if _, err := parseTargets(bad); !errors.Is(err, ErrInternal) { t.Errorf("%s: %v, want ErrInternal", name, err) @@ -90,7 +91,11 @@ func TestParseTargetsReadsEachEntry(t *testing.T) { // operator is one se_targets operators entry. func operator(name, fidelity, arity, index string) vcvalue.Object { - return vcvalue.Object{{Key: "operator", Value: name}, {Key: "fidelity", Value: fidelity}, {Key: "arity", Value: arity}, {Key: "index", Value: index}} + input := "value" + if arity == "pair" { + input = "pair" + } + return vcvalue.Object{{Key: "operator", Value: name}, {Key: "fidelity", Value: fidelity}, {Key: "arity", Value: arity}, {Key: "index", Value: index}, {Key: "input", Value: input}} } // withOperators is a well-formed TextEq entry whose operators are value. @@ -107,10 +112,12 @@ func TestParseQueryTermTakesOnlyWhatTheEngineWrites(t *testing.T) { term := func(operator, fidelity, index string) vcvalue.Object { return vcvalue.Object{ {Key: "operator", Value: operator}, {Key: "fidelity", Value: fidelity}, {Key: "index", Value: index}, - {Key: "domain", Value: nil}, {Key: "term", Value: []byte{1, 2}}, + {Key: "domain", Value: nil}, {Key: "encoding", Value: "bytes"}, {Key: "term", Value: []byte{1, 2}}, } } - for _, name := range []string{"eq", "lt", "lte", "gt", "gte", "between", "matches"} { + for _, name := range []string{"eq", "lt", "lte", "gt", "gte", "between", "matches", + "contains", "contained_by", "eq_at", "lt_at", "lte_at", "gt_at", "gte_at", "between_at", + "selector", "subtree", "children", "child_count"} { got, err := parseQueryTerm(term(name, "exact", "ore")) if err != nil || got.Operator != name || got.Fidelity != record.Exact || got.Index != record.Ore { t.Errorf("%s: %+v, %v", name, got, err) @@ -123,7 +130,7 @@ func TestParseQueryTermTakesOnlyWhatTheEngineWrites(t *testing.T) { "unknown operator": term("like", "exact", "eq"), "operator case": term("Eq", "exact", "eq"), "unknown fidelity": term("eq", "approximate", "eq"), - "unknown index": term("eq", "exact", "json"), + "unknown index": term("eq", "exact", "jsonb"), "misspelled index": term("eq", "exact", "equality"), } { if _, err := parseQueryTerm(bad); err == nil { diff --git a/languages/golang/internal/record/record.go b/languages/golang/internal/record/record.go index 7f7b4b2e1..78b2f6dd4 100644 --- a/languages/golang/internal/record/record.go +++ b/languages/golang/internal/record/record.go @@ -75,12 +75,16 @@ type Output string // The outputs. Passthrough is not one a Plan's Fields ask for (see the // package doc); it is the context field's, which Wire writes itself. const ( - Ciphertext Output = "c" - Equality Output = "eq" - Match Output = "match" - Ore Output = "ore" - Ope Output = "ope" - OreBlock Output = "ore_block" + Ciphertext Output = "c" + Equality Output = "eq" + Match Output = "match" + Ore Output = "ore" + Ope Output = "ope" + OreBlock Output = "ore_block" + // JSONIndex is the JSON index, which a target that stores a JSON + // document carries and answers a document's operators through. No + // field asks for it as an output. + JSONIndex Output = "json" Passthrough Output = "passthrough" ) @@ -560,7 +564,9 @@ const ( // lists it. type Operator struct { // Name is the operator's name, as a query names it: eq, lt, lte, gt, - // gte, between or matches. + // gte, between or matches; for a field that stores a JSON document, + // contains, contained_by, eq_at, lt_at, lte_at, gt_at, gte_at, + // between_at, selector, subtree, children or child_count. Name string // Fidelity is how far the server's answer can be trusted. Fidelity Fidelity @@ -569,4 +575,9 @@ type Operator struct { Pair bool // Index is the index that answers it: the field's, or its EQL type's. Index Output + // Input is the shape of the operator's input, as the engine names it: + // value (one value of the field's kind), pair (two, low then high), + // json, path_json, path_pair, path or path_bound (objects of named + // inputs, for a document's operators). + Input string } diff --git a/packages/eql/crates/eql-bindings/CHANGELOG.md b/packages/eql/crates/eql-bindings/CHANGELOG.md index 355f6b817..6d23b2923 100644 --- a/packages/eql/crates/eql-bindings/CHANGELOG.md +++ b/packages/eql/crates/eql-bindings/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **`Target::leaf_query_sql_domain`** (`targets` feature, #1350): the + domain a document type's ordering at a path binds as, + `eql_v4.query_json_tree_leaf` for `EncryptedJson` and `None` for every + other type, which a resolver hands Stack Encrypt + (`TargetDescriptor::with_leaf_query_sql_domain`). + - **`EncryptedJson`, the `json_tree` document, by name** (`targets` feature, #1348, #1349). The target table has a row for it, generated from the catalog's document family (`eql_domains::DOCUMENT_FAMILIES`): diff --git a/packages/eql/crates/eql-bindings/src/encryption/targets.rs b/packages/eql/crates/eql-bindings/src/encryption/targets.rs index 4bf82085c..867b9838b 100644 --- a/packages/eql/crates/eql-bindings/src/encryption/targets.rs +++ b/packages/eql/crates/eql-bindings/src/encryption/targets.rs @@ -142,6 +142,16 @@ impl Target { self.plaintext.and_then(|name| name.parse().ok()) } + /// The PostgreSQL domain of the type's ordering operand at a path, for + /// a type that stores a JSON document (`EncryptedJson`: + /// `eql_v4.query_json_tree_leaf`); `None` for every other type. Stack + /// Encrypt binds an ordering at a path as it + /// (`TargetDescriptor::with_leaf_query_sql_domain`). + pub fn leaf_query_sql_domain(&self) -> Option<&'static str> { + use crate::v4::DomainType; + document(self.name).map(|_| crate::v4::json_tree::JsonTreeLeafQuery::sql_domain_static()) + } + /// The options this type's match index is built under, for a type that /// carries one (`TextMatch`): [`EqlMatch`](super::EqlMatch)'s, the one /// match configuration every EQL type's filter is built under. `None` diff --git a/packages/eql/tests/encryption/tests/json_tree_data_plan.rs b/packages/eql/tests/encryption/tests/json_tree_data_plan.rs index 0b348cb9a..a54ea32dc 100644 --- a/packages/eql/tests/encryption/tests/json_tree_data_plan.rs +++ b/packages/eql/tests/encryption/tests/json_tree_data_plan.rs @@ -36,7 +36,7 @@ use stack_encrypt::{CipherText, Encrypt, KeysetCipher, Label, NonEmpty, Pending, struct Eql; fn describe(target: &Target) -> TargetDescriptor { - TargetDescriptor::new( + let descriptor = TargetDescriptor::new( target.name, target.family, target.suffix, @@ -47,7 +47,11 @@ fn describe(target: &Target) -> TargetDescriptor { target.query_sql_domain.map(str::to_owned), target.producible, target.reason.map(str::to_owned), - ) + ); + match target.leaf_query_sql_domain() { + Some(domain) => descriptor.with_leaf_query_sql_domain(domain), + None => descriptor, + } } /// As the guest converts: a document refusal keeps its code. @@ -334,3 +338,613 @@ async fn text_that_is_not_a_document_is_refused_before_any_key() { "no key was requested" ); } + +// ============================================================================= +// Operators: the data path's operands against the typed chain's +// ============================================================================= + +/// A typed plan over the same column and options, as a Rust program +/// queries a document: the engine's document target under the label +/// `users/profile`, the same context as the EQL identifier. +fn typed_plan() -> stack_encrypt::plan::ValuePlan< + Value, + stack_encrypt::plan::Laid, +> { + stack_encrypt::Plan::value::() + .context("users/profile") + .encrypt_with_layout(stack_encrypt::target::Json::new(options())) + .build() + .unwrap() +} + +/// An operator's input as a binding writes it: an object of named inputs. +fn input(entries: &[(&str, Value)]) -> ValueFrame { + let object: serde_json::Map = entries + .iter() + .map(|(k, v)| ((*k).to_owned(), v.clone())) + .collect(); + ValueFrame::from_bytes(&dynamic::frame::serde::to_vec(&object).unwrap()).unwrap() +} + +/// The data path's term for one operator and input on `profile`. +async fn data_term( + keyset: &KeysetCipher<'_, common::Observed>, + plan: &record::Plan, + operator: &str, + entries: &[(&str, Value)], +) -> dynamic::QueryTerm { + dynamic::query_term( + keyset, + plan, + "profile", + dynamic::OpSpec::parse(operator).unwrap(), + input(entries), + &Eql, + ) + .unwrap() + .await + .unwrap() +} + +fn one(term: &dynamic::QueryTerm) -> Value { + let dynamic::Operand::One(bytes) = term.operand() else { + panic!("{}: one operand", term.operator()); + }; + serde_json::from_slice(bytes).unwrap() +} + +/// The operators a document field answers, with the input each takes: +/// the orderings at a path only when the field orders a path. +#[test] +fn a_document_field_answers_the_json_operators() { + let names = |plan: &record::Plan| -> Vec<(&'static str, &'static str)> { + dynamic::field_operators(plan, "profile", &Eql) + .unwrap() + .into_iter() + .map(|(op, index)| { + assert_eq!(index.key(), "json"); + (op.name(), op.input()) + }) + .collect() + }; + assert_eq!( + names(&plan()), + [ + ("contains", "json"), + ("contained_by", "json"), + ("eq_at", "path_json"), + ("lt_at", "path_json"), + ("lte_at", "path_json"), + ("gt_at", "path_json"), + ("gte_at", "path_json"), + ("between_at", "path_pair"), + ("selector", "path"), + ("subtree", "path_bound"), + ("children", "path"), + ("child_count", "path"), + ] + ); + let unordered = record::plan_with( + &dynamic::frame::serde::to_vec(&json!({ + "profile": {"context": ["users", "profile"], "target": "EncryptedJson"} + })) + .unwrap(), + &Eql, + ) + .unwrap(); + assert_eq!( + names(&unordered) + .into_iter() + .map(|(name, _)| name) + .collect::>(), + [ + "contains", + "contained_by", + "eq_at", + "selector", + "subtree", + "children", + "child_count" + ], + "no ordering without an ordered path" + ); +} + +/// Every operand the data path derives is the typed chain's, in the EQL +/// wire form, bound as the domain the target names for it. +#[tokio::test] +async fn every_operand_is_the_typed_chains() { + use stack_encrypt::op; + let (cipher, calls) = common::cipher().await; + let keyset = cipher.default_keyset(); + let plan = plan(); + let typed = typed_plan(); + let needle = json!({"visits": [{"at": 3}], "name": "Alice"}); + + for (operator, typed_op) in [("contains", true), ("contained_by", false)] { + let term = data_term( + &keyset, + &plan, + operator, + &[("json", json!(needle.to_string()))], + ) + .await; + let typed = if typed_op { + cipher + .query(op::Contains, &needle) + .on(&typed) + .await + .unwrap() + } else { + cipher + .query(op::ContainedBy, &needle) + .on(&typed) + .await + .unwrap() + }; + assert_eq!( + one(&term), + serde_json::to_value(&typed).unwrap(), + "{operator}" + ); + assert_eq!(term.domain(), Some("eql_v4.query_json_tree")); + assert_eq!(term.encoding(), "eql"); + } + + let eq = data_term( + &keyset, + &plan, + "eq_at", + &[("path", json!("$.name")), ("json", json!("\"Alice\""))], + ) + .await; + let typed_eq = cipher + .query(op::AtPath(op::Eq), &("$.name", "Alice")) + .on(&typed) + .await + .unwrap(); + assert_eq!(one(&eq), serde_json::to_value(&typed_eq).unwrap()); + assert_eq!(eq.domain(), Some("eql_v4.query_json_tree")); + + let lt = data_term( + &keyset, + &plan, + "lt_at", + &[("path", json!("$.age")), ("json", json!("40"))], + ) + .await; + let typed_lt = cipher + .query(op::AtPath(op::Lt), &("$.age", 40)) + .on(&typed) + .await + .unwrap(); + assert_eq!(one(<), serde_json::to_value(&typed_lt).unwrap()); + assert_eq!(lt.domain(), Some("eql_v4.query_json_tree_leaf")); + + let between = data_term( + &keyset, + &plan, + "between_at", + &[ + ("path", json!("$.age")), + ("low", json!("18")), + ("high", json!("40")), + ], + ) + .await; + let dynamic::Operand::Range { lo, hi } = between.operand() else { + panic!("a range"); + }; + let gte = data_term( + &keyset, + &plan, + "gte_at", + &[("path", json!("$.age")), ("json", json!("18"))], + ) + .await; + assert_eq!( + serde_json::from_slice::(lo).unwrap(), + one(>e), + "the low bound" + ); + assert_eq!( + serde_json::from_slice::(hi).unwrap(), + one(<), + "the high bound" + ); + assert_eq!(between.domain(), Some("eql_v4.query_json_tree_leaf")); + + let path = stack_encrypt::json::JsonPath::parse("$.visits[*].at").unwrap(); + let selector = data_term( + &keyset, + &plan, + "selector", + &[("path", json!("$.visits[*].at"))], + ) + .await; + assert_eq!( + selector.operand(), + &dynamic::Operand::Text( + keyset + .json_selector(&path, column()) + .await + .unwrap() + .to_string() + ) + ); + assert_eq!( + (selector.domain(), selector.encoding()), + (Some("text"), "text") + ); + + let subtree = data_term( + &keyset, + &plan, + "subtree", + &[ + ("path", json!("$.visits[*].at")), + ("wildcard_bound", json!(4)), + ], + ) + .await; + let key = keyset + .json_subtree_key_with_bound(&path, &options(), 4, column()) + .await + .unwrap(); + assert_eq!(subtree.operand(), &dynamic::Operand::TextList(key.tokens())); + assert_eq!(key.tokens().len(), 5, "four elements and a probe"); + assert_eq!( + (subtree.domain(), subtree.encoding()), + (Some("text[]"), "text_array") + ); + + let children = data_term(&keyset, &plan, "children", &[("path", json!("$.visits"))]).await; + let visits = keyset + .json_subtree_key( + &stack_encrypt::json::JsonPath::parse("$.visits").unwrap(), + &options(), + column(), + ) + .await + .unwrap(); + assert_eq!( + children.operand(), + &dynamic::Operand::Text(visits.tokens()[0].clone()) + ); + let count = data_term( + &keyset, + &plan, + "child_count", + &[("path", json!("$.visits"))], + ) + .await; + assert_eq!(count.operand(), children.operand()); + + assert!( + calls.lock().unwrap().generate.is_empty(), + "a query mints no key" + ); +} + +/// The refusals a JSON operator makes before it derives anything, each +/// naming the field and a reason or an engine error, never the input. +#[test] +fn a_json_operator_refuses_what_it_cannot_answer() { + let plan = plan(); + let check = |operator: &str, entries: &[(&str, Value)]| { + dynamic::check_query( + &plan, + "profile", + dynamic::OpSpec::parse(operator).unwrap(), + input(entries), + &Eql, + ) + }; + let reason = |operator: &str, entries: &[(&str, Value)]| { + let error = check(operator, entries).unwrap_err(); + assert!(!format!("{error}").contains("secret"), "{error}"); + match error.reason() { + // An input refusal names the field; the engine's own (a term + // error) is the same refusal the typed chain makes. + Some(reason) => { + assert_eq!(error.field(), Some("profile"), "{error}"); + reason.as_str() + } + None => "engine", + } + }; + use dynamic::Reason; + assert_eq!(reason("contains", &[("needle", json!("{}"))]), "json_input"); + assert_eq!( + reason("contains", &[("json", json!(r#"{"secret":1,"secret":1}"#))]), + "json_duplicate_key" + ); + assert_eq!( + reason("contains", &[("json", json!("{\"secret\""))]), + "json_syntax" + ); + assert_eq!( + reason("eq_at", &[("path", json!("$.a[2]")), ("json", json!("1"))]), + "json_path_position" + ); + assert_eq!( + reason("selector", &[("path", json!("$.a[0].b"))]), + "json_path_position" + ); + assert_eq!( + reason("children", &[("path", json!("$.a[*]"))]), + "json_path_wildcard" + ); + // The engine's own refusals: a container at a path, a path the field + // does not order, a value no ordering takes, a path that is not one. + for (operator, entries) in [ + ( + "eq_at", + vec![("path", json!("$.a")), ("json", json!(r#"{"secret":1}"#))], + ), + ( + "lt_at", + vec![("path", json!("$.name")), ("json", json!("1"))], + ), + ( + "lt_at", + vec![("path", json!("$.age")), ("json", json!("true"))], + ), + ( + "lt_at", + vec![("path", json!("$.visits[2].at")), ("json", json!("1"))], + ), + ("selector", vec![("path", json!("secret"))]), + ] { + assert_eq!(reason(operator, &entries), "engine", "{operator}"); + } + // `eq` and friends are not a document's operators. + let error = + dynamic::check_query(&plan, "profile", dynamic::OpSpec::Eq, input(&[]), &Eql).unwrap_err(); + assert_eq!(error.reason(), Some(Reason::OperatorNotAnswered)); +} + +// ============================================================================= +// Reads: rows opened against the request that prepared them +// ============================================================================= + +/// The engine's document under a stored EQL document: its entries, `h`, `sv` +/// and `q` without the envelope. +fn engine_document(stored: &JsonTreeDocument) -> stack_encrypt::json::JsonDocument { + let mut value = serde_json::to_value(stored).unwrap(); + let object = value.as_object_mut().unwrap(); + let _ = object.remove("v"); + let _ = object.remove("i"); + serde_json::from_value(value).unwrap() +} + +/// The rows `json_tree_subtree(doc, tokens, 0)` returns, as the SQL writes +/// them: each node with its header, ordinal and depth grafted. +fn subtree_rows(doc: &stack_encrypt::json::JsonDocument, tokens: &[String]) -> Vec { + use base64::Engine; + let tokens: Vec<[u8; 16]> = tokens + .iter() + .map(|t| { + base64::engine::general_purpose::STANDARD + .decode(t) + .unwrap() + .try_into() + .unwrap() + }) + .collect(); + doc.subtree(&tokens) + .into_iter() + .map(|(j, _, d)| { + let mut wire = serde_json::to_value(doc.node(j).unwrap()).unwrap(); + wire["d"] = json!(d); + wire.to_string() + }) + .collect() +} + +/// The entry `doc -> selector` returns: the first entry carrying it. +fn extracted_row(doc: &stack_encrypt::json::JsonDocument, selector: &str) -> Option { + use base64::Engine; + let selector: [u8; 16] = base64::engine::general_purpose::STANDARD + .decode(selector) + .unwrap() + .try_into() + .unwrap(); + let j = doc + .entries() + .iter() + .position(|e| e.selector() == &selector)?; + Some( + serde_json::to_value(doc.node(j).unwrap()) + .unwrap() + .to_string(), + ) +} + +async fn open( + keyset: &KeysetCipher<'_, common::Observed>, + plan: &record::Plan, + request: Value, + groups: Vec>, +) -> Result>, stack_encrypt::Error> { + let request = + ValueFrame::from_bytes(&dynamic::frame::serde::to_vec(&request).unwrap()).unwrap(); + let rows = frame(&groups); + let opened = dynamic::json_open( + keyset, + Scope::Keyset(keyset.clone()), + plan, + "profile", + request, + rows, + &Eql, + ) + .unwrap() + .await?; + let groups: Vec> = + dynamic::frame::serde::from_slice(opened.risky_ref()).unwrap(); + Ok(groups + .into_iter() + .map(|g| { + g.into_iter() + .map(|b| serde_json::from_slice(&b.0).unwrap()) + .collect() + }) + .collect()) +} + +/// Two documents, read through the data path: `->` by selector, a concrete +/// subtree, a wildcard subtree, and a row with no node there, in one key +/// request; each group of rows opens on its own. +#[tokio::test] +async fn reads_open_against_the_request_that_prepared_them() { + let (cipher, calls) = common::cipher().await; + let keyset = cipher.default_keyset(); + let plan = plan(); + let documents = [ + json!({"name": "Alice", "age": 34, "visits": [{"at": 3}, {"at": 1, "where": "x"}]}), + json!({"name": "Bob", "visits": []}), + ]; + let mut stored = Vec::new(); + for document in &documents { + let text = document.to_string(); + let sealed = record::encrypt_with( + &keyset, + frame(BTreeMap::from([("profile", text.as_str())])), + &plan, + &Eql, + ) + .unwrap() + .await + .unwrap(); + stored.push(engine_document(&stored_of(&sealed))); + } + + // `->` on `$.name` in each row, and a row with no `$.age` (NULL). + let name = data_term(&keyset, &plan, "selector", &[("path", json!("$.name"))]).await; + let dynamic::Operand::Text(name) = name.operand() else { + panic!() + }; + let age = data_term(&keyset, &plan, "selector", &[("path", json!("$.age"))]).await; + let dynamic::Operand::Text(age) = age.operand() else { + panic!() + }; + calls.lock().unwrap().retrieve.clear(); + let names = open( + &keyset, + &plan, + json!({"operation": "selector", "path": "$.name"}), + stored + .iter() + .map(|d| extracted_row(d, name).into_iter().collect()) + .collect(), + ) + .await + .unwrap(); + assert_eq!(names, [vec![json!("Alice")], vec![json!("Bob")]]); + assert_eq!( + calls.lock().unwrap().retrieve.len(), + 1, + "one key request for every row" + ); + let ages = open( + &keyset, + &plan, + json!({"operation": "selector", "path": "$.age"}), + stored + .iter() + .map(|d| extracted_row(d, age).into_iter().collect()) + .collect(), + ) + .await + .unwrap(); + assert_eq!( + ages, + [vec![json!(34)], vec![]], + "absence is no value, not null" + ); + + // A wildcard read: every element's `at` below the bound, per row. + let tokens = data_term( + &keyset, + &plan, + "subtree", + &[("path", json!("$.visits[*].at"))], + ) + .await; + let dynamic::Operand::TextList(tokens) = tokens.operand() else { + panic!() + }; + let ats = open( + &keyset, + &plan, + json!({"operation": "subtree", "path": "$.visits[*].at"}), + stored.iter().map(|d| subtree_rows(d, tokens)).collect(), + ) + .await + .unwrap(); + assert_eq!(ats, [vec![json!(3), json!(1)], vec![]]); + + // The rows of one read opened against another's request are refused: + // the request, not the rows, says which path they are for. + let wrong = open( + &keyset, + &plan, + json!({"operation": "subtree", "path": "$.name"}), + vec![subtree_rows(&stored[0], tokens)], + ) + .await; + assert!( + matches!(wrong, Err(stack_encrypt::Error::JsonWrongEntry { .. })), + "{:?}", + wrong.err() + ); + let wrong = open( + &keyset, + &plan, + json!({"operation": "selector", "path": "$.age"}), + vec![extracted_row(&stored[0], name).into_iter().collect()], + ) + .await; + assert!( + matches!(wrong, Err(stack_encrypt::Error::JsonWrongEntry { .. })), + "{:?}", + wrong.err() + ); + + // A wildcard read of an array longer than its bound fails rather than + // opening to part of the answer. + let short = data_term( + &keyset, + &plan, + "subtree", + &[ + ("path", json!("$.visits[*].at")), + ("wildcard_bound", json!(1)), + ], + ) + .await; + let dynamic::Operand::TextList(short) = short.operand() else { + panic!() + }; + let truncated = open( + &keyset, + &plan, + json!({"operation": "subtree", "path": "$.visits[*].at", "wildcard_bound": 1}), + vec![subtree_rows(&stored[0], short)], + ) + .await; + assert!( + matches!( + truncated, + Err(stack_encrypt::Error::JsonWildcardTruncated { .. }) + ), + "{:?}", + truncated.err() + ); +} + +/// The stored document a sealed row holds for `profile`. +fn stored_of(row: &stack_encrypt::StackCipherText) -> JsonTreeDocument { + stored(row) +} diff --git a/packages/eql/tests/encryption/tests/typed_queries.rs b/packages/eql/tests/encryption/tests/typed_queries.rs index b29d939d5..fbce6687f 100644 --- a/packages/eql/tests/encryption/tests/typed_queries.rs +++ b/packages/eql/tests/encryption/tests/typed_queries.rs @@ -290,7 +290,13 @@ async fn typed_and_input( #[test] fn the_typed_chain_answers_what_the_data_path_answers() { let mut gap = Vec::new(); - for target in eql_targets::targets() { + // A document's typed chain is the engine's own document target, not + // `op::Answers` on the EQL type: tests/json_tree_data_plan.rs holds the + // two to the same operands. + for target in eql_targets::targets() + .iter() + .filter(|t| t.plaintext != Some("json")) + { let data: Vec<&str> = describe(target) .operators() .into_iter() diff --git a/packages/stack-encrypt/CHANGELOG.md b/packages/stack-encrypt/CHANGELOG.md index e7f4a9614..1590fc7f6 100644 --- a/packages/stack-encrypt/CHANGELOG.md +++ b/packages/stack-encrypt/CHANGELOG.md @@ -9,6 +9,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Breaking +- **`OpSpec`, `Operand` and the query term's wire form grow for documents** + (#1350). `OpSpec` gains twelve operators and `Operand` gains `Text` and + `TextList`; a binding that matches on either adds the arms. The query + term an `se_query_term`-style export writes has a sixth key, + `encoding` (`bytes`, `eql`, `text`, `text_array`), and an operator entry + a fifth, `input` (`value`, `pair`, `json`, `path_json`, `path_pair`, + `path`, `path_bound`); a reader that checks every key reads the new + ones. `TargetDescriptor` has a `leaf_query_sql_domain`, set with + `with_leaf_query_sql_domain`. + - **`TargetResolver::encrypt` takes the field's target options** (#1349): `encrypt(name, options: Option<&TargetOptions>, keyset, label, plaintext)`. A resolver implemented outside this crate adds the @@ -31,6 +41,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **A document field's queries and reads through a data plan** (#1350, + #1352). A field naming `EncryptedJson` answers `contains`, + `contained_by`, `eq_at`, `selector`, `subtree`, `children` and + `child_count`, and, when it orders a path, `lt_at`, `lte_at`, `gt_at`, + `gte_at` and `between_at`, each taking an object of named inputs + (`{"json"}`, `{"path", "json"}`, `{"path", "low", "high"}`, + `{"path"}`, `{"path", "wildcard_bound"}`; JSON values as JSON text). + `query_term` derives the typed chain's operands, bit for bit: the + containment needle and the leaf ordering operand as EQL JSON (bound as + `eql_v4.query_json_tree` and `eql_v4.query_json_tree_leaf`), the + selector as `text` and a read's tokens as `text[]`. A position in a + path is refused where the operand is a selector (`json_path_position`), + a wildcard where the operator names one node (`json_path_wildcard`). + `json_open` opens the rows a read returned against the request that + prepared it: the engine derives the selector or the read key again from + the request, the field's options and the keyset, checks every row with + the index's own checks, and opens each database row's group on its own, + in one key request. `check_json_open` is its cipher-free check. + - **A JSON document through a data plan** (#1349). A plan field may declare the new `json` value kind and name a target that stores a document, `EncryptedJson`, with the JSON index's options: diff --git a/packages/stack-encrypt/src/dynamic/json_query.rs b/packages/stack-encrypt/src/dynamic/json_query.rs new file mode 100644 index 000000000..3a81e51d8 --- /dev/null +++ b/packages/stack-encrypt/src/dynamic/json_query.rs @@ -0,0 +1,525 @@ +//! The JSON operators of a document field, for a binding: queries named by +//! operator, and reads opened against the request that prepared them. +//! +//! A field that names a target storing a document (`EncryptedJson`) runs +//! under the JSON index's options its plan resolved +//! ([`TargetOptions::Json`](super::TargetOptions::Json)). Each operator +//! takes an object of named inputs and gives a tagged operand +//! ([`Operand`]): +//! +//! | operator | input | operand | +//! |---|---|---| +//! | `contains`, `contained_by` | `{"json"}`: a JSON needle | EQL: the needle's projection (`eql_v4.query_json_tree`) | +//! | `eq_at` | `{"path", "json"}`: a scalar at a path | EQL: the needle holding the scalar there | +//! | `lt_at`, `lte_at`, `gt_at`, `gte_at` | `{"path", "json"}`: a number or string at an ordered path | EQL: the path's selector and the leaf's ordering term (`eql_v4.query_json_tree_leaf`) | +//! | `between_at` | `{"path", "low", "high"}` | EQL, two of them: the low bound's then the high's | +//! | `selector` | `{"path"}` | text: the selector `->` takes | +//! | `subtree` | `{"path", "wildcard_bound"?}` | text array: the tokens `json_tree_subtree` takes | +//! | `children`, `child_count` | `{"path"}`: a path with no wildcard | text: the node's token | +//! +//! Every JSON value in an input is UTF-8 JSON text, parsed by +//! [`parse_text`](crate::json::parse_text)'s rules. A path is JSONPath text +//! ([`JsonPath::parse`]). A position in a path (`[2]`) is refused for the +//! operators whose operand is a selector (`eq_at`, the orderings, +//! `selector`): a selector reads every position as the wildcard, so the +//! query would compare whichever element the database found first. A +//! positional read is a `subtree` read. Equality at a wildcard path matches +//! when any element matches; an ordering or `->` through one compares the +//! first. Two `eq_at` predicates on one row need not hold of the same +//! array element; one containment needle holding both does. +//! +//! # Reads +//! +//! [`json_open`] opens the rows a read returned, against the request that +//! prepared the read: the operator, the path and the bound. The engine +//! derives the selector or the read key again from that request, the +//! field's definition and the keyset, and checks every row against it with +//! the index's own checks ([`JsonSelector::open`], +//! [`JsonSubtreeKey::open`](crate::json::JsonSubtreeKey::open)). The rows +//! never say which path they are for. Each group of rows is one database +//! row's, and opens on its own: entries of two documents are never one walk. +//! +//! What opening proves is limited, as the index documents: a row of the +//! column other than the one asked for can answer, an element under one +//! array element can stand for the same field under another, and the +//! database can leave whole nodes out. Opening does not prove the result is +//! complete. + +use serde::de::{self, Deserializer, Visitor}; +use serde::Deserialize; +use serde_json::Value; +use vitaminc_protected::Protected; +use zeroize::Zeroizing; + +use super::query::{OpSpec, Operand}; +use super::{Error, Reason, Scope, ValueFrame}; +use crate::json::{ + parse_text, wipe_value, JsonExtracted, JsonLeafQuery, JsonNode, JsonOptions, JsonPath, + JsonQuery, JsonSubtree, JsonSubtreeKey, +}; +use crate::sem::TermError; +use crate::target::{AeadContext, DeclaredContext, Index, Json, Owned}; +use crate::{KeysetCipher, Label, NonEmpty, Pending}; + +/// The SQL type a JSON operand binds as. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Binds { + /// The target's needle domain (`eql_v4.query_json_tree`). + Needle, + /// The target's leaf operand domain (`eql_v4.query_json_tree_leaf`). + Leaf, + /// `text`. + Text, + /// `text[]`. + TextArray, +} + +/// What a JSON operator's operand binds as. +pub(super) fn binds(op: OpSpec) -> Binds { + match op { + OpSpec::LtAt | OpSpec::LteAt | OpSpec::GtAt | OpSpec::GteAt | OpSpec::BetweenAt => { + Binds::Leaf + } + OpSpec::Selector | OpSpec::Children | OpSpec::ChildCount => Binds::Text, + OpSpec::Subtree => Binds::TextArray, + _ => Binds::Needle, + } +} + +/// UTF-8 JSON text in an input: a text or bytes leaf, held in a buffer +/// wiped on drop. +struct JsonText(Zeroizing>); + +impl<'de> Deserialize<'de> for JsonText { + fn deserialize>(d: D) -> Result { + struct V; + impl Visitor<'_> for V { + type Value = JsonText; + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("JSON text, as text or bytes") + } + fn visit_str(self, v: &str) -> Result { + Ok(JsonText(Zeroizing::new(v.as_bytes().to_vec()))) + } + fn visit_bytes(self, v: &[u8]) -> Result { + Ok(JsonText(Zeroizing::new(v.to_vec()))) + } + } + d.deserialize_any(V) + } +} + +/// A document parsed from an input, wiped when it drops. +struct Parsed(Value); + +impl Drop for Parsed { + fn drop(&mut self) { + wipe_value(&mut self.0); + } +} + +impl JsonText { + fn parse(&self) -> Result { + parse_text(&self.0) + .map(Parsed) + .map_err(|error| Error::Source { + field: None, + reason: json_reason(error.code()), + }) + } +} + +/// The reason a JSON text refusal is reported with, from its code. +fn json_reason(code: &str) -> Reason { + match code { + "json_duplicate_key" => Reason::JsonDuplicateKey, + "json_integer_range" => Reason::JsonIntegerRange, + _ => Reason::JsonSyntax, + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct NeedleInput { + json: JsonText, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct AtInput { + path: String, + json: JsonText, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct BetweenInput { + path: String, + low: JsonText, + high: JsonText, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct PathInput { + path: String, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ReadInput { + path: String, + #[serde(default)] + wildcard_bound: Option, +} + +/// An operator's input, read from its frame and checked: everything +/// [`derive`] would refuse before it derives anything. +enum Input { + Needle(Parsed), + At(JsonPath, Parsed), + Between(JsonPath, Parsed, Parsed), + Path(JsonPath), + Read(JsonPath, u32), +} + +fn read_frame Deserialize<'de>>(input: &ValueFrame) -> Result { + vitaminc_aead_frame::serde::from_slice(input.risky_ref()).map_err(|_| Error::Source { + field: None, + reason: Reason::JsonInput, + }) +} + +fn path(text: &str) -> Result { + JsonPath::parse(text).map_err(|error| Error::Cipher(TermError::JsonPath(error).into())) +} + +/// A path whose operand is a selector, which reads a position as the +/// wildcard. +fn selector_path(text: &str) -> Result { + let path = path(text)?; + if path.names_a_position() { + return Err(Error::bad_source(Reason::JsonPathPosition)); + } + Ok(path) +} + +/// A path that names one node: no wildcard. +fn node_path(text: &str) -> Result { + let path = path(text)?; + if path.is_positional() { + Ok(path) + } else { + Err(Error::bad_source(Reason::JsonPathWildcard)) + } +} + +fn term(error: TermError) -> Error { + Error::Cipher(error.into()) +} + +/// Read and check an operator's input against the field's options. +fn read_input(op: OpSpec, options: &JsonOptions, input: &ValueFrame) -> Result { + Ok(match op { + OpSpec::Contains | OpSpec::ContainedBy => { + Input::Needle(read_frame::(input)?.json.parse()?) + } + OpSpec::EqAt => { + let at: AtInput = read_frame(input)?; + let path = selector_path(&at.path)?; + let value = at.json.parse()?; + if matches!(value.0, Value::Object(_) | Value::Array(_)) { + return Err(term(TermError::JsonValueNotScalar)); + } + Input::At(path, value) + } + OpSpec::LtAt | OpSpec::LteAt | OpSpec::GtAt | OpSpec::GteAt => { + let at: AtInput = read_frame(input)?; + let path = ordered_path(options, &at.path)?; + let value = at.json.parse()?; + leaf(&value.0)?; + Input::At(path, value) + } + OpSpec::BetweenAt => { + let between: BetweenInput = read_frame(input)?; + let path = ordered_path(options, &between.path)?; + let (low, high) = (between.low.parse()?, between.high.parse()?); + leaf(&low.0)?; + leaf(&high.0)?; + Input::Between(path, low, high) + } + OpSpec::Selector => Input::Path(selector_path(&read_frame::(input)?.path)?), + OpSpec::Children | OpSpec::ChildCount => { + Input::Path(node_path(&read_frame::(input)?.path)?) + } + OpSpec::Subtree => { + let read: ReadInput = read_frame(input)?; + Input::Read( + path(&read.path)?, + read.wildcard_bound.unwrap_or(JsonSubtreeKey::DEFAULT_BOUND), + ) + } + _ => return Err(Error::Internal), + }) +} + +/// A path the index orders, which names no position. +fn ordered_path(options: &JsonOptions, text: &str) -> Result { + let path = path(text)?; + if path.names_a_position() { + Err(term(TermError::JsonOrderingAtPosition)) + } else if options.orders(&path) { + Ok(path) + } else { + Err(term(TermError::JsonPathNotOrdered)) + } +} + +/// A leaf the index orders: a number or a string. +fn leaf(value: &Value) -> Result<(), Error> { + match value { + Value::Number(_) | Value::String(_) => Ok(()), + _ => Err(term(TermError::JsonLeafNotOrdered)), + } +} + +/// Check a JSON operator's input without a cipher: what [`derive`] +/// refuses before it derives anything. +pub(super) fn check(op: OpSpec, options: &JsonOptions, input: &ValueFrame) -> Result<(), Error> { + read_input(op, options, input).map(drop) +} + +/// The JSON index's terms of `value` under the column, as the stored +/// document's were derived. +fn terms<'a, K: 'static>( + keyset: &'a KeysetCipher<'_, K>, + options: &JsonOptions, + label: &Label, + value: Value, +) -> Pending<'a, crate::json::JsonTerms, K> { + keyset.run( + Index::::operation::(&Json::new(options.clone())) + .under(NonEmpty::from(label.clone())), + value, + DeclaredContext::default(), + ) +} + +fn eql(value: &T) -> Result, crate::Error> { + serde_json::to_vec(value).map_err(|error| crate::Error::Other(Box::new(error))) +} + +/// Derive a JSON operator's operand for one input, under the field's +/// column and options. Derived locally: the pending carries no request. +pub(super) fn derive<'a, K: 'static>( + keyset: &'a KeysetCipher<'_, K>, + label: &Label, + options: &JsonOptions, + op: OpSpec, + input: &ValueFrame, +) -> Result, Error> { + let column = || NonEmpty::from(label.clone()); + Ok(match read_input(op, options, input)? { + Input::Needle(needle) => terms(keyset, options, label, needle.0.clone()) + .try_map(|terms| eql(&JsonQuery::of_terms(&terms)).map(Operand::One)), + Input::At(path, value) if op == OpSpec::EqAt => { + terms(keyset, options, label, path.needle(value.0.clone())) + .try_map(|terms| eql(&JsonQuery::at_path(&terms)).map(Operand::One)) + } + Input::At(path, value) => terms(keyset, options, label, path.needle(value.0.clone())) + .try_map(|terms| eql(&JsonLeafQuery::of_leaf(&terms)).map(Operand::One)), + Input::Between(path, low, high) => { + terms(keyset, options, label, path.needle(low.0.clone())) + .zip(terms(keyset, options, label, path.needle(high.0.clone()))) + .try_map(|(lo, hi)| { + Ok(Operand::Range { + lo: eql(&JsonLeafQuery::of_leaf(&lo))?, + hi: eql(&JsonLeafQuery::of_leaf(&hi))?, + }) + }) + } + Input::Path(path) if op == OpSpec::Selector => { + let selector = keyset.json_selector_now(&path, column())?; + Pending::ready(keyset, Ok(Operand::Text(selector.to_string()))) + } + Input::Path(path) => { + // A path with no wildcard names one node: its key holds the + // one token, and nothing that covers a node above it. + let key = keyset.json_subtree_key_now( + &path, + options, + JsonSubtreeKey::DEFAULT_BOUND, + column(), + )?; + let [token]: [String; 1] = key.tokens().try_into().map_err(|_| Error::Internal)?; + Pending::ready(keyset, Ok(Operand::Text(token))) + } + Input::Read(path, bound) => { + let key = keyset.json_subtree_key_now(&path, options, bound, column())?; + Pending::ready(keyset, Ok(Operand::TextList(key.tokens()))) + } + }) +} + +/// A read to open, as the request that prepared it names it: the +/// operator, the path and, for a subtree read, the wildcard bound. What a +/// binding keeps from preparing the read and hands back with its rows; the +/// engine derives the selector or the key again from it. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ReadRequest { + operation: String, + path: String, + #[serde(default)] + wildcard_bound: Option, +} + +/// The groups of rows a read returned: one list of entries per database +/// row, each entry the `public.eql_v4_json_tree_entry` JSON the SQL +/// returned, as text or bytes. +#[derive(Deserialize)] +#[serde(transparent)] +struct Groups(Vec>); + +/// A read as its request names it, checked. +enum Read { + Selector(JsonPath), + Subtree(JsonPath, u32), +} + +/// Read and check a read's request and its rows: everything [`open`] +/// refuses before it derives anything. A selector's group holds one entry +/// or none. +fn read_open(request: &ValueFrame, rows: &ValueFrame) -> Result<(Read, Vec>), Error> { + let request: ReadRequest = read_frame(request)?; + let read = match OpSpec::parse(&request.operation) { + Some(OpSpec::Selector) if request.wildcard_bound.is_none() => { + Read::Selector(selector_path(&request.path)?) + } + Some(OpSpec::Subtree) => Read::Subtree( + path(&request.path)?, + request + .wildcard_bound + .unwrap_or(JsonSubtreeKey::DEFAULT_BOUND), + ), + _ => return Err(Error::bad_source(Reason::JsonInput)), + }; + let groups: Groups = read_frame(rows)?; + let groups = groups + .0 + .into_iter() + .map(|group| { + group + .iter() + .map(|row| serde_json::from_slice::(&row.0)) + .collect::, _>>() + .map_err(|_| Error::bad_source(Reason::JsonInput)) + }) + .collect::, _>>()?; + if matches!(read, Read::Selector(_)) && groups.iter().any(|group| group.len() > 1) { + return Err(Error::bad_source(Reason::JsonInput)); + } + Ok((read, groups)) +} + +/// Check a read's request and rows without a cipher: what [`open`] refuses +/// before it derives anything. +pub(super) fn check_open(request: &ValueFrame, rows: &ValueFrame) -> Result<(), Error> { + read_open(request, rows).map(drop) +} + +/// What a read opens with: the selector or the subtree key, derived again +/// from the request. +enum Opener { + Selector(crate::json::JsonSelector), + Subtree(JsonSubtreeKey), +} + +/// Open the rows a read returned against the request that prepared it, +/// under the field's column and options and through `keyset`, the keyset +/// whose index key derived the read; the rows open through `scope`. Each +/// group of rows is one database row's: for `selector`, the entry `->` +/// returned or none (SQL `NULL`); for `subtree`, the rows +/// `json_tree_subtree` returned. Every group opens to a list of JSON +/// texts: for `selector`, the scalar or none, and for `subtree`, one value +/// per concrete path the read named that the row has, in document order. +/// One key request for every group. +/// +/// # Errors +/// +/// [`Error::Source`] for a request or rows that do not parse +/// ([`Reason::JsonInput`]), an operator that is not a read +/// ([`Reason::OperatorNotAnswered`]), or a selector group of more than one +/// entry; [`Error::Cipher`] for the read's own refusals (a path that does +/// not parse, a wildcard that expands too far). A row that is not the one +/// the read asked for fails the pending +/// ([`crate::Error::JsonWrongEntry`]), as does a wildcard read of an array +/// longer than the bound ([`crate::Error::JsonWildcardTruncated`]). +pub(super) fn open<'a, K: 'static>( + keyset: &'a KeysetCipher<'_, K>, + scope: Scope<'a, K>, + label: &Label, + options: &JsonOptions, + request: &ValueFrame, + rows: &ValueFrame, +) -> Result, Error> { + let (read, groups) = read_open(request, rows)?; + let column = || NonEmpty::from(label.clone()); + let opener = match read { + Read::Selector(path) => Opener::Selector(keyset.json_selector_now(&path, column())?), + Read::Subtree(path, bound) => { + Opener::Subtree(keyset.json_subtree_key_now(&path, options, bound, column())?) + } + }; + let context = || AeadContext::from(NonEmpty::from(label.clone())); + // Opened through the client, confined to the scope's keyset when it is + // one, as a record's leaves are: a document sealed under another + // keyset is refused before any key is retrieved. + let (opener_cipher, confine) = match &scope { + Scope::Client(cipher) => (*cipher, None), + Scope::Keyset(keyset) => (keyset.cipher(), Some(keyset.keyset_id())), + }; + let confined = |pending: Pending<'a, Vec, K>| match confine { + Some(keyset) => pending.scoped_to(keyset), + None => pending, + }; + let mut pendings: Vec, K>> = Vec::with_capacity(groups.len()); + for nodes in groups { + pendings.push(match &opener { + Opener::Selector(selector) => match <[JsonNode; 1]>::try_from(nodes) { + Ok([node]) => confined( + opener_cipher + .decrypt_as::(selector.open(node), context()) + .map(|value| vec![value]), + ), + // `read_open` allows a selector group one entry or none. + Err(_) => Pending::ready(keyset, Ok(Vec::new())), + }, + Opener::Subtree(key) => confined( + opener_cipher.decrypt_as::, JsonSubtree>(key.open(nodes), context()), + ), + }); + } + Ok(Pending::all(keyset, pendings).try_map(opened_frame)) +} + +/// The opened groups as a frame: a list per group of each value's JSON +/// text, as bytes. Each value is wiped once its text is written, and each +/// text once the frame holds it. +fn opened_frame(mut groups: Vec>) -> Result { + let texts: Result>>>, crate::Error> = groups + .iter() + .map(|group| { + group + .iter() + .map(|value| { + crate::json::to_text(value) + .map(Protected::new) + .map_err(|error| crate::Error::Other(Box::new(error))) + }) + .collect() + }) + .collect(); + groups.iter_mut().flatten().for_each(wipe_value); + ValueFrame::of(&texts?).map_err(|error| crate::Error::Other(Box::new(error))) +} diff --git a/packages/stack-encrypt/src/dynamic/mod.rs b/packages/stack-encrypt/src/dynamic/mod.rs index 1f82a9bb5..cb73b96ef 100644 --- a/packages/stack-encrypt/src/dynamic/mod.rs +++ b/packages/stack-encrypt/src/dynamic/mod.rs @@ -89,6 +89,7 @@ //! reason, given on the type. mod context; mod json; +mod json_query; mod kind; mod query; pub mod record; @@ -107,8 +108,8 @@ pub use json::{json_frame, read_json, JsonPlaintext, JsonValueError}; /// a kind means to this engine. pub use kind::{admits, not_admitted, read, ParseValueKindError, ValueKind}; pub use query::{ - check_query, field_operators, query_term, target_limit, verify, Fidelity, OpSpec, Operand, - OperatorEntry, QueryTerm, + check_json_open, check_query, field_operators, json_open, query_term, target_limit, verify, + Fidelity, OpSpec, Operand, OperatorEntry, QueryTerm, }; pub use record::{FieldPlan, NullMode, Output, Plan}; pub use target::{NoTargets, TargetDescriptor, TargetError, TargetOptions, TargetResolver}; @@ -618,6 +619,25 @@ reasons! { /// Statistics are not a row count with an optional histogram and /// confidence ([`Stats::from_frame`](crate::op::Stats::from_frame)). NotStats => "not_stats", "the statistics are not a row count with an optional histogram and confidence"; + /// A JSON operator's input or a read's request or rows are not the + /// object the operator takes ([`json_open`], the JSON operators of + /// [`OpSpec`]): a missing or unknown key, a value of another kind, + /// a row that is not an entry, or more than one entry for a + /// selector. + JsonInput => "json_input", "the input is not the object the JSON operator takes"; + /// A JSON value in an input is not JSON text the engine reads. + JsonSyntax => "json_syntax", "a JSON value is not JSON text"; + /// A JSON value in an input names an object member twice. + JsonDuplicateKey => "json_duplicate_key", "a JSON object names a member twice"; + /// A JSON value in an input has an integer literal outside the 64-bit + /// range. + JsonIntegerRange => "json_integer_range", "a JSON integer literal is outside the 64-bit range"; + /// A path names an array position where the operand is a selector, + /// which reads every position as the wildcard: read a position with + /// a subtree read. + JsonPathPosition => "json_path_position", "a selector reads a position as the wildcard; read a position with subtree"; + /// A path has a wildcard where the operator names one node. + JsonPathWildcard => "json_path_wildcard", "the operator names one node, and a wildcard names many"; /// `"null"` is not `"sealed"` or `"sql"`. UnknownNull => "unknown_null", r#""null" is not "sealed" or "sql""#; /// A passthrough field says how to store a null: it carries one as diff --git a/packages/stack-encrypt/src/dynamic/query.rs b/packages/stack-encrypt/src/dynamic/query.rs index 224c49023..c76c9a162 100644 --- a/packages/stack-encrypt/src/dynamic/query.rs +++ b/packages/stack-encrypt/src/dynamic/query.rs @@ -83,7 +83,10 @@ use super::term::OneScalar; use super::term::{derive, Use}; use super::value_frame::Parts; use super::ValueKind; -use super::{admits, read, Error, Reason, TargetDescriptor, TargetResolver, ValueFrame}; +use super::{ + admits, read, Error, Reason, Scope, TargetDescriptor, TargetError, TargetOptions, + TargetResolver, ValueFrame, +}; use crate::op::{Limit, Stats}; use crate::sem::{match_limit, MatchOptions, QueryTokens, TermError}; use crate::target::IndexSpec; @@ -116,12 +119,46 @@ pub enum OpSpec { /// `"matches"`: [`op::Matches`](struct@crate::op::Matches), in the case /// the field's match index fixes. Matches, + /// `"contains"`: [`op::Contains`](crate::op::Contains), a stored + /// document containing a JSON needle (`@>`). Answered by the JSON index + /// of a field that stores a document; see [`json_query`](super::json_query) + /// for the inputs and operands of this and the operators below. + Contains, + /// `"contained_by"`: [`op::ContainedBy`](crate::op::ContainedBy), a + /// stored document contained by a JSON needle (`<@`). + ContainedBy, + /// `"eq_at"`: [`op::AtPath`](crate::op::AtPath)`(op::Eq)`, a scalar at + /// a path. + EqAt, + /// `"lt_at"`: `op::AtPath(op::Lt)`, at a path the index orders. + LtAt, + /// `"lte_at"`: `op::AtPath(op::Lte)`. + LteAt, + /// `"gt_at"`: `op::AtPath(op::Gt)`. + GtAt, + /// `"gte_at"`: `op::AtPath(op::Gte)`. + GteAt, + /// `"between_at"`: `op::AtPath(op::Between)`, two bounds, inclusive. + BetweenAt, + /// `"selector"`: the selector of a path, which `->` and the path + /// functions take ([`KeysetCipher::json_selector`](crate::KeysetCipher::json_selector)). + Selector, + /// `"subtree"`: the tokens of a read of the node at a path, wildcards + /// expanded to a bound, which `json_tree_subtree` takes + /// ([`KeysetCipher::json_subtree_key_with_bound`](crate::KeysetCipher::json_subtree_key_with_bound)). + Subtree, + /// `"children"`: the token of the node at a path with no wildcard, + /// which `json_tree_children` takes. + Children, + /// `"child_count"`: the same token, which `json_tree_child_count` + /// takes. + ChildCount, } impl OpSpec { /// Every operator, in declaration order: the order a catalog lists them /// in. - pub const ALL: [OpSpec; 7] = [ + pub const ALL: [OpSpec; 19] = [ OpSpec::Eq, OpSpec::Lt, OpSpec::Lte, @@ -129,6 +166,18 @@ impl OpSpec { OpSpec::Gte, OpSpec::Between, OpSpec::Matches, + OpSpec::Contains, + OpSpec::ContainedBy, + OpSpec::EqAt, + OpSpec::LtAt, + OpSpec::LteAt, + OpSpec::GtAt, + OpSpec::GteAt, + OpSpec::BetweenAt, + OpSpec::Selector, + OpSpec::Subtree, + OpSpec::Children, + OpSpec::ChildCount, ]; /// The operator a name names, or `None` for a name that is not one. @@ -147,6 +196,60 @@ impl OpSpec { OpSpec::Gte => "gte", OpSpec::Between => "between", OpSpec::Matches => "matches", + OpSpec::Contains => "contains", + OpSpec::ContainedBy => "contained_by", + OpSpec::EqAt => "eq_at", + OpSpec::LtAt => "lt_at", + OpSpec::LteAt => "lte_at", + OpSpec::GtAt => "gt_at", + OpSpec::GteAt => "gte_at", + OpSpec::BetweenAt => "between_at", + OpSpec::Selector => "selector", + OpSpec::Subtree => "subtree", + OpSpec::Children => "children", + OpSpec::ChildCount => "child_count", + } + } + + /// Whether the JSON index answers this operator: the document + /// operators, whose input is an object of named JSON inputs + /// ([`json_query`](super::json_query)). + pub fn is_json(self) -> bool { + matches!( + self, + OpSpec::Contains + | OpSpec::ContainedBy + | OpSpec::EqAt + | OpSpec::LtAt + | OpSpec::LteAt + | OpSpec::GtAt + | OpSpec::GteAt + | OpSpec::BetweenAt + | OpSpec::Selector + | OpSpec::Subtree + | OpSpec::Children + | OpSpec::ChildCount + ) + } + + /// The shape of the operator's input, as a catalog names it: `value` + /// (one value of the field's kind), `pair` (two, low then high), + /// `json` (`{"json"}`), `path_json` (`{"path", "json"}`), `path_pair` + /// (`{"path", "low", "high"}`), `path` (`{"path"}`) or `path_bound` + /// (`{"path", "wildcard_bound"?}`). + pub fn input(self) -> &'static str { + match self { + OpSpec::Between => "pair", + OpSpec::Contains | OpSpec::ContainedBy => "json", + OpSpec::EqAt | OpSpec::LtAt | OpSpec::LteAt | OpSpec::GtAt | OpSpec::GteAt => { + "path_json" + } + OpSpec::BetweenAt => "path_pair", + OpSpec::Selector | OpSpec::Children | OpSpec::ChildCount => "path", + OpSpec::Subtree => "path_bound", + OpSpec::Eq | OpSpec::Lt | OpSpec::Lte | OpSpec::Gt | OpSpec::Gte | OpSpec::Matches => { + "value" + } } } @@ -162,7 +265,7 @@ impl OpSpec { /// Whether the operator takes a pair of values, low then high, rather /// than one: `between` does. pub fn takes_pair(self) -> bool { - self == OpSpec::Between + matches!(self, OpSpec::Between | OpSpec::BetweenAt) } /// The index among `indexes` that answers this operator for a field of @@ -182,6 +285,21 @@ impl OpSpec { /// ``` pub fn answered_by(self, kind: ValueKind, indexes: &[IndexSpec]) -> Option<&IndexSpec> { let declared = |wanted: fn(&IndexSpec) -> bool| indexes.iter().find(|index| wanted(index)); + if self.is_json() { + // A document field's JSON index answers the document operators; + // the orderings at a path only when it orders some path. + if kind != ValueKind::Json { + return None; + } + let ordering = matches!( + self, + OpSpec::LtAt | OpSpec::LteAt | OpSpec::GtAt | OpSpec::GteAt | OpSpec::BetweenAt + ); + return indexes.iter().find(|index| match index { + IndexSpec::Json(options) => !ordering || !options.ordered.is_empty(), + _ => false, + }); + } match self { OpSpec::Eq => declared(|index| matches!(index, IndexSpec::Equality)), OpSpec::Matches => declared(|index| matches!(index, IndexSpec::Match(_))), @@ -193,6 +311,7 @@ impl OpSpec { .or_else(|| declared(|index| matches!(index, IndexSpec::OreBlock))) .or_else(|| declared(|index| matches!(index, IndexSpec::Ope))) } + _ => None, } .filter(|index| admits(kind, index)) } @@ -282,9 +401,11 @@ pub(super) fn target_operators(target: &TargetDescriptor) -> Vec<(OpSpec, IndexS }, ) .collect(); + // `between` through an EQL type is not answered yet; `between_at` on a + // document is (its bounds are two leaf operands). OpSpec::ALL .into_iter() - .filter(|op| !op.takes_pair()) + .filter(|op| !op.takes_pair() || op.is_json()) .filter_map(|op| { op.answered_by(kind, &indexes) .map(|index| (op, index.clone())) @@ -313,13 +434,20 @@ pub enum Operand { /// One term: an index term's frozen bytes ([`TermBytes`](super::TermBytes)), or an EQL /// query value's JSON bytes. One(Vec), - /// A range's two terms, each the index term of one bound, inclusive. + /// A range's two terms, each the index term of one bound, inclusive: + /// index terms' bytes, or for `between_at` two EQL leaf operands' JSON. Range { /// The low bound's term. lo: Vec, /// The high bound's term. hi: Vec, }, + /// Text: a JSON path's selector (`selector`), or one node's token + /// (`children`, `child_count`). Binds as SQL `text`. + Text(String), + /// A list of text: the tokens of a subtree read (`subtree`), in the + /// read's order. Binds as SQL `text[]`. + TextList(Vec), } /// The term in its wire form: an object with every key present, an @@ -330,8 +458,9 @@ pub enum Operand { /// | `operator` | string: the [operator's name](OpSpec::name), `"matches"` | /// | `fidelity` | string: `"exact"` or `"probabilistic"` ([`Fidelity::name`]) | /// | `index` | string: the [key](IndexSpec::key) of the index that answers, `"match"` | -/// | `domain` | string or null: the query value's PostgreSQL domain, for a field that names an EQL type | -/// | `term` | bytes: the term, or for `between` a list of two byte strings, the low bound's term then the high's | +/// | `domain` | string or null: the SQL type the operand binds as: an EQL query domain for a field that names an EQL type, `text` or `text[]` for a selector or tokens, null for an index term's bytes | +/// | `encoding` | string: what `term` holds: `bytes` (an index term), `eql` (an EQL query value's JSON), `text` or `text_array` | +/// | `term` | bytes, or for a range a list of two (the low bound's then the high's); a string for `text`; a list of strings for `text_array` | impl Serialize for QueryTerm { fn serialize(&self, s: S) -> Result { #[derive(Serialize)] @@ -340,6 +469,7 @@ impl Serialize for QueryTerm { fidelity: &'static str, index: &'static str, domain: Option<&'a str>, + encoding: &'static str, term: Term<'a>, } #[derive(Serialize)] @@ -347,15 +477,20 @@ impl Serialize for QueryTerm { enum Term<'a> { One(WireBytes<'a>), Range([WireBytes<'a>; 2]), + Text(&'a str), + TextList(&'a [String]), } Wire { operator: self.operator.name(), fidelity: self.fidelity().name(), index: self.index.key(), domain: self.domain.as_deref(), + encoding: self.encoding(), term: match &self.operand { Operand::One(term) => Term::One(WireBytes(term)), Operand::Range { lo, hi } => Term::Range([WireBytes(lo), WireBytes(hi)]), + Operand::Text(text) => Term::Text(text), + Operand::TextList(texts) => Term::TextList(texts), }, } .serialize(s) @@ -388,6 +523,18 @@ impl QueryTerm { pub fn operand(&self) -> &Operand { &self.operand } + + /// What the operand holds, as the wire's `encoding` names it: `bytes` + /// for an index term, `eql` for an EQL query value's JSON, `text` for a + /// selector or a token, `text_array` for a read's tokens. + pub fn encoding(&self) -> &'static str { + match &self.operand { + Operand::Text(_) => "text", + Operand::TextList(_) => "text_array", + Operand::One(_) | Operand::Range { .. } if self.domain.is_some() => "eql", + Operand::One(_) | Operand::Range { .. } => "bytes", + } + } } /// The field a query is about, and the index that answers its operator. @@ -465,6 +612,17 @@ fn answers( }; if let Some(name) = field.target() { let target = resolver.resolve(name).map_err(Error::Target)?; + // A document's operators are its JSON index's under the options + // the plan resolved for the field: the orderings at a path only + // when it orders one. + if let Some(TargetOptions::Json(options)) = field.target_options() { + let index = [IndexSpec::Json(options.clone())]; + let operators = OpSpec::ALL + .into_iter() + .filter_map(|op| op.answered_by(kind, &index).map(|i| (op, i.clone()))) + .collect(); + return Ok((operators, Some(target))); + } return Ok((target_operators(&target), Some(target))); } let indexes = field.indexes(); @@ -528,12 +686,14 @@ pub fn field_operators( /// | `fidelity` | string: `exact` or `probabilistic` ([`Fidelity::name`]) | /// | `arity` | string: `one`, or `pair` for an operator whose input is two values, low then high | /// | `index` | string: the [key](IndexSpec::key) of the index that answers it, `eq` | +/// | `input` | string: the shape of the operator's input ([`OpSpec::input`]): `value`, `pair`, `json`, `path_json`, `path_pair`, `path` or `path_bound` | #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] pub struct OperatorEntry { operator: &'static str, fidelity: &'static str, arity: &'static str, index: &'static str, + input: &'static str, } impl OperatorEntry { @@ -544,6 +704,7 @@ impl OperatorEntry { fidelity: operator.fidelity().name(), arity: if operator.takes_pair() { "pair" } else { "one" }, index: index.key(), + input: operator.input(), } } } @@ -729,10 +890,152 @@ pub fn check_query( ) -> Result<(), Error> { let answer = answer(plan, field, operator, resolver)?; has_context(plan, field)?; + if operator.is_json() { + let input = ValueFrame::of(input).map_err(|_| json_input(field))?; + return super::json_query::check(operator, json_options(&answer)?, &input) + .map_err(|error| error.in_field(field)); + } let input = read_input(&answer, operator, input)?; check_input(&answer.index, &input) } +/// The JSON index options a document field's plan resolved. +fn json_options<'p>(answer: &Answer<'p>) -> Result<&'p crate::json::JsonOptions, Error> { + match answer.field.target_options() { + Some(TargetOptions::Json(options)) => Ok(options), + // A JSON operator is answered only through a document target, + // whose options the plan always resolves. + _ => Err(Error::Internal), + } +} + +fn json_input(field: &str) -> Error { + Error::Source { + field: Some(field.to_owned()), + reason: Reason::JsonInput, + } +} + +/// A JSON operator's term: the operand [`json_query`](super::json_query) +/// derives, bound as the type its target names for it. +fn json_term<'a, K: 'static>( + cipher: &'a KeysetCipher<'_, K>, + answer: Answer<'_>, + operator: OpSpec, + input: impl Encrypt, +) -> Result, Error> { + use super::json_query::Binds; + let field = answer.field.name(); + let input = ValueFrame::of(input).map_err(|_| json_input(field))?; + let target = answer.target.as_ref().ok_or(Error::Internal)?; + // The SQL type each operand binds as, from the target: a resolver + // that names no leaf domain for a document answers no ordering. + let domain = |binds: Binds| match binds { + Binds::Needle => target.query_sql_domain.clone(), + Binds::Leaf => target.leaf_query_sql_domain.clone(), + Binds::Text => Some("text".to_owned()), + Binds::TextArray => Some("text[]".to_owned()), + }; + let binds = super::json_query::binds(operator); + let domain = domain(binds).ok_or_else(|| { + Error::Target(TargetError::NoQuery { + name: target.name.clone(), + }) + })?; + let index = answer.index.clone(); + let derived = super::json_query::derive( + cipher, + answer.field.label(), + json_options(&answer)?, + operator, + &input, + ) + .map_err(|error| error.in_field(field))?; + Ok(derived.map(move |operand| QueryTerm { + operator, + index, + domain: Some(domain), + operand, + })) +} + +/// Check a read of a document field without a cipher: everything +/// [`json_open`] refuses before it derives anything. A binding calls this +/// at its boundary. +/// +/// # Errors +/// +/// As [`json_open`]. +pub fn check_json_open( + plan: &Plan, + field: &str, + request: impl Encrypt, + rows: impl Encrypt, + resolver: &(impl TargetResolver + ?Sized), +) -> Result<(), Error> { + let _ = answer(plan, field, OpSpec::Selector, resolver)?; + has_context(plan, field)?; + let request = ValueFrame::of(request).map_err(|_| json_input(field))?; + let rows = ValueFrame::of(rows).map_err(|_| json_input(field))?; + super::json_query::check_open(&request, &rows).map_err(|error| error.in_field(field)) +} + +/// Open the rows a read of a document field returned, against the request +/// that prepared it: the runtime form of [`JsonSelector::open`] and +/// [`JsonSubtreeKey::open`], for a binding. `request` is the read as the +/// binding prepared it, `{"operation": "selector" | "subtree", "path", +/// "wildcard_bound"?}`, and `rows` a list per database row of the entries +/// the SQL returned for it (`public.eql_v4_json_tree_entry` JSON, as text +/// or bytes): for `selector`, the one entry `->` returned or none, and for +/// `subtree`, the rows `json_tree_subtree` returned. The engine derives +/// the selector or the read key again, under `cipher`'s keyset and the +/// field's column and options, and checks every row against it; the rows +/// open through `scope`. The result is a list per database row of JSON +/// texts, as bytes: for `selector` the scalar or nothing, for `subtree` +/// one value per concrete path the read named that the row has, in +/// document order. One key request for every row. +/// +/// [`JsonSelector::open`]: crate::json::JsonSelector::open +/// [`JsonSubtreeKey::open`]: crate::json::JsonSubtreeKey::open +/// +/// # Errors +/// +/// As [`query_term`] refuses a field that does not answer the read's +/// operator, and [`Error::Source`] naming the field for a request or rows +/// that do not parse ([`Reason::JsonInput`]). A row that is not one the +/// read asked for fails the pending +/// ([`Error::JsonWrongEntry`](crate::Error::JsonWrongEntry)), as does a +/// wildcard read of an array longer than its bound +/// ([`Error::JsonWildcardTruncated`](crate::Error::JsonWildcardTruncated)). +#[allow(clippy::too_many_arguments)] +pub fn json_open<'a, K>( + cipher: &'a KeysetCipher<'_, K>, + scope: Scope<'a, K>, + plan: &Plan, + field: &str, + request: impl Encrypt, + rows: impl Encrypt, + resolver: &(impl TargetResolver + ?Sized), +) -> Result, Error> +where + K: DataKeySource + Sync + 'static, +{ + // Answering `selector` is what makes a field a document field. + let answer = answer(plan, field, OpSpec::Selector, resolver)?; + has_context(plan, field)?; + let request = ValueFrame::of(request).map_err(|_| json_input(field))?; + let rows = ValueFrame::of(rows).map_err(|_| json_input(field))?; + super::json_query::open( + cipher, + scope, + answer.field.label(), + json_options(&answer)?, + &request, + &rows, + ) + .map_err(|error| error.in_field(field)) +} + /// Derive the query term for one field of a plan, an operator and its /// input: one value for most operators, a list of two for /// [`between`](OpSpec::Between). @@ -799,6 +1102,9 @@ where { let answer = answer(plan, field, operator, resolver)?; has_context(plan, field)?; + if operator.is_json() { + return json_term(cipher, answer, operator, input); + } let input = read_input(&answer, operator, input)?; check_input(&answer.index, &input)?; let index = answer.index.clone(); @@ -1170,7 +1476,7 @@ mod tests { let names: std::collections::BTreeSet<&str> = OpSpec::ALL.iter().map(|op| op.name()).collect(); assert_eq!(names.len(), OpSpec::ALL.len(), "names are distinct"); - for not_one in ["Eq", "=", "", "contains", "at_path", "matches "] { + for not_one in ["Eq", "=", "", "Contains", "at_path", "matches ", "eq-at"] { assert_eq!(OpSpec::parse(not_one), None, "{not_one:?}"); } } @@ -1183,7 +1489,11 @@ mod tests { op == OpSpec::Matches, "{op}" ); - assert_eq!(op.takes_pair(), op == OpSpec::Between, "{op}"); + assert_eq!( + op.takes_pair(), + matches!(op, OpSpec::Between | OpSpec::BetweenAt), + "{op}" + ); } assert_eq!(Fidelity::Exact.name(), "exact"); assert_eq!(Fidelity::Probabilistic.name(), "probabilistic"); @@ -2202,16 +2512,20 @@ mod tests { panic!("an object"); }; let keys: Vec<&str> = entries.iter().map(|(k, _)| k.as_str()).collect(); - assert_eq!(keys, ["operator", "fidelity", "index", "domain", "term"]); + assert_eq!( + keys, + ["operator", "fidelity", "index", "domain", "encoding", "term"] + ); assert!(matches!(&entries[0].1, Tree::String(s) if s.as_bytes() == b"between")); assert!(matches!(&entries[1].1, Tree::String(s) if s.as_bytes() == b"exact")); assert!(matches!(&entries[2].1, Tree::String(s) if s.as_bytes() == b"ore")); assert!(matches!(&entries[3].1, Tree::Null)); + assert!(matches!(&entries[4].1, Tree::String(s) if s.as_bytes() == b"bytes")); let Operand::Range { lo, hi } = term.operand() else { panic!("a range"); }; assert!(matches!( - &entries[4].1, + &entries[5].1, Tree::Array(bounds) if matches!( bounds.as_slice(), [Tree::Bytes(l), Tree::Bytes(h)] if l.as_slice() == lo && h.as_slice() == hi @@ -2309,7 +2623,8 @@ mod tests { ("operator", b"lt".as_slice()), ("fidelity", b"exact".as_slice()), ("arity", b"one".as_slice()), - ("index", b"ore".as_slice()) + ("index", b"ore".as_slice()), + ("input", b"value".as_slice()) ] ); } @@ -2410,7 +2725,8 @@ mod tests { ("operator", b"between".as_slice()), ("fidelity", b"exact".as_slice()), ("arity", b"pair".as_slice()), - ("index", b"ope".as_slice()) + ("index", b"ope".as_slice()), + ("input", b"pair".as_slice()) ] ); let Tree::Object(entries) = @@ -2739,6 +3055,7 @@ mod wire_bytes { }), IndexSpec::Ore, IndexSpec::Ope, + IndexSpec::Json(crate::json::JsonOptions::default()), ] } @@ -2762,9 +3079,10 @@ mod wire_bytes { ) } - /// Every index, and every operator entry under it; three query terms, - /// one a range and one with a domain; four limits; three pages of - /// verdicts; four catalog entries. + /// Every index, the JSON index's included, and every operator entry + /// under it; five query terms, one a range, one with a domain, one a + /// selector's text and one a read's tokens; four limits; three pages + /// of verdicts; four catalog entries. #[test] fn every_answer_writes_its_pinned_bytes() { let mut written = Vec::new(); @@ -2796,6 +3114,18 @@ mod wire_bytes { domain: None, operand: Operand::One(Vec::new()), }, + QueryTerm { + operator: OpSpec::Selector, + index: IndexSpec::Json(crate::json::JsonOptions::default()), + domain: Some("text".to_owned()), + operand: Operand::Text("c2VsZWN0b3I=".to_owned()), + }, + QueryTerm { + operator: OpSpec::Subtree, + index: IndexSpec::Json(crate::json::JsonOptions::default()), + domain: Some("text[]".to_owned()), + operand: Operand::TextList(vec!["dA==".to_owned(), "dQ==".to_owned()]), + }, ] { written.push(hex(&term)); } @@ -2822,58 +3152,152 @@ mod wire_bytes { assert_eq!(written, PINNED); } - const PINNED: [&str; 62] = [ + const PINNED: [&str; 156] = [ "0a020000006571", - "f104000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a020000006571", - "f104000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a020000006571", - "f104000000080000006f70657261746f720a030000006c746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a020000006571", - "f104000000080000006f70657261746f720a02000000677408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a020000006571", - "f104000000080000006f70657261746f720a0300000067746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a020000006571", - "f104000000080000006f70657261746f720a070000006265747765656e08000000666964656c6974790a0500000065786163740500000061726974790a040000007061697205000000696e6465780a020000006571", - "f104000000080000006f70657261746f720a070000006d61746368657308000000666964656c6974790a0d00000070726f626162696c69737469630500000061726974790a030000006f6e6505000000696e6465780a020000006571", + "f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a030000006c746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a02000000677408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a0300000067746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a070000006265747765656e08000000666964656c6974790a0500000065786163740500000061726974790a040000007061697205000000696e6465780a02000000657105000000696e7075740a0400000070616972", + "f105000000080000006f70657261746f720a070000006d61746368657308000000666964656c6974790a0d00000070726f626162696c69737469630500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a08000000636f6e7461696e7308000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a040000006a736f6e", + "f105000000080000006f70657261746f720a0c000000636f6e7461696e65645f627908000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a040000006a736f6e", + "f105000000080000006f70657261746f720a0500000065715f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a050000006c745f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a060000006c74655f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a0500000067745f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a060000006774655f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a0a0000006265747765656e5f617408000000666964656c6974790a0500000065786163740500000061726974790a040000007061697205000000696e6465780a02000000657105000000696e7075740a09000000706174685f70616972", + "f105000000080000006f70657261746f720a0800000073656c6563746f7208000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0400000070617468", + "f105000000080000006f70657261746f720a070000007375627472656508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0a000000706174685f626f756e64", + "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0400000070617468", + "f105000000080000006f70657261746f720a0b0000006368696c645f636f756e7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0400000070617468", "0a050000006d61746368", - "f104000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d61746368", - "f104000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d61746368", - "f104000000080000006f70657261746f720a030000006c746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d61746368", - "f104000000080000006f70657261746f720a02000000677408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d61746368", - "f104000000080000006f70657261746f720a0300000067746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d61746368", - "f104000000080000006f70657261746f720a070000006265747765656e08000000666964656c6974790a0500000065786163740500000061726974790a040000007061697205000000696e6465780a050000006d61746368", - "f104000000080000006f70657261746f720a070000006d61746368657308000000666964656c6974790a0d00000070726f626162696c69737469630500000061726974790a030000006f6e6505000000696e6465780a050000006d61746368", + "f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a030000006c746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a02000000677408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a0300000067746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a070000006265747765656e08000000666964656c6974790a0500000065786163740500000061726974790a040000007061697205000000696e6465780a050000006d6174636805000000696e7075740a0400000070616972", + "f105000000080000006f70657261746f720a070000006d61746368657308000000666964656c6974790a0d00000070726f626162696c69737469630500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a08000000636f6e7461696e7308000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a040000006a736f6e", + "f105000000080000006f70657261746f720a0c000000636f6e7461696e65645f627908000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a040000006a736f6e", + "f105000000080000006f70657261746f720a0500000065715f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a050000006c745f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a060000006c74655f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a0500000067745f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a060000006774655f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a0a0000006265747765656e5f617408000000666964656c6974790a0500000065786163740500000061726974790a040000007061697205000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f70616972", + "f105000000080000006f70657261746f720a0800000073656c6563746f7208000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", + "f105000000080000006f70657261746f720a070000007375627472656508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0a000000706174685f626f756e64", + "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", + "f105000000080000006f70657261746f720a0b0000006368696c645f636f756e7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", "f101000000050000006d61746368f10400000009000000746f6b656e697a65720a080000007374616e6461726408000000646f776e6361736502010000006b070400000000000000010000006d070002000000000000", - "f104000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d61746368", - "f104000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d61746368", - "f104000000080000006f70657261746f720a030000006c746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d61746368", - "f104000000080000006f70657261746f720a02000000677408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d61746368", - "f104000000080000006f70657261746f720a0300000067746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d61746368", - "f104000000080000006f70657261746f720a070000006265747765656e08000000666964656c6974790a0500000065786163740500000061726974790a040000007061697205000000696e6465780a050000006d61746368", - "f104000000080000006f70657261746f720a070000006d61746368657308000000666964656c6974790a0d00000070726f626162696c69737469630500000061726974790a030000006f6e6505000000696e6465780a050000006d61746368", + "f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a030000006c746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a02000000677408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a0300000067746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a070000006265747765656e08000000666964656c6974790a0500000065786163740500000061726974790a040000007061697205000000696e6465780a050000006d6174636805000000696e7075740a0400000070616972", + "f105000000080000006f70657261746f720a070000006d61746368657308000000666964656c6974790a0d00000070726f626162696c69737469630500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a08000000636f6e7461696e7308000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a040000006a736f6e", + "f105000000080000006f70657261746f720a0c000000636f6e7461696e65645f627908000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a040000006a736f6e", + "f105000000080000006f70657261746f720a0500000065715f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a050000006c745f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a060000006c74655f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a0500000067745f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a060000006774655f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a0a0000006265747765656e5f617408000000666964656c6974790a0500000065786163740500000061726974790a040000007061697205000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f70616972", + "f105000000080000006f70657261746f720a0800000073656c6563746f7208000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", + "f105000000080000006f70657261746f720a070000007375627472656508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0a000000706174685f626f756e64", + "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", + "f105000000080000006f70657261746f720a0b0000006368696c645f636f756e7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", "f101000000050000006d61746368f10400000009000000746f6b656e697a6572f101000000050000006e6772616d07040000000000000008000000646f776e6361736503010000006b070300000000000000010000006d070001000000000000", - "f104000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d61746368", - "f104000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d61746368", - "f104000000080000006f70657261746f720a030000006c746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d61746368", - "f104000000080000006f70657261746f720a02000000677408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d61746368", - "f104000000080000006f70657261746f720a0300000067746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d61746368", - "f104000000080000006f70657261746f720a070000006265747765656e08000000666964656c6974790a0500000065786163740500000061726974790a040000007061697205000000696e6465780a050000006d61746368", - "f104000000080000006f70657261746f720a070000006d61746368657308000000666964656c6974790a0d00000070726f626162696c69737469630500000061726974790a030000006f6e6505000000696e6465780a050000006d61746368", + "f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a030000006c746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a02000000677408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a0300000067746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a070000006265747765656e08000000666964656c6974790a0500000065786163740500000061726974790a040000007061697205000000696e6465780a050000006d6174636805000000696e7075740a0400000070616972", + "f105000000080000006f70657261746f720a070000006d61746368657308000000666964656c6974790a0d00000070726f626162696c69737469630500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a08000000636f6e7461696e7308000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a040000006a736f6e", + "f105000000080000006f70657261746f720a0c000000636f6e7461696e65645f627908000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a040000006a736f6e", + "f105000000080000006f70657261746f720a0500000065715f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a050000006c745f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a060000006c74655f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a0500000067745f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a060000006774655f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a0a0000006265747765656e5f617408000000666964656c6974790a0500000065786163740500000061726974790a040000007061697205000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f70616972", + "f105000000080000006f70657261746f720a0800000073656c6563746f7208000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", + "f105000000080000006f70657261746f720a070000007375627472656508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0a000000706174685f626f756e64", + "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", + "f105000000080000006f70657261746f720a0b0000006368696c645f636f756e7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", "0a030000006f7265", - "f104000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f7265", - "f104000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f7265", - "f104000000080000006f70657261746f720a030000006c746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f7265", - "f104000000080000006f70657261746f720a02000000677408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f7265", - "f104000000080000006f70657261746f720a0300000067746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f7265", - "f104000000080000006f70657261746f720a070000006265747765656e08000000666964656c6974790a0500000065786163740500000061726974790a040000007061697205000000696e6465780a030000006f7265", - "f104000000080000006f70657261746f720a070000006d61746368657308000000666964656c6974790a0d00000070726f626162696c69737469630500000061726974790a030000006f6e6505000000696e6465780a030000006f7265", + "f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a030000006c746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a02000000677408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a0300000067746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a070000006265747765656e08000000666964656c6974790a0500000065786163740500000061726974790a040000007061697205000000696e6465780a030000006f726505000000696e7075740a0400000070616972", + "f105000000080000006f70657261746f720a070000006d61746368657308000000666964656c6974790a0d00000070726f626162696c69737469630500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a08000000636f6e7461696e7308000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a040000006a736f6e", + "f105000000080000006f70657261746f720a0c000000636f6e7461696e65645f627908000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a040000006a736f6e", + "f105000000080000006f70657261746f720a0500000065715f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a050000006c745f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a060000006c74655f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a0500000067745f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a060000006774655f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a0a0000006265747765656e5f617408000000666964656c6974790a0500000065786163740500000061726974790a040000007061697205000000696e6465780a030000006f726505000000696e7075740a09000000706174685f70616972", + "f105000000080000006f70657261746f720a0800000073656c6563746f7208000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0400000070617468", + "f105000000080000006f70657261746f720a070000007375627472656508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0a000000706174685f626f756e64", + "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0400000070617468", + "f105000000080000006f70657261746f720a0b0000006368696c645f636f756e7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0400000070617468", "0a030000006f7065", - "f104000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f7065", - "f104000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f7065", - "f104000000080000006f70657261746f720a030000006c746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f7065", - "f104000000080000006f70657261746f720a02000000677408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f7065", - "f104000000080000006f70657261746f720a0300000067746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f7065", - "f104000000080000006f70657261746f720a070000006265747765656e08000000666964656c6974790a0500000065786163740500000061726974790a040000007061697205000000696e6465780a030000006f7065", - "f104000000080000006f70657261746f720a070000006d61746368657308000000666964656c6974790a0d00000070726f626162696c69737469630500000061726974790a030000006f6e6505000000696e6465780a030000006f7065", - "f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a05000000657861637405000000696e6465780a02000000657106000000646f6d61696e00040000007465726d0b03000000010203", - "f105000000080000006f70657261746f720a070000006265747765656e08000000666964656c6974790a05000000657861637405000000696e6465780a030000006f706506000000646f6d61696e0a1000000065716c5f76342e71756572795f696e74040000007465726df0020000000b01000000040b020000000506", - "f105000000080000006f70657261746f720a070000006d61746368657308000000666964656c6974790a0d00000070726f626162696c697374696305000000696e6465780a050000006d6174636806000000646f6d61696e00040000007465726d0b00000000", + "f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a030000006c746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a02000000677408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a0300000067746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a070000006265747765656e08000000666964656c6974790a0500000065786163740500000061726974790a040000007061697205000000696e6465780a030000006f706505000000696e7075740a0400000070616972", + "f105000000080000006f70657261746f720a070000006d61746368657308000000666964656c6974790a0d00000070726f626162696c69737469630500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a08000000636f6e7461696e7308000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a040000006a736f6e", + "f105000000080000006f70657261746f720a0c000000636f6e7461696e65645f627908000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a040000006a736f6e", + "f105000000080000006f70657261746f720a0500000065715f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a050000006c745f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a060000006c74655f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a0500000067745f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a060000006774655f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a0a0000006265747765656e5f617408000000666964656c6974790a0500000065786163740500000061726974790a040000007061697205000000696e6465780a030000006f706505000000696e7075740a09000000706174685f70616972", + "f105000000080000006f70657261746f720a0800000073656c6563746f7208000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0400000070617468", + "f105000000080000006f70657261746f720a070000007375627472656508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0a000000706174685f626f756e64", + "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0400000070617468", + "f105000000080000006f70657261746f720a0b0000006368696c645f636f756e7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0400000070617468", + "0a040000006a736f6e", + "f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a030000006c746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a02000000677408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a0300000067746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a070000006265747765656e08000000666964656c6974790a0500000065786163740500000061726974790a040000007061697205000000696e6465780a040000006a736f6e05000000696e7075740a0400000070616972", + "f105000000080000006f70657261746f720a070000006d61746368657308000000666964656c6974790a0d00000070726f626162696c69737469630500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0500000076616c7565", + "f105000000080000006f70657261746f720a08000000636f6e7461696e7308000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a040000006a736f6e", + "f105000000080000006f70657261746f720a0c000000636f6e7461696e65645f627908000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a040000006a736f6e", + "f105000000080000006f70657261746f720a0500000065715f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a050000006c745f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a060000006c74655f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a0500000067745f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a060000006774655f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a09000000706174685f6a736f6e", + "f105000000080000006f70657261746f720a0a0000006265747765656e5f617408000000666964656c6974790a0500000065786163740500000061726974790a040000007061697205000000696e6465780a040000006a736f6e05000000696e7075740a09000000706174685f70616972", + "f105000000080000006f70657261746f720a0800000073656c6563746f7208000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0400000070617468", + "f105000000080000006f70657261746f720a070000007375627472656508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0a000000706174685f626f756e64", + "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0400000070617468", + "f105000000080000006f70657261746f720a0b0000006368696c645f636f756e7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0400000070617468", + "f106000000080000006f70657261746f720a02000000657108000000666964656c6974790a05000000657861637405000000696e6465780a02000000657106000000646f6d61696e0008000000656e636f64696e670a050000006279746573040000007465726d0b03000000010203", + "f106000000080000006f70657261746f720a070000006265747765656e08000000666964656c6974790a05000000657861637405000000696e6465780a030000006f706506000000646f6d61696e0a1000000065716c5f76342e71756572795f696e7408000000656e636f64696e670a0300000065716c040000007465726df0020000000b01000000040b020000000506", + "f106000000080000006f70657261746f720a070000006d61746368657308000000666964656c6974790a0d00000070726f626162696c697374696305000000696e6465780a050000006d6174636806000000646f6d61696e0008000000656e636f64696e670a050000006279746573040000007465726d0b00000000", + "f106000000080000006f70657261746f720a0800000073656c6563746f7208000000666964656c6974790a05000000657861637405000000696e6465780a040000006a736f6e06000000646f6d61696e0a040000007465787408000000656e636f64696e670a0400000074657874040000007465726d0a0c000000633256735a574e306233493d", + "f106000000080000006f70657261746f720a070000007375627472656508000000666964656c6974790a05000000657861637405000000696e6465780a040000006a736f6e06000000646f6d61696e0a06000000746578745b5d08000000656e636f64696e670a0a000000746578745f6172726179040000007465726df0020000000a0400000064413d3d0a0400000064513d3d", "070000000000000000", "070b00000000000000", "07ffffffffffffffff", @@ -2881,9 +3305,9 @@ mod wire_bytes { "f000000000", "f00100000003", "f003000000030203", - "f10b000000040000006e616d650a04000000536f6d650600000066616d696c790a04000000736f6d65060000007375666669780a0000000009000000706c61696e746578740a06000000737472696e670a00000073716c5f646f6d61696e0a0b0000007075626c69632e736f6d6507000000696e6465786573f0020000000a0200000065710a050000006d617463680500000071756572790a09000000536f6d6551756572791000000071756572795f73716c5f646f6d61696e0a1100000065716c5f76332e71756572795f736f6d650a00000070726f64756369626c650306000000726561736f6e00090000006f70657261746f7273f001000000f104000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a020000006571", - "f10b000000040000006e616d650a04000000536f6d650600000066616d696c790a04000000736f6d65060000007375666669780a0000000009000000706c61696e746578740a06000000737472696e670a00000073716c5f646f6d61696e0a0b0000007075626c69632e736f6d6507000000696e6465786573f0020000000a0200000065710a050000006d617463680500000071756572790a09000000536f6d6551756572791000000071756572795f73716c5f646f6d61696e0a1100000065716c5f76332e71756572795f736f6d650a00000070726f64756369626c650306000000726561736f6e00090000006f70657261746f7273f002000000f104000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a020000006571f104000000080000006f70657261746f720a070000006d61746368657308000000666964656c6974790a0d00000070726f626162696c69737469630500000061726974790a030000006f6e6505000000696e6465780a050000006d61746368", - "f10b000000040000006e616d650a04000000536f6d650600000066616d696c790a04000000736f6d65060000007375666669780a0000000009000000706c61696e746578740a05000000696e7436340a00000073716c5f646f6d61696e0a0b0000007075626c69632e736f6d6507000000696e6465786573f0010000000a030000006f72650500000071756572790a09000000536f6d6551756572791000000071756572795f73716c5f646f6d61696e0a1100000065716c5f76332e71756572795f736f6d650a00000070726f64756369626c650306000000726561736f6e00090000006f70657261746f7273f004000000f104000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f7265f104000000080000006f70657261746f720a030000006c746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f7265f104000000080000006f70657261746f720a02000000677408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f7265f104000000080000006f70657261746f720a0300000067746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f7265", + "f10b000000040000006e616d650a04000000536f6d650600000066616d696c790a04000000736f6d65060000007375666669780a0000000009000000706c61696e746578740a06000000737472696e670a00000073716c5f646f6d61696e0a0b0000007075626c69632e736f6d6507000000696e6465786573f0020000000a0200000065710a050000006d617463680500000071756572790a09000000536f6d6551756572791000000071756572795f73716c5f646f6d61696e0a1100000065716c5f76332e71756572795f736f6d650a00000070726f64756369626c650306000000726561736f6e00090000006f70657261746f7273f001000000f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0500000076616c7565", + "f10b000000040000006e616d650a04000000536f6d650600000066616d696c790a04000000736f6d65060000007375666669780a0000000009000000706c61696e746578740a06000000737472696e670a00000073716c5f646f6d61696e0a0b0000007075626c69632e736f6d6507000000696e6465786573f0020000000a0200000065710a050000006d617463680500000071756572790a09000000536f6d6551756572791000000071756572795f73716c5f646f6d61696e0a1100000065716c5f76332e71756572795f736f6d650a00000070726f64756369626c650306000000726561736f6e00090000006f70657261746f7273f002000000f105000000080000006f70657261746f720a02000000657108000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0500000076616c7565f105000000080000006f70657261746f720a070000006d61746368657308000000666964656c6974790a0d00000070726f626162696c69737469630500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0500000076616c7565", + "f10b000000040000006e616d650a04000000536f6d650600000066616d696c790a04000000736f6d65060000007375666669780a0000000009000000706c61696e746578740a05000000696e7436340a00000073716c5f646f6d61696e0a0b0000007075626c69632e736f6d6507000000696e6465786573f0010000000a030000006f72650500000071756572790a09000000536f6d6551756572791000000071756572795f73716c5f646f6d61696e0a1100000065716c5f76332e71756572795f736f6d650a00000070726f64756369626c650306000000726561736f6e00090000006f70657261746f7273f004000000f105000000080000006f70657261746f720a020000006c7408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0500000076616c7565f105000000080000006f70657261746f720a030000006c746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0500000076616c7565f105000000080000006f70657261746f720a02000000677408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0500000076616c7565f105000000080000006f70657261746f720a0300000067746508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0500000076616c7565", "f10b000000040000006e616d650a04000000536f6d650600000066616d696c790a04000000736f6d65060000007375666669780a0000000009000000706c61696e74657874000a00000073716c5f646f6d61696e0a0b0000007075626c69632e736f6d6507000000696e6465786573f000000000050000007175657279001000000071756572795f73716c5f646f6d61696e000a00000070726f64756369626c650206000000726561736f6e0a070000006e6f7420796574090000006f70657261746f7273f000000000", ]; } diff --git a/packages/stack-encrypt/src/dynamic/target.rs b/packages/stack-encrypt/src/dynamic/target.rs index 3593c2c3e..e70441fbd 100644 --- a/packages/stack-encrypt/src/dynamic/target.rs +++ b/packages/stack-encrypt/src/dynamic/target.rs @@ -103,6 +103,13 @@ pub struct TargetDescriptor { /// [`with_match_options`]: Self::with_match_options /// [`verify`]: super::verify pub match_options: Option, + /// The PostgreSQL domain of the type's ordering operand at a path, for + /// a type that stores a JSON document + /// ([`with_leaf_query_sql_domain`](Self::with_leaf_query_sql_domain)): + /// what an ordering at a path binds as, beside the containment needle + /// [`query_sql_domain`](Self::query_sql_domain) names. Not on the + /// wire; a generator reads it from the operators' terms. + pub leaf_query_sql_domain: Option, } /// The descriptor in its wire form: the object a guest's `se_targets` @@ -170,9 +177,18 @@ impl TargetDescriptor { producible, reason, match_options: None, + leaf_query_sql_domain: None, } } + /// The descriptor of a type that stores a JSON document, whose + /// ordering operand at a path is the domain `domain`. + #[must_use] + pub fn with_leaf_query_sql_domain(mut self, domain: impl Into) -> Self { + self.leaf_query_sql_domain = Some(domain.into()); + self + } + /// The descriptor of a type whose match index is built under `options`: /// such a type answers `matches` through its query twin /// ([`operators`](Self::operators)). A type with a match index and no diff --git a/packages/stack-encrypt/src/json/derive.rs b/packages/stack-encrypt/src/json/derive.rs index 1b4753cd3..bbc81a017 100644 --- a/packages/stack-encrypt/src/json/derive.rs +++ b/packages/stack-encrypt/src/json/derive.rs @@ -1880,16 +1880,26 @@ impl KeysetCipher<'_, K> { path: &JsonPath, descriptor: NonEmpty>, ) -> Pending<'_, JsonSelector, K> { + Pending::ready(self, self.json_selector_now(path, descriptor)) + } + + /// [`json_selector`](Self::json_selector), derived and returned now: + /// for the engine's own callers, which derive locally and want a + /// refusal as a plain error. + pub(crate) fn json_selector_now<'c>( + &self, + path: &JsonPath, + descriptor: NonEmpty>, + ) -> Result { let context = descriptor.into_prf_context().into_owned(); - let selector = prf16( + prf16( self.prf(), SELECTOR_DOMAIN, &context, &path.selector_bytes(), ) .map(JsonSelector) - .map_err(Error::from); - Pending::ready(self, selector) + .map_err(Error::from) } /// The key that reads the node at `path`, under the field `descriptor` @@ -1940,10 +1950,24 @@ impl KeysetCipher<'_, K> { bound: u32, descriptor: NonEmpty>, ) -> Pending<'_, JsonSubtreeKey, K> { + Pending::ready( + self, + self.json_subtree_key_now(path, options, bound, descriptor), + ) + } + + /// [`json_subtree_key_with_bound`](Self::json_subtree_key_with_bound), + /// derived and returned now, for the engine's own callers. + pub(crate) fn json_subtree_key_now<'c>( + &self, + path: &JsonPath, + options: &JsonOptions, + bound: u32, + descriptor: NonEmpty>, + ) -> Result { let context = descriptor.into_prf_context().into_owned(); let selectors = Selectors::new(Arc::clone(self.state()), context); - let key = subtree_key_for(selectors, options, path, bound); - Pending::ready(self, key.map_err(Error::from)) + subtree_key_for(selectors, options, path, bound).map_err(Error::from) } } diff --git a/packages/stack-encrypt/src/json/query.rs b/packages/stack-encrypt/src/json/query.rs index c7d9d4075..cffb4f94b 100644 --- a/packages/stack-encrypt/src/json/query.rs +++ b/packages/stack-encrypt/src/json/query.rs @@ -245,7 +245,7 @@ pub struct JsonLeafQuery { } impl JsonLeafQuery { - fn of_leaf(terms: &JsonTerms) -> Self { + pub(crate) fn of_leaf(terms: &JsonTerms) -> Self { // `check` admitted only a number or a string, both of which carry a // term; an empty term here would be a leaf that cannot be. let leaf = terms.leaf(); diff --git a/packages/stack-encrypt/tests/query_operators.rs b/packages/stack-encrypt/tests/query_operators.rs index 71c76870c..11cf45f6a 100644 --- a/packages/stack-encrypt/tests/query_operators.rs +++ b/packages/stack-encrypt/tests/query_operators.rs @@ -116,6 +116,8 @@ fn json_of_term(term: &QueryTerm) -> Json { "term": match term.operand() { Operand::One(bytes) => json!(hex(bytes)), Operand::Range { lo, hi } => json!([hex(lo), hex(hi)]), + Operand::Text(text) => json!(text), + Operand::TextList(texts) => json!(texts), }, }) } diff --git a/packages/stack-encrypt/tests/ui/op_at_path_not_a_leaf.stderr b/packages/stack-encrypt/tests/ui/op_at_path_not_a_leaf.stderr index f9b886cc1..5a1eb22c2 100644 --- a/packages/stack-encrypt/tests/ui/op_at_path_not_a_leaf.stderr +++ b/packages/stack-encrypt/tests/ui/op_at_path_not_a_leaf.stderr @@ -1,4 +1,4 @@ -error[E0277]: `Contains` is not a leaf operator, so it cannot be applied at a path +error[E0277]: `stack_encrypt::op::Contains` is not a leaf operator, so it cannot be applied at a path --> tests/ui/op_at_path_not_a_leaf.rs:9:25 | 9 | operator(op::AtPath(op::Contains)); @@ -6,7 +6,7 @@ error[E0277]: `Contains` is not a leaf operator, so it cannot be applied at a pa | | | required by a bound introduced by this call | - = help: the trait `Leaf` is not implemented for `Contains` + = help: the trait `Leaf` is not implemented for `stack_encrypt::op::Contains` = note: `op::AtPath` applies `op::Eq`, `Lt`, `Lte`, `Gt`, `Gte`, `Between` or `Matches` at a path; `op::Contains` and `op::ContainedBy` compare whole documents, and an accessor does not nest = help: the following other types implement trait `Leaf`: stack_encrypt::op::Between @@ -16,7 +16,7 @@ error[E0277]: `Contains` is not a leaf operator, so it cannot be applied at a pa stack_encrypt::op::Lt stack_encrypt::op::Lte stack_encrypt::op::Matches - = note: required for `AtPath` to implement `Operator` + = note: required for `AtPath` to implement `Operator` note: required by a bound in `operator` --> tests/ui/op_at_path_not_a_leaf.rs:6:17 | diff --git a/packages/stack-encrypt/tests/ui/op_json_not_answered.stderr b/packages/stack-encrypt/tests/ui/op_json_not_answered.stderr index 04abbdf46..913dd894e 100644 --- a/packages/stack-encrypt/tests/ui/op_json_not_answered.stderr +++ b/packages/stack-encrypt/tests/ui/op_json_not_answered.stderr @@ -1,11 +1,11 @@ -error[E0277]: `Contains` has no default index given `std::string::String` +error[E0277]: `stack_encrypt::op::Contains` has no default index given `std::string::String` --> tests/ui/op_json_not_answered.rs:14:46 | 14 | let _ = cipher.query(op::Contains, "{}").on(&profile_plan); | ^^ no default index answers this operator here | = note: `op::Eq` defaults to `Equality`; `op::Matches` defaults to `Match` with the default configuration, which is case-insensitive and over text only; `op::Lt`, `Lte`, `Gt`, `Gte` and `Between` default to `Ore` over an integer, and `Between` takes a pair; name another index, such as `Ope`, with `.via(index)`; `op::Contains`, `op::ContainedBy` and `op::AtPath` default to `Json` over a `serde_json::Value` -help: the trait `DefaultIndex` is not implemented for `Contains` +help: the trait `DefaultIndex` is not implemented for `stack_encrypt::op::Contains` but trait `DefaultIndex` is implemented for it --> src/json/query.rs | @@ -15,7 +15,7 @@ help: the trait `DefaultIndex` is not implemented for `Cont | containment!(Contains, ContainedBy); | ----------------------------------- in this macro invocation = help: for that trait implementation, expected `serde_json::value::Value`, found `std::string::String` - = note: required for `ValuePlan>` to implement `Chooses` + = note: required for `ValuePlan>` to implement `Chooses` = note: this error originates in the macro `containment` (in Nightly builds, run with -Z macro-backtrace for more info) error[E0271]: type mismatch resolving `<(&str, &str) as Input>>::Plaintext == String` diff --git a/packages/stack-encrypt/tests/value_kinds.rs b/packages/stack-encrypt/tests/value_kinds.rs index 8680e2630..52905fc57 100644 --- a/packages/stack-encrypt/tests/value_kinds.rs +++ b/packages/stack-encrypt/tests/value_kinds.rs @@ -335,7 +335,7 @@ async fn query( .expect("the term derives"); match term.operand() { Operand::One(bytes) => bytes.clone(), - Operand::Range { .. } => panic!("one term"), + other => panic!("one term, not {other:?}"), } } From eb47bad0678028cf67d1f672ffd85609347a92f4 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 01:17:36 +1100 Subject: [PATCH 02/26] feat(go): generate encrypted JSON document fields A json.RawMessage field tagged encrypt_into=EncryptedJson, with optional ordered paths and sealed_root, is sealed by the engine from the field's data plan and gets generated methods for every json_tree query and read: Contains, ContainedBy, EqAt, Selector, Subtree, Children, ChildCount, and an At accessor per ordered path with Lt, Lte, Gt, Gte and Between. Selector and subtree reads return handles (encrypt.JSONSelection, encrypt.JSONRead) that keep the request they were prepared from and open the rows the database returns only as answers to it. TestJSONDocumentsQueryAndReadInPostgres runs every method against the EQL v4 bundle from the tree and joins go:test:postgres. Refs #1347, #1351, #1352 Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/CHANGELOG.md | 13 + languages/golang/cmd/stashgen/README.md | 31 +- languages/golang/cmd/stashgen/main_test.go | 66 ++++ languages/golang/encrypt/README.md | 30 +- languages/golang/encrypt/eql_postgres_test.go | 4 + languages/golang/encrypt/gensupport/codec.go | 2 +- .../golang/encrypt/gensupport/declaration.go | 45 +++ .../gensupport/gensupport_internal_test.go | 2 +- languages/golang/encrypt/gensupport/json.go | 118 +++++++ languages/golang/encrypt/gensupport/widen.go | 12 + .../internal/testusers/customer_stash.go | 283 +++++++++++++++++ .../encrypt/internal/testusers/customers.go | 15 + languages/golang/encrypt/json.go | 207 ++++++++++++ languages/golang/encrypt/json_fields_test.go | 133 ++++++++ .../golang/encrypt/json_postgres_test.go | 297 ++++++++++++++++++ languages/golang/encrypt/term.go | 18 +- languages/golang/stashgen/declaration.go | 11 +- languages/golang/stashgen/emit.go | 237 +++++++++++++- languages/golang/stashgen/engine.go | 7 +- languages/golang/stashgen/read.go | 11 +- languages/golang/stashgen/tag.go | 95 ++++++ .../testdata/stubsdk/encrypt/encrypt.go | 33 ++ .../stubsdk/encrypt/gensupport/gensupport.go | 42 +++ mise.toml | 6 +- 24 files changed, 1700 insertions(+), 18 deletions(-) create mode 100644 languages/golang/encrypt/gensupport/json.go create mode 100644 languages/golang/encrypt/internal/testusers/customer_stash.go create mode 100644 languages/golang/encrypt/internal/testusers/customers.go create mode 100644 languages/golang/encrypt/json_fields_test.go create mode 100644 languages/golang/encrypt/json_postgres_test.go diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index aec2688b0..273c7b01c 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -12,6 +12,19 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. ### Added +- **Encrypted JSON documents with `stashgen`** (#1351, #1352). A + `json.RawMessage` field tagged `encrypt_into=EncryptedJson` (options: + `ordered=$.a;$.b[*].c`, `sealed_root=false`) is stored as an + `eql.EncryptedJSON` in a `public.eql_v4_json_tree` column; a + `*json.RawMessage` field takes `null=sql`. Its `Fields` entry has + `Contains`, `ContainedBy`, `EqAt`, `Selector`, `Subtree`, `Children` and + `ChildCount`, and `At.` per ordered path with `Lt`, `Lte`, `Gt`, + `Gte` and `Between`. `encrypt.JSONSelection` and `encrypt.JSONRead` open + the rows `->` and `eql_v4.json_tree_subtree` return against the request + that prepared them; `encrypt.JSONSelector`, `JSONToken` and `JSONTokens` + bind as `text` and `text[]`. `gensupport.JSON`, `TargetOptions`, + `Declaration.Options` and the `JSON*` helpers are the generated code's. + - **Query and read an encrypted JSON document, at the record level** (#1350, #1352). `Cipher.QueryTerm` derives a document field's operands: `QueryTerm.Encoding` says what the term holds, and `QueryTerm.Text` diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index 8a20c7b2e..8386a75ce 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -74,7 +74,8 @@ The first part of a tag is the field's name, which is the column name in a datab | `stash:"notes,encrypt"` | seal the field, with no index | | `stash:"email,encrypt,index=equality;match"` | seal the field, and derive each index beside it | | `stash:"score,index=ore"` | derive the index alone; no ciphertext is stored, so `Decrypt` leaves the field at its zero value | -| `stash:"attrs,index=json"` | refused in this build: the engine does not derive the `json` index yet | +| `stash:"profile,encrypt_into=EncryptedJson(ordered=$.age)"` | on a `json.RawMessage`: store the field as an encrypted JSON document; see "Encrypted JSON documents" | +| `stash:"attrs,index=json"` | refused: a JSON document is stored with `encrypt_into=EncryptedJson`, not as an index beside a ciphertext | | `stash:"id,passthrough"` | store the field as it is | | `stash:"tenant,context_field"` | the field's value is the context of every other field in the struct, in place of a `context=` tag; stored as it is | | `stash:"-"` | leave the field out | @@ -108,7 +109,33 @@ An index takes its options in parentheses after its name, separated by commas: ` This build refuses an index with options: a query term is derived with the default options only, so a stored term with other options would never match one. These words are the same as the Rust API's words for the same behaviour, with two that only Go has. `opaque` seals the whole struct as one `bytes` field holding a storage blob (see "What crosses the binding"). -`json` names the JSON index, which the Rust API does not declare yet; this build refuses it. +`json` names the JSON index, which a field reaches only through `encrypt_into=EncryptedJson`. + +### Encrypted JSON documents + +A `json.RawMessage` field with `encrypt_into=EncryptedJson` is stored as one encrypted JSON document, an `eql.EncryptedJSON` in a `public.eql_v4_json_tree` column. +The field holds the document's JSON text; the engine parses it, so an object that names a member twice, an integer outside the 64-bit range and text that is not JSON are refused with `encrypt.ErrEncoding`. +Values come back as JSON text holding the same values; key order and the spelling of numbers are not kept. +A `*json.RawMessage` field is optional and must say `null=sql`: a document has no fixed length to hide a missing one behind. A non-nil pointer to `null` is a stored JSON `null`. + +The type takes options in parentheses: `ordered=` names the paths whose numbers and strings carry an ordering term, separated by `;`, and `sealed_root=false` turns off the sealed root (on by default). +`encrypt_into=EncryptedJson(ordered=$.age;$.visits[*].at)` orders two paths. +A path in a tag is plain JSONPath: `$`, then `.key` steps of letters, digits and underscores, and `[n]` or `[*]` steps; an ordered path names no position. The engine checks every path when `stashgen` runs. + +The field's `Fields` entry has a method per operation: + +| Method | Returns | SQL | +|---|---|---| +| `Contains(ctx, c, needle)`, `ContainedBy(ctx, c, needle)` | `eql.EncryptedJSONQuery` | `col @> $1::eql_v4.query_json_tree`, `col <@ $1::eql_v4.query_json_tree` | +| `EqAt(ctx, c, path, v)` | `eql.EncryptedJSONQuery` | `col @> $1::eql_v4.query_json_tree` | +| `At.Age.Lt(ctx, c, v)`, `Lte`, `Gt`, `Gte`, `Between` | `eql.EncryptedJSONLeafQuery` | `(col -> $sel::text) < $1::eql_v4.query_json_tree_leaf` | +| `Selector(ctx, c, path)` | `encrypt.JSONSelection`: `Selector()` and `Open` | `col -> $1::text` | +| `Subtree(ctx, c, path, opts)` | `encrypt.JSONRead`: `Tokens()`, `Open` and `OpenOne` | `eql_v4.json_tree_subtree(col, $1::text[], 64)` | +| `Children(ctx, c, path)`, `ChildCount(ctx, c, path)` | `encrypt.JSONToken` | `eql_v4.json_tree_children(col, $1::text)`, `eql_v4.json_tree_child_count(col, $1::text)` | + +`At` has one accessor per ordered path, named for its keys (`$.visits[*].at` is `At.VisitsAt`); two paths with one name stop `stashgen`. +The needle and the values are `json.RawMessage`: write the JSON you mean, such as `json.RawMessage(`+"`"+`{"active":true}`+"`"+`)`. +A position in a path is refused for `EqAt` and `Selector`, whose selector reads every position as the wildcard; read a position with `Subtree`. ### Optional fields diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index 298a67a2b..5ed46be0f 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -276,3 +276,69 @@ func TestModelFlagsRepeat(t *testing.T) { t.Fatalf("String = %q", got) } } + +// A json.RawMessage field stored as an encrypted JSON document, against the +// embedded engine with the EQL types: the generated entry has the +// document's methods and an accessor per ordered path, and every refusal +// names the field before anything is written. +func TestRunGeneratesEncryptedJSONFields(t *testing.T) { + checker, err := encrypt.NewChecker(context.Background()) + if err != nil { + t.Skip(err) + } + _ = checker.Close() + source := func(fields string) string { + return "package users\n\nimport \"encoding/json\"\n\nvar _ json.RawMessage\n\ntype Customer struct {\n\t_ struct{} `stash:\"context=customers\"`\n\tID int64 `stash:\"id,passthrough\"`\n" + fields + "\n}\n" + } + generate := func(fields string) (string, string, int) { + dir := writeModule(t, map[string]string{"model.go": source(fields)}) + var stdout, stderr bytes.Buffer + code := run([]string{"-type", "Customer"}, dir, &stdout, &stderr, stashgen.GuestEngine) + written, _ := os.ReadFile(filepath.Join(dir, "customer_stash.go")) + return string(written), stderr.String(), code + } + written, stderr, code := generate("\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$.age;$.visits[*].at,sealed_root=false)\"`\n\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=EncryptedJson,null=sql\"`") + if code != 0 { + t.Fatalf("exit %d\n%s", code, stderr) + } + for _, want := range []string{ + "Profile eql.EncryptedJSON", + "Prefs *eql.EncryptedJSON", + `EncryptInto("profile", gensupport.JSON, "EncryptedJson").`, + `Options("profile", gensupport.TargetOptions{SealedRoot: gensupport.Ptr(false), Ordered: []string{"$.age", "$.visits[*].at"}})`, + "func (f ProfileField) Contains(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.EncryptedJSONQuery, error)", + "func (f ProfileField) Subtree(ctx context.Context, c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[eql.EncryptedJSONEntry], error)", + "Age ProfileFieldPath // $.age", + "VisitsAt ProfileFieldPath // $.visits[*].at", + "func (p ProfileFieldPath) Between(ctx context.Context, c *encrypt.Cipher, lo, hi json.RawMessage) (encrypt.JSONRange[eql.EncryptedJSONLeafQuery], error)", + } { + if !strings.Contains(written, want) { + t.Errorf("the generated file lacks %q", want) + } + } + if strings.Contains(written, "PrefsFieldAt") { + t.Error("Prefs orders no path, and got an At") + } + for _, c := range []struct{ name, fields, want string }{ + {"a sealed null", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=EncryptedJson,null=sealed\"`", "Customer.Prefs"}, + {"a nil document that does not say null=sql", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=EncryptedJson\"`", "null=sql"}, + {"a position in an ordered path", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$.scores[2])\"`", "the engine refuses the declaration"}, + {"a quoted key in a tag", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$[\\\"a b\\\"])\"`", "not a plain JSONPath"}, + {"an unknown option", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(bound=3)\"`", "no option \"bound\""}, + {"the Go name in the tag", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJSON\"`", "as the engine does, EncryptedJson"}, + {"two paths, one accessor", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$.a_b;$.aB)\"`", "both name the accessor At.AB"}, + {"a document with no target", "\tProfile json.RawMessage `stash:\"profile,encrypt\"`", "the engine refuses the declaration"}, + {"a string into a document", "\tProfile string `stash:\"profile,encrypt_into=EncryptedJson\"`", "EncryptedJson seals a json"}, + {"options on a type that takes none", "\tEmail string `stash:\"email,encrypt_into=TextEq(ordered=$.a)\"`", "the engine refuses the declaration"}, + } { + t.Run(c.name, func(t *testing.T) { + written, stderr, code := generate(c.fields) + if code != 1 || written != "" { + t.Fatalf("exit %d, wrote %d bytes\n%s", code, len(written), stderr) + } + if !strings.Contains(stderr, c.want) { + t.Fatalf("stderr = %q, want %q", stderr, c.want) + } + }) + } +} diff --git a/languages/golang/encrypt/README.md b/languages/golang/encrypt/README.md index f68e8f2fb..5ba5516a5 100644 --- a/languages/golang/encrypt/README.md +++ b/languages/golang/encrypt/README.md @@ -576,7 +576,7 @@ the installer disables these domains, and the `Ord` types are the ones to use. Text whose fold is longer than 1023 bytes, or ends in NUL, has no block ORE term and is refused, stored or queried. -The rest of the catalog (ordered dates, timestamps and decimals, JSON) is listed in +The rest of the catalog (ordered dates, timestamps and decimals, the v3 JSON types) is listed in `eql.Types` with the reason it is not produced yet, and `stashgen` refuses it with that reason. The guest builds the value: the generated code imports `encrypt/eql`, which embeds the build of the engine that holds the @@ -591,6 +591,34 @@ with a tenant part refuses a struct with an `encrypt_into` field: an EQL value is stored under a table and a column, and the extended label has no column. +### Encrypted JSON documents + +`EncryptedJson` stores a whole JSON document in one `public.eql_v4_json_tree` +column, from a `json.RawMessage` field (`stash:"profile,encrypt_into=EncryptedJson(ordered=$.age)"`; +see the stashgen README). One entry per node is encrypted, so the database +can test containment, equality and ordering at a path, return one node with +`->`, and walk a subtree, without decrypting: + +```go +q, err := users.Fields.Profile.Contains(ctx, cipher, json.RawMessage(`{"plan":"pro"}`)) +rows, err := db.QueryContext(ctx, `SELECT id FROM users WHERE profile @> $1::eql_v4.query_json_tree`, q) + +read, err := users.Fields.Profile.Subtree(ctx, cipher, "$.visits[*].at", encrypt.JSONReadOptions{}) +// SELECT e FROM users, eql_v4.json_tree_subtree(profile, $1::text[], 64) AS e WHERE id = $2 +// with read.Tokens() as $1; collect each row's entries, then: +values, err := read.Open(ctx, entriesPerRow) +``` + +A selection or read opens rows only as answers to the request that prepared +it: the engine derives the selector or the read again from the field, the +path and the cipher's keyset, and refuses a row that is not one it asked +for. It does not prove the database returned every row, or rows of the +database row you meant. `encrypt.JSONTokens` binds as a `text[]` literal, +and a selector or token as `text`; cast each parameter, as above, so +PostgreSQL picks EQL's operator and not `jsonb`'s. Numbers order as `float64`, +so integers past 2^53 can tie at an ordered path; containment and `EqAt` +compare them exactly. + ## Building The package embeds `wasm/stack_encrypt_guest.wasm`, a build artefact of the diff --git a/languages/golang/encrypt/eql_postgres_test.go b/languages/golang/encrypt/eql_postgres_test.go index 9772237cc..af7136dc2 100644 --- a/languages/golang/encrypt/eql_postgres_test.go +++ b/languages/golang/encrypt/eql_postgres_test.go @@ -148,6 +148,10 @@ func TestEQLTypesRoundTripAndQueryInPostgres(t *testing.T) { ds[i] = d } runEQLType(h, gensupport.Decimal, ds, nil) + case "json": + // A document has operators of its own, not one query per + // index: TestJSONDocumentsQueryAndReadInPostgres runs them. + return default: t.Fatalf("no plaintexts for wire kind %q: add a case so %s is tested", typ.Plaintext, typ.Name) } diff --git a/languages/golang/encrypt/gensupport/codec.go b/languages/golang/encrypt/gensupport/codec.go index ab6aee541..4eba52782 100644 --- a/languages/golang/encrypt/gensupport/codec.go +++ b/languages/golang/encrypt/gensupport/codec.go @@ -26,7 +26,7 @@ type Output struct { Match encrypt.MatchTerm Ore encrypt.OreTerm Ope encrypt.OpeTerm - JSON encrypt.JSONTerm + JSON encrypt.JSONTerm //nolint:staticcheck // the deprecated output, kept so a declaration naming it fails into a Go type EQL []byte } diff --git a/languages/golang/encrypt/gensupport/declaration.go b/languages/golang/encrypt/gensupport/declaration.go index f110bb8ca..f74c054cf 100644 --- a/languages/golang/encrypt/gensupport/declaration.go +++ b/languages/golang/encrypt/gensupport/declaration.go @@ -48,6 +48,9 @@ const ( Timestamp Kind = "timestamp" String Kind = "string" Bytes Kind = "bytes" + // JSON is a json.RawMessage field stored as an EncryptedJSON document: + // it crosses as its JSON text, which the engine parses. + JSON Kind = "json" ) // OpaqueField is the one field of an opaque declaration: the whole struct, @@ -78,6 +81,19 @@ type field struct { indexes []encrypt.Index eqlType string null NullMode + options *TargetOptions +} + +// TargetOptions are the options an EncryptInto field gives its EQL type: +// for EncryptedJson, the JSON index's. The engine checks them when the +// declaration is first used (stashgen asked it when the file was written). +type TargetOptions struct { + // SealedRoot says whether the document's root is sealed; nil is the + // default, sealed. + SealedRoot *bool + // Ordered are the JSONPath paths whose number and string leaves carry + // an ordering term. + Ordered []string } // NullMode is how a sealed field declared over a pointer (*T) stores nil: @@ -170,6 +186,28 @@ func (d Declaration) EncryptInto(name string, kind Kind, eqlType string) Declara return d.add(field{name: name, kind: kind, verb: verbEncryptInto, eqlType: eqlType}) } +// Options gives the named EncryptInto field's EQL type its options: the +// `encrypt_into=EncryptedJson(ordered=...)` tag. +func (d Declaration) Options(name string, options TargetOptions) Declaration { + if d.err != nil { + return d + } + for i := range d.fields { + if d.fields[i].name != name { + continue + } + if d.fields[i].verb != verbEncryptInto { + d.err = fmt.Errorf("gensupport: Options(%q): only an EncryptInto field's EQL type takes options", name) + return d + } + d.fields = slices.Clone(d.fields) + d.fields[i].options = &options + return d + } + d.err = fmt.Errorf("gensupport: Options(%q): no such field", name) + return d +} + // Null says how the named field, declared over a pointer, stores nil. The // engine decides whether the field can, when the declaration is first used: // a sealed null needs a field whose only index is equality, of a kind of @@ -293,6 +331,9 @@ func (d Declaration) plan() (*record.Plan, error) { // is the engine's to say, at the first call (stashgen asked it // when the file was written). rf.Target = f.eqlType + if f.options != nil { + rf.TargetOptions = &record.TargetOptions{SealedRoot: f.options.SealedRoot, Ordered: slices.Clone(f.options.Ordered)} + } case verbEncrypt: rf.Outputs = []record.Output{record.Ciphertext} case verbEncryptIndex: @@ -320,3 +361,7 @@ func (d Declaration) plan() (*record.Plan, error) { var termOutputs = map[record.Output]struct{}{ record.Equality: {}, record.Match: {}, record.Ore: {}, record.Ope: {}, } + +// Ptr is a pointer to v: what a generated TargetOptions literal spells an +// option set to false with. +func Ptr[T any](v T) *T { return &v } diff --git a/languages/golang/encrypt/gensupport/gensupport_internal_test.go b/languages/golang/encrypt/gensupport/gensupport_internal_test.go index 0c2655799..a109e5c17 100644 --- a/languages/golang/encrypt/gensupport/gensupport_internal_test.go +++ b/languages/golang/encrypt/gensupport/gensupport_internal_test.go @@ -73,7 +73,7 @@ func TestDeclarationRefusals(t *testing.T) { "identity of no field": Declare("u").Encrypt("a", String).Identity("b", "x"), "seals nothing": Declare("u").Passthrough("id"), "EQL type unnamed": Declare("u").EncryptInto("email", String, ""), - "json index": Declare("u").EncryptIndex("a", String, encrypt.JSON()), + "json index": Declare("u").EncryptIndex("a", String, encrypt.JSON()), //nolint:staticcheck // the deprecated index is still refused "name not a label": Declare("u").Encrypt("1a", String), } for name, d := range cases { diff --git a/languages/golang/encrypt/gensupport/json.go b/languages/golang/encrypt/gensupport/json.go new file mode 100644 index 000000000..f04391529 --- /dev/null +++ b/languages/golang/encrypt/gensupport/json.go @@ -0,0 +1,118 @@ +package gensupport + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/vitaminc/bindings/go/vcvalue" +) + +// The operators of a field stored as an EncryptedJSON document, for the +// generated methods of its Fields entry. Each sends the operator's named +// inputs, JSON values as their JSON text, and checks the engine answered +// in the encoding the method returns. + +// jsonTerm derives one JSON operator's term on the field. +func (f Field[T]) jsonTerm(ctx context.Context, c *encrypt.Cipher, operator, encoding string, input vcvalue.Object) (encrypt.QueryTerm, error) { + if f.err != nil { + return encrypt.QueryTerm{}, f.err + } + if c == nil { + return encrypt.QueryTerm{}, fmt.Errorf("%w: gensupport: %s: a query needs a cipher", encrypt.ErrEncoding, f.name) + } + term, err := c.QueryTerm(ctx, f.plan, f.name, operator, input) + if err != nil { + return encrypt.QueryTerm{}, err + } + if term.Encoding != encoding || term.Fidelity != "exact" { + return encrypt.QueryTerm{}, fmt.Errorf("gensupport: %s: %s came back as a %s %s term, not an exact %s one", f.name, operator, term.Fidelity, term.Encoding, encoding) + } + return term, nil +} + +// jsonText is a JSON value as an input carries it: its text, as bytes. +func jsonText(v json.RawMessage) []byte { return []byte(v) } + +// JSONQuery derives a containment operator's query on a JSON field +// (contains, contained_by): the needle's projection, Q. +func JSONQuery[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, operator string, needle json.RawMessage) (Q, error) { + term, err := f.jsonTerm(ctx, c, operator, "eql", vcvalue.Object{{Key: "json", Value: jsonText(needle)}}) + if err != nil || len(term.Terms) != 1 { + return nil, oneTermErr(f.name, operator, err) + } + return Q(term.Terms[0]), nil +} + +// JSONQueryAt derives an operator at a path on a JSON field (eq_at, lt_at, +// lte_at, gt_at, gte_at): Q is the needle for eq_at and the leaf operand +// for the orderings. +func JSONQueryAt[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, operator, path string, v json.RawMessage) (Q, error) { + term, err := f.jsonTerm(ctx, c, operator, "eql", vcvalue.Object{{Key: "path", Value: path}, {Key: "json", Value: jsonText(v)}}) + if err != nil || len(term.Terms) != 1 { + return nil, oneTermErr(f.name, operator, err) + } + return Q(term.Terms[0]), nil +} + +// JSONBetweenAt derives between_at on a JSON field: the two leaf operands, +// low then high, inclusive. +func JSONBetweenAt[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, path string, lo, hi json.RawMessage) (encrypt.JSONRange[Q], error) { + term, err := f.jsonTerm(ctx, c, "between_at", "eql", vcvalue.Object{{Key: "path", Value: path}, {Key: "low", Value: jsonText(lo)}, {Key: "high", Value: jsonText(hi)}}) + if err != nil { + return encrypt.JSONRange[Q]{}, err + } + if len(term.Terms) != 2 { + return encrypt.JSONRange[Q]{}, fmt.Errorf("gensupport: %s: between_at came back as %d terms, not 2", f.name, len(term.Terms)) + } + return encrypt.JSONRange[Q]{Low: Q(term.Terms[0]), High: Q(term.Terms[1])}, nil +} + +// JSONSelect prepares `->` at a path on a JSON field: the selector to +// bind, and what opens the entries it returns. E is the field's entry type. +func JSONSelect[E ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, path string) (encrypt.JSONSelection[E], error) { + term, err := f.jsonTerm(ctx, c, "selector", "text", vcvalue.Object{{Key: "path", Value: path}}) + if err == nil && len(term.Text) != 1 { + err = fmt.Errorf("gensupport: %s: selector came back as %d texts, not 1", f.name, len(term.Text)) + } + if err != nil { + return encrypt.JSONSelection[E]{}, err + } + return encrypt.NewJSONSelection[E](c, f.plan, f.name, path, term.Text[0]), nil +} + +// JSONSubtree prepares a subtree read at a path on a JSON field: the +// tokens to bind, and what opens the rows they return. +func JSONSubtree[E ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[E], error) { + input := vcvalue.Object{{Key: "path", Value: path}} + if opts.WildcardBound != 0 { + input = append(input, vcvalue.Field{Key: "wildcard_bound", Value: opts.WildcardBound}) + } + term, err := f.jsonTerm(ctx, c, "subtree", "text_array", input) + if err != nil { + return encrypt.JSONRead[E]{}, err + } + return encrypt.NewJSONRead[E](c, f.plan, f.name, path, opts.WildcardBound, term.Text), nil +} + +// JSONNodeToken derives the token of the node at a path with no wildcard +// (children, child_count), which json_tree_children and +// json_tree_child_count take. +func JSONNodeToken[T any](ctx context.Context, f Field[T], c *encrypt.Cipher, operator, path string) (encrypt.JSONToken, error) { + term, err := f.jsonTerm(ctx, c, operator, "text", vcvalue.Object{{Key: "path", Value: path}}) + if err == nil && len(term.Text) != 1 { + err = fmt.Errorf("gensupport: %s: %s came back as %d texts, not 1", f.name, operator, len(term.Text)) + } + if err != nil { + return "", err + } + return encrypt.JSONToken(term.Text[0]), nil +} + +func oneTermErr(field, operator string, err error) error { + if err != nil { + return err + } + return fmt.Errorf("gensupport: %s: %s came back as other than one term", field, operator) +} diff --git a/languages/golang/encrypt/gensupport/widen.go b/languages/golang/encrypt/gensupport/widen.go index 02e13c735..ca9f0564e 100644 --- a/languages/golang/encrypt/gensupport/widen.go +++ b/languages/golang/encrypt/gensupport/widen.go @@ -1,6 +1,7 @@ package gensupport import ( + "encoding/json" "fmt" "math" "reflect" @@ -31,6 +32,17 @@ func asKind(kind Kind, v any) (any, error) { if err := checkValue(v); err != nil { return nil, err } + if kind == JSON { + // A document crosses as its JSON text, in a bytes value; the + // engine parses it. + switch raw := v.(type) { + case json.RawMessage: + return []byte(raw), nil + case []byte: + return raw, nil + } + return nil, fmt.Errorf("%w: a JSON field takes a json.RawMessage, not a %T", encrypt.ErrEncoding, v) + } if !isInteger(kind) { return v, nil } diff --git a/languages/golang/encrypt/internal/testusers/customer_stash.go b/languages/golang/encrypt/internal/testusers/customer_stash.go new file mode 100644 index 000000000..b8ebbf437 --- /dev/null +++ b/languages/golang/encrypt/internal/testusers/customer_stash.go @@ -0,0 +1,283 @@ +// Code generated by stashgen. DO NOT EDIT. + +package testusers + +import ( + "context" + "encoding/json" + "log/slog" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/eql" + "github.com/cipherstash/stack/languages/golang/encrypt/gensupport" +) + +// Stops compiling when the library does not accept this version of generated file. +const _ = gensupport.GeneratedVersion1 + +// Customer prints its sealed fields in the clear: it has no String or LogValue +// method. Write them, or run stashgen with -redact. + +type EncryptedCustomer struct { + ID int64 + Profile eql.EncryptedJSON + Prefs *eql.EncryptedJSON +} + +func (e EncryptedCustomer) String() string { + return gensupport.Redacted("EncryptedCustomer", map[string]any{"ID": e.ID}, "Profile", "Prefs") +} + +func (e EncryptedCustomer) LogValue() slog.Value { + return gensupport.RedactedLog(map[string]any{"ID": e.ID}, "Profile", "Prefs") +} + +// Stops compiling when Customer gains, loses, reorders or retypes a field. +var _ = customerShape(Customer{}) + +type customerShape struct { + _ struct{} + ID int64 + Profile json.RawMessage + Prefs *json.RawMessage +} + +var customerDeclaration = gensupport.Declare("customers"). + Passthrough("id"). + EncryptInto("profile", gensupport.JSON, "EncryptedJson"). + Options("profile", gensupport.TargetOptions{Ordered: []string{"$.age", "$.visits[*].at"}}). + EncryptInto("prefs", gensupport.JSON, "EncryptedJson"). + Null("prefs", gensupport.NullSQL) + +var customerCodec = gensupport.New(gensupport.Generated[Customer, EncryptedCustomer]{ + TypeName: "Customer", + Declaration: customerDeclaration, + PrintsPlaintext: true, + Source: func(v Customer) gensupport.Values { + return gensupport.Values{ + "id": v.ID, + "profile": v.Profile, + "prefs": gensupport.Deref(v.Prefs), + } + }, + Seal: func(rec gensupport.Record) (EncryptedCustomer, error) { + var e EncryptedCustomer + var err error + if e.ID, err = gensupport.Passthrough[int64](rec, "id"); err != nil { + return EncryptedCustomer{}, err + } + e.Profile = eql.EncryptedJSON(rec["profile"].EQL) + if o, ok := rec["prefs"]; ok { + v := eql.EncryptedJSON(o.EQL) + e.Prefs = &v + } + return e, nil + }, + Open: func(e EncryptedCustomer) gensupport.Record { + rec := gensupport.Record{ + "id": {Value: e.ID}, + "profile": {EQL: e.Profile}, + } + if e.Prefs != nil { + rec["prefs"] = gensupport.Output{EQL: *e.Prefs} + } + return rec + }, + Value: func(e EncryptedCustomer, vals gensupport.Values) (Customer, error) { + var v Customer + var err error + if v.ID, err = gensupport.Get[int64](vals, "id"); err != nil { + return Customer{}, err + } + if v.Profile, err = gensupport.Get[json.RawMessage](vals, "profile"); err != nil { + return Customer{}, err + } + if v.Prefs, err = gensupport.GetPtr[json.RawMessage](vals, "prefs"); err != nil { + return Customer{}, err + } + return v, nil + }, +}) + +// EncryptCustomer seals each Customer, with one ZeroKMS request for each 500 +// sealed values, plus one the first time a keyset is used. The result has one +// element for each input, in the same order. +func EncryptCustomer(ctx context.Context, cipher *encrypt.Cipher, values []Customer) ([]EncryptedCustomer, error) { + return customerCodec.Encrypt(ctx, cipher, values) +} + +// DecryptCustomer opens each EncryptedCustomer, with one ZeroKMS request for +// each 500 sealed values, plus one the first time a keyset is used. +func DecryptCustomer(ctx context.Context, d encrypt.Decrypter, encrypted []EncryptedCustomer) ([]Customer, error) { + return customerCodec.Decrypt(ctx, d, encrypted) +} + +var CustomerFields = struct { + Profile CustomerProfileField + Prefs CustomerPrefsField +}{ + Profile: newCustomerProfileField(gensupport.NewField[json.RawMessage](customerDeclaration, "profile")), + Prefs: newCustomerPrefsField(gensupport.NewField[json.RawMessage](customerDeclaration, "prefs")), +} + +type CustomerProfileField struct { + field gensupport.Field[json.RawMessage] + // At has one accessor per path the field orders, whose methods + // are the orderings there. + At CustomerProfileFieldAt +} + +func newCustomerProfileField(f gensupport.Field[json.RawMessage]) CustomerProfileField { + return CustomerProfileField{field: f, At: CustomerProfileFieldAt{ + Age: CustomerProfileFieldPath{field: f, path: "$.age"}, + VisitsAt: CustomerProfileFieldPath{field: f, path: "$.visits[*].at"}, + }} +} + +func (f CustomerProfileField) Encrypt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.EncryptedJSON, error) { + out, err := f.field.Encrypt(ctx, c, v) + return eql.EncryptedJSON(out.EQL), err +} + +// Contains derives the query for documents that contain needle, a JSON value: +// col @> query. Each array element of the needle must sit inside one element of +// the document's array. +func (f CustomerProfileField) Contains(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.EncryptedJSONQuery, error) { + return gensupport.JSONQuery[eql.EncryptedJSONQuery](ctx, f.field, c, "contains", needle) +} + +// ContainedBy derives the query for documents contained by needle, a JSON +// value: col <@ query. +func (f CustomerProfileField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.EncryptedJSONQuery, error) { + return gensupport.JSONQuery[eql.EncryptedJSONQuery](ctx, f.field, c, "contained_by", needle) +} + +// EqAt derives the query for documents holding the scalar v at path: col @> +// query. A wildcard step matches when any element holds it; a position ([2]) is +// refused, since the stored selector reads every position as the wildcard. +func (f CustomerProfileField) EqAt(ctx context.Context, c *encrypt.Cipher, path string, v json.RawMessage) (eql.EncryptedJSONQuery, error) { + return gensupport.JSONQueryAt[eql.EncryptedJSONQuery](ctx, f.field, c, "eq_at", path, v) +} + +// Selector prepares `->` at path: bind Selector() as col -> $1::text, and open +// the entries it returns with Open. A position ([2]) is refused; read one with +// Subtree. +func (f CustomerProfileField) Selector(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONSelection[eql.EncryptedJSONEntry], error) { + return gensupport.JSONSelect[eql.EncryptedJSONEntry](ctx, f.field, c, path) +} + +// Subtree prepares a read of the value at path: bind Tokens() as +// eql_v4.json_tree_subtree(col, $1::text[], budget), and open the rows it +// returns with Open or OpenOne. +func (f CustomerProfileField) Subtree(ctx context.Context, c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[eql.EncryptedJSONEntry], error) { + return gensupport.JSONSubtree[eql.EncryptedJSONEntry](ctx, f.field, c, path, opts) +} + +// Children derives the token of the node at path, a path with no wildcard, +// which eql_v4.json_tree_children(col, $1::text) takes. Its rows are encrypted +// entries one level down. +func (f CustomerProfileField) Children(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONToken, error) { + return gensupport.JSONNodeToken(ctx, f.field, c, "children", path) +} + +// ChildCount derives the token of the node at path, which +// eql_v4.json_tree_child_count(col, $1::text) takes: zero for a scalar or an +// empty container, NULL where the row has no node there or for the sealed root. +func (f CustomerProfileField) ChildCount(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONToken, error) { + return gensupport.JSONNodeToken(ctx, f.field, c, "child_count", path) +} + +type CustomerProfileFieldAt struct { + Age CustomerProfileFieldPath // $.age + VisitsAt CustomerProfileFieldPath // $.visits[*].at +} + +// CustomerProfileFieldPath is one ordered path of Profile: its orderings +// compare the first value `->` extracts there, by the leaf's ordering term. +// Numbers order as float64, so integers past 2^53 can tie; text orders case- +// and accent-folded. +type CustomerProfileFieldPath struct { + field gensupport.Field[json.RawMessage] + path string +} + +func (p CustomerProfileFieldPath) Lt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.EncryptedJSONLeafQuery, error) { + return gensupport.JSONQueryAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, "lt_at", p.path, v) +} + +func (p CustomerProfileFieldPath) Lte(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.EncryptedJSONLeafQuery, error) { + return gensupport.JSONQueryAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, "lte_at", p.path, v) +} + +func (p CustomerProfileFieldPath) Gt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.EncryptedJSONLeafQuery, error) { + return gensupport.JSONQueryAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, "gt_at", p.path, v) +} + +func (p CustomerProfileFieldPath) Gte(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.EncryptedJSONLeafQuery, error) { + return gensupport.JSONQueryAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, "gte_at", p.path, v) +} + +func (p CustomerProfileFieldPath) Between(ctx context.Context, c *encrypt.Cipher, lo, hi json.RawMessage) (encrypt.JSONRange[eql.EncryptedJSONLeafQuery], error) { + return gensupport.JSONBetweenAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, p.path, lo, hi) +} + +type CustomerPrefsField struct { + field gensupport.Field[json.RawMessage] +} + +func newCustomerPrefsField(f gensupport.Field[json.RawMessage]) CustomerPrefsField { + return CustomerPrefsField{field: f} +} + +func (f CustomerPrefsField) Encrypt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.EncryptedJSON, error) { + out, err := f.field.Encrypt(ctx, c, v) + return eql.EncryptedJSON(out.EQL), err +} + +// Contains derives the query for documents that contain needle, a JSON value: +// col @> query. Each array element of the needle must sit inside one element of +// the document's array. +func (f CustomerPrefsField) Contains(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.EncryptedJSONQuery, error) { + return gensupport.JSONQuery[eql.EncryptedJSONQuery](ctx, f.field, c, "contains", needle) +} + +// ContainedBy derives the query for documents contained by needle, a JSON +// value: col <@ query. +func (f CustomerPrefsField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.EncryptedJSONQuery, error) { + return gensupport.JSONQuery[eql.EncryptedJSONQuery](ctx, f.field, c, "contained_by", needle) +} + +// EqAt derives the query for documents holding the scalar v at path: col @> +// query. A wildcard step matches when any element holds it; a position ([2]) is +// refused, since the stored selector reads every position as the wildcard. +func (f CustomerPrefsField) EqAt(ctx context.Context, c *encrypt.Cipher, path string, v json.RawMessage) (eql.EncryptedJSONQuery, error) { + return gensupport.JSONQueryAt[eql.EncryptedJSONQuery](ctx, f.field, c, "eq_at", path, v) +} + +// Selector prepares `->` at path: bind Selector() as col -> $1::text, and open +// the entries it returns with Open. A position ([2]) is refused; read one with +// Subtree. +func (f CustomerPrefsField) Selector(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONSelection[eql.EncryptedJSONEntry], error) { + return gensupport.JSONSelect[eql.EncryptedJSONEntry](ctx, f.field, c, path) +} + +// Subtree prepares a read of the value at path: bind Tokens() as +// eql_v4.json_tree_subtree(col, $1::text[], budget), and open the rows it +// returns with Open or OpenOne. +func (f CustomerPrefsField) Subtree(ctx context.Context, c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[eql.EncryptedJSONEntry], error) { + return gensupport.JSONSubtree[eql.EncryptedJSONEntry](ctx, f.field, c, path, opts) +} + +// Children derives the token of the node at path, a path with no wildcard, +// which eql_v4.json_tree_children(col, $1::text) takes. Its rows are encrypted +// entries one level down. +func (f CustomerPrefsField) Children(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONToken, error) { + return gensupport.JSONNodeToken(ctx, f.field, c, "children", path) +} + +// ChildCount derives the token of the node at path, which +// eql_v4.json_tree_child_count(col, $1::text) takes: zero for a scalar or an +// empty container, NULL where the row has no node there or for the sealed root. +func (f CustomerPrefsField) ChildCount(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONToken, error) { + return gensupport.JSONNodeToken(ctx, f.field, c, "child_count", path) +} diff --git a/languages/golang/encrypt/internal/testusers/customers.go b/languages/golang/encrypt/internal/testusers/customers.go new file mode 100644 index 000000000..f6a330798 --- /dev/null +++ b/languages/golang/encrypt/internal/testusers/customers.go @@ -0,0 +1,15 @@ +package testusers + +import "encoding/json" + +//go:generate go run github.com/cipherstash/stack/languages/golang/cmd/stashgen -type Customer -name Customer + +// Customer holds two encrypted JSON documents (the EQL type EncryptedJson, +// stored in public.eql_v4_json_tree): a profile whose $.age and every +// visit's $.at order, and optional preferences, stored as SQL NULL when nil. +type Customer struct { + _ struct{} `stash:"context=customers"` + ID int64 `stash:"id,passthrough"` + Profile json.RawMessage `stash:"profile,encrypt_into=EncryptedJson(ordered=$.age;$.visits[*].at)"` + Prefs *json.RawMessage `stash:"prefs,encrypt_into=EncryptedJson,null=sql"` +} diff --git a/languages/golang/encrypt/json.go b/languages/golang/encrypt/json.go index 988a50547..ca17273ac 100644 --- a/languages/golang/encrypt/json.go +++ b/languages/golang/encrypt/json.go @@ -2,7 +2,10 @@ package encrypt import ( "context" + "database/sql/driver" + "encoding/json" "fmt" + "strings" "github.com/cipherstash/stack/languages/golang/internal/record" "github.com/cipherstash/vitaminc/bindings/go/vcffi" @@ -91,3 +94,207 @@ func (cph *Cipher) JSONOpen(ctx context.Context, plan *record.Plan, field string } return opened, nil } + +// JSONSelector is a JSON path's selector: what `->`, `->>` and the path +// functions of an EncryptedJSON column take. It binds as SQL text; cast it, +// `col -> $1::text`, so PostgreSQL picks EQL's operator and not jsonb's. +// It is column-wide: the same path has the same selector in every row. +type JSONSelector string + +// Value binds the selector as text. +func (s JSONSelector) Value() (driver.Value, error) { return string(s), nil } + +// JSONToken is one node's token: what eql_v4.json_tree_children and +// eql_v4.json_tree_child_count take. It binds as SQL text. A token is a +// capability over the node's subtree in every row of the column: send it +// to the database for a query, and do not log it. +type JSONToken string + +// Value binds the token as text. +func (t JSONToken) Value() (driver.Value, error) { return string(t), nil } + +// JSONTokens are a subtree read's tokens: what eql_v4.json_tree_subtree +// takes. They bind as a PostgreSQL text[] literal; cast the parameter, +// `$2::text[]`. Like a [JSONToken], each is a capability: do not log them. +type JSONTokens []string + +// Value binds the tokens as a text[] literal. A token is base64, so it +// needs no escape inside the quotes. +func (t JSONTokens) Value() (driver.Value, error) { + var b strings.Builder + b.WriteByte('{') + for i, token := range t { + if i > 0 { + b.WriteByte(',') + } + b.WriteByte('"') + b.WriteString(token) + b.WriteByte('"') + } + b.WriteByte('}') + return b.String(), nil +} + +// JSONValue is one value a read reached, or none. Found is false for no +// value at the path (the row has no node there, or SQL NULL); a JSON null +// that is there is Found, with JSON "null". JSON is the value's JSON text: +// plaintext, which the caller owns. +type JSONValue struct { + Found bool + JSON json.RawMessage +} + +// JSONReadOptions are a subtree read's options. +type JSONReadOptions struct { + // WildcardBound is how many elements each [*] in the path is expanded + // to; 0 is the engine's default, 64. A read of an array longer than the + // bound fails rather than open to part of it ([ErrEncoding], with the + // diagnostic code stack_encrypt::json_wildcard_truncated). Two wildcard + // levels send bound² + bound + 1 tokens, and the engine refuses a read + // past 8,192. + WildcardBound uint32 +} + +// JSONRange is both bounds of a between at a path, each an ordering +// operand Q, low then high, inclusive. +type JSONRange[Q ~[]byte] struct { + Low Q + High Q +} + +// JSONSelection is a prepared `->` on an EncryptedJSON field: the selector +// to bind, and what opens the entries the database returns for it. It keeps +// the request it was prepared from (the field's declaration, the path and +// the cipher's keyset), so the entries open only as answers to it: the +// engine derives the selector again and refuses an entry that does not +// carry it. E is the column's entry type, eql.EncryptedJSONEntry. +type JSONSelection[E ~[]byte] struct { + cipher *Cipher + plan *record.Plan + field string + path string + selector JSONSelector +} + +// NewJSONSelection is a selection prepared by [Cipher.QueryTerm]'s +// selector operator. For generated code. +func NewJSONSelection[E ~[]byte](c *Cipher, plan *record.Plan, field, path, selector string) JSONSelection[E] { + return JSONSelection[E]{cipher: c, plan: plan, field: field, path: path, selector: JSONSelector(selector)} +} + +// Selector is the selector to bind: `col -> $1::text`. +func (s JSONSelection[E]) Selector() JSONSelector { return s.selector } + +// Open opens the entries `col -> selector` returned, one per database row, +// nil for a row where it returned SQL NULL, with one ZeroKMS request. Each +// opens to the scalar it holds (Found), or for the sealed root at "$" the +// whole document; a nil opens to no value. A container's entry is refused: +// read a container with a subtree read. An entry that is not the selector's +// is [ErrEncoding]. +func (s JSONSelection[E]) Open(ctx context.Context, rows []*E) ([]JSONValue, error) { + if s.cipher == nil { + return nil, fmt.Errorf("%w: a zero JSONSelection has no request to open against", ErrEncoding) + } + groups := make([][][]byte, len(rows)) + for i, row := range rows { + if row != nil { + groups[i] = [][]byte{[]byte(*row)} + } + } + opened, err := s.cipher.JSONOpen(ctx, s.plan, s.field, vcvalue.Object{{Key: "operation", Value: "selector"}, {Key: "path", Value: s.path}}, groups) + if err != nil { + return nil, err + } + out := make([]JSONValue, len(opened)) + for i, values := range opened { + if len(values) == 1 { + out[i] = JSONValue{Found: true, JSON: values[0]} + } + } + return out, nil +} + +// JSONRead is a prepared subtree read of an EncryptedJSON field: the tokens +// to bind, and what opens the rows eql_v4.json_tree_subtree returns for +// them. It keeps the request it was prepared from (the field's declaration, +// the path, the bound and the cipher's keyset); the engine derives the read +// again from it, and checks every row against it. E is the column's entry +// type, eql.EncryptedJSONEntry. +type JSONRead[E ~[]byte] struct { + cipher *Cipher + plan *record.Plan + field string + path string + bound uint32 + tokens JSONTokens +} + +// NewJSONRead is a read prepared by [Cipher.QueryTerm]'s subtree operator. +// For generated code. +func NewJSONRead[E ~[]byte](c *Cipher, plan *record.Plan, field, path string, bound uint32, tokens []string) JSONRead[E] { + return JSONRead[E]{cipher: c, plan: plan, field: field, path: path, bound: bound, tokens: JSONTokens(tokens)} +} + +// Tokens are the tokens to bind: +// `eql_v4.json_tree_subtree(col, $1::text[], 64)`. +func (r JSONRead[E]) Tokens() JSONTokens { return r.tokens } + +// Open opens the rows the read returned, one group per database row, with +// one ZeroKMS request: per row, the value at each concrete path the read +// named that the row has, in document order. A row with no node there +// opens to no value. Past its walk budget the database returns the sealed +// root instead, and the read's paths are evaluated over the whole document +// to the same values; only those values come back. A row that is not one +// the read asked for, or a wildcard read of an array longer than the bound, +// is [ErrEncoding]. +// +// Opening checks each row against the read, not that the database +// returned every row it should have, nor that the rows came from the row +// the caller meant. +func (r JSONRead[E]) Open(ctx context.Context, rows [][]E) ([][]json.RawMessage, error) { + if r.cipher == nil { + return nil, fmt.Errorf("%w: a zero JSONRead has no request to open against", ErrEncoding) + } + groups := make([][][]byte, len(rows)) + for i, row := range rows { + groups[i] = make([][]byte, len(row)) + for j, entry := range row { + groups[i][j] = []byte(entry) + } + } + request := vcvalue.Object{{Key: "operation", Value: "subtree"}, {Key: "path", Value: r.path}} + if r.bound != 0 { + request = append(request, vcvalue.Field{Key: "wildcard_bound", Value: r.bound}) + } + opened, err := r.cipher.JSONOpen(ctx, r.plan, r.field, request, groups) + if err != nil { + return nil, err + } + out := make([][]json.RawMessage, len(opened)) + for i, values := range opened { + out[i] = make([]json.RawMessage, len(values)) + for j, v := range values { + out[i][j] = v + } + } + return out, nil +} + +// OpenOne opens one row's rows to the one value at the read's path: Found +// with its JSON, or not Found where the row has no node there. A read that +// reaches more than one value (a wildcard) is [ErrEncoding]; open it with +// [JSONRead.Open]. +func (r JSONRead[E]) OpenOne(ctx context.Context, rows []E) (JSONValue, error) { + opened, err := r.Open(ctx, [][]E{rows}) + if err != nil { + return JSONValue{}, err + } + switch values := opened[0]; len(values) { + case 0: + return JSONValue{}, nil + case 1: + return JSONValue{Found: true, JSON: values[0]}, nil + default: + return JSONValue{}, fmt.Errorf("%w: the read reached %d values, not one; open it with Open", ErrEncoding, len(values)) + } +} diff --git a/languages/golang/encrypt/json_fields_test.go b/languages/golang/encrypt/json_fields_test.go new file mode 100644 index 000000000..12fab79b0 --- /dev/null +++ b/languages/golang/encrypt/json_fields_test.go @@ -0,0 +1,133 @@ +package encrypt_test + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/internal/testusers" +) + +// Generated code for json.RawMessage fields stored as EncryptedJson: +// testusers.Customer's Profile, ordered at $.age and $.visits[*].at, and its +// optional Prefs. + +func rawPtr(s string) *json.RawMessage { + r := json.RawMessage(s) + return &r +} + +// sameJSON reports whether two JSON texts hold the same value: the +// document is sealed as values, so its text is not promised back byte for +// byte. +func sameJSON(t *testing.T, a, b []byte) bool { + t.Helper() + var x, y any + if err := json.Unmarshal(a, &x); err != nil { + t.Fatalf("%s: %v", a, err) + } + if err := json.Unmarshal(b, &y); err != nil { + t.Fatalf("%s: %v", b, err) + } + return reflect.DeepEqual(x, y) +} + +func TestACustomersDocumentsRoundTrip(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + in := []testusers.Customer{ + {ID: 1, Profile: json.RawMessage(`{"name":"Alice","age":34,"visits":[{"at":3},{"at":1}]}`), Prefs: rawPtr(`{"theme":"dark"}`)}, + {ID: 2, Profile: json.RawMessage(`[1,2,3]`)}, + {ID: 3, Profile: json.RawMessage(`null`), Prefs: rawPtr(`null`)}, + } + encrypted, err := testusers.EncryptCustomer(ctx, c.DefaultKeyset(), in) + if err != nil { + t.Fatal(err) + } + if encrypted[1].Prefs != nil || encrypted[2].Prefs == nil { + t.Fatalf("a nil document is SQL NULL, a JSON null is a document: %v %v", encrypted[1].Prefs, encrypted[2].Prefs) + } + if s := encrypted[0].String(); strings.Contains(s, "Alice") { + t.Fatalf("String shows the document: %s", s) + } + back, err := testusers.DecryptCustomer(ctx, c, encrypted) + if err != nil { + t.Fatal(err) + } + for i := range in { + if back[i].ID != in[i].ID || !sameJSON(t, back[i].Profile, in[i].Profile) { + t.Fatalf("row %d: %s", i, back[i].Profile) + } + if (back[i].Prefs == nil) != (in[i].Prefs == nil) || (in[i].Prefs != nil && !sameJSON(t, *back[i].Prefs, *in[i].Prefs)) { + t.Fatalf("row %d prefs: %v", i, back[i].Prefs) + } + } + // Empty bytes are not a document: a required field takes JSON. + if _, err := testusers.EncryptCustomer(ctx, c.DefaultKeyset(), []testusers.Customer{{ID: 4}}); !errors.Is(err, encrypt.ErrEncoding) { + t.Fatalf("an empty document: %v", err) + } +} + +func TestACustomersFieldsDeriveEachQuery(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + cipher := c.DefaultKeyset() + f := testusers.CustomerFields.Profile + contains, err := f.Contains(ctx, cipher, json.RawMessage(`{"name":"Alice"}`)) + if err != nil || len(contains) == 0 { + t.Fatalf("Contains: %v", err) + } + if v, err := contains.Value(); err != nil || !strings.Contains(v.(string), `"sv"`) { + t.Fatalf("Contains binds as JSON: %v %v", v, err) + } + if _, err := f.ContainedBy(ctx, cipher, json.RawMessage(`{"name":"Alice","age":34}`)); err != nil { + t.Fatal(err) + } + if _, err := f.EqAt(ctx, cipher, "$.name", json.RawMessage(`"Alice"`)); err != nil { + t.Fatal(err) + } + if _, err := f.At.Age.Lt(ctx, cipher, json.RawMessage(`40`)); err != nil { + t.Fatal(err) + } + rng, err := f.At.VisitsAt.Between(ctx, cipher, json.RawMessage(`1`), json.RawMessage(`2`)) + if err != nil || len(rng.Low) == 0 || len(rng.High) == 0 { + t.Fatalf("Between: %+v %v", rng, err) + } + sel, err := f.Selector(ctx, cipher, "$.name") + if err != nil || sel.Selector() == "" { + t.Fatalf("Selector: %v", err) + } + read, err := f.Subtree(ctx, cipher, "$.visits[*]", encrypt.JSONReadOptions{WildcardBound: 2}) + if err != nil || len(read.Tokens()) != 3 { + t.Fatalf("Subtree: %v %v", read.Tokens(), err) + } + if v, err := read.Tokens().Value(); err != nil || !strings.HasPrefix(v.(string), `{"`) { + t.Fatalf("Tokens bind as a text[] literal: %v %v", v, err) + } + if _, err := f.Children(ctx, cipher, "$.visits"); err != nil { + t.Fatal(err) + } + // What the engine refuses: an ordering on a value no ordering takes, a + // container at a path, a position in a selector, a wildcard where one + // node is named. Each is a returned error naming no value. + for name, err := range map[string]error{ + "bool ordering": func() error { _, err := f.At.Age.Gt(ctx, cipher, json.RawMessage(`true`)); return err }(), + "container eq": func() error { _, err := f.EqAt(ctx, cipher, "$.name", json.RawMessage(`{"secret":1}`)); return err }(), + "position": func() error { _, err := f.Selector(ctx, cipher, "$.visits[1].at"); return err }(), + "wildcard node": func() error { _, err := f.ChildCount(ctx, cipher, "$.visits[*]"); return err }(), + "not JSON": func() error { _, err := f.Contains(ctx, cipher, json.RawMessage(`{"secret"`)); return err }(), + } { + if err == nil || strings.Contains(err.Error(), "secret") { + t.Errorf("%s: %v", name, err) + } + } + // Prefs orders no path: its entry has no At, and the engine answers no + // ordering on it. + if _, ok := reflect.TypeOf(testusers.CustomerFields.Prefs).FieldByName("At"); ok { + t.Fatal("Prefs has an At") + } +} diff --git a/languages/golang/encrypt/json_postgres_test.go b/languages/golang/encrypt/json_postgres_test.go new file mode 100644 index 000000000..946f937af --- /dev/null +++ b/languages/golang/encrypt/json_postgres_test.go @@ -0,0 +1,297 @@ +package encrypt_test + +import ( + "context" + "database/sql" + "encoding/json" + "os" + "slices" + "testing" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/eql" + "github.com/cipherstash/stack/languages/golang/encrypt/internal/testusers" +) + +// Encrypted JSON documents (EncryptedJson, public.eql_v4_json_tree) the Go +// SDK stores, found by the queries its generated methods derive and read by +// its selections and reads, inside PostgreSQL with EQL v4 installed. Each +// query's rows are the ones the plaintexts say. Skipped unless DATABASE_URL +// is set, as eql_postgres_test.go is. + +var jsonCustomers = []testusers.Customer{ + {ID: 1, Profile: json.RawMessage(`{"name":"Alice","age":34,"visits":[{"at":3},{"at":1}],"tags":["a"]}`), Prefs: rawPtr(`{"theme":"dark"}`)}, + {ID: 2, Profile: json.RawMessage(`{"name":"Bob","age":7,"visits":[{"at":5}]}`)}, + {ID: 3, Profile: json.RawMessage(`{"name":"Carol","age":61,"visits":[]}`), Prefs: rawPtr(`null`)}, + {ID: 4, Profile: json.RawMessage(`{"name":"Dan"}`)}, +} + +func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { + url := os.Getenv("DATABASE_URL") + if url == "" { + t.Skip("DATABASE_URL is not set") + } + ctx := context.Background() + c := deterministicEQLClient(t) + cipher := c.DefaultKeyset() + conn := eqlPostgresConn(t, ctx, url) + exec := func(q string, args ...any) { + t.Helper() + if _, err := conn.ExecContext(ctx, q, args...); err != nil { + t.Fatalf("%s: %v", q, err) + } + } + exec(`CREATE TEMP TABLE customers (id bigint PRIMARY KEY, profile public.eql_v4_json_tree NOT NULL, prefs public.eql_v4_json_tree)`) + t.Cleanup(func() { _, _ = conn.ExecContext(context.Background(), `DROP TABLE IF EXISTS customers`) }) + encrypted, err := testusers.EncryptCustomer(ctx, cipher, jsonCustomers) + if err != nil { + t.Fatal(err) + } + for _, e := range encrypted { + // Each value binds through its driver.Valuer; a nil Prefs is NULL. + exec(`INSERT INTO customers VALUES ($1, $2, $3)`, e.ID, e.Profile, e.Prefs) + } + ids := func(q string, args ...any) []int64 { + t.Helper() + rows, err := conn.QueryContext(ctx, q, args...) + if err != nil { + t.Fatalf("%s: %v", q, err) + } + defer rows.Close() + var out []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + t.Fatal(err) + } + out = append(out, id) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return out + } + want := func(name string, got []int64, want ...int64) { + t.Helper() + if !slices.Equal(got, want) { + t.Errorf("%s: rows %v, want %v", name, got, want) + } + } + f := testusers.CustomerFields.Profile + must := func(v []byte, err error) []byte { + t.Helper() + if err != nil { + t.Fatal(err) + } + return v + } + + // Stored values read back, re-serialised by PostgreSQL, open as they were. + rows, err := conn.QueryContext(ctx, `SELECT id, profile, prefs FROM customers ORDER BY id`) + if err != nil { + t.Fatal(err) + } + var stored []testusers.EncryptedCustomer + for rows.Next() { + var e testusers.EncryptedCustomer + if err := rows.Scan(&e.ID, &e.Profile, &e.Prefs); err != nil { + t.Fatal(err) + } + stored = append(stored, e) + } + _ = rows.Close() + back, err := testusers.DecryptCustomer(ctx, c, stored) + if err != nil { + t.Fatal(err) + } + for i := range jsonCustomers { + if !sameJSON(t, back[i].Profile, jsonCustomers[i].Profile) || (back[i].Prefs == nil) != (jsonCustomers[i].Prefs == nil) { + t.Fatalf("row %d read back as %s %v", i, back[i].Profile, back[i].Prefs) + } + } + + // Containment, both ways. + q := eql.EncryptedJSONQuery(must(f.Contains(ctx, cipher, json.RawMessage(`{"name":"Alice"}`)))) + want("contains name", ids(`SELECT id FROM customers WHERE profile @> $1::eql_v4.query_json_tree ORDER BY id`, q), 1) + q = eql.EncryptedJSONQuery(must(f.Contains(ctx, cipher, json.RawMessage(`{"visits":[{"at":5}]}`)))) + want("contains an element", ids(`SELECT id FROM customers WHERE profile @> $1::eql_v4.query_json_tree ORDER BY id`, q), 2) + q = eql.EncryptedJSONQuery(must(f.ContainedBy(ctx, cipher, json.RawMessage(`{"name":"Dan","x":1}`)))) + want("contained by", ids(`SELECT id FROM customers WHERE profile <@ $1::eql_v4.query_json_tree ORDER BY id`, q), 4) + // Equality at a wildcard path matches any element. + q = eql.EncryptedJSONQuery(must(f.EqAt(ctx, cipher, "$.visits[*].at", json.RawMessage(`1`)))) + want("eq_at", ids(`SELECT id FROM customers WHERE profile @> $1::eql_v4.query_json_tree ORDER BY id`, q), 1) + // Containment on the optional column: SQL NULL rows match nothing. + pq, err := testusers.CustomerFields.Prefs.Contains(ctx, cipher, json.RawMessage(`{"theme":"dark"}`)) + if err != nil { + t.Fatal(err) + } + want("prefs contains", ids(`SELECT id FROM customers WHERE prefs @> $1::eql_v4.query_json_tree ORDER BY id`, pq), 1) + + // Orderings at an ordered path compare the leaf `->` extracts there. + age, err := f.Selector(ctx, cipher, "$.age") + if err != nil { + t.Fatal(err) + } + lt, err := f.At.Age.Lt(ctx, cipher, json.RawMessage(`40`)) + if err != nil { + t.Fatal(err) + } + want("age < 40", ids(`SELECT id FROM customers WHERE (profile -> $1::text) < $2::eql_v4.query_json_tree_leaf ORDER BY id`, age.Selector(), lt), 1, 2) + rng, err := f.At.Age.Between(ctx, cipher, json.RawMessage(`30`), json.RawMessage(`70`)) + if err != nil { + t.Fatal(err) + } + want("age between", ids(`SELECT id FROM customers WHERE (profile -> $1::text) >= $2::eql_v4.query_json_tree_leaf AND (profile -> $1::text) <= $3::eql_v4.query_json_tree_leaf ORDER BY id`, age.Selector(), rng.Low, rng.High), 1, 3) + at, err := f.Selector(ctx, cipher, "$.visits[*].at") + if err != nil { + t.Fatal(err) + } + gt, err := f.At.VisitsAt.Gt(ctx, cipher, json.RawMessage(`2`)) + if err != nil { + t.Fatal(err) + } + want("first visit > 2", ids(`SELECT id FROM customers WHERE (profile -> $1::text) > $2::eql_v4.query_json_tree_leaf ORDER BY id`, at.Selector(), gt), 1, 2) + + // `->` opened against the selection that asked for it: each row's + // value, and no value where the row has none. + selected := func(sel encrypt.JSONSelection[eql.EncryptedJSONEntry]) []encrypt.JSONValue { + t.Helper() + rows, err := conn.QueryContext(ctx, `SELECT profile -> $1::text FROM customers ORDER BY id`, sel.Selector()) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var entries []*eql.EncryptedJSONEntry + for rows.Next() { + var e *eql.EncryptedJSONEntry + if err := rows.Scan(&e); err != nil { + t.Fatal(err) + } + entries = append(entries, e) + } + values, err := sel.Open(ctx, entries) + if err != nil { + t.Fatal(err) + } + return values + } + name, err := f.Selector(ctx, cipher, "$.name") + if err != nil { + t.Fatal(err) + } + var names []string + for _, v := range selected(name) { + names = append(names, string(v.JSON)) + } + if !slices.Equal(names, []string{`"Alice"`, `"Bob"`, `"Carol"`, `"Dan"`}) { + t.Errorf("->: %v", names) + } + ages := selected(age) + if !ages[0].Found || string(ages[0].JSON) != "34" || ages[3].Found { + t.Errorf("-> $.age: %+v", ages) + } + + // A subtree read: per row, every visit's `at`, in document order, with + // the walk and with the sealed-root fallback a budget of 1 forces. + read, err := f.Subtree(ctx, cipher, "$.visits[*].at", encrypt.JSONReadOptions{}) + if err != nil { + t.Fatal(err) + } + walk := func(budget int) [][]json.RawMessage { + t.Helper() + groups := make([][]eql.EncryptedJSONEntry, len(jsonCustomers)) + for i, cust := range jsonCustomers { + rows, err := conn.QueryContext(ctx, `SELECT e FROM customers, eql_v4.json_tree_subtree(profile, $2::text[], $3) AS e WHERE id = $1`, cust.ID, read.Tokens(), budget) + if err != nil { + t.Fatal(err) + } + for rows.Next() { + var e eql.EncryptedJSONEntry + if err := rows.Scan(&e); err != nil { + t.Fatal(err) + } + groups[i] = append(groups[i], e) + } + _ = rows.Close() + } + opened, err := read.Open(ctx, groups) + if err != nil { + t.Fatal(err) + } + return opened + } + text := func(groups [][]json.RawMessage) [][]string { + out := make([][]string, len(groups)) + for i, g := range groups { + out[i] = []string{} + for _, v := range g { + out[i] = append(out[i], string(v)) + } + } + return out + } + wantAts := [][]string{{"3", "1"}, {"5"}, {}, {}} + for _, budget := range []int{64, 1} { + if got := text(walk(budget)); !slicesEqual2(got, wantAts) { + t.Errorf("subtree with budget %d: %v, want %v", budget, got, wantAts) + } + } + + // A concrete read of a container, opened to one value or none. + visits, err := f.Subtree(ctx, cipher, "$.visits", encrypt.JSONReadOptions{}) + if err != nil { + t.Fatal(err) + } + var entries []eql.EncryptedJSONEntry + rs, err := conn.QueryContext(ctx, `SELECT e FROM customers, eql_v4.json_tree_subtree(profile, $1::text[]) AS e WHERE id = 1`, visits.Tokens()) + if err != nil { + t.Fatal(err) + } + for rs.Next() { + var e eql.EncryptedJSONEntry + if err := rs.Scan(&e); err != nil { + t.Fatal(err) + } + entries = append(entries, e) + } + _ = rs.Close() + one, err := visits.OpenOne(ctx, entries) + if err != nil || !one.Found || !sameJSON(t, one.JSON, []byte(`[{"at":3},{"at":1}]`)) { + t.Errorf("OpenOne $.visits: %+v %v", one, err) + } + + // Child counts: two visits, one, an empty array, and none. + token, err := f.ChildCount(ctx, cipher, "$.visits") + if err != nil { + t.Fatal(err) + } + cr, err := conn.QueryContext(ctx, `SELECT eql_v4.json_tree_child_count(profile, $1::text) FROM customers ORDER BY id`, token) + if err != nil { + t.Fatal(err) + } + var counts []string + for cr.Next() { + var n sql.NullInt64 + if err := cr.Scan(&n); err != nil { + t.Fatal(err) + } + if n.Valid { + counts = append(counts, string(rune('0'+n.Int64))) + } else { + counts = append(counts, "NULL") + } + } + _ = cr.Close() + if !slices.Equal(counts, []string{"2", "1", "0", "NULL"}) { + t.Errorf("child counts: %v", counts) + } + children, err := f.Children(ctx, cipher, "$.visits") + if err != nil { + t.Fatal(err) + } + want("children rows", ids(`SELECT id FROM customers, eql_v4.json_tree_children(profile, $1::text) AS e ORDER BY id`, children), 1, 1, 2) +} + +func slicesEqual2(a, b [][]string) bool { + return slices.EqualFunc(a, b, func(x, y []string) bool { return slices.Equal(x, y) }) +} diff --git a/languages/golang/encrypt/term.go b/languages/golang/encrypt/term.go index 8e3bf55a0..e30d1c97d 100644 --- a/languages/golang/encrypt/term.go +++ b/languages/golang/encrypt/term.go @@ -70,9 +70,12 @@ type JSONOption interface { jsonOption() } -// JSON is the index over a JSON document. The engine does not derive it -// yet: a declaration that names it is refused by stashgen, and at run time -// by the engine. +// JSON is the index over a JSON document as an output beside a ciphertext, +// which the engine does not derive: a declaration that names it is refused +// by stashgen, and at run time by the engine. +// +// Deprecated: store a JSON document as a json.RawMessage field with +// `encrypt_into=EncryptedJson`, whose EQL type carries the JSON index. func JSON(options ...JSONOption) Index { _ = options return index("json") @@ -151,9 +154,12 @@ func (t OpeTerm) Compare(other OpeTerm) int { return bytes.Compare(t, other) } // Less reports whether t's plaintext orders before other's. func (t OpeTerm) Less(other OpeTerm) bool { return t.Compare(other) < 0 } -// JSONTerm is the term of the json index. The engine does not derive it -// yet; the type exists so a generated declaration that names [JSON] has a -// Go type to fail into. +// JSONTerm is the term of the json index as an output. The engine does not +// derive it; the type exists so a generated declaration that names [JSON] +// has a Go type to fail into. +// +// Deprecated: an EncryptedJson field's queries are eql.EncryptedJSONQuery +// and eql.EncryptedJSONLeafQuery. type JSONTerm []byte // compareCLLW is cllw-ore's compare_lex: compare_slice over the common diff --git a/languages/golang/stashgen/declaration.go b/languages/golang/stashgen/declaration.go index d19d612e7..de10e50eb 100644 --- a/languages/golang/stashgen/declaration.go +++ b/languages/golang/stashgen/declaration.go @@ -3,6 +3,8 @@ package stashgen import ( "fmt" "strings" + + "github.com/cipherstash/stack/languages/golang/internal/record" ) // Declaration is how each field of one struct is encrypted: the context of @@ -43,6 +45,9 @@ type Field struct { Indexes []Index // EQLType is the EQL type an EncryptInto field seals into. EQLType string + // TargetOptions are the EQL type's options, for a type that takes them + // (EncryptedJson); nil for its defaults. + TargetOptions *record.TargetOptions // Identity is the field's part of its context when it differs from Name: // a column that was renamed keeps the identity it was first written // under. "" means Name. Only a policy sets it. @@ -270,6 +275,9 @@ const ( KindTimestamp // KindDecimal is encrypt.Decimal. KindDecimal + // KindJSON is json.RawMessage: a JSON document, stored as an + // EncryptedJSON value. + KindJSON ) var kindNames = [...]string{ @@ -289,6 +297,7 @@ var kindNames = [...]string{ KindDate: "date", KindTimestamp: "timestamp", KindDecimal: "decimal", + KindJSON: "json", } // String returns the kind's name. @@ -303,7 +312,7 @@ func (k Kind) String() string { func (k Kind) Scalar() bool { switch k { case KindString, KindBool, KindInt, KindUint, KindFloat, KindBytes, - KindInt128, KindUint128, KindDate, KindTimestamp, KindDecimal: + KindInt128, KindUint128, KindDate, KindTimestamp, KindDecimal, KindJSON: return true } return false diff --git a/languages/golang/stashgen/emit.go b/languages/golang/stashgen/emit.go index e91a6d6d3..04849b653 100644 --- a/languages/golang/stashgen/emit.go +++ b/languages/golang/stashgen/emit.go @@ -263,6 +263,8 @@ func kindConst(k record.Kind) string { return "String" case record.Bytes: return "Bytes" + case record.JSON: + return "JSON" } return "Untyped" } @@ -295,8 +297,9 @@ func (w *writer) declaration(f *genFile) { } } for i, fld := range fields { + last := i == len(fields)-1 end := "." - if i == len(fields)-1 && fld.Identity == "" && fld.Null == NotNull { + if last && fld.Identity == "" && fld.Null == NotNull && fld.TargetOptions == nil { end = "" } switch fld.Verb { @@ -312,6 +315,13 @@ func (w *writer) declaration(f *genFile) { kind = "gensupport." + kindConst(wire) } w.p("\tEncryptInto(%q, %s, %q)%s", fld.Name, kind, fld.EQLType, end) + if fld.TargetOptions != nil { + end = "." + if last && fld.Identity == "" && fld.Null == NotNull { + end = "" + } + w.p("\tOptions(%q, %s)%s", fld.Name, targetOptionsExpr(fld.TargetOptions), end) + } case VerbEncryptIndex, VerbIndex: call := "EncryptIndex" if fld.Verb == VerbIndex { @@ -340,6 +350,23 @@ func (w *writer) declaration(f *genFile) { } } +// targetOptionsExpr is an EQL type's options as a gensupport.TargetOptions +// literal. +func targetOptionsExpr(o *record.TargetOptions) string { + var parts []string + if o.SealedRoot != nil { + parts = append(parts, fmt.Sprintf("SealedRoot: gensupport.Ptr(%t)", *o.SealedRoot)) + } + if o.Ordered != nil { + quoted := make([]string, len(o.Ordered)) + for i, p := range o.Ordered { + quoted[i] = fmt.Sprintf("%q", p) + } + parts = append(parts, "Ordered: []string{"+strings.Join(quoted, ", ")+"}") + } + return "gensupport.TargetOptions{" + strings.Join(parts, ", ") + "}" +} + // nullExpr is a null mode's constant in gensupport. func nullExpr(n Null) string { if n == NullSQL { @@ -638,11 +665,19 @@ func (w *writer) fieldEntries(f *genFile) { } w.p("}{") for _, g := range sealed { + if g.isJSON() { + w.p("\t%s: new%s(gensupport.NewField[%s](%s, %q)),", g.GoName, g.fieldType, g.valueExpr, f.declVar, g.Name) + continue + } w.p("\t%s: %s{gensupport.NewField[%s](%s, %q)},", g.GoName, g.fieldType, g.valueExpr, f.declVar, g.Name) } w.p("}") for _, g := range sealed { w.nl() + if g.isJSON() { + w.jsonEntry(g) + continue + } w.p("type %s struct {", g.fieldType) w.p("\tfield gensupport.Field[%s]", g.valueExpr) w.p("}") @@ -716,6 +751,14 @@ func planQueries(f *genFile) error { for i := range f.fields { g := &f.fields[i] g.queries = nil + if g.isJSON() { + // A document's operators have methods of their own: see + // jsonEntry. + if err := planJSON(f, g); err != nil { + return err + } + continue + } for _, op := range g.operators { q, err := queryFor(g, op) if err != nil { @@ -927,3 +970,195 @@ func joinNames(names []string) string { } return strings.Join(names[:len(names)-1], ", ") + " and " + names[len(names)-1] } + +// isJSON reports whether the field is stored as an EncryptedJSON document: +// a json.RawMessage sealed into an EQL type. +func (g genField) isJSON() bool { + return g.Verb == VerbEncryptInto && g.GoType.Kind == KindJSON +} + +// jsonOperators are the operators a document field answers, and what each +// is a method of. +var jsonOperators = map[string]bool{ + "contains": true, "contained_by": true, "eq_at": true, "selector": true, + "subtree": true, "children": true, "child_count": true, + "lt_at": true, "lte_at": true, "gt_at": true, "gte_at": true, "between_at": true, +} + +// planJSON checks a document field's operators against the methods the +// generator writes for them, and names each ordered path's accessor. +func planJSON(f *genFile, g *genField) error { + for _, op := range g.operators { + if !jsonOperators[op.Name] { + return fieldErr(f.typeName, g.GoName, "the engine answers %s on %s, which the generator has no method for", op.Name, g.EQLType) + } + } + g.jsonPaths = nil + seen := map[string]string{} + if g.TargetOptions != nil { + for _, path := range g.TargetOptions.Ordered { + name, err := jsonPathName(path) + if err != nil { + return fieldErr(f.typeName, g.GoName, "%v", err) + } + if prior, ok := seen[name]; ok { + return fieldErr(f.typeName, g.GoName, "the ordered paths %s and %s both name the accessor At.%s; order one of them only", prior, path, name) + } + seen[name] = path + g.jsonPaths = append(g.jsonPaths, jsonPath{name: name, path: path}) + } + } + f.imports.add("encoding/json", "json") + return nil +} + +// jsonPathName is an ordered path's accessor name: its keys in Go case, +// array steps dropped. $.age is Age, $.visits[*].at is VisitsAt, $ is Root. +func jsonPathName(path string) (string, error) { + var b strings.Builder + for _, step := range strings.FieldsFunc(strings.TrimPrefix(path, "$"), func(r rune) bool { return r == '.' || r == '[' || r == ']' }) { + if step == "*" || strings.Trim(step, "0123456789") == "" { + continue + } + for _, word := range strings.Split(step, "_") { + if word != "" { + b.WriteString(strings.ToUpper(word[:1]) + word[1:]) + } + } + } + name := b.String() + if name == "" { + name = "Root" + } + if !isIdent(name) || name[0] < 'A' || name[0] > 'Z' { + return "", fmt.Errorf("the ordered path %s has no Go accessor name: a key starts with a digit", path) + } + return name, nil +} + +// jsonPath is one ordered path's accessor on a document field's entry. +type jsonPath struct { + name string // "Age" + path string // "$.age" +} + +// jsonEntry writes a document field's Fields entry: its query methods, a +// method for each read, and an accessor under At for each ordered path, +// whose methods are the orderings there. +func (w *writer) jsonEntry(g genField) { + answered := map[string]bool{} + for _, op := range g.operators { + answered[op.Name] = true + } + eqlName := strings.TrimPrefix(g.outputType, "eql.") + needle := g.queryType + leaf := "eql." + eqlName + "LeafQuery" + entry := "eql." + eqlName + "Entry" + ordered := len(g.jsonPaths) > 0 && answered["lt_at"] + field := "gensupport.Field[" + g.valueExpr + "]" + + w.p("type %s struct {", g.fieldType) + w.p("\tfield %s", field) + if ordered { + w.p("\t// At has one accessor per path the field orders, whose methods") + w.p("\t// are the orderings there.") + w.p("\tAt %sAt", g.fieldType) + } + w.p("}") + w.nl() + w.p("func new%s(f %s) %s {", g.fieldType, field, g.fieldType) + if ordered { + w.p("\treturn %s{field: f, At: %sAt{", g.fieldType, g.fieldType) + for _, p := range g.jsonPaths { + w.p("\t\t%s: %sPath{field: f, path: %q},", p.name, g.fieldType, p.path) + } + w.p("\t}}") + } else { + w.p("\treturn %s{field: f}", g.fieldType) + } + w.p("}") + w.nl() + w.p("func (f %s) Encrypt(ctx context.Context, c *encrypt.Cipher, v %s) (%s, error) {", g.fieldType, g.valueExpr, g.outputType) + w.sealedOutput(g, "f.field.Encrypt(ctx, c, v)") + w.p("}") + recv := "func (f " + g.fieldType + ") " + if answered["contains"] { + w.nl() + w.b.WriteString(wrapComment("Contains derives the query for documents that contain needle, a JSON value: col @> query. Each array element of the needle must sit inside one element of the document's array.", 80)) + w.p("%sContains(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (%s, error) {", recv, needle) + w.p("\treturn gensupport.JSONQuery[%s](ctx, f.field, c, \"contains\", needle)", needle) + w.p("}") + } + if answered["contained_by"] { + w.nl() + w.b.WriteString(wrapComment("ContainedBy derives the query for documents contained by needle, a JSON value: col <@ query.", 80)) + w.p("%sContainedBy(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (%s, error) {", recv, needle) + w.p("\treturn gensupport.JSONQuery[%s](ctx, f.field, c, \"contained_by\", needle)", needle) + w.p("}") + } + if answered["eq_at"] { + w.nl() + w.b.WriteString(wrapComment("EqAt derives the query for documents holding the scalar v at path: col @> query. A wildcard step matches when any element holds it; a position ([2]) is refused, since the stored selector reads every position as the wildcard.", 80)) + w.p("%sEqAt(ctx context.Context, c *encrypt.Cipher, path string, v json.RawMessage) (%s, error) {", recv, needle) + w.p("\treturn gensupport.JSONQueryAt[%s](ctx, f.field, c, \"eq_at\", path, v)", needle) + w.p("}") + } + if answered["selector"] { + w.nl() + w.b.WriteString(wrapComment("Selector prepares `->` at path: bind Selector() as col -> $1::text, and open the entries it returns with Open. A position ([2]) is refused; read one with Subtree.", 80)) + w.p("%sSelector(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONSelection[%s], error) {", recv, entry) + w.p("\treturn gensupport.JSONSelect[%s](ctx, f.field, c, path)", entry) + w.p("}") + } + if answered["subtree"] { + w.nl() + w.b.WriteString(wrapComment("Subtree prepares a read of the value at path: bind Tokens() as eql_v4.json_tree_subtree(col, $1::text[], budget), and open the rows it returns with Open or OpenOne.", 80)) + w.p("%sSubtree(ctx context.Context, c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[%s], error) {", recv, entry) + w.p("\treturn gensupport.JSONSubtree[%s](ctx, f.field, c, path, opts)", entry) + w.p("}") + } + if answered["children"] { + w.nl() + w.b.WriteString(wrapComment("Children derives the token of the node at path, a path with no wildcard, which eql_v4.json_tree_children(col, $1::text) takes. Its rows are encrypted entries one level down.", 80)) + w.p("%sChildren(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONToken, error) {", recv) + w.p("\treturn gensupport.JSONNodeToken(ctx, f.field, c, \"children\", path)") + w.p("}") + } + if answered["child_count"] { + w.nl() + w.b.WriteString(wrapComment("ChildCount derives the token of the node at path, which eql_v4.json_tree_child_count(col, $1::text) takes: zero for a scalar or an empty container, NULL where the row has no node there or for the sealed root.", 80)) + w.p("%sChildCount(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONToken, error) {", recv) + w.p("\treturn gensupport.JSONNodeToken(ctx, f.field, c, \"child_count\", path)") + w.p("}") + } + if !ordered { + return + } + w.nl() + w.p("type %sAt struct {", g.fieldType) + for _, p := range g.jsonPaths { + w.p("\t%s %sPath // %s", p.name, g.fieldType, p.path) + } + w.p("}") + w.nl() + w.b.WriteString(wrapComment(fmt.Sprintf("%sPath is one ordered path of %s: its orderings compare the first value `->` extracts there, by the leaf's ordering term. Numbers order as float64, so integers past 2^53 can tie; text orders case- and accent-folded.", g.fieldType, g.GoName), 80)) + w.p("type %sPath struct {", g.fieldType) + w.p("\tfield %s", field) + w.p("\tpath string") + w.p("}") + for _, op := range []struct{ name, method string }{{"lt_at", "Lt"}, {"lte_at", "Lte"}, {"gt_at", "Gt"}, {"gte_at", "Gte"}} { + if !answered[op.name] { + continue + } + w.nl() + w.p("func (p %sPath) %s(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (%s, error) {", g.fieldType, op.method, leaf) + w.p("\treturn gensupport.JSONQueryAt[%s](ctx, p.field, c, %q, p.path, v)", leaf, op.name) + w.p("}") + } + if answered["between_at"] { + w.nl() + w.p("func (p %sPath) Between(ctx context.Context, c *encrypt.Cipher, lo, hi json.RawMessage) (encrypt.JSONRange[%s], error) {", g.fieldType, leaf) + w.p("\treturn gensupport.JSONBetweenAt[%s](ctx, p.field, c, p.path, lo, hi)", leaf) + w.p("}") + } +} diff --git a/languages/golang/stashgen/engine.go b/languages/golang/stashgen/engine.go index b99a197ae..3609e4dbf 100644 --- a/languages/golang/stashgen/engine.go +++ b/languages/golang/stashgen/engine.go @@ -250,6 +250,7 @@ func lowerDeclaration(d Declaration, eqlTypes []EQLType) (*record.Plan, error) { return nil, &FieldError{Type: d.Type, Field: f.GoName, Reason: fmt.Sprintf("%s is stored under a table and a column, and the context %q has %d segments, not one", f.EQLType, d.Context, len(plan.Context))} } rf.Target = f.EQLType + rf.TargetOptions = f.TargetOptions case VerbEncrypt, VerbEncryptIndex: rf.Outputs = append(rf.Outputs, record.Ciphertext) } @@ -281,7 +282,7 @@ var outputOfIndex = map[IndexName]record.Output{ // no entry in outputOfIndex, so no tag asks for it. var indexOfOutput = map[record.Output]IndexName{ record.Equality: IndexEquality, record.Match: IndexMatch, record.Ore: IndexOre, record.Ope: IndexOpe, - record.OreBlock: IndexOreBlock, + record.OreBlock: IndexOreBlock, record.JSONIndex: IndexJSON, } // wireKind is the record kind of a Go type: the same mapping the emitter @@ -330,6 +331,8 @@ func wireKind(t GoType) record.Kind { return record.Timestamp case KindDecimal: return record.Decimal + case KindJSON: + return record.JSON } return record.Untyped } @@ -360,6 +363,8 @@ func kindOfWire(k record.Kind) Kind { return KindTimestamp case record.Decimal: return KindDecimal + case record.JSON: + return KindJSON } return KindOther } diff --git a/languages/golang/stashgen/read.go b/languages/golang/stashgen/read.go index 663b36191..841554f9c 100644 --- a/languages/golang/stashgen/read.go +++ b/languages/golang/stashgen/read.go @@ -105,6 +105,7 @@ type genField struct { queries []queryMethod // the field's query methods, one for each operator fieldType string // "EmailField" pathType string // the type with its package path, for model checks + jsonPaths []jsonPath // a document field's ordered paths, each with its accessor } // definedScalar reports whether the field's type is a named type over a @@ -708,7 +709,7 @@ func (r *reader) buildFields(c *collected) error { return fieldErr(typeName, cf.goName, "two fields write the name %q: %s and %s", name, prev, cf.goName) } seen[name] = cf.goName - field := Field{Name: name, GoName: cf.goName, GoType: r.goType(cf.typ), Verb: t.Verb, Indexes: t.Indexes, EQLType: t.EQLType, Identity: t.Identity} + field := Field{Name: name, GoName: cf.goName, GoType: r.goType(cf.typ), Verb: t.Verb, Indexes: t.Indexes, EQLType: t.EQLType, TargetOptions: t.EQLOptions, Identity: t.Identity} if t.Omit { field.Verb = VerbOmit } @@ -760,6 +761,11 @@ func (r *reader) buildFields(c *collected) error { if len(r.eql) == 0 { return fieldErr(typeName, cf.goName, "EQL types are not available yet; this build of the engine produces none") } + for _, known := range r.eql { + if EQLGoName(known.Name) == field.EQLType && known.Name != field.EQLType { + return fieldErr(typeName, cf.goName, "encrypt_into names the EQL type as the engine does, %s; eql.%s is its Go type", known.Name, field.EQLType) + } + } return fieldErr(typeName, cf.goName, "the engine has no EQL type %s", field.EQLType) } if !eqlType.Producible { @@ -954,6 +960,9 @@ func valueKind(t types.Type) (Kind, bool) { if path == "time" && name == "Time" { return KindTimestamp, true } + if path == "encoding/json" && name == "RawMessage" { + return KindJSON, true + } if path != encryptPath && path != vcvaluePath { continue } diff --git a/languages/golang/stashgen/tag.go b/languages/golang/stashgen/tag.go index 655419ba4..925b5a093 100644 --- a/languages/golang/stashgen/tag.go +++ b/languages/golang/stashgen/tag.go @@ -4,7 +4,10 @@ import ( "errors" "fmt" "reflect" + "strconv" "strings" + + "github.com/cipherstash/stack/languages/golang/internal/record" ) // The key of the struct tag the generator reads. @@ -23,6 +26,8 @@ const tagKey = "stash" // stash:"tenant,context_field" Name, VerbContextField // stash:",passthrough" VerbPassthrough with no Name, on an embedded struct // stash:"dob,encrypt_into=Date,null=sql" Null, on a sealed field declared over a pointer +// stash:"profile,encrypt_into=EncryptedJson(ordered=$.age;$.visits[*].at,sealed_root=false)" +// EQLType and EQLOptions type tag struct { Omit bool Context string @@ -31,6 +36,9 @@ type tag struct { Verb Verb Indexes []Index EQLType string + // EQLOptions are the EQL type's options, from the parentheses after its + // name; nil when the tag gives none. + EQLOptions *record.TargetOptions // Null is how a sealed field declared over a pointer stores nil, when // the tag says; the generator picks NullSealed where it can. Null Null @@ -96,6 +104,16 @@ func parseTagValue(value string) (tag, error) { } sawInto = true t.EQLType = strings.TrimPrefix(item, "encrypt_into=") + if open := strings.IndexByte(t.EQLType, '('); open >= 0 { + if !strings.HasSuffix(t.EQLType, ")") { + return tag{}, fmt.Errorf("tag %q: encrypt_into's options open with ( and do not close", value) + } + t.EQLOptions, err = parseEQLOptions(t.EQLType[open+1 : len(t.EQLType)-1]) + if err != nil { + return tag{}, fmt.Errorf("tag %q: %w", value, err) + } + t.EQLType = t.EQLType[:open] + } if !isIdent(t.EQLType) { return tag{}, fmt.Errorf("tag %q: encrypt_into names an EQL type, and %q is not a type name", value, t.EQLType) } @@ -184,6 +202,83 @@ func parseContextTag(items []string) (tag, error) { return t, nil } +// parseEQLOptions reads an EQL type's options: `ordered=$.age;$.visits[*].at` +// and `sealed_root=true|false`, separated by commas. The engine checks the +// paths when the declaration is checked. A path in a tag is plain JSONPath: +// keys of letters, digits and underscores, [n] and [*]; a quoted key, or a +// key holding a comma, semicolon or parenthesis, has no tag spelling. +func parseEQLOptions(list string) (*record.TargetOptions, error) { + if list == "" { + return nil, errors.New("encrypt_into has empty options; drop the parentheses") + } + options := &record.TargetOptions{} + seen := map[string]bool{} + for _, opt := range strings.Split(list, ",") { + key, value, ok := strings.Cut(opt, "=") + if !ok || value == "" { + return nil, fmt.Errorf("encrypt_into option %q is not key=value", opt) + } + if seen[key] { + return nil, fmt.Errorf("encrypt_into option %s is given twice", key) + } + seen[key] = true + switch key { + case "ordered": + for _, path := range strings.Split(value, ";") { + if !plainPath(path) { + return nil, fmt.Errorf("encrypt_into ordered path %q is not a plain JSONPath ($ then .key, [n] or [*] steps, keys of letters, digits and underscores)", path) + } + options.Ordered = append(options.Ordered, path) + } + case "sealed_root": + b, err := strconv.ParseBool(value) + if err != nil || (value != "true" && value != "false") { + return nil, fmt.Errorf("encrypt_into sealed_root is true or false, not %q", value) + } + options.SealedRoot = &b + default: + return nil, fmt.Errorf("encrypt_into has no option %q; the options are ordered and sealed_root", key) + } + } + return options, nil +} + +// plainPath reports whether s is a JSONPath a tag can spell: $, then .key +// steps (letters, digits, underscores) and [n] or [*] steps. +func plainPath(s string) bool { + if !strings.HasPrefix(s, "$") { + return false + } + for i := 1; i < len(s); { + switch s[i] { + case '.': + j := i + 1 + for j < len(s) && (s[j] == '_' || s[j] >= 'a' && s[j] <= 'z' || s[j] >= 'A' && s[j] <= 'Z' || s[j] >= '0' && s[j] <= '9') { + j++ + } + if j == i+1 { + return false + } + i = j + case '[': + end := strings.IndexByte(s[i:], ']') + if end < 0 { + return false + } + inner := s[i+1 : i+end] + if inner != "*" { + if _, err := strconv.ParseUint(inner, 10, 32); err != nil { + return false + } + } + i += end + 1 + default: + return false + } + } + return true +} + // parseIndexes reads `equality;match(k=3,m=2048)`. func parseIndexes(list string) ([]Index, error) { parts, err := splitOutsideParens(list, ';') diff --git a/languages/golang/stashgen/testdata/stubsdk/encrypt/encrypt.go b/languages/golang/stashgen/testdata/stubsdk/encrypt/encrypt.go index 97273a058..6ddfbe7bc 100644 --- a/languages/golang/stashgen/testdata/stubsdk/encrypt/encrypt.go +++ b/languages/golang/stashgen/testdata/stubsdk/encrypt/encrypt.go @@ -119,3 +119,36 @@ type ( Negative bool } ) + +// JSONSelector is the selector `col -> $1::text` binds. +type JSONSelector string + +// JSONToken is one node's token. +type JSONToken string + +// JSONTokens are a subtree read's tokens. +type JSONTokens []string + +// JSONValue is one value a read reached, or none. +type JSONValue struct { + Found bool + JSON []byte +} + +// JSONReadOptions are a subtree read's options. +type JSONReadOptions struct{ WildcardBound uint32 } + +// JSONRange is both bounds of a between at a path. +type JSONRange[Q ~[]byte] struct{ Low, High Q } + +// JSONSelection is a prepared `->` on an EncryptedJSON field. +type JSONSelection[E ~[]byte] struct{ _ E } + +func (JSONSelection[E]) Selector() JSONSelector { return "" } +func (JSONSelection[E]) Open(context.Context, []*E) ([]JSONValue, error) { return nil, nil } + +// JSONRead is a prepared subtree read on an EncryptedJSON field. +type JSONRead[E ~[]byte] struct{ _ E } + +func (JSONRead[E]) Tokens() JSONTokens { return nil } +func (JSONRead[E]) OpenOne(context.Context, []E) (JSONValue, error) { return JSONValue{}, nil } diff --git a/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go b/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go index 83abf8f0f..76ae60221 100644 --- a/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go +++ b/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go @@ -7,6 +7,7 @@ package gensupport import ( "context" + "encoding/json" "log/slog" "github.com/cipherstash/stack/languages/golang/encrypt" @@ -44,6 +45,7 @@ const ( Timestamp Kind = "timestamp" String Kind = "string" Bytes Kind = "bytes" + JSON Kind = "json" ) // Redacted formats a value with its sealed fields hidden. @@ -87,6 +89,7 @@ func (d Declaration) Encrypt(name string, kind Kind) Declaration func (d Declaration) EncryptIndex(name string, kind Kind, idx ...encrypt.Index) Declaration { return d } func (d Declaration) Index(name string, kind Kind, idx ...encrypt.Index) Declaration { return d } func (d Declaration) EncryptInto(name string, kind Kind, eqlType string) Declaration { return d } +func (d Declaration) Options(name string, options TargetOptions) Declaration { return d } func (d Declaration) Omit(name string) Declaration { return d } func (d Declaration) Err() error { return nil } @@ -196,3 +199,42 @@ func (f Field[T]) NullTerm(context.Context, *encrypt.Cipher, string) ([]byte, er // GetPtr reads one opened field declared over a pointer. func GetPtr[T any](Values, string) (*T, error) { return nil, nil } + +// TargetOptions are the options an EncryptInto field gives its EQL type. +type TargetOptions struct { + SealedRoot *bool + Ordered []string +} + +// Ptr is a pointer to v. +func Ptr[T any](v T) *T { return &v } + +// JSONQuery derives a containment operator's query on a JSON field. +func JSONQuery[Q ~[]byte, T any](context.Context, Field[T], *encrypt.Cipher, string, json.RawMessage) (Q, error) { + return nil, nil +} + +// JSONQueryAt derives an operator at a path on a JSON field. +func JSONQueryAt[Q ~[]byte, T any](context.Context, Field[T], *encrypt.Cipher, string, string, json.RawMessage) (Q, error) { + return nil, nil +} + +// JSONBetweenAt derives between_at on a JSON field. +func JSONBetweenAt[Q ~[]byte, T any](context.Context, Field[T], *encrypt.Cipher, string, json.RawMessage, json.RawMessage) (encrypt.JSONRange[Q], error) { + return encrypt.JSONRange[Q]{}, nil +} + +// JSONSelect prepares `->` at a path on a JSON field. +func JSONSelect[E ~[]byte, T any](context.Context, Field[T], *encrypt.Cipher, string) (encrypt.JSONSelection[E], error) { + return encrypt.JSONSelection[E]{}, nil +} + +// JSONSubtree prepares a subtree read at a path on a JSON field. +func JSONSubtree[E ~[]byte, T any](context.Context, Field[T], *encrypt.Cipher, string, encrypt.JSONReadOptions) (encrypt.JSONRead[E], error) { + return encrypt.JSONRead[E]{}, nil +} + +// JSONNodeToken derives the token of the node at a path. +func JSONNodeToken[T any](context.Context, Field[T], *encrypt.Cipher, string, string) (encrypt.JSONToken, error) { + return "", nil +} diff --git a/mise.toml b/mise.toml index da66a306a..930072ba3 100644 --- a/mise.toml +++ b/mise.toml @@ -368,7 +368,7 @@ description = "Format check, vet and test the Go module (languages/golang: encry run = "scripts/go-binding-test.sh languages/golang" [tasks."go:test:postgres"] -description = "Install the EQL v4 bundle built from the tree into the database EQL_TEST_DATABASE_URL names and run the Go module's PostgreSQL tests against it: rows the Go SDK stores in every EQL type it produces, found by the queries it derives, and random values in every ordered type, ordered as their plaintexts (EQL_ORDER_SEED replays a seed); needs `wasm:guest:build:deterministic` and `wasm:guest:build:eql:deterministic` first" +description = "Install the EQL v4 bundle built from the tree into the database EQL_TEST_DATABASE_URL names and run the Go module's PostgreSQL tests against it: rows the Go SDK stores in every EQL type it produces, found by the queries it derives, encrypted JSON documents queried and read through their generated methods, and random values in every ordered type, ordered as their plaintexts (EQL_ORDER_SEED replays a seed); needs `wasm:guest:build:deterministic` and `wasm:guest:build:eql:deterministic` first" # The bundle is built from packages/eql, as test:encryption:postgres builds # it: the copy under crates/eql-bindings/sql is the RELEASED bundle and moves # only at the lockstep bump, so it cannot show a SQL change in the same pull @@ -384,7 +384,7 @@ cd languages/golang # The tests skip without DATABASE_URL and without their guest builds; -v and # the grep turn either skip into a failure here, where both are provided. DATABASE_URL="$EQL_TEST_DATABASE_URL" CGO_ENABLED=0 go test -count=1 -v \ - -run '^(TestEQLTypesRoundTripAndQueryInPostgres|TestEQLOrderMatchesPlaintextInPostgres|TestQueriesRunAgainstPostgres)$' ./encrypt \ + -run '^(TestEQLTypesRoundTripAndQueryInPostgres|TestEQLOrderMatchesPlaintextInPostgres|TestQueriesRunAgainstPostgres|TestJSONDocumentsQueryAndReadInPostgres)$' ./encrypt \ | tee "${TMPDIR:-/tmp}/go-test-postgres.log" if grep -q -- '--- SKIP' "${TMPDIR:-/tmp}/go-test-postgres.log"; then echo "error: a PostgreSQL test skipped; build the deterministic guests first" >&2 @@ -392,7 +392,7 @@ if grep -q -- '--- SKIP' "${TMPDIR:-/tmp}/go-test-postgres.log"; then fi # A -run pattern that matches no test (one renamed, say) prints "no tests # to run" and PASS, so each test has to be seen to pass by name. -for test in TestEQLTypesRoundTripAndQueryInPostgres TestEQLOrderMatchesPlaintextInPostgres TestQueriesRunAgainstPostgres; do +for test in TestEQLTypesRoundTripAndQueryInPostgres TestEQLOrderMatchesPlaintextInPostgres TestQueriesRunAgainstPostgres TestJSONDocumentsQueryAndReadInPostgres; do if ! grep -q -- "^--- PASS: $test " "${TMPDIR:-/tmp}/go-test-postgres.log"; then echo "error: $test did not run and pass; was it renamed?" >&2 exit 1 From e1b21cecc001bc55aa8b5504278e9873a082900c Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 02:09:58 +1100 Subject: [PATCH 03/26] test(stack-encrypt): cover the JSON operators and reads inside the crate CI's rustdoc, CRAP and coverage gates failed on this branch: two public docs linked the private json_query module, and the JSON operators were exercised only from the EQL workspace, so stack-encrypt's own coverage of read_input, derive, open and read_open was zero. - OpSpec's docs link OpSpec::input, query_term and json_open instead. - tests/json_operators.rs runs every operator, the input refusals, the read refusals and reads through a resolver backed by the engine's own JSON index, so the crate measures its own code. - read_input is split per operator; its complexity alone (31) passed the CRAP threshold at full coverage. Refs #1347 Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- .../stack-encrypt/src/dynamic/json_query.rs | 81 +- packages/stack-encrypt/src/dynamic/query.rs | 7 +- .../stack-encrypt/tests/json_operators.rs | 713 ++++++++++++++++++ 3 files changed, 764 insertions(+), 37 deletions(-) create mode 100644 packages/stack-encrypt/tests/json_operators.rs diff --git a/packages/stack-encrypt/src/dynamic/json_query.rs b/packages/stack-encrypt/src/dynamic/json_query.rs index 3a81e51d8..4efd6be84 100644 --- a/packages/stack-encrypt/src/dynamic/json_query.rs +++ b/packages/stack-encrypt/src/dynamic/json_query.rs @@ -220,47 +220,60 @@ fn term(error: TermError) -> Error { /// Read and check an operator's input against the field's options. fn read_input(op: OpSpec, options: &JsonOptions, input: &ValueFrame) -> Result { - Ok(match op { - OpSpec::Contains | OpSpec::ContainedBy => { - Input::Needle(read_frame::(input)?.json.parse()?) - } - OpSpec::EqAt => { - let at: AtInput = read_frame(input)?; - let path = selector_path(&at.path)?; - let value = at.json.parse()?; - if matches!(value.0, Value::Object(_) | Value::Array(_)) { - return Err(term(TermError::JsonValueNotScalar)); - } - Input::At(path, value) - } + match op { + OpSpec::Contains | OpSpec::ContainedBy => Ok(Input::Needle( + read_frame::(input)?.json.parse()?, + )), + OpSpec::EqAt => read_eq_at(input), OpSpec::LtAt | OpSpec::LteAt | OpSpec::GtAt | OpSpec::GteAt => { - let at: AtInput = read_frame(input)?; - let path = ordered_path(options, &at.path)?; - let value = at.json.parse()?; - leaf(&value.0)?; - Input::At(path, value) - } - OpSpec::BetweenAt => { - let between: BetweenInput = read_frame(input)?; - let path = ordered_path(options, &between.path)?; - let (low, high) = (between.low.parse()?, between.high.parse()?); - leaf(&low.0)?; - leaf(&high.0)?; - Input::Between(path, low, high) - } - OpSpec::Selector => Input::Path(selector_path(&read_frame::(input)?.path)?), - OpSpec::Children | OpSpec::ChildCount => { - Input::Path(node_path(&read_frame::(input)?.path)?) + read_ordering(options, input) } + OpSpec::BetweenAt => read_between(options, input), + OpSpec::Selector => Ok(Input::Path(selector_path( + &read_frame::(input)?.path, + )?)), + OpSpec::Children | OpSpec::ChildCount => Ok(Input::Path(node_path( + &read_frame::(input)?.path, + )?)), OpSpec::Subtree => { let read: ReadInput = read_frame(input)?; - Input::Read( + Ok(Input::Read( path(&read.path)?, read.wildcard_bound.unwrap_or(JsonSubtreeKey::DEFAULT_BOUND), - ) + )) } - _ => return Err(Error::Internal), - }) + _ => Err(Error::Internal), + } +} + +/// `eq_at`'s input: a scalar at a path read through a selector. +fn read_eq_at(input: &ValueFrame) -> Result { + let at: AtInput = read_frame(input)?; + let path = selector_path(&at.path)?; + let value = at.json.parse()?; + if matches!(value.0, Value::Object(_) | Value::Array(_)) { + return Err(term(TermError::JsonValueNotScalar)); + } + Ok(Input::At(path, value)) +} + +/// An ordering's input: a leaf the index orders at an ordered path. +fn read_ordering(options: &JsonOptions, input: &ValueFrame) -> Result { + let at: AtInput = read_frame(input)?; + let path = ordered_path(options, &at.path)?; + let value = at.json.parse()?; + leaf(&value.0)?; + Ok(Input::At(path, value)) +} + +/// `between_at`'s input: two leaves the index orders at an ordered path. +fn read_between(options: &JsonOptions, input: &ValueFrame) -> Result { + let between: BetweenInput = read_frame(input)?; + let path = ordered_path(options, &between.path)?; + let (low, high) = (between.low.parse()?, between.high.parse()?); + leaf(&low.0)?; + leaf(&high.0)?; + Ok(Input::Between(path, low, high)) } /// A path the index orders, which names no position. diff --git a/packages/stack-encrypt/src/dynamic/query.rs b/packages/stack-encrypt/src/dynamic/query.rs index c76c9a162..b56963830 100644 --- a/packages/stack-encrypt/src/dynamic/query.rs +++ b/packages/stack-encrypt/src/dynamic/query.rs @@ -121,8 +121,9 @@ pub enum OpSpec { Matches, /// `"contains"`: [`op::Contains`](crate::op::Contains), a stored /// document containing a JSON needle (`@>`). Answered by the JSON index - /// of a field that stores a document; see [`json_query`](super::json_query) - /// for the inputs and operands of this and the operators below. + /// of a field that stores a document. The inputs of this and the + /// operators below are named by [`OpSpec::input`]; [`query_term`] + /// derives their operands and [`json_open`] opens their reads. Contains, /// `"contained_by"`: [`op::ContainedBy`](crate::op::ContainedBy), a /// stored document contained by a JSON needle (`<@`). @@ -213,7 +214,7 @@ impl OpSpec { /// Whether the JSON index answers this operator: the document /// operators, whose input is an object of named JSON inputs - /// ([`json_query`](super::json_query)). + /// ([`OpSpec::input`]). pub fn is_json(self) -> bool { matches!( self, diff --git a/packages/stack-encrypt/tests/json_operators.rs b/packages/stack-encrypt/tests/json_operators.rs new file mode 100644 index 000000000..16420e878 --- /dev/null +++ b/packages/stack-encrypt/tests/json_operators.rs @@ -0,0 +1,713 @@ +//! A document field's JSON operators and reads through the data plan, with +//! the engine's own JSON index behind the target: what a binding names by +//! operator (`contains`, `eq_at`, the orderings at a path, `selector`, +//! `subtree`, `children`, `child_count`), the operand each derives and the +//! type it binds as, the refusals made before anything is derived, and +//! reads opened against the request that prepared them. +//! +//! The resolver here stores the engine's `JsonDocument` as JSON, so the +//! whole path runs inside this crate. `packages/eql/tests/encryption/tests/ +//! json_tree_data_plan.rs` runs the same path through eql-bindings' table +//! and holds every operand to the typed chain's. +#![cfg(feature = "dynamic")] + +mod common; + +use std::collections::BTreeMap; + +use base64ct::{Base64, Encoding}; +use serde_json::{json, Value}; +use stack_encrypt::dynamic::{ + self, record, OpSpec, Operand, QueryTerm, Reason, Scope, TargetDescriptor, TargetError, + TargetOptions, TargetResolver, ValueFrame, ValueKind, +}; +use stack_encrypt::json::{JsonDocument, Tag}; +use stack_encrypt::target::{Borrowed, CallerContext, IndexSpec}; +use stack_encrypt::{CipherText, Encrypt, KeysetCipher, Label, NonEmpty, Pending, StackCipher}; +use stack_kms::FakeDataKeySource; + +type Keyset<'c> = KeysetCipher<'c, FakeDataKeySource>; + +/// A resolver holding one type, `EncryptedJson`, sealed with the engine's +/// JSON index under the options the plan resolved. It stores the +/// `JsonDocument` as JSON and needs no query twin: the JSON operators are +/// derived by the engine, not the resolver. +struct Documents; + +impl TargetResolver for Documents { + fn targets(&self) -> Vec { + vec![TargetDescriptor::new( + "EncryptedJson", + "json", + "", + Some(ValueKind::Json), + "public.eql_v4_json_tree", + vec!["json".to_string()], + Some("EncryptedJsonQuery".to_string()), + Some("eql_v4.query_json_tree".to_string()), + true, + None, + ) + .with_leaf_query_sql_domain("eql_v4.query_json_tree_leaf")] + } + + fn encrypt<'a, K: 'static>( + &self, + _: &str, + options: Option<&TargetOptions>, + keyset: &'a KeysetCipher<'_, K>, + label: &Label, + plaintext: ValueFrame, + ) -> Result, K>, TargetError> { + let options = options + .and_then(TargetOptions::json) + .cloned() + .unwrap_or_default(); + let document = + dynamic::read_json(&plaintext).map_err(|e| TargetError::Other(Box::new(e)))?; + Ok(keyset + .run( + stack_encrypt::target::json_document::<_, Borrowed>(options), + document.value(), + CallerContext::from(NonEmpty::from(label.clone())), + ) + .try_map(|doc| { + serde_json::to_vec(&doc).map_err(|e| stack_encrypt::Error::Other(Box::new(e))) + })) + } + + fn decrypt<'a, K: 'static>( + &self, + _: &str, + _: &'a StackCipher, + _: &Label, + _: &[u8], + ) -> Result, TargetError> { + unreachable!("these tests open documents through reads") + } + + fn query<'a, K: 'static>( + &self, + _: &str, + _: &IndexSpec, + _: &'a KeysetCipher<'_, K>, + _: &Label, + _: ValueFrame, + ) -> Result, K>, TargetError> { + unreachable!("the JSON operators are the engine's") + } +} + +/// `profile` under `users/profile`, ordering `$.age` and every visit's +/// `at`, as a binding writes it. +fn plan() -> record::Plan { + plan_of(json!({ + "name": "EncryptedJson", + "options": {"ordered": ["$.age", "$.visits[*].at"]} + })) +} + +fn plan_of(target: Value) -> record::Plan { + let wire = json!({ + "profile": {"context": ["users", "profile"], "type": "json", "null": "sql", "target": target} + }); + record::plan_with(&dynamic::frame::serde::to_vec(&wire).unwrap(), &Documents).unwrap() +} + +fn frame(value: T) -> ValueFrame { + ValueFrame::of(value).unwrap() +} + +/// A JSON value as a frame: what a binding sends as an input or a request. +fn json_frame(value: &Value) -> ValueFrame { + ValueFrame::from_bytes(&dynamic::frame::serde::to_vec(value).unwrap()).unwrap() +} + +async fn term(keyset: &Keyset<'_>, plan: &record::Plan, operator: &str, input: Value) -> QueryTerm { + dynamic::query_term( + keyset, + plan, + "profile", + OpSpec::parse(operator).unwrap(), + json_frame(&input), + &Documents, + ) + .unwrap() + .await + .unwrap() +} + +/// A document sealed through the plan, as the engine's `JsonDocument`. +async fn seal(keyset: &Keyset<'_>, plan: &record::Plan, document: &Value) -> JsonDocument { + let text = document.to_string(); + let row = record::encrypt_with( + keyset, + frame(BTreeMap::from([("profile", text.as_str())])), + plan, + &Documents, + ) + .unwrap() + .await + .unwrap(); + let CipherText::Map(fields) = &row else { + panic!("a row is a map") + }; + let Some((_, CipherText::Map(outputs))) = fields.iter().find(|(name, _)| name == "profile") + else { + panic!("profile is a map of outputs") + }; + let Some((_, CipherText::Passthrough(payload))) = outputs.iter().find(|(key, _)| key == "eql") + else { + panic!("profile's stored value is a passthrough") + }; + let bytes: bytes::Bytes = dynamic::frame::serde::from_slice(&payload.0).unwrap(); + serde_json::from_slice(&bytes.0).unwrap() +} + +fn tag(text: &str) -> Tag { + Base64::decode_vec(text).unwrap().try_into().unwrap() +} + +/// The entry `doc -> selector` returns: the first carrying the selector. +fn extracted(doc: &JsonDocument, selector: &str) -> Vec { + let selector = tag(selector); + doc.entries() + .iter() + .position(|entry| entry.selector() == &selector) + .map(|j| { + serde_json::to_value(doc.node(j).unwrap()) + .unwrap() + .to_string() + }) + .into_iter() + .collect() +} + +/// The rows `json_tree_subtree(doc, tokens, 0)` returns: each node with +/// its depth grafted. +fn subtree(doc: &JsonDocument, tokens: &[String]) -> Vec { + let tokens: Vec = tokens.iter().map(|t| tag(t)).collect(); + doc.subtree(&tokens) + .into_iter() + .map(|(j, _, d)| { + let mut wire = serde_json::to_value(doc.node(j).unwrap()).unwrap(); + wire["d"] = json!(d); + wire.to_string() + }) + .collect() +} + +async fn open( + keyset: &Keyset<'_>, + plan: &record::Plan, + request: Value, + groups: Vec>, +) -> Result>, stack_encrypt::Error> { + let opened = dynamic::json_open( + keyset, + Scope::Keyset(keyset.clone()), + plan, + "profile", + json_frame(&request), + frame(&groups), + &Documents, + ) + .unwrap() + .await?; + let groups: Vec> = + dynamic::frame::serde::from_slice(opened.risky_ref()).unwrap(); + Ok(groups + .into_iter() + .map(|g| { + g.into_iter() + .map(|b| serde_json::from_slice(&b.0).unwrap()) + .collect() + }) + .collect()) +} + +/// A byte string read out of a frame. +mod bytes { + pub struct Bytes(pub Vec); + impl<'de> serde::Deserialize<'de> for Bytes { + fn deserialize>(d: D) -> Result { + struct V; + impl serde::de::Visitor<'_> for V { + type Value = Bytes; + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("bytes") + } + fn visit_bytes(self, v: &[u8]) -> Result { + Ok(Bytes(v.to_vec())) + } + fn visit_byte_buf(self, v: Vec) -> Result { + Ok(Bytes(v)) + } + } + d.deserialize_bytes(V) + } + } +} + +/// Each operator derives an operand in the encoding it names, bound as the +/// domain the target names for it: a needle for containment and `eq_at`, +/// a leaf operand for an ordering, text for a selector or a node's token, +/// and a text array for a subtree read. +#[tokio::test] +async fn every_operator_derives_its_tagged_operand() { + let cipher = common::stack_cipher().await; + let keyset = cipher.default_keyset(); + let plan = plan(); + let needle = |term: &QueryTerm| match term.operand() { + Operand::One(bytes) => serde_json::from_slice::(bytes).unwrap(), + other => panic!("{}: {other:?}", term.operator()), + }; + + for (operator, input) in [ + ("contains", json!({"json": r#"{"name": "Alice"}"#})), + ( + "contained_by", + json!({"json": r#"{"name": "Alice", "age": 1}"#}), + ), + ("eq_at", json!({"path": "$.visits[*].at", "json": "3"})), + ] { + let term = term(&keyset, &plan, operator, input).await; + assert_eq!(term.encoding(), "eql", "{operator}"); + assert_eq!(term.domain(), Some("eql_v4.query_json_tree"), "{operator}"); + assert!(needle(&term).is_object(), "{operator}"); + } + for operator in ["lt_at", "lte_at", "gt_at", "gte_at"] { + let term = term( + &keyset, + &plan, + operator, + json!({"path": "$.age", "json": "40"}), + ) + .await; + assert_eq!(term.domain(), Some("eql_v4.query_json_tree_leaf")); + assert!(needle(&term).is_object(), "{operator}"); + } + let between = term( + &keyset, + &plan, + "between_at", + json!({"path": "$.visits[*].at", "low": "1", "high": "\"z\""}), + ) + .await; + assert!(matches!(between.operand(), Operand::Range { .. })); + assert_eq!(between.domain(), Some("eql_v4.query_json_tree_leaf")); + + let selector = term(&keyset, &plan, "selector", json!({"path": "$.name"})).await; + assert_eq!( + (selector.encoding(), selector.domain()), + ("text", Some("text")) + ); + let Operand::Text(selector) = selector.operand() else { + panic!("a selector is text") + }; + assert_eq!(tag(selector).len(), 16); + + let tokens = term( + &keyset, + &plan, + "subtree", + json!({"path": "$.visits[*].at", "wildcard_bound": 3}), + ) + .await; + assert_eq!( + (tokens.encoding(), tokens.domain()), + ("text_array", Some("text[]")) + ); + let Operand::TextList(tokens) = tokens.operand() else { + panic!("a subtree read is a text list") + }; + assert_eq!( + tokens.len(), + 3 + 1, + "one token per element below the bound, and one past it that shows a longer array" + ); + + let children = term(&keyset, &plan, "children", json!({"path": "$.visits"})).await; + let count = term(&keyset, &plan, "child_count", json!({"path": "$.visits"})).await; + assert_eq!( + children.operand(), + count.operand(), + "both take the node's token" + ); + assert!(matches!(children.operand(), Operand::Text(_))); +} + +/// A field with no ordered path answers no ordering; a target given +/// options it does not take is refused when the plan is built. +#[test] +fn the_orderings_follow_the_resolved_options() { + let names = |plan: &record::Plan| -> Vec<&'static str> { + dynamic::field_operators(plan, "profile", &Documents) + .unwrap() + .into_iter() + .map(|(op, _)| op.name()) + .collect() + }; + assert!(names(&plan()).contains(&"lt_at")); + let unordered = plan_of(json!("EncryptedJson")); + assert_eq!( + names(&unordered), + [ + "contains", + "contained_by", + "eq_at", + "selector", + "subtree", + "children", + "child_count" + ] + ); + let error = dynamic::check_query( + &unordered, + "profile", + OpSpec::LtAt, + json_frame(&json!({"path": "$.age", "json": "1"})), + &Documents, + ) + .unwrap_err(); + assert_eq!(error.reason(), Some(Reason::OperatorNotAnswered)); +} + +/// Inputs refused before anything is derived, each with its own reason and +/// never a byte of the input; and the engine's own refusals of a path or a +/// value it cannot answer. +#[test] +fn malformed_inputs_are_refused_with_their_reason() { + let plan = plan(); + let check = |operator: OpSpec, input: Value| { + dynamic::check_query(&plan, "profile", operator, json_frame(&input), &Documents) + }; + let reason = |operator: OpSpec, input: Value| { + let error = check(operator, input).unwrap_err(); + assert!(!error.to_string().contains("secret"), "{error}"); + error.reason().map(Reason::as_str).unwrap_or("engine") + }; + for (operator, input, want) in [ + (OpSpec::Contains, json!({"needle": "{}"}), "json_input"), + (OpSpec::Contains, json!({"json": 1}), "json_input"), + (OpSpec::Contains, json!("{}"), "json_input"), + (OpSpec::EqAt, json!({"path": "$.a"}), "json_input"), + ( + OpSpec::BetweenAt, + json!({"path": "$.age", "low": "1"}), + "json_input", + ), + ( + OpSpec::Subtree, + json!({"path": "$.a", "wildcard_bound": "x"}), + "json_input", + ), + ( + OpSpec::Contains, + json!({"json": r#"{"secret": 1, "secret": 2}"#}), + "json_duplicate_key", + ), + ( + OpSpec::Contains, + json!({"json": r#"{"secret""#}), + "json_syntax", + ), + ( + OpSpec::Contains, + json!({"json": "99999999999999999999"}), + "json_integer_range", + ), + ( + OpSpec::EqAt, + json!({"path": "$.a[2]", "json": "1"}), + "json_path_position", + ), + ( + OpSpec::LtAt, + json!({"path": "$.visits[0].at", "json": "1"}), + // A position is not the ordered wildcard path: the engine + // refuses the path as one the field does not order. + "engine", + ), + ( + OpSpec::Selector, + json!({"path": "$.a[0].b"}), + "json_path_position", + ), + ( + OpSpec::Children, + json!({"path": "$.a[*]"}), + "json_path_wildcard", + ), + ( + OpSpec::ChildCount, + json!({"path": "$.a[*].b"}), + "json_path_wildcard", + ), + // The engine's own: a container at a path, a path the field does + // not order, a value no ordering takes, a path that is not one. + ( + OpSpec::EqAt, + json!({"path": "$.a", "json": r#"{"secret": 1}"#}), + "engine", + ), + ( + OpSpec::LtAt, + json!({"path": "$.name", "json": "1"}), + "engine", + ), + ( + OpSpec::GtAt, + json!({"path": "$.age", "json": "true"}), + "engine", + ), + (OpSpec::Selector, json!({"path": "secret"}), "engine"), + ] { + assert_eq!( + reason(operator, input.clone()), + want, + "{operator:?} {input}" + ); + } + for (operator, input) in [ + (OpSpec::Contains, json!({"json": "[1, {\"a\": null}]"})), + (OpSpec::EqAt, json!({"path": "$.visits[*].at", "json": "1"})), + ( + OpSpec::BetweenAt, + json!({"path": "$.age", "low": "1", "high": "2"}), + ), + (OpSpec::Subtree, json!({"path": "$.visits[1]"})), + (OpSpec::ChildCount, json!({"path": "$.visits"})), + ] { + assert!( + check(operator, input.clone()).is_ok(), + "{operator:?} {input}" + ); + } +} + +/// Two documents read through the data path: `->` by selector, absence as +/// no value, a wildcard subtree per row, and rows refused against another +/// read's request or past the bound, each group opening on its own. +#[tokio::test] +async fn reads_open_against_the_request_that_prepared_them() { + let cipher = common::stack_cipher().await; + let keyset = cipher.default_keyset(); + let plan = plan(); + let docs = [ + seal( + &keyset, + &plan, + &json!({"name": "Alice", "age": 34, "visits": [{"at": 3}, {"at": 1, "where": "x"}]}), + ) + .await, + seal(&keyset, &plan, &json!({"name": "Bob", "visits": []})).await, + ]; + + let selector = |t: QueryTerm| match t.operand() { + Operand::Text(text) => text.clone(), + other => panic!("{other:?}"), + }; + let name = selector(term(&keyset, &plan, "selector", json!({"path": "$.name"})).await); + let age = selector(term(&keyset, &plan, "selector", json!({"path": "$.age"})).await); + let names = open( + &keyset, + &plan, + json!({"operation": "selector", "path": "$.name"}), + docs.iter().map(|d| extracted(d, &name)).collect(), + ) + .await + .unwrap(); + assert_eq!(names, [vec![json!("Alice")], vec![json!("Bob")]]); + let ages = open( + &keyset, + &plan, + json!({"operation": "selector", "path": "$.age"}), + docs.iter().map(|d| extracted(d, &age)).collect(), + ) + .await + .unwrap(); + assert_eq!( + ages, + [vec![json!(34)], vec![]], + "absence is no value, not null" + ); + + let tokens = |t: QueryTerm| match t.operand() { + Operand::TextList(list) => list.clone(), + other => panic!("{other:?}"), + }; + let ats = tokens(term(&keyset, &plan, "subtree", json!({"path": "$.visits[*].at"})).await); + let read = open( + &keyset, + &plan, + json!({"operation": "subtree", "path": "$.visits[*].at"}), + docs.iter().map(|d| subtree(d, &ats)).collect(), + ) + .await + .unwrap(); + assert_eq!(read, [vec![json!(3), json!(1)], vec![]]); + + // Rows opened against another read's request are refused: the + // request, not the rows, says which path they are for. + let wrong = open( + &keyset, + &plan, + json!({"operation": "subtree", "path": "$.name"}), + vec![subtree(&docs[0], &ats)], + ) + .await; + assert!( + matches!(wrong, Err(stack_encrypt::Error::JsonWrongEntry { .. })), + "{wrong:?}" + ); + let wrong = open( + &keyset, + &plan, + json!({"operation": "selector", "path": "$.age"}), + vec![extracted(&docs[0], &name)], + ) + .await; + assert!( + matches!(wrong, Err(stack_encrypt::Error::JsonWrongEntry { .. })), + "{wrong:?}" + ); + + // A wildcard read of an array longer than its bound fails rather than + // open to part of the answer. + let short = tokens( + term( + &keyset, + &plan, + "subtree", + json!({"path": "$.visits[*].at", "wildcard_bound": 1}), + ) + .await, + ); + let truncated = open( + &keyset, + &plan, + json!({"operation": "subtree", "path": "$.visits[*].at", "wildcard_bound": 1}), + vec![subtree(&docs[0], &short)], + ) + .await; + assert!( + matches!( + truncated, + Err(stack_encrypt::Error::JsonWildcardTruncated { .. }) + ), + "{truncated:?}" + ); + + // Through the client rather than the keyset, the same rows open. + let opened = dynamic::json_open( + &keyset, + Scope::Client(&cipher), + &plan, + "profile", + json_frame(&json!({"operation": "selector", "path": "$.name"})), + frame(vec![extracted(&docs[1], &name)]), + &Documents, + ) + .unwrap() + .await; + assert!(opened.is_ok()); +} + +/// A read's request and rows are checked before anything is derived. +#[test] +fn a_malformed_read_is_refused_before_any_key() { + let plan = plan(); + let check = |request: Value, rows: Value| { + dynamic::check_json_open( + &plan, + "profile", + json_frame(&request), + json_frame(&rows), + &Documents, + ) + }; + let refused = |request: Value, rows: Value| { + let error = check(request.clone(), rows).unwrap_err(); + match error.reason() { + Some(reason) => { + assert_eq!(error.field(), Some("profile"), "{request}"); + reason.as_str() + } + None => "engine", + } + }; + assert!(check( + json!({"operation": "selector", "path": "$.a"}), + json!([[], []]) + ) + .is_ok()); + assert!(check( + json!({"operation": "subtree", "path": "$.a[*]", "wildcard_bound": 2}), + json!([[]]) + ) + .is_ok()); + for (request, rows, want) in [ + ( + json!({"operation": "contains", "path": "$.a"}), + json!([]), + "json_input", + ), + ( + json!({"operation": "nope", "path": "$.a"}), + json!([]), + "json_input", + ), + ( + json!({"operation": "selector", "path": "$.a", "wildcard_bound": 2}), + json!([]), + "json_input", + ), + (json!({"path": "$.a"}), json!([]), "json_input"), + ( + json!({"operation": "selector", "path": "$.a"}), + json!([["not an entry"]]), + "json_input", + ), + ( + json!({"operation": "selector", "path": "$.a"}), + json!({"rows": 1}), + "json_input", + ), + ( + json!({"operation": "selector", "path": "$.a[1]"}), + json!([]), + "json_path_position", + ), + ( + json!({"operation": "subtree", "path": "secret"}), + json!([]), + "engine", + ), + ] { + assert_eq!(refused(request.clone(), rows), want, "{request}"); + } + + // A selector group holds one entry or none: `->` returns one. + let entry = json!({"s": "AAAAAAAAAAAAAAAAAAAAAA==", "c": "AA=="}).to_string(); + let two = check( + json!({"operation": "selector", "path": "$.a"}), + json!([[entry.clone(), entry]]), + ); + assert!(two.is_err()); + + // A field that does not store a document has no reads. + let wire = + json!({"email": {"context": ["users", "email"], "type": "string", "outputs": ["c"]}}); + let scalar = + record::plan_with(&dynamic::frame::serde::to_vec(&wire).unwrap(), &Documents).unwrap(); + let error = dynamic::check_json_open( + &scalar, + "email", + json_frame(&json!({"operation": "selector", "path": "$.a"})), + json_frame(&json!([])), + &Documents, + ) + .unwrap_err(); + assert_eq!(error.reason(), Some(Reason::OperatorNotAnswered)); +} From b2b5e1d032026802c85e8a22f6c482865f1a4237 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 03:11:47 +1100 Subject: [PATCH 04/26] test(stack-encrypt): pin eq_at against the orderings and a token against a selector The mutants gate's last survivors in derive: eq_at's needle and an ordering's leaf operand were only checked to be objects, and children's token only against child_count's. The test now holds eq_at to contains' shape and an ordering to another, and a node's token apart from its selector. Parsed's Drop and the JsonText visitor's expecting are excluded with their reasons, as for the PR below. Refs #1347 Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- .cargo/mutants.toml | 6 +++++ .../stack-encrypt/tests/json_operators.rs | 23 +++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index 331962238..11912592b 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -127,6 +127,12 @@ exclude_re = [ # none. What it runs, `wipe_value`, is pinned by # `wipe_value_leaves_no_plaintext_in_the_value`. 'stack-encrypt/src/dynamic/json\.rs:\d+:\d+: replace ::drop with \(\)$', + # The same two for a JSON operator's input: `Parsed`'s Drop wipes a + # document no safe code can read once dropped, and the `JsonText` + # visitor's `expecting` writes a message `read_frame` discards for its + # fixed `Reason::JsonInput`. + 'stack-encrypt/src/dynamic/json_query\.rs:\d+:\d+: replace ::drop with \(\)$', + 'stack-encrypt/src/dynamic/json_query\.rs:\d+:\d+: replace for JsonText>::deserialize:: for V>::expecting -> std::fmt::Result with Ok\(Default::default\(\)\)$', ] # Headroom over the measured baseline before a slow-but-correct mutant is diff --git a/packages/stack-encrypt/tests/json_operators.rs b/packages/stack-encrypt/tests/json_operators.rs index 16420e878..6b49a70d3 100644 --- a/packages/stack-encrypt/tests/json_operators.rs +++ b/packages/stack-encrypt/tests/json_operators.rs @@ -276,6 +276,23 @@ async fn every_operator_derives_its_tagged_operand() { assert_eq!(term.domain(), Some("eql_v4.query_json_tree"), "{operator}"); assert!(needle(&term).is_object(), "{operator}"); } + // Equality at a path is a containment needle, the shape `contains` + // derives; an ordering at the same path is a leaf operand, another. + let keys = |v: &Value| -> Vec { v.as_object().unwrap().keys().cloned().collect() }; + let at_age = json!({"path": "$.age", "json": "40"}); + let contains = needle( + &term( + &keyset, + &plan, + "contains", + json!({"json": r#"{"age": 40}"#}), + ) + .await, + ); + let eq = needle(&term(&keyset, &plan, "eq_at", at_age.clone()).await); + let lt = needle(&term(&keyset, &plan, "lt_at", at_age).await); + assert_eq!(keys(&eq), keys(&contains), "eq_at is a needle: {eq}"); + assert_ne!(keys(<), keys(&eq), "an ordering is a leaf operand: {lt}"); for operator in ["lt_at", "lte_at", "gt_at", "gte_at"] { let term = term( &keyset, @@ -329,6 +346,12 @@ async fn every_operator_derives_its_tagged_operand() { let children = term(&keyset, &plan, "children", json!({"path": "$.visits"})).await; let count = term(&keyset, &plan, "child_count", json!({"path": "$.visits"})).await; + let selector = term(&keyset, &plan, "selector", json!({"path": "$.visits"})).await; + assert_ne!( + children.operand(), + selector.operand(), + "a node's token is not its selector" + ); assert_eq!( children.operand(), count.operand(), From abdc767efb5a78a0ac09062b7e5345c4a3f31c7c Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 05:20:53 +1100 Subject: [PATCH 05/26] fix(stack-encrypt,go): close the JSON operators review findings - A document field lists an operator only when its target names the SQL type the operand binds as, so field_operators lists exactly what query_term derives. - json_open drops its scope parameter, which had no effect: a read opens through the keyset that derived it, and that keyset alone. - between_at's arity is one (its input is one object); takes_pair is between alone. Pinned wire bytes re-pinned. - json_query::binds is exhaustive over OpSpec and defines is_json; answered_by lists the document operators in their own arm; json_reason matches JsonTextError's variants. - Go's parseQueryTerm refuses a term whose encoding, shape and domain do not agree as the engine writes them. - Parsed's Drop is no longer excluded from mutation: heap_hygiene's a_json_operators_input_leaves_no_plaintext_on_the_heap fails without it. - Docs: the guest's parse helpers and query_term (with encoding), Rust query_term's inputs and refusals, QueryTerm::domain, Cipher.QueryTerm, parseOperatorName, open()'s refusal reasons. Refs #1347 Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- .cargo/mutants.toml | 7 +- languages/golang/encrypt/checker.go | 6 +- languages/golang/encrypt/guest/src/ops.rs | 34 +++--- languages/golang/encrypt/query.go | 39 ++++-- languages/golang/encrypt/targets_test.go | 38 ++++++ .../encryption/tests/json_tree_data_plan.rs | 1 - .../stack-encrypt/src/dynamic/json_query.rs | 77 +++++++----- packages/stack-encrypt/src/dynamic/query.rs | 112 ++++++++++-------- packages/stack-encrypt/tests/heap_hygiene.rs | 93 +++++++++++++++ .../stack-encrypt/tests/json_operators.rs | 19 +-- 10 files changed, 292 insertions(+), 134 deletions(-) diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index 11912592b..8dc163c8d 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -127,11 +127,8 @@ exclude_re = [ # none. What it runs, `wipe_value`, is pinned by # `wipe_value_leaves_no_plaintext_in_the_value`. 'stack-encrypt/src/dynamic/json\.rs:\d+:\d+: replace ::drop with \(\)$', - # The same two for a JSON operator's input: `Parsed`'s Drop wipes a - # document no safe code can read once dropped, and the `JsonText` - # visitor's `expecting` writes a message `read_frame` discards for its - # fixed `Reason::JsonInput`. - 'stack-encrypt/src/dynamic/json_query\.rs:\d+:\d+: replace ::drop with \(\)$', + # A JSON operator's input: the `JsonText` visitor's `expecting` writes a + # message `read_frame` discards for its fixed `Reason::JsonInput`. 'stack-encrypt/src/dynamic/json_query\.rs:\d+:\d+: replace for JsonText>::deserialize:: for V>::expecting -> std::fmt::Result with Ok\(Default::default\(\)\)$', ] diff --git a/languages/golang/encrypt/checker.go b/languages/golang/encrypt/checker.go index d79f22fa8..96bf65c7e 100644 --- a/languages/golang/encrypt/checker.go +++ b/languages/golang/encrypt/checker.go @@ -279,8 +279,10 @@ func parseOperators(value any) ([]record.Operator, error) { } // parseOperatorName reads an operator's name as the engine writes it: one -// of eq, lt, lte, gt, gte, between and matches, the names the engine's -// OpSpec::parse accepts. The catalog (se_targets) and se_query_term are +// of the names the engine's OpSpec::parse accepts, the scalar operators +// (eq, lt, lte, gt, gte, between, matches) and a document's (contains, +// contained_by, eq_at, the orderings at a path, between_at, selector, +// subtree, children, child_count). The catalog (se_targets) and se_query_term are // read through this, parseFidelity and parseOperatorIndex, so the two // readers refuse the same values. func parseOperatorName(s string) (string, error) { diff --git a/languages/golang/encrypt/guest/src/ops.rs b/languages/golang/encrypt/guest/src/ops.rs index 90bcf9047..3f82111df 100644 --- a/languages/golang/encrypt/guest/src/ops.rs +++ b/languages/golang/encrypt/guest/src/ops.rs @@ -236,17 +236,21 @@ where /// /// ```text /// { "operator": "matches", "fidelity": "probabilistic", "index": "match", -/// "domain": null, "term": } +/// "domain": null, "encoding": "bytes", "term": } /// ``` /// /// The input is one value for every operator but `between`, whose input is -/// a list of two, low then high. It is read as the field's declared kind -/// first, so a host's float `34.0` queries a `uint64` field. A field with -/// indexes derives the term of the index that answers the operator, as -/// bytes; `between` derives one per bound, as a list of two. A field that -/// names an EQL type runs the type's query twin through this build's +/// a list of two, low then high, and a document's operators, whose input +/// is an object of named inputs (`{"json"}`, `{"path", "json"}`, ...; see +/// [`dynamic::OpSpec::input`]). A value is read as the field's declared +/// kind first, so a host's float `34.0` queries a `uint64` field. A field +/// with indexes derives the term of the index that answers the operator, +/// as bytes; `between` derives one per bound, as a list of two. A field +/// that names an EQL type runs the type's query twin through this build's /// resolver, and `term` holds the twin's JSON bytes and `domain` its -/// PostgreSQL domain. No data key is minted. +/// PostgreSQL domain. A document's operator gives `encoding` `eql` (a +/// needle or leaf operand), `text` (a selector or node token) or +/// `text_array` (a subtree read's tokens). No data key is minted. pub async fn query_term( cipher: &KeysetCipher<'_, K>, input: &[u8], @@ -296,7 +300,6 @@ where let (plan, field) = parse_field(plan, field)?; let opened = dynamic::json_open( keyset, - Scope::Keyset(keyset.clone()), &plan, field, parse_frame(request)?, @@ -369,11 +372,8 @@ pub fn target_limit( encode_answer(&limit.rows()) } -/// The plan, field name and operator a query names: the plan parsed against -/// this build's resolver, the field's name UTF-8, and the operator one the -/// engine knows ([`dynamic::OpSpec::parse`]); an unknown operator is -/// refused naming the field, as an operator the field does not answer is. -/// A plan and one of its fields' names. +/// A plan and one of its fields' names: the plan parsed against this +/// build's resolver, and the field's name UTF-8. fn parse_field<'f>(plan: &[u8], field: &'f [u8]) -> Result<(dynamic::record::Plan, &'f str), u32> { let plan = parse_plan(plan)?; let field = std::str::from_utf8(field) @@ -381,14 +381,16 @@ fn parse_field<'f>(plan: &[u8], field: &'f [u8]) -> Result<(dynamic::record::Pla Ok((plan, field)) } +/// The plan, field name and operator a query names: [`parse_field`]'s two, +/// and the operator one the engine knows ([`dynamic::OpSpec::parse`]); an +/// unknown operator is refused naming the field, as an operator the field +/// does not answer is. fn parse_query<'f>( plan: &[u8], field: &'f [u8], operator: &[u8], ) -> Result<(dynamic::record::Plan, &'f str, dynamic::OpSpec), u32> { - let plan = parse_plan(plan)?; - let field = std::str::from_utf8(field) - .map_err(|_| last_error::malformed("the field name is not UTF-8"))?; + let (plan, field) = parse_field(plan, field)?; let operator = std::str::from_utf8(operator) .ok() .and_then(dynamic::OpSpec::parse) diff --git a/languages/golang/encrypt/query.go b/languages/golang/encrypt/query.go index 4e4f948c0..115f246c7 100644 --- a/languages/golang/encrypt/query.go +++ b/languages/golang/encrypt/query.go @@ -111,7 +111,9 @@ func (l Limit) FetchOr(page uint64) uint64 { // QueryTerm derives the query term for one field of the plan, an operator // and its input under this cipher's keyset: one value for every operator -// but between, whose input is a two-element slice, low then high. The +// but between, whose input is a two-element slice, low then high, and a +// document's operators (contains, eq_at, selector, ...), whose input is a +// vcvalue.Object of named inputs with JSON values as JSON text. The // engine reads the input as the field's declared type, finds the index // that answers the operator, and derives that index's term under the // field's context; for a field that names an EQL type it runs the type's @@ -158,15 +160,19 @@ func (cph *Cipher) QueryTerm(ctx context.Context, plan *record.Plan, field, oper } // parseQueryTerm reads se_query_term's object: exactly the keys operator, -// fidelity, index, domain (a string or null) and term (bytes, or a list of -// two for between). The operator, fidelity and index take only the values -// the engine writes, as the catalog reader's do. +// fidelity, index, domain (a string or null), encoding and term, in the +// shapes the engine writes and no other: encoding bytes is a term of bytes +// (a list of two for between) with no domain; eql is bytes (two for +// between_at) with a domain; text is one string; text_array is a list of +// strings, perhaps empty. The operator, fidelity and index take only the +// values the engine writes, as the catalog reader's do. func parseQueryTerm(decoded any) (QueryTerm, error) { obj, ok := decoded.(vcvalue.Object) if !ok { return QueryTerm{}, fmt.Errorf("came back as %T, not an object", decoded) } var t QueryTerm + var shape string seen := map[string]bool{} for _, f := range obj { if seen[f.Key] { @@ -206,9 +212,12 @@ func parseQueryTerm(decoded any) (QueryTerm, error) { switch v := f.Value.(type) { case []byte: t.Terms = [][]byte{v} + shape = "bytes" case string: t.Text = []string{v} + shape = "text" case []any: + shape = "list" for _, item := range v { switch x := item.(type) { case []byte: @@ -239,13 +248,21 @@ func parseQueryTerm(decoded any) (QueryTerm, error) { if len(seen) != 6 { return QueryTerm{}, fmt.Errorf("%d keys, not 6", len(seen)) } - // The encoding and the term agree: text for the text encodings, - // bytes for the others. - if text := t.Encoding == "text" || t.Encoding == "text_array"; text != (t.Text != nil) { - return QueryTerm{}, fmt.Errorf("a %s term that holds %d texts and %d byte strings", t.Encoding, len(t.Text), len(t.Terms)) - } - if t.Encoding == "text" && len(t.Text) != 1 { - return QueryTerm{}, fmt.Errorf("a text term of %d texts", len(t.Text)) + // The encoding, the term's shape and the domain agree, as the engine + // writes them. + var fits bool + switch t.Encoding { + case "bytes": + fits = t.Terms != nil && t.Domain == "" + case "eql": + fits = t.Terms != nil && t.Domain != "" + case "text": + fits = shape == "text" && t.Domain != "" + case "text_array": + fits = shape == "list" && t.Terms == nil && t.Domain != "" + } + if !fits { + return QueryTerm{}, fmt.Errorf("a %s term of %d texts and %d byte strings (a %s), domain set %t", t.Encoding, len(t.Text), len(t.Terms), shape, t.Domain != "") } return t, nil } diff --git a/languages/golang/encrypt/targets_test.go b/languages/golang/encrypt/targets_test.go index 6b07edb11..8fe6b87c7 100644 --- a/languages/golang/encrypt/targets_test.go +++ b/languages/golang/encrypt/targets_test.go @@ -138,3 +138,41 @@ func TestParseQueryTermTakesOnlyWhatTheEngineWrites(t *testing.T) { } } } + +// The encoding, the term's shape and the domain must agree as the engine +// writes them; a term that mixes them is a wire bug, refused. +func TestParseQueryTermRefusesAMixedShape(t *testing.T) { + domain := "eql_v4.query_json_tree" + shaped := func(encoding string, domain any, term any) vcvalue.Object { + return vcvalue.Object{ + {Key: "operator", Value: "contains"}, {Key: "fidelity", Value: "exact"}, {Key: "index", Value: "json"}, + {Key: "domain", Value: domain}, {Key: "encoding", Value: encoding}, {Key: "term", Value: term}, + } + } + for name, ok := range map[string]vcvalue.Object{ + "bytes": shaped("bytes", nil, []byte{1}), + "bytes range": shaped("bytes", nil, []any{[]byte{1}, []byte{2}}), + "eql": shaped("eql", domain, []byte{1}), + "eql range": shaped("eql", domain, []any{[]byte{1}, []byte{2}}), + "text": shaped("text", "text", "c2VsZWN0b3I="), + "text array": shaped("text_array", "text[]", []any{"dA==", "dQ=="}), + "no tokens": shaped("text_array", "text[]", []any{}), + } { + if _, err := parseQueryTerm(ok); err != nil { + t.Errorf("%s: %v", name, err) + } + } + for name, bad := range map[string]vcvalue.Object{ + "text array as one string": shaped("text_array", "text[]", "dA=="), + "text as a list": shaped("text", "text", []any{"dA=="}), + "eql with no domain": shaped("eql", nil, []byte{1}), + "bytes with a domain": shaped("bytes", domain, []byte{1}), + "text with no domain": shaped("text", nil, "dA=="), + "text array of bytes": shaped("text_array", "text[]", []any{[]byte{1}, []byte{2}}), + "eql as text": shaped("eql", domain, "dA=="), + } { + if _, err := parseQueryTerm(bad); err == nil { + t.Errorf("%s: read, want refused", name) + } + } +} diff --git a/packages/eql/tests/encryption/tests/json_tree_data_plan.rs b/packages/eql/tests/encryption/tests/json_tree_data_plan.rs index 8b9a8aefc..94fc445bf 100644 --- a/packages/eql/tests/encryption/tests/json_tree_data_plan.rs +++ b/packages/eql/tests/encryption/tests/json_tree_data_plan.rs @@ -772,7 +772,6 @@ async fn open( let rows = frame(&groups); let opened = dynamic::json_open( keyset, - Scope::Keyset(keyset.clone()), plan, "profile", request, diff --git a/packages/stack-encrypt/src/dynamic/json_query.rs b/packages/stack-encrypt/src/dynamic/json_query.rs index 4efd6be84..8432e7a56 100644 --- a/packages/stack-encrypt/src/dynamic/json_query.rs +++ b/packages/stack-encrypt/src/dynamic/json_query.rs @@ -52,7 +52,7 @@ use vitaminc_protected::Protected; use zeroize::Zeroizing; use super::query::{OpSpec, Operand}; -use super::{Error, Reason, Scope, ValueFrame}; +use super::{Error, Reason, ValueFrame}; use crate::json::{ parse_text, wipe_value, JsonExtracted, JsonLeafQuery, JsonNode, JsonOptions, JsonPath, JsonQuery, JsonSubtree, JsonSubtreeKey, @@ -74,15 +74,36 @@ pub(super) enum Binds { TextArray, } -/// What a JSON operator's operand binds as. -pub(super) fn binds(op: OpSpec) -> Binds { +/// What a JSON operator's operand binds as, or `None` for an operator +/// that is not one: the one table that says which operators the JSON +/// index answers ([`OpSpec::is_json`]). Exhaustive, so a new operator +/// must be placed. +pub(super) fn binds(op: OpSpec) -> Option { match op { + OpSpec::Contains | OpSpec::ContainedBy | OpSpec::EqAt => Some(Binds::Needle), OpSpec::LtAt | OpSpec::LteAt | OpSpec::GtAt | OpSpec::GteAt | OpSpec::BetweenAt => { - Binds::Leaf + Some(Binds::Leaf) } - OpSpec::Selector | OpSpec::Children | OpSpec::ChildCount => Binds::Text, - OpSpec::Subtree => Binds::TextArray, - _ => Binds::Needle, + OpSpec::Selector | OpSpec::Children | OpSpec::ChildCount => Some(Binds::Text), + OpSpec::Subtree => Some(Binds::TextArray), + OpSpec::Eq + | OpSpec::Matches + | OpSpec::Lt + | OpSpec::Lte + | OpSpec::Gt + | OpSpec::Gte + | OpSpec::Between => None, + } +} + +/// The SQL type an operand of `binds` binds as for `target`: its needle +/// or leaf domain, which a resolver may not name, or text. +pub(super) fn domain(target: &super::TargetDescriptor, binds: Binds) -> Option { + match binds { + Binds::Needle => target.query_sql_domain.clone(), + Binds::Leaf => target.leaf_query_sql_domain.clone(), + Binds::Text => Some("text".to_owned()), + Binds::TextArray => Some("text[]".to_owned()), } } @@ -124,17 +145,18 @@ impl JsonText { .map(Parsed) .map_err(|error| Error::Source { field: None, - reason: json_reason(error.code()), + reason: json_reason(error), }) } } -/// The reason a JSON text refusal is reported with, from its code. -fn json_reason(code: &str) -> Reason { - match code { - "json_duplicate_key" => Reason::JsonDuplicateKey, - "json_integer_range" => Reason::JsonIntegerRange, - _ => Reason::JsonSyntax, +/// The reason a JSON text refusal is reported with. +fn json_reason(error: crate::json::JsonTextError) -> Reason { + use crate::json::JsonTextError as E; + match error { + E::Syntax { .. } => Reason::JsonSyntax, + E::DuplicateKey { .. } => Reason::JsonDuplicateKey, + E::IntegerRange => Reason::JsonIntegerRange, } } @@ -450,7 +472,8 @@ enum Opener { /// Open the rows a read returned against the request that prepared it, /// under the field's column and options and through `keyset`, the keyset -/// whose index key derived the read; the rows open through `scope`. Each +/// whose index key derived the read, and that keyset alone: a document +/// sealed under another is refused before any key is retrieved. Each /// group of rows is one database row's: for `selector`, the entry `->` /// returned or none (SQL `NULL`); for `subtree`, the rows /// `json_tree_subtree` returned. Every group opens to a list of JSON @@ -460,9 +483,9 @@ enum Opener { /// /// # Errors /// -/// [`Error::Source`] for a request or rows that do not parse -/// ([`Reason::JsonInput`]), an operator that is not a read -/// ([`Reason::OperatorNotAnswered`]), or a selector group of more than one +/// [`Error::Source`] ([`Reason::JsonInput`]) for a request or rows that +/// do not parse, an operation that is not `selector` or `subtree`, a +/// `wildcard_bound` on a `selector`, or a selector group of more than one /// entry; [`Error::Cipher`] for the read's own refusals (a path that does /// not parse, a wildcard that expands too far). A row that is not the one /// the read asked for fails the pending @@ -470,7 +493,6 @@ enum Opener { /// longer than the bound ([`crate::Error::JsonWildcardTruncated`]). pub(super) fn open<'a, K: 'static>( keyset: &'a KeysetCipher<'_, K>, - scope: Scope<'a, K>, label: &Label, options: &JsonOptions, request: &ValueFrame, @@ -485,17 +507,12 @@ pub(super) fn open<'a, K: 'static>( } }; let context = || AeadContext::from(NonEmpty::from(label.clone())); - // Opened through the client, confined to the scope's keyset when it is - // one, as a record's leaves are: a document sealed under another - // keyset is refused before any key is retrieved. - let (opener_cipher, confine) = match &scope { - Scope::Client(cipher) => (*cipher, None), - Scope::Keyset(keyset) => (keyset.cipher(), Some(keyset.keyset_id())), - }; - let confined = |pending: Pending<'a, Vec, K>| match confine { - Some(keyset) => pending.scoped_to(keyset), - None => pending, - }; + // Opened through the client, confined to the deriving keyset, as a + // record's leaves are under `Scope::Keyset`: the read's key is that + // keyset's, so a document sealed under another cannot answer it. + let opener_cipher = keyset.cipher(); + let confine = keyset.keyset_id(); + let confined = |pending: Pending<'a, Vec, K>| pending.scoped_to(confine); let mut pendings: Vec, K>> = Vec::with_capacity(groups.len()); for nodes in groups { pendings.push(match &opener { diff --git a/packages/stack-encrypt/src/dynamic/query.rs b/packages/stack-encrypt/src/dynamic/query.rs index b56963830..d66450905 100644 --- a/packages/stack-encrypt/src/dynamic/query.rs +++ b/packages/stack-encrypt/src/dynamic/query.rs @@ -84,8 +84,8 @@ use super::term::{derive, Use}; use super::value_frame::Parts; use super::ValueKind; use super::{ - admits, read, Error, Reason, Scope, TargetDescriptor, TargetError, TargetOptions, - TargetResolver, ValueFrame, + admits, read, Error, Reason, TargetDescriptor, TargetError, TargetOptions, TargetResolver, + ValueFrame, }; use crate::op::{Limit, Stats}; use crate::sem::{match_limit, MatchOptions, QueryTokens, TermError}; @@ -216,21 +216,7 @@ impl OpSpec { /// operators, whose input is an object of named JSON inputs /// ([`OpSpec::input`]). pub fn is_json(self) -> bool { - matches!( - self, - OpSpec::Contains - | OpSpec::ContainedBy - | OpSpec::EqAt - | OpSpec::LtAt - | OpSpec::LteAt - | OpSpec::GtAt - | OpSpec::GteAt - | OpSpec::BetweenAt - | OpSpec::Selector - | OpSpec::Subtree - | OpSpec::Children - | OpSpec::ChildCount - ) + super::json_query::binds(self).is_some() } /// The shape of the operator's input, as a catalog names it: `value` @@ -264,9 +250,10 @@ impl OpSpec { } /// Whether the operator takes a pair of values, low then high, rather - /// than one: `between` does. + /// than one: `between` does. `between_at` takes one object holding its + /// path and both bounds (its [`input`](Self::input) is `path_pair`). pub fn takes_pair(self) -> bool { - matches!(self, OpSpec::Between | OpSpec::BetweenAt) + matches!(self, OpSpec::Between) } /// The index among `indexes` that answers this operator for a field of @@ -312,7 +299,19 @@ impl OpSpec { .or_else(|| declared(|index| matches!(index, IndexSpec::OreBlock))) .or_else(|| declared(|index| matches!(index, IndexSpec::Ope))) } - _ => None, + // Answered above, by the JSON index alone. + OpSpec::Contains + | OpSpec::ContainedBy + | OpSpec::EqAt + | OpSpec::LtAt + | OpSpec::LteAt + | OpSpec::GtAt + | OpSpec::GteAt + | OpSpec::BetweenAt + | OpSpec::Selector + | OpSpec::Subtree + | OpSpec::Children + | OpSpec::ChildCount => None, } .filter(|index| admits(kind, index)) } @@ -406,7 +405,7 @@ pub(super) fn target_operators(target: &TargetDescriptor) -> Vec<(OpSpec, IndexS // document is (its bounds are two leaf operands). OpSpec::ALL .into_iter() - .filter(|op| !op.takes_pair() || op.is_json()) + .filter(|op| !op.takes_pair()) .filter_map(|op| { op.answered_by(kind, &indexes) .map(|index| (op, index.clone())) @@ -514,8 +513,9 @@ impl QueryTerm { &self.index } - /// The PostgreSQL domain the operand is cast to, for a field that names - /// an EQL type; `None` for a field with indexes, whose terms are bytes. + /// The SQL type the operand is bound as: the PostgreSQL domain of an EQL + /// type's query, `text` or `text[]` for a document's selector, token or + /// tokens; `None` for a field with indexes, whose terms are bytes. pub fn domain(&self) -> Option<&str> { self.domain.as_deref() } @@ -616,10 +616,18 @@ fn answers( // A document's operators are its JSON index's under the options // the plan resolved for the field: the orderings at a path only // when it orders one. + // An operator is listed only when the target names the SQL type + // its operand binds as, so every one listed is one `query_term` + // derives. if let Some(TargetOptions::Json(options)) = field.target_options() { let index = [IndexSpec::Json(options.clone())]; let operators = OpSpec::ALL .into_iter() + .filter(|op| { + super::json_query::binds(*op) + .and_then(|binds| super::json_query::domain(&target, binds)) + .is_some() + }) .filter_map(|op| op.answered_by(kind, &index).map(|i| (op, i.clone()))) .collect(); return Ok((operators, Some(target))); @@ -925,20 +933,13 @@ fn json_term<'a, K: 'static>( operator: OpSpec, input: impl Encrypt, ) -> Result, Error> { - use super::json_query::Binds; let field = answer.field.name(); let input = ValueFrame::of(input).map_err(|_| json_input(field))?; let target = answer.target.as_ref().ok_or(Error::Internal)?; - // The SQL type each operand binds as, from the target: a resolver - // that names no leaf domain for a document answers no ordering. - let domain = |binds: Binds| match binds { - Binds::Needle => target.query_sql_domain.clone(), - Binds::Leaf => target.leaf_query_sql_domain.clone(), - Binds::Text => Some("text".to_owned()), - Binds::TextArray => Some("text[]".to_owned()), - }; - let binds = super::json_query::binds(operator); - let domain = domain(binds).ok_or_else(|| { + // The SQL type each operand binds as, from the target: `answer` lists + // no operator whose domain the target does not name. + let binds = super::json_query::binds(operator).ok_or(Error::Internal)?; + let domain = super::json_query::domain(target, binds).ok_or_else(|| { Error::Target(TargetError::NoQuery { name: target.name.clone(), }) @@ -991,7 +992,9 @@ pub fn check_json_open( /// `subtree`, the rows `json_tree_subtree` returned. The engine derives /// the selector or the read key again, under `cipher`'s keyset and the /// field's column and options, and checks every row against it; the rows -/// open through `scope`. The result is a list per database row of JSON +/// open through that keyset alone, so a document sealed under another is +/// refused before any key is retrieved. The result is a list per database +/// row of JSON /// texts, as bytes: for `selector` the scalar or nothing, for `subtree` /// one value per concrete path the read named that the row has, in /// document order. One key request for every row. @@ -1008,10 +1011,8 @@ pub fn check_json_open( /// ([`Error::JsonWrongEntry`](crate::Error::JsonWrongEntry)), as does a /// wildcard read of an array longer than its bound /// ([`Error::JsonWildcardTruncated`](crate::Error::JsonWildcardTruncated)). -#[allow(clippy::too_many_arguments)] pub fn json_open<'a, K>( cipher: &'a KeysetCipher<'_, K>, - scope: Scope<'a, K>, plan: &Plan, field: &str, request: impl Encrypt, @@ -1028,7 +1029,6 @@ where let rows = ValueFrame::of(rows).map_err(|_| json_input(field))?; super::json_query::open( cipher, - scope, answer.field.label(), json_options(&answer)?, &request, @@ -1039,7 +1039,11 @@ where /// Derive the query term for one field of a plan, an operator and its /// input: one value for most operators, a list of two for -/// [`between`](OpSpec::Between). +/// [`between`](OpSpec::Between), and for a document's operators an object +/// of named inputs ([`OpSpec::input`]), whose operand the engine derives +/// with the JSON index under the options the plan resolved for the field, +/// tagged with its encoding ([`QueryTerm::encoding`]) and the SQL type it +/// binds as ([`QueryTerm::domain`]). /// /// The input is read as the field's declared kind first ([`read`]), so a /// host's `34` as a float queries a `uint64` field as the `u64` it stored. @@ -1083,7 +1087,15 @@ where /// that context. [`Error::Source`] naming the field for an operator the /// field does not answer ([`Reason::OperatorNotAnswered`]), an input that /// cannot be read as the field's kind ([`Reason::FieldType`]) and a -/// `between` input that is not two values ([`Reason::NotARange`]). +/// `between` input that is not two values ([`Reason::NotARange`]). For a +/// document's operator: an input that is not its object +/// ([`Reason::JsonInput`]), JSON text that is not a document +/// ([`Reason::JsonSyntax`], [`Reason::JsonDuplicateKey`], +/// [`Reason::JsonIntegerRange`]), a position where the operand is a +/// selector ([`Reason::JsonPathPosition`]) and a wildcard where it names +/// one node ([`Reason::JsonPathWildcard`]); [`Error::Cipher`] for what the +/// JSON index refuses (a path that does not parse, a container at a path, +/// a path the field does not order). /// [`Error::Cipher`] for the query refusals the typed path makes: match /// text that cuts no token ([`TermError::EmptyTermText`]), and text the /// index cannot normalize ([`TermError::Normalize`]). A range whose low @@ -1490,11 +1502,7 @@ mod tests { op == OpSpec::Matches, "{op}" ); - assert_eq!( - op.takes_pair(), - matches!(op, OpSpec::Between | OpSpec::BetweenAt), - "{op}" - ); + assert_eq!(op.takes_pair(), op == OpSpec::Between, "{op}"); } assert_eq!(Fidelity::Exact.name(), "exact"); assert_eq!(Fidelity::Probabilistic.name(), "probabilistic"); @@ -3169,7 +3177,7 @@ mod wire_bytes { "f105000000080000006f70657261746f720a060000006c74655f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a09000000706174685f6a736f6e", "f105000000080000006f70657261746f720a0500000067745f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a09000000706174685f6a736f6e", "f105000000080000006f70657261746f720a060000006774655f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a09000000706174685f6a736f6e", - "f105000000080000006f70657261746f720a0a0000006265747765656e5f617408000000666964656c6974790a0500000065786163740500000061726974790a040000007061697205000000696e6465780a02000000657105000000696e7075740a09000000706174685f70616972", + "f105000000080000006f70657261746f720a0a0000006265747765656e5f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a09000000706174685f70616972", "f105000000080000006f70657261746f720a0800000073656c6563746f7208000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a070000007375627472656508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0a000000706174685f626f756e64", "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a02000000657105000000696e7075740a0400000070617468", @@ -3189,7 +3197,7 @@ mod wire_bytes { "f105000000080000006f70657261746f720a060000006c74655f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f6a736f6e", "f105000000080000006f70657261746f720a0500000067745f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f6a736f6e", "f105000000080000006f70657261746f720a060000006774655f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f6a736f6e", - "f105000000080000006f70657261746f720a0a0000006265747765656e5f617408000000666964656c6974790a0500000065786163740500000061726974790a040000007061697205000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f70616972", + "f105000000080000006f70657261746f720a0a0000006265747765656e5f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f70616972", "f105000000080000006f70657261746f720a0800000073656c6563746f7208000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a070000007375627472656508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0a000000706174685f626f756e64", "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", @@ -3209,7 +3217,7 @@ mod wire_bytes { "f105000000080000006f70657261746f720a060000006c74655f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f6a736f6e", "f105000000080000006f70657261746f720a0500000067745f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f6a736f6e", "f105000000080000006f70657261746f720a060000006774655f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f6a736f6e", - "f105000000080000006f70657261746f720a0a0000006265747765656e5f617408000000666964656c6974790a0500000065786163740500000061726974790a040000007061697205000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f70616972", + "f105000000080000006f70657261746f720a0a0000006265747765656e5f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f70616972", "f105000000080000006f70657261746f720a0800000073656c6563746f7208000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a070000007375627472656508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0a000000706174685f626f756e64", "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", @@ -3229,7 +3237,7 @@ mod wire_bytes { "f105000000080000006f70657261746f720a060000006c74655f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f6a736f6e", "f105000000080000006f70657261746f720a0500000067745f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f6a736f6e", "f105000000080000006f70657261746f720a060000006774655f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f6a736f6e", - "f105000000080000006f70657261746f720a0a0000006265747765656e5f617408000000666964656c6974790a0500000065786163740500000061726974790a040000007061697205000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f70616972", + "f105000000080000006f70657261746f720a0a0000006265747765656e5f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a09000000706174685f70616972", "f105000000080000006f70657261746f720a0800000073656c6563746f7208000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a070000007375627472656508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0a000000706174685f626f756e64", "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a050000006d6174636805000000696e7075740a0400000070617468", @@ -3249,7 +3257,7 @@ mod wire_bytes { "f105000000080000006f70657261746f720a060000006c74655f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a09000000706174685f6a736f6e", "f105000000080000006f70657261746f720a0500000067745f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a09000000706174685f6a736f6e", "f105000000080000006f70657261746f720a060000006774655f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a09000000706174685f6a736f6e", - "f105000000080000006f70657261746f720a0a0000006265747765656e5f617408000000666964656c6974790a0500000065786163740500000061726974790a040000007061697205000000696e6465780a030000006f726505000000696e7075740a09000000706174685f70616972", + "f105000000080000006f70657261746f720a0a0000006265747765656e5f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a09000000706174685f70616972", "f105000000080000006f70657261746f720a0800000073656c6563746f7208000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a070000007375627472656508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0a000000706174685f626f756e64", "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f726505000000696e7075740a0400000070617468", @@ -3269,7 +3277,7 @@ mod wire_bytes { "f105000000080000006f70657261746f720a060000006c74655f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a09000000706174685f6a736f6e", "f105000000080000006f70657261746f720a0500000067745f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a09000000706174685f6a736f6e", "f105000000080000006f70657261746f720a060000006774655f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a09000000706174685f6a736f6e", - "f105000000080000006f70657261746f720a0a0000006265747765656e5f617408000000666964656c6974790a0500000065786163740500000061726974790a040000007061697205000000696e6465780a030000006f706505000000696e7075740a09000000706174685f70616972", + "f105000000080000006f70657261746f720a0a0000006265747765656e5f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a09000000706174685f70616972", "f105000000080000006f70657261746f720a0800000073656c6563746f7208000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a070000007375627472656508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0a000000706174685f626f756e64", "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a030000006f706505000000696e7075740a0400000070617468", @@ -3289,7 +3297,7 @@ mod wire_bytes { "f105000000080000006f70657261746f720a060000006c74655f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a09000000706174685f6a736f6e", "f105000000080000006f70657261746f720a0500000067745f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a09000000706174685f6a736f6e", "f105000000080000006f70657261746f720a060000006774655f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a09000000706174685f6a736f6e", - "f105000000080000006f70657261746f720a0a0000006265747765656e5f617408000000666964656c6974790a0500000065786163740500000061726974790a040000007061697205000000696e6465780a040000006a736f6e05000000696e7075740a09000000706174685f70616972", + "f105000000080000006f70657261746f720a0a0000006265747765656e5f617408000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a09000000706174685f70616972", "f105000000080000006f70657261746f720a0800000073656c6563746f7208000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0400000070617468", "f105000000080000006f70657261746f720a070000007375627472656508000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0a000000706174685f626f756e64", "f105000000080000006f70657261746f720a080000006368696c6472656e08000000666964656c6974790a0500000065786163740500000061726974790a030000006f6e6505000000696e6465780a040000006a736f6e05000000696e7075740a0400000070617468", diff --git a/packages/stack-encrypt/tests/heap_hygiene.rs b/packages/stack-encrypt/tests/heap_hygiene.rs index 0908d3b25..8bb886744 100644 --- a/packages/stack-encrypt/tests/heap_hygiene.rs +++ b/packages/stack-encrypt/tests/heap_hygiene.rs @@ -322,6 +322,99 @@ fn verify_leaves_no_plaintext_on_the_heap() { }); } +/// A resolver holding `EncryptedJson` alone, for checking a document +/// field's operators: a check derives nothing, so nothing here runs. +struct Documents; + +impl stack_encrypt::dynamic::TargetResolver for Documents { + fn targets(&self) -> Vec { + vec![stack_encrypt::dynamic::TargetDescriptor::new( + "EncryptedJson", + "json", + "", + Some(stack_encrypt::dynamic::ValueKind::Json), + "public.eql_v4_json_tree", + vec!["json".to_string()], + Some("EncryptedJsonQuery".to_string()), + Some("eql_v4.query_json_tree".to_string()), + true, + None, + ) + .with_leaf_query_sql_domain("eql_v4.query_json_tree_leaf")] + } + fn encrypt<'a, K: 'static>( + &self, + _: &str, + _: Option<&stack_encrypt::dynamic::TargetOptions>, + _: &'a stack_encrypt::KeysetCipher<'_, K>, + _: &stack_encrypt::Label, + _: ValueFrame, + ) -> Result, K>, stack_encrypt::dynamic::TargetError> { + unreachable!("a check seals nothing") + } + fn decrypt<'a, K: 'static>( + &self, + _: &str, + _: &'a stack_encrypt::StackCipher, + _: &stack_encrypt::Label, + _: &[u8], + ) -> Result, stack_encrypt::dynamic::TargetError> + { + unreachable!("a check opens nothing") + } + fn query<'a, K: 'static>( + &self, + _: &str, + _: &stack_encrypt::target::IndexSpec, + _: &'a stack_encrypt::KeysetCipher<'_, K>, + _: &stack_encrypt::Label, + _: ValueFrame, + ) -> Result, K>, stack_encrypt::dynamic::TargetError> { + unreachable!("a document's operators are the engine's") + } +} + +/// A document operator's JSON input is parsed and checked, and the parsed +/// needle wiped when the check drops it: a check derives nothing, so what +/// it frees is the input's alone. +#[test] +fn a_json_operators_input_leaves_no_plaintext_on_the_heap() { + let wire = serde_json::json!({ + "profile": {"context": ["users", "profile"], "type": "json", "null": "sql", + "target": {"name": "EncryptedJson", "options": {"ordered": ["$.age"]}}} + }); + let plan = record::plan_with( + &stack_encrypt::dynamic::frame::serde::to_vec(&wire).unwrap(), + &Documents, + ) + .unwrap(); + let mark = marked("needle"); + let pad = "x".repeat(160); + let needle = format!(r#"{{"{mark}": ["{mark}\n{pad}"]}}"#); + let at = format!(r#""{mark}\n""#); + let inputs = [ + (OpSpec::Contains, serde_json::json!({"json": needle})), + ( + OpSpec::EqAt, + serde_json::json!({"path": "$.name", "json": at}), + ), + ( + OpSpec::GtAt, + serde_json::json!({"path": "$.age", "json": at}), + ), + ] + .map(|(op, input)| { + let bytes = stack_encrypt::dynamic::frame::serde::to_vec(&input).unwrap(); + (op, ValueFrame::from_bytes(&bytes).unwrap()) + }); + for (operator, input) in &inputs { + watched(&format!("check_query ({operator:?})"), || { + stack_encrypt::dynamic::check_query(&plan, "profile", *operator, input, &Documents) + .expect("a document operator's input"); + }); + } +} + /// The spy itself: a plain `String` holding the marker, freed while /// watched, fails the watch. Without this a spy that saw nothing would pass /// every test above. diff --git a/packages/stack-encrypt/tests/json_operators.rs b/packages/stack-encrypt/tests/json_operators.rs index 6b49a70d3..7dc6aecc9 100644 --- a/packages/stack-encrypt/tests/json_operators.rs +++ b/packages/stack-encrypt/tests/json_operators.rs @@ -18,8 +18,8 @@ use std::collections::BTreeMap; use base64ct::{Base64, Encoding}; use serde_json::{json, Value}; use stack_encrypt::dynamic::{ - self, record, OpSpec, Operand, QueryTerm, Reason, Scope, TargetDescriptor, TargetError, - TargetOptions, TargetResolver, ValueFrame, ValueKind, + self, record, OpSpec, Operand, QueryTerm, Reason, TargetDescriptor, TargetError, TargetOptions, + TargetResolver, ValueFrame, ValueKind, }; use stack_encrypt::json::{JsonDocument, Tag}; use stack_encrypt::target::{Borrowed, CallerContext, IndexSpec}; @@ -205,7 +205,6 @@ async fn open( ) -> Result>, stack_encrypt::Error> { let opened = dynamic::json_open( keyset, - Scope::Keyset(keyset.clone()), plan, "profile", json_frame(&request), @@ -621,20 +620,6 @@ async fn reads_open_against_the_request_that_prepared_them() { ), "{truncated:?}" ); - - // Through the client rather than the keyset, the same rows open. - let opened = dynamic::json_open( - &keyset, - Scope::Client(&cipher), - &plan, - "profile", - json_frame(&json!({"operation": "selector", "path": "$.name"})), - frame(vec![extracted(&docs[1], &name)]), - &Documents, - ) - .unwrap() - .await; - assert!(opened.is_ok()); } /// A read's request and rows are checked before anything is derived. From 8a2488e9c570735d9b130956eb9318b50826ac0c Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 05:45:50 +1100 Subject: [PATCH 06/26] test(stack-encrypt): frame the operator heap test's inputs with the test writer CI's Linux run flagged a freed hash table in vitaminc's frame writer that held the marker in bytes left by the test's own input building (serde_json and a growing format! string), not by the engine. The inputs are now joined into strings sized before they are written and framed by the suite's object writer. The test still fails when Parsed's Drop is a no-op. Refs #1347 Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- packages/stack-encrypt/tests/heap_hygiene.rs | 24 ++++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/packages/stack-encrypt/tests/heap_hygiene.rs b/packages/stack-encrypt/tests/heap_hygiene.rs index 6744e0a1a..dde1da6d4 100644 --- a/packages/stack-encrypt/tests/heap_hygiene.rs +++ b/packages/stack-encrypt/tests/heap_hygiene.rs @@ -390,23 +390,27 @@ fn a_json_operators_input_leaves_no_plaintext_on_the_heap() { .unwrap(); let mark = marked("needle"); let pad = "x".repeat(160); - let needle = format!(r#"{{"{mark}": ["{mark}\n{pad}"]}}"#); - let at = format!(r#""{mark}\n""#); + let needle = joined(&[r#"{""#, &mark, r#"": [""#, &mark, r"\n", &pad, r#""]}"#]); + let at = joined(&[r#"""#, &mark, r#"\n""#]); + // Framed by the test's own writer, which leaves no copy behind, so a + // block the watch sees is the engine's. let inputs = [ - (OpSpec::Contains, serde_json::json!({"json": needle})), + ( + OpSpec::Contains, + object(vec![("json", frame(needle.as_str()))]), + ), ( OpSpec::EqAt, - serde_json::json!({"path": "$.name", "json": at}), + object(vec![ + ("path", frame("$.name")), + ("json", frame(at.as_str())), + ]), ), ( OpSpec::GtAt, - serde_json::json!({"path": "$.age", "json": at}), + object(vec![("path", frame("$.age")), ("json", frame(at.as_str()))]), ), - ] - .map(|(op, input)| { - let bytes = stack_encrypt::dynamic::frame::serde::to_vec(&input).unwrap(); - (op, ValueFrame::from_bytes(&bytes).unwrap()) - }); + ]; for (operator, input) in &inputs { watched(&format!("check_query ({operator:?})"), || { stack_encrypt::dynamic::check_query(&plan, "profile", *operator, input, &Documents) From 3ae71660c53b2913596c9d4db575d3dfcc519719 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 06:06:03 +1100 Subject: [PATCH 07/26] fix(go): close the encrypted JSON fields review findings - A json.RawMessage is a document only under encrypt_into; sealed or indexed any other way it stays bytes, so existing models generate as before. - An ordered path's accessor has its own Selector, so an ordering and the value it compares come from one path; the README says why. - JSONRead keeps the subtree input it was prepared from and sends it back when it opens; OpenOne refuses a wildcard path before opening anything. - JSONSelection.Open makes no request for a page with no entries and refuses more than one value for a row; a zero JSONSelector or JSONToken is refused rather than bound as ''. - gensupport's JSON helpers check the term's encoding, fidelity (record.Exact) and part count in one place. - stashgen: imports are re-checked after planning adds encoding/json; a document field needs the engine to answer selector on it; an ordered path given twice, or with no name, is refused by name; a key of digits names its accessor; index=json points at encrypt_into=EncryptedJson. - Tests check rows.Err() and no longer assert a driver value's type. Refs #1347 Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/CHANGELOG.md | 11 ++-- languages/golang/cmd/stashgen/README.md | 7 +- languages/golang/cmd/stashgen/main_test.go | 17 ++++- languages/golang/encrypt/gensupport/json.go | 65 ++++++++----------- .../internal/testusers/customer_stash.go | 7 ++ languages/golang/encrypt/json.go | 54 +++++++++++---- languages/golang/encrypt/json_fields_test.go | 31 ++++++++- .../golang/encrypt/json_postgres_test.go | 22 ++++++- languages/golang/stashgen/emit.go | 40 ++++++++++-- languages/golang/stashgen/engine.go | 6 +- .../golang/stashgen/enginetest/enginetest.go | 2 +- languages/golang/stashgen/read.go | 11 ++++ languages/golang/stashgen/refusal_test.go | 2 +- 13 files changed, 202 insertions(+), 73 deletions(-) diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index 4da8a5e57..5fad839ff 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -19,10 +19,13 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. `*json.RawMessage` field takes `null=sql`. Its `Fields` entry has `Contains`, `ContainedBy`, `EqAt`, `Selector`, `Subtree`, `Children` and `ChildCount`, and `At.` per ordered path with `Lt`, `Lte`, `Gt`, - `Gte` and `Between`. `encrypt.JSONSelection` and `encrypt.JSONRead` open - the rows `->` and `eql_v4.json_tree_subtree` return against the request - that prepared them; `encrypt.JSONSelector`, `JSONToken` and `JSONTokens` - bind as `text` and `text[]`. `gensupport.JSON`, `TargetOptions`, + `Gte`, `Between` and the path's own `Selector`. `encrypt.JSONSelection` + and `encrypt.JSONRead` open the rows `->` and `eql_v4.json_tree_subtree` + return against the request that prepared them (`OpenOne` refuses a + wildcard read up front); `encrypt.JSONSelector`, `JSONToken` and + `JSONTokens` bind as `text` and `text[]`, and a zero selector or token is + refused rather than bound. A `json.RawMessage` sealed any other way stays + bytes. `gensupport.JSON`, `TargetOptions`, `Declaration.Options` and the `JSON*` helpers are the generated code's. - **Query and read an encrypted JSON document, at the record level** diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index fe859e01c..17ca31938 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -117,6 +117,7 @@ A `json.RawMessage` field with `encrypt_into=EncryptedJson` is stored as one enc The field holds the document's JSON text; the engine parses it, so an object that names a member twice, an integer outside the 64-bit range and text that is not JSON are refused with `encrypt.ErrEncoding`. Values come back as JSON text holding the same values; key order and the spelling of numbers are not kept. A `*json.RawMessage` field is optional and must say `null=sql`: a document has no fixed length to hide a missing one behind. A non-nil pointer to `null` is a stored JSON `null`. +A `json.RawMessage` sealed any other way (`encrypt`, `index=equality`) is bytes, as any type defined over `[]byte` is. The type takes options in parentheses: `ordered=` names the paths whose numbers and strings carry an ordering term, separated by `;`, and `sealed_root=false` turns off the sealed root (on by default). `encrypt_into=EncryptedJson(ordered=$.age;$.visits[*].at)` orders two paths. @@ -128,12 +129,14 @@ The field's `Fields` entry has a method per operation: |---|---|---| | `Contains(ctx, c, needle)`, `ContainedBy(ctx, c, needle)` | `eql.EncryptedJSONQuery` | `col @> $1::eql_v4.query_json_tree`, `col <@ $1::eql_v4.query_json_tree` | | `EqAt(ctx, c, path, v)` | `eql.EncryptedJSONQuery` | `col @> $1::eql_v4.query_json_tree` | -| `At.Age.Lt(ctx, c, v)`, `Lte`, `Gt`, `Gte`, `Between` | `eql.EncryptedJSONLeafQuery` | `(col -> $sel::text) < $1::eql_v4.query_json_tree_leaf` | +| `At.Age.Lt(ctx, c, v)`, `Lte`, `Gt`, `Gte`, `Between` | `eql.EncryptedJSONLeafQuery` | `(col -> $sel::text) < $1::eql_v4.query_json_tree_leaf`, `$sel` from `At.Age.Selector(ctx, c)` | | `Selector(ctx, c, path)` | `encrypt.JSONSelection`: `Selector()` and `Open` | `col -> $1::text` | | `Subtree(ctx, c, path, opts)` | `encrypt.JSONRead`: `Tokens()`, `Open` and `OpenOne` | `eql_v4.json_tree_subtree(col, $1::text[], 64)` | | `Children(ctx, c, path)`, `ChildCount(ctx, c, path)` | `encrypt.JSONToken` | `eql_v4.json_tree_children(col, $1::text)`, `eql_v4.json_tree_child_count(col, $1::text)` | -`At` has one accessor per ordered path, named for its keys (`$.visits[*].at` is `At.VisitsAt`); two paths with one name stop `stashgen`. +`At` has one accessor per ordered path, named for its keys (`$.visits[*].at` is `At.VisitsAt`, `$.a.1` is `At.A1`); two paths with one name, or one path given twice, stop `stashgen`. +An ordering compares the value `->` extracts at its path, and EQL's `<` compares ordering terms alone, so take the selector from the same accessor (`At.Age.Selector`) rather than spelling the path again. +`OpenOne` reads one value and refuses a read whose path has a wildcard before anything is opened; open a wildcard read with `Open`. The needle and the values are `json.RawMessage`: write the JSON you mean, such as `json.RawMessage(`+"`"+`{"active":true}`+"`"+`)`. A position in a path is refused for `EqAt` and `Selector`, whose selector reads every position as the wildcard; read a position with `Subtree`. diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index ac3cb928d..f2f27d73c 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -333,6 +333,20 @@ func TestRunGeneratesEncryptedJSONFields(t *testing.T) { if strings.Contains(written, "PrefsFieldAt") { t.Error("Prefs orders no path, and got an At") } + if !strings.Contains(written, "func (p ProfileFieldPath) Selector(ctx context.Context, c *encrypt.Cipher) (encrypt.JSONSelection[eql.EncryptedJSONEntry], error)") { + t.Error("an ordered path has no Selector of its own") + } + // A json.RawMessage sealed any other way is bytes, as it was before + // documents; an object key of digits names its accessor. + written, stderr, code = generate("\tData json.RawMessage `stash:\"data,encrypt,index=equality\"`\n\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$.a.1)\"`") + if code != 0 { + t.Fatalf("exit %d\n%s", code, stderr) + } + for _, want := range []string{`EncryptIndex("data", gensupport.Bytes`, "A1 ProfileFieldPath // $.a.1"} { + if !strings.Contains(written, want) { + t.Errorf("the generated file lacks %q", want) + } + } for _, c := range []struct{ name, fields, want string }{ {"a sealed null", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=EncryptedJson,null=sealed\"`", "Customer.Prefs"}, {"a nil document that does not say null=sql", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=EncryptedJson\"`", "null=sql"}, @@ -341,7 +355,8 @@ func TestRunGeneratesEncryptedJSONFields(t *testing.T) { {"an unknown option", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(bound=3)\"`", "no option \"bound\""}, {"the Go name in the tag", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJSON\"`", "as the engine does, EncryptedJson"}, {"two paths, one accessor", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$.a_b;$.aB)\"`", "both name the accessor At.AB"}, - {"a document with no target", "\tProfile json.RawMessage `stash:\"profile,encrypt\"`", "the engine refuses the declaration"}, + {"the same ordered path twice", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$.a;$.a)\"`", "the ordered path $.a is given twice"}, + {"a path with no accessor name", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$.__)\"`", "no Go accessor name"}, {"a string into a document", "\tProfile string `stash:\"profile,encrypt_into=EncryptedJson\"`", "EncryptedJson seals a json"}, {"options on a type that takes none", "\tEmail string `stash:\"email,encrypt_into=TextEq(ordered=$.a)\"`", "the engine refuses the declaration"}, } { diff --git a/languages/golang/encrypt/gensupport/json.go b/languages/golang/encrypt/gensupport/json.go index f04391529..717e1821b 100644 --- a/languages/golang/encrypt/gensupport/json.go +++ b/languages/golang/encrypt/gensupport/json.go @@ -6,16 +6,20 @@ import ( "fmt" "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/internal/record" "github.com/cipherstash/vitaminc/bindings/go/vcvalue" ) // The operators of a field stored as an EncryptedJSON document, for the // generated methods of its Fields entry. Each sends the operator's named -// inputs, JSON values as their JSON text, and checks the engine answered -// in the encoding the method returns. +// inputs, JSON values as their JSON text ([]byte of the json.RawMessage), +// and checks the engine answered in the encoding and with the number of +// parts the method returns. -// jsonTerm derives one JSON operator's term on the field. -func (f Field[T]) jsonTerm(ctx context.Context, c *encrypt.Cipher, operator, encoding string, input vcvalue.Object) (encrypt.QueryTerm, error) { +// jsonTerm derives one JSON operator's term on the field: an exact term in +// encoding, of parts byte strings ("eql") or texts ("text"); parts 0 takes +// any number of texts ("text_array"). +func (f Field[T]) jsonTerm(ctx context.Context, c *encrypt.Cipher, operator, encoding string, parts int, input vcvalue.Object) (encrypt.QueryTerm, error) { if f.err != nil { return encrypt.QueryTerm{}, f.err } @@ -26,21 +30,22 @@ func (f Field[T]) jsonTerm(ctx context.Context, c *encrypt.Cipher, operator, enc if err != nil { return encrypt.QueryTerm{}, err } - if term.Encoding != encoding || term.Fidelity != "exact" { - return encrypt.QueryTerm{}, fmt.Errorf("gensupport: %s: %s came back as a %s %s term, not an exact %s one", f.name, operator, term.Fidelity, term.Encoding, encoding) + got := len(term.Terms) + if encoding == "text" || encoding == "text_array" { + got = len(term.Text) + } + if term.Encoding != encoding || term.Fidelity != record.Exact || (parts != 0 && got != parts) { + return encrypt.QueryTerm{}, fmt.Errorf("gensupport: %s: %s came back as a %s %s term of %d parts, not an exact %s one of %d", f.name, operator, term.Fidelity, term.Encoding, got, encoding, parts) } return term, nil } -// jsonText is a JSON value as an input carries it: its text, as bytes. -func jsonText(v json.RawMessage) []byte { return []byte(v) } - // JSONQuery derives a containment operator's query on a JSON field // (contains, contained_by): the needle's projection, Q. func JSONQuery[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, operator string, needle json.RawMessage) (Q, error) { - term, err := f.jsonTerm(ctx, c, operator, "eql", vcvalue.Object{{Key: "json", Value: jsonText(needle)}}) - if err != nil || len(term.Terms) != 1 { - return nil, oneTermErr(f.name, operator, err) + term, err := f.jsonTerm(ctx, c, operator, "eql", 1, vcvalue.Object{{Key: "json", Value: []byte(needle)}}) + if err != nil { + return nil, err } return Q(term.Terms[0]), nil } @@ -49,9 +54,9 @@ func JSONQuery[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cip // lte_at, gt_at, gte_at): Q is the needle for eq_at and the leaf operand // for the orderings. func JSONQueryAt[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, operator, path string, v json.RawMessage) (Q, error) { - term, err := f.jsonTerm(ctx, c, operator, "eql", vcvalue.Object{{Key: "path", Value: path}, {Key: "json", Value: jsonText(v)}}) - if err != nil || len(term.Terms) != 1 { - return nil, oneTermErr(f.name, operator, err) + term, err := f.jsonTerm(ctx, c, operator, "eql", 1, vcvalue.Object{{Key: "path", Value: path}, {Key: "json", Value: []byte(v)}}) + if err != nil { + return nil, err } return Q(term.Terms[0]), nil } @@ -59,23 +64,17 @@ func JSONQueryAt[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.C // JSONBetweenAt derives between_at on a JSON field: the two leaf operands, // low then high, inclusive. func JSONBetweenAt[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, path string, lo, hi json.RawMessage) (encrypt.JSONRange[Q], error) { - term, err := f.jsonTerm(ctx, c, "between_at", "eql", vcvalue.Object{{Key: "path", Value: path}, {Key: "low", Value: jsonText(lo)}, {Key: "high", Value: jsonText(hi)}}) + term, err := f.jsonTerm(ctx, c, "between_at", "eql", 2, vcvalue.Object{{Key: "path", Value: path}, {Key: "low", Value: []byte(lo)}, {Key: "high", Value: []byte(hi)}}) if err != nil { return encrypt.JSONRange[Q]{}, err } - if len(term.Terms) != 2 { - return encrypt.JSONRange[Q]{}, fmt.Errorf("gensupport: %s: between_at came back as %d terms, not 2", f.name, len(term.Terms)) - } return encrypt.JSONRange[Q]{Low: Q(term.Terms[0]), High: Q(term.Terms[1])}, nil } // JSONSelect prepares `->` at a path on a JSON field: the selector to // bind, and what opens the entries it returns. E is the field's entry type. func JSONSelect[E ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, path string) (encrypt.JSONSelection[E], error) { - term, err := f.jsonTerm(ctx, c, "selector", "text", vcvalue.Object{{Key: "path", Value: path}}) - if err == nil && len(term.Text) != 1 { - err = fmt.Errorf("gensupport: %s: selector came back as %d texts, not 1", f.name, len(term.Text)) - } + term, err := f.jsonTerm(ctx, c, "selector", "text", 1, vcvalue.Object{{Key: "path", Value: path}}) if err != nil { return encrypt.JSONSelection[E]{}, err } @@ -83,36 +82,28 @@ func JSONSelect[E ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Ci } // JSONSubtree prepares a subtree read at a path on a JSON field: the -// tokens to bind, and what opens the rows they return. +// tokens to bind, and what opens the rows they return. The request is +// built once, here, and the read keeps it: opening sends the engine the +// same request the tokens were derived from. func JSONSubtree[E ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[E], error) { input := vcvalue.Object{{Key: "path", Value: path}} if opts.WildcardBound != 0 { input = append(input, vcvalue.Field{Key: "wildcard_bound", Value: opts.WildcardBound}) } - term, err := f.jsonTerm(ctx, c, "subtree", "text_array", input) + term, err := f.jsonTerm(ctx, c, "subtree", "text_array", 0, input) if err != nil { return encrypt.JSONRead[E]{}, err } - return encrypt.NewJSONRead[E](c, f.plan, f.name, path, opts.WildcardBound, term.Text), nil + return encrypt.NewJSONRead[E](c, f.plan, f.name, path, input, term.Text), nil } // JSONNodeToken derives the token of the node at a path with no wildcard // (children, child_count), which json_tree_children and // json_tree_child_count take. func JSONNodeToken[T any](ctx context.Context, f Field[T], c *encrypt.Cipher, operator, path string) (encrypt.JSONToken, error) { - term, err := f.jsonTerm(ctx, c, operator, "text", vcvalue.Object{{Key: "path", Value: path}}) - if err == nil && len(term.Text) != 1 { - err = fmt.Errorf("gensupport: %s: %s came back as %d texts, not 1", f.name, operator, len(term.Text)) - } + term, err := f.jsonTerm(ctx, c, operator, "text", 1, vcvalue.Object{{Key: "path", Value: path}}) if err != nil { return "", err } return encrypt.JSONToken(term.Text[0]), nil } - -func oneTermErr(field, operator string, err error) error { - if err != nil { - return err - } - return fmt.Errorf("gensupport: %s: %s came back as other than one term", field, operator) -} diff --git a/languages/golang/encrypt/internal/testusers/customer_stash.go b/languages/golang/encrypt/internal/testusers/customer_stash.go index a6afbe086..50b266aa1 100644 --- a/languages/golang/encrypt/internal/testusers/customer_stash.go +++ b/languages/golang/encrypt/internal/testusers/customer_stash.go @@ -221,6 +221,13 @@ func (p CustomerProfileFieldPath) Between(ctx context.Context, c *encrypt.Cipher return gensupport.JSONBetweenAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, p.path, lo, hi) } +// Selector prepares `->` at this path: the value an ordering here compares is +// col -> Selector(), so bind it beside the ordering's operand, from the same +// path. +func (p CustomerProfileFieldPath) Selector(ctx context.Context, c *encrypt.Cipher) (encrypt.JSONSelection[eql.EncryptedJSONEntry], error) { + return gensupport.JSONSelect[eql.EncryptedJSONEntry](ctx, p.field, c, p.path) +} + type CustomerPrefsField struct { field gensupport.Field[json.RawMessage] } diff --git a/languages/golang/encrypt/json.go b/languages/golang/encrypt/json.go index ca17273ac..37ed84527 100644 --- a/languages/golang/encrypt/json.go +++ b/languages/golang/encrypt/json.go @@ -101,8 +101,14 @@ func (cph *Cipher) JSONOpen(ctx context.Context, plan *record.Plan, field string // It is column-wide: the same path has the same selector in every row. type JSONSelector string -// Value binds the selector as text. -func (s JSONSelector) Value() (driver.Value, error) { return string(s), nil } +// Value binds the selector as text. A zero JSONSelector (one never +// prepared) is refused rather than bound as ”, which matches no row. +func (s JSONSelector) Value() (driver.Value, error) { + if s == "" { + return nil, fmt.Errorf("%w: an empty JSONSelector: prepare one with the field's Selector", ErrEncoding) + } + return string(s), nil +} // JSONToken is one node's token: what eql_v4.json_tree_children and // eql_v4.json_tree_child_count take. It binds as SQL text. A token is a @@ -110,8 +116,14 @@ func (s JSONSelector) Value() (driver.Value, error) { return string(s), nil } // to the database for a query, and do not log it. type JSONToken string -// Value binds the token as text. -func (t JSONToken) Value() (driver.Value, error) { return string(t), nil } +// Value binds the token as text. A zero JSONToken is refused, as a zero +// JSONSelector is. +func (t JSONToken) Value() (driver.Value, error) { + if t == "" { + return nil, fmt.Errorf("%w: an empty JSONToken: derive one with the field's Children or ChildCount", ErrEncoding) + } + return string(t), nil +} // JSONTokens are a subtree read's tokens: what eql_v4.json_tree_subtree // takes. They bind as a PostgreSQL text[] literal; cast the parameter, @@ -196,19 +208,29 @@ func (s JSONSelection[E]) Open(ctx context.Context, rows []*E) ([]JSONValue, err return nil, fmt.Errorf("%w: a zero JSONSelection has no request to open against", ErrEncoding) } groups := make([][][]byte, len(rows)) + entries := false for i, row := range rows { if row != nil { groups[i] = [][]byte{[]byte(*row)} + entries = true } } + // No row has an entry: nothing to open, and no key to ask for. + if !entries { + return make([]JSONValue, len(rows)), nil + } opened, err := s.cipher.JSONOpen(ctx, s.plan, s.field, vcvalue.Object{{Key: "operation", Value: "selector"}, {Key: "path", Value: s.path}}, groups) if err != nil { return nil, err } out := make([]JSONValue, len(opened)) for i, values := range opened { - if len(values) == 1 { + switch len(values) { + case 0: + case 1: out[i] = JSONValue{Found: true, JSON: values[0]} + default: + return nil, fmt.Errorf("%w: se_json_open opened %d values for row %d of a selector read", ErrInternal, len(values), i) } } return out, nil @@ -225,14 +247,16 @@ type JSONRead[E ~[]byte] struct { plan *record.Plan field string path string - bound uint32 + // input is the subtree operator's input the tokens were derived from; + // opening sends it back as the read's request, unchanged. + input vcvalue.Object tokens JSONTokens } -// NewJSONRead is a read prepared by [Cipher.QueryTerm]'s subtree operator. -// For generated code. -func NewJSONRead[E ~[]byte](c *Cipher, plan *record.Plan, field, path string, bound uint32, tokens []string) JSONRead[E] { - return JSONRead[E]{cipher: c, plan: plan, field: field, path: path, bound: bound, tokens: JSONTokens(tokens)} +// NewJSONRead is a read prepared by [Cipher.QueryTerm]'s subtree operator +// from input, its {"path", "wildcard_bound"?} object. For generated code. +func NewJSONRead[E ~[]byte](c *Cipher, plan *record.Plan, field, path string, input vcvalue.Object, tokens []string) JSONRead[E] { + return JSONRead[E]{cipher: c, plan: plan, field: field, path: path, input: input, tokens: JSONTokens(tokens)} } // Tokens are the tokens to bind: @@ -262,10 +286,7 @@ func (r JSONRead[E]) Open(ctx context.Context, rows [][]E) ([][]json.RawMessage, groups[i][j] = []byte(entry) } } - request := vcvalue.Object{{Key: "operation", Value: "subtree"}, {Key: "path", Value: r.path}} - if r.bound != 0 { - request = append(request, vcvalue.Field{Key: "wildcard_bound", Value: r.bound}) - } + request := append(vcvalue.Object{{Key: "operation", Value: "subtree"}}, r.input...) opened, err := r.cipher.JSONOpen(ctx, r.plan, r.field, request, groups) if err != nil { return nil, err @@ -285,6 +306,11 @@ func (r JSONRead[E]) Open(ctx context.Context, rows [][]E) ([][]json.RawMessage, // reaches more than one value (a wildcard) is [ErrEncoding]; open it with // [JSONRead.Open]. func (r JSONRead[E]) OpenOne(ctx context.Context, rows []E) (JSONValue, error) { + // A wildcard reaches a value per element: refused before anything is + // opened, not only for the rows that happen to hold two. + if strings.Contains(r.path, "[*]") { + return JSONValue{}, fmt.Errorf("%w: OpenOne reads one value, and %s has a wildcard; open it with Open", ErrEncoding, r.path) + } opened, err := r.Open(ctx, [][]E{rows}) if err != nil { return JSONValue{}, err diff --git a/languages/golang/encrypt/json_fields_test.go b/languages/golang/encrypt/json_fields_test.go index 12fab79b0..976ad14c5 100644 --- a/languages/golang/encrypt/json_fields_test.go +++ b/languages/golang/encrypt/json_fields_test.go @@ -4,11 +4,13 @@ import ( "context" "encoding/json" "errors" + "fmt" "reflect" "strings" "testing" "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/eql" "github.com/cipherstash/stack/languages/golang/encrypt/internal/testusers" ) @@ -81,7 +83,7 @@ func TestACustomersFieldsDeriveEachQuery(t *testing.T) { if err != nil || len(contains) == 0 { t.Fatalf("Contains: %v", err) } - if v, err := contains.Value(); err != nil || !strings.Contains(v.(string), `"sv"`) { + if v, err := contains.Value(); err != nil || !strings.Contains(fmt.Sprint(v), `"sv"`) { t.Fatalf("Contains binds as JSON: %v %v", v, err) } if _, err := f.ContainedBy(ctx, cipher, json.RawMessage(`{"name":"Alice","age":34}`)); err != nil { @@ -105,12 +107,37 @@ func TestACustomersFieldsDeriveEachQuery(t *testing.T) { if err != nil || len(read.Tokens()) != 3 { t.Fatalf("Subtree: %v %v", read.Tokens(), err) } - if v, err := read.Tokens().Value(); err != nil || !strings.HasPrefix(v.(string), `{"`) { + if v, err := read.Tokens().Value(); err != nil || !strings.HasPrefix(fmt.Sprint(v), `{"`) { t.Fatalf("Tokens bind as a text[] literal: %v %v", v, err) } if _, err := f.Children(ctx, cipher, "$.visits"); err != nil { t.Fatal(err) } + // An ordered path's own selector is the field's at that path. + pathSel, err := f.At.VisitsAt.Selector(ctx, cipher) + if err != nil { + t.Fatal(err) + } + if fieldSel, _ := f.Selector(ctx, cipher, "$.visits[*].at"); pathSel.Selector() != fieldSel.Selector() { + t.Fatal("the accessor's selector is another path's") + } + // OpenOne refuses a wildcard read before anything is opened, whatever + // the rows hold; a zero selector or token binds nothing. + if _, err := read.OpenOne(ctx, nil); !errors.Is(err, encrypt.ErrEncoding) { + t.Fatalf("OpenOne on a wildcard read: %v", err) + } + if _, err := (encrypt.JSONSelector("")).Value(); !errors.Is(err, encrypt.ErrEncoding) { + t.Fatalf("a zero selector binds: %v", err) + } + if _, err := (encrypt.JSONToken("")).Value(); !errors.Is(err, encrypt.ErrEncoding) { + t.Fatalf("a zero token binds: %v", err) + } + // A selection over rows with no entry opens to no value without a + // request. + none, err := sel.Open(ctx, make([]*eql.EncryptedJSONEntry, 3)) + if err != nil || len(none) != 3 || none[0].Found || none[2].Found { + t.Fatalf("an empty page: %+v %v", none, err) + } // What the engine refuses: an ordering on a value no ordering takes, a // container at a path, a position in a selector, a wildcard where one // node is named. Each is a returned error naming no value. diff --git a/languages/golang/encrypt/json_postgres_test.go b/languages/golang/encrypt/json_postgres_test.go index 946f937af..ba9864ee2 100644 --- a/languages/golang/encrypt/json_postgres_test.go +++ b/languages/golang/encrypt/json_postgres_test.go @@ -99,6 +99,9 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { } stored = append(stored, e) } + if err := rows.Err(); err != nil { + t.Fatal(err) + } _ = rows.Close() back, err := testusers.DecryptCustomer(ctx, c, stored) if err != nil { @@ -127,8 +130,9 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { } want("prefs contains", ids(`SELECT id FROM customers WHERE prefs @> $1::eql_v4.query_json_tree ORDER BY id`, pq), 1) - // Orderings at an ordered path compare the leaf `->` extracts there. - age, err := f.Selector(ctx, cipher, "$.age") + // Orderings at an ordered path compare the leaf `->` extracts there, + // selected by the accessor's own Selector, so the path is named once. + age, err := f.At.Age.Selector(ctx, cipher) if err != nil { t.Fatal(err) } @@ -142,7 +146,7 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { t.Fatal(err) } want("age between", ids(`SELECT id FROM customers WHERE (profile -> $1::text) >= $2::eql_v4.query_json_tree_leaf AND (profile -> $1::text) <= $3::eql_v4.query_json_tree_leaf ORDER BY id`, age.Selector(), rng.Low, rng.High), 1, 3) - at, err := f.Selector(ctx, cipher, "$.visits[*].at") + at, err := f.At.VisitsAt.Selector(ctx, cipher) if err != nil { t.Fatal(err) } @@ -169,6 +173,9 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { } entries = append(entries, e) } + if err := rows.Err(); err != nil { + t.Fatal(err) + } values, err := sel.Open(ctx, entries) if err != nil { t.Fatal(err) @@ -212,6 +219,9 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { } groups[i] = append(groups[i], e) } + if err := rows.Err(); err != nil { + t.Fatal(err) + } _ = rows.Close() } opened, err := read.Open(ctx, groups) @@ -254,6 +264,9 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { } entries = append(entries, e) } + if err := rs.Err(); err != nil { + t.Fatal(err) + } _ = rs.Close() one, err := visits.OpenOne(ctx, entries) if err != nil || !one.Found || !sameJSON(t, one.JSON, []byte(`[{"at":3},{"at":1}]`)) { @@ -281,6 +294,9 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { counts = append(counts, "NULL") } } + if err := cr.Err(); err != nil { + t.Fatal(err) + } _ = cr.Close() if !slices.Equal(counts, []string{"2", "1", "0", "NULL"}) { t.Errorf("child counts: %v", counts) diff --git a/languages/golang/stashgen/emit.go b/languages/golang/stashgen/emit.go index 04253b0c1..314781003 100644 --- a/languages/golang/stashgen/emit.go +++ b/languages/golang/stashgen/emit.go @@ -25,6 +25,11 @@ func emit(f *genFile) ([]byte, error) { if err := planQueries(f); err != nil { return nil, err } + // Planning adds imports (encoding/json for a document field): a clash + // it makes is refused here, before anything is written. + if f.imports.err != nil { + return nil, f.imports.err + } w := &writer{} w.header(f) w.encryptedType(f) @@ -989,15 +994,27 @@ var jsonOperators = map[string]bool{ // planJSON checks a document field's operators against the methods the // generator writes for them, and names each ordered path's accessor. func planJSON(f *genFile, g *genField) error { + document := false for _, op := range g.operators { if !jsonOperators[op.Name] { return fieldErr(f.typeName, g.GoName, "the engine answers %s on %s, which the generator has no method for", op.Name, g.EQLType) } + document = document || op.Name == "selector" + } + // The engine answers selector for a field whose type stores a json_tree + // document; the leaf and entry types below are that family's. + if !document { + return fieldErr(f.typeName, g.GoName, "the engine answers no document operator on %s, and the generator writes json.RawMessage fields of EncryptedJson only", g.EQLType) } g.jsonPaths = nil seen := map[string]string{} + given := map[string]bool{} if g.TargetOptions != nil { for _, path := range g.TargetOptions.Ordered { + if given[path] { + return fieldErr(f.typeName, g.GoName, "the ordered path %s is given twice", path) + } + given[path] = true name, err := jsonPathName(path) if err != nil { return fieldErr(f.typeName, g.GoName, "%v", err) @@ -1014,11 +1031,16 @@ func planJSON(f *genFile, g *genField) error { } // jsonPathName is an ordered path's accessor name: its keys in Go case, -// array steps dropped. $.age is Age, $.visits[*].at is VisitsAt, $ is Root. +// wildcard steps dropped. $.age is Age, $.visits[*].at is VisitsAt, $.a.1 +// is A1, and $ is Root. An ordered path names no position (the engine +// refuses one), so a step of digits is an object key. func jsonPathName(path string) (string, error) { + if path == "$" { + return "Root", nil + } var b strings.Builder for _, step := range strings.FieldsFunc(strings.TrimPrefix(path, "$"), func(r rune) bool { return r == '.' || r == '[' || r == ']' }) { - if step == "*" || strings.Trim(step, "0123456789") == "" { + if step == "*" { continue } for _, word := range strings.Split(step, "_") { @@ -1028,11 +1050,8 @@ func jsonPathName(path string) (string, error) { } } name := b.String() - if name == "" { - name = "Root" - } - if !isIdent(name) || name[0] < 'A' || name[0] > 'Z' { - return "", fmt.Errorf("the ordered path %s has no Go accessor name: a key starts with a digit", path) + if name == "" || !isIdent(name) || name[0] < 'A' || name[0] > 'Z' { + return "", fmt.Errorf("the ordered path %s has no Go accessor name: its keys give no name that starts with a letter", path) } return name, nil } @@ -1162,4 +1181,11 @@ func (w *writer) jsonEntry(g genField) { w.p("\treturn gensupport.JSONBetweenAt[%s](ctx, p.field, c, p.path, lo, hi)", leaf) w.p("}") } + if answered["selector"] { + w.nl() + w.b.WriteString(wrapComment("Selector prepares `->` at this path: the value an ordering here compares is col -> Selector(), so bind it beside the ordering's operand, from the same path.", 80)) + w.p("func (p %sPath) Selector(ctx context.Context, c *encrypt.Cipher) (encrypt.JSONSelection[%s], error) {", g.fieldType, entry) + w.p("\treturn gensupport.JSONSelect[%s](ctx, p.field, c, p.path)", entry) + w.p("}") + } } diff --git a/languages/golang/stashgen/engine.go b/languages/golang/stashgen/engine.go index db598931c..08275aaa1 100644 --- a/languages/golang/stashgen/engine.go +++ b/languages/golang/stashgen/engine.go @@ -269,7 +269,11 @@ func lowerDeclaration(d Declaration, eqlTypes []EQLType) (*record.Plan, error) { } out, ok := outputOfIndex[idx.Name] if !ok { - return nil, &FieldError{Type: d.Type, Field: f.GoName, Reason: fmt.Sprintf("the engine does not derive the %s index yet", idx.Name)} + reason := fmt.Sprintf("the engine does not derive the %s index yet", idx.Name) + if idx.Name == IndexJSON { + reason = "the json index is not an output beside a ciphertext: store a JSON document as a json.RawMessage field with encrypt_into=EncryptedJson, whose EQL type carries the index" + } + return nil, &FieldError{Type: d.Type, Field: f.GoName, Reason: reason} } rf.Outputs = append(rf.Outputs, out) } diff --git a/languages/golang/stashgen/enginetest/enginetest.go b/languages/golang/stashgen/enginetest/enginetest.go index 2e5905a74..2f91c56d3 100644 --- a/languages/golang/stashgen/enginetest/enginetest.go +++ b/languages/golang/stashgen/enginetest/enginetest.go @@ -278,7 +278,7 @@ func indexApplies(name stashgen.IndexName, t stashgen.GoType) error { return nil } case stashgen.IndexJSON: - return fmt.Errorf("the engine cannot derive the json index yet") + return fmt.Errorf("the json index is not an output beside a ciphertext: store a JSON document with encrypt_into=EncryptedJson") } return fmt.Errorf("%s does not apply to %s", name, t) } diff --git a/languages/golang/stashgen/read.go b/languages/golang/stashgen/read.go index 810bdd8bf..2a98fe518 100644 --- a/languages/golang/stashgen/read.go +++ b/languages/golang/stashgen/read.go @@ -714,6 +714,17 @@ func (r *reader) buildFields(c *collected) error { if t.Omit { field.Verb = VerbOmit } + // A json.RawMessage is a document only where it names an EQL type + // that stores one; sealed or indexed any other way it is bytes, as + // it was before documents were. + for g := &field.GoType; g != nil; g = g.Elem { + if g.Kind == KindJSON && field.Verb != VerbEncryptInto { + g.Kind, g.Basic = KindBytes, "[]byte" + } + if g.Kind != KindPointer { + break + } + } // A sealed field declared over a pointer to one scalar may be nil: // it is that scalar's field, and says how it stores nil (#1298). pointer := field.Sealed() && field.GoType.Kind == KindPointer && field.GoType.Elem != nil && field.GoType.Elem.Kind.Scalar() diff --git a/languages/golang/stashgen/refusal_test.go b/languages/golang/stashgen/refusal_test.go index 67f161491..3297fdc9c 100644 --- a/languages/golang/stashgen/refusal_test.go +++ b/languages/golang/stashgen/refusal_test.go @@ -113,7 +113,7 @@ func TestRefusals(t *testing.T) { {"an opaque struct with a map keyed by a struct", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tM map[hidden]int") + "\ntype hidden struct{ A int }\n", stashgen.Request{Type: "User"}, "M", "map key that is not a string or an integer"}, {"an EQL type the engine cannot produce yet", user(ctx + "\tAge int64 `stash:\"age,encrypt_into=BigintOrdOre\"`"), stashgen.Request{Type: "User"}, "Age", "has no EQL type BigintOrdOre"}, {"an index on a composite", user(ctx + "\tAttrs map[string]string `stash:\"attrs,encrypt,index=equality\"`"), stashgen.Request{Type: "User"}, "Attrs", "seals only as part of an opaque struct"}, - {"the json index, not in the engine yet", user(ctx + "\tAttrs string `stash:\"attrs,index=json\"`"), stashgen.Request{Type: "User"}, "Attrs", "cannot derive the json index yet"}, + {"the json index as an output", user(ctx + "\tAttrs string `stash:\"attrs,index=json\"`"), stashgen.Request{Type: "User"}, "Attrs", "encrypt_into=EncryptedJson"}, {"an index option a query term cannot use", user(ctx + "\tEmail string `stash:\"email,encrypt,index=match(k=3)\"`"), stashgen.Request{Type: "User"}, "Email", "a query term uses only the default index options"}, {"a passthrough field that has an index", user(ctx + "\tID int64 `stash:\"id,passthrough,index=equality\"`"), stashgen.Request{Type: "User"}, "ID", "passthrough field has no index"}, {"an embedded struct from another package with no tag", user(ctx + "\tgorm.Model\n\tEmail string `stash:\"email,encrypt\"`"), stashgen.Request{Type: "User"}, "Model", "cannot carry tags; tag the field"}, From 3ddb62a0aa39c818d6cd6685242ab92787d7e501 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Sun, 25 Oct 2026 23:41:16 +0000 Subject: [PATCH 08/26] feat(stack-encrypt,eql): answer JsonSearch's JSON operators under the v3 names The JSON operators and reads this branch adds named the document target `EncryptedJson` and bound their operands as `eql_v4.query_json_tree` and `eql_v4.query_json_tree_leaf`. With the base's rename those are `JsonSearch`, `eql_v4.query_json` and `eql_v4.query_json_leaf`, and a read opens `public.eql_v4_json_entry` rows. The resolvers the tests build, the pinned operand domains, the guest's and Go's operator tests, the operations changeset and the CHANGELOGs now say so. Refs #1350, #1352 Claude-Session: https://claude.ai/code/session_01HLuZxys1NDRmAkv3uhnuoJ --- .changeset/eql-encrypted-json-operations.md | 2 +- languages/golang/encrypt/guest/src/ops.rs | 2 +- .../golang/encrypt/guest/tests/native_ops.rs | 2 +- languages/golang/encrypt/json.go | 2 +- languages/golang/encrypt/json_tree_test.go | 10 ++++----- languages/golang/encrypt/query_test.go | 6 ++--- languages/golang/encrypt/targets_test.go | 2 +- packages/eql/crates/eql-bindings/CHANGELOG.md | 4 ++-- .../eql-bindings/src/encryption/targets.rs | 4 ++-- .../encryption/tests/json_tree_data_plan.rs | 10 ++++----- packages/stack-encrypt/CHANGELOG.md | 4 ++-- .../stack-encrypt/src/dynamic/json_query.rs | 12 +++++----- packages/stack-encrypt/src/dynamic/query.rs | 2 +- packages/stack-encrypt/tests/heap_hygiene.rs | 14 ++++++------ .../stack-encrypt/tests/json_operators.rs | 22 +++++++++---------- 15 files changed, 49 insertions(+), 49 deletions(-) diff --git a/.changeset/eql-encrypted-json-operations.md b/.changeset/eql-encrypted-json-operations.md index 3edfbb491..bf9576dbb 100644 --- a/.changeset/eql-encrypted-json-operations.md +++ b/.changeset/eql-encrypted-json-operations.md @@ -2,4 +2,4 @@ "@cipherstash/eql": minor --- -**`EncryptedJson` answers its JSON operators by name.** `eql-bindings`' target table row now tells Stack Encrypt the domain an ordering at a path binds as (`Target::leaf_query_sql_domain`, `eql_v4.query_json_tree_leaf`), so a data plan's `lt_at`, `lte_at`, `gt_at`, `gte_at` and `between_at` bind their operand as that domain, beside `contains`, `contained_by` and `eq_at`, which bind the needle as `eql_v4.query_json_tree`. No SQL changes. +**`JsonSearch` answers its JSON operators by name.** `eql-bindings`' target table row now tells Stack Encrypt the domain an ordering at a path binds as (`Target::leaf_query_sql_domain`, `eql_v4.query_json_leaf`), so a data plan's `lt_at`, `lte_at`, `gt_at`, `gte_at` and `between_at` bind their operand as that domain, beside `contains`, `contained_by` and `eq_at`, which bind the needle as `eql_v4.query_json`. No SQL changes. diff --git a/languages/golang/encrypt/guest/src/ops.rs b/languages/golang/encrypt/guest/src/ops.rs index 3f82111df..59f604291 100644 --- a/languages/golang/encrypt/guest/src/ops.rs +++ b/languages/golang/encrypt/guest/src/ops.rs @@ -278,7 +278,7 @@ where /// Open the rows a read of a document field returned /// ([`dynamic::json_open`]): a codec-encoded list per database row of the -/// entries the SQL returned (`public.eql_v4_json_tree_entry` JSON, text or +/// entries the SQL returned (`public.eql_v4_json_entry` JSON, text or /// bytes), the plan, the field's name and the codec-encoded read request /// (`{"operation": "selector" | "subtree", "path", "wildcard_bound"?}`) in; /// a codec-encoded list per row of JSON texts (bytes) out, which is diff --git a/languages/golang/encrypt/guest/tests/native_ops.rs b/languages/golang/encrypt/guest/tests/native_ops.rs index 434e689c6..67610f7b6 100644 --- a/languages/golang/encrypt/guest/tests/native_ops.rs +++ b/languages/golang/encrypt/guest/tests/native_ops.rs @@ -1845,7 +1845,7 @@ fn the_catalog_lists_each_types_operators() { ]), // A document's operators under the index's defaults: the // orderings at a path only for a field that orders one. - "EncryptedJson" => { + "JsonSearch" => { let json = |operator: &'static str, arity: &'static str, input: &'static str| { vec![ ("operator", operator), diff --git a/languages/golang/encrypt/json.go b/languages/golang/encrypt/json.go index 988a50547..e2e4752ab 100644 --- a/languages/golang/encrypt/json.go +++ b/languages/golang/encrypt/json.go @@ -15,7 +15,7 @@ import ( // rows holds one group per database row: for selector, the entry col -> // selector returned, or none for SQL NULL; for subtree, the rows // eql_v4.json_tree_subtree returned. Each entry is the -// public.eql_v4_json_tree_entry JSON the database sent. +// public.eql_v4_json_entry JSON the database sent. // // The engine derives the selector or the read's key again from the request, // the field's declaration and this cipher's keyset, and checks every row diff --git a/languages/golang/encrypt/json_tree_test.go b/languages/golang/encrypt/json_tree_test.go index 8f568a06f..3eb7837cb 100644 --- a/languages/golang/encrypt/json_tree_test.go +++ b/languages/golang/encrypt/json_tree_test.go @@ -148,11 +148,11 @@ func TestJSONOperatorsDeriveTaggedTerms(t *testing.T) { input obj parts int }{ - {"contains", "eql", "eql_v4.query_json_tree", obj{{Key: "json", Value: []byte(`{"age":34}`)}}, 1}, - {"contained_by", "eql", "eql_v4.query_json_tree", obj{{Key: "json", Value: `{"age":34}`}}, 1}, - {"eq_at", "eql", "eql_v4.query_json_tree", obj{{Key: "path", Value: "$.name"}, {Key: "json", Value: []byte(`"Alice"`)}}, 1}, - {"lt_at", "eql", "eql_v4.query_json_tree_leaf", obj{{Key: "path", Value: "$.age"}, {Key: "json", Value: []byte(`40`)}}, 1}, - {"between_at", "eql", "eql_v4.query_json_tree_leaf", obj{{Key: "path", Value: "$.age"}, {Key: "low", Value: []byte(`18`)}, {Key: "high", Value: []byte(`40`)}}, 2}, + {"contains", "eql", "eql_v4.query_json", obj{{Key: "json", Value: []byte(`{"age":34}`)}}, 1}, + {"contained_by", "eql", "eql_v4.query_json", obj{{Key: "json", Value: `{"age":34}`}}, 1}, + {"eq_at", "eql", "eql_v4.query_json", obj{{Key: "path", Value: "$.name"}, {Key: "json", Value: []byte(`"Alice"`)}}, 1}, + {"lt_at", "eql", "eql_v4.query_json_leaf", obj{{Key: "path", Value: "$.age"}, {Key: "json", Value: []byte(`40`)}}, 1}, + {"between_at", "eql", "eql_v4.query_json_leaf", obj{{Key: "path", Value: "$.age"}, {Key: "low", Value: []byte(`18`)}, {Key: "high", Value: []byte(`40`)}}, 2}, {"selector", "text", "text", obj{{Key: "path", Value: "$.name"}}, 1}, {"subtree", "text_array", "text[]", obj{{Key: "path", Value: "$.visits[*]"}, {Key: "wildcard_bound", Value: uint32(3)}}, 4}, {"children", "text", "text", obj{{Key: "path", Value: "$.visits"}}, 1}, diff --git a/languages/golang/encrypt/query_test.go b/languages/golang/encrypt/query_test.go index 9b21aa00a..b2dc90dc0 100644 --- a/languages/golang/encrypt/query_test.go +++ b/languages/golang/encrypt/query_test.go @@ -457,18 +457,18 @@ func TestTheCatalogListsEachTypesOperators(t *testing.T) { if !slices.Equal(target.Operators, want) { t.Fatalf("%s answers %+v, want %+v", target.Name, target.Operators, want) } - case target.Name == "EncryptedJson": + case target.Name == "JsonSearch": // A document's operators, under the index's defaults: no // ordering without an ordered path. Each through the JSON index. var names []string for _, op := range target.Operators { if op.Index != record.JSONIndex || op.Fidelity != record.Exact { - t.Fatalf("EncryptedJson answers %+v", op) + t.Fatalf("JsonSearch answers %+v", op) } names = append(names, op.Name) } if want := []string{"contains", "contained_by", "eq_at", "selector", "subtree", "children", "child_count"}; !slices.Equal(names, want) { - t.Fatalf("EncryptedJson answers %v, want %v", names, want) + t.Fatalf("JsonSearch answers %v, want %v", names, want) } case len(target.Operators) != 0: // Every other type is storage-only or not produced. diff --git a/languages/golang/encrypt/targets_test.go b/languages/golang/encrypt/targets_test.go index 8fe6b87c7..4cdb97784 100644 --- a/languages/golang/encrypt/targets_test.go +++ b/languages/golang/encrypt/targets_test.go @@ -142,7 +142,7 @@ func TestParseQueryTermTakesOnlyWhatTheEngineWrites(t *testing.T) { // The encoding, the term's shape and the domain must agree as the engine // writes them; a term that mixes them is a wire bug, refused. func TestParseQueryTermRefusesAMixedShape(t *testing.T) { - domain := "eql_v4.query_json_tree" + domain := "eql_v4.query_json" shaped := func(encoding string, domain any, term any) vcvalue.Object { return vcvalue.Object{ {Key: "operator", Value: "contains"}, {Key: "fidelity", Value: "exact"}, {Key: "index", Value: "json"}, diff --git a/packages/eql/crates/eql-bindings/CHANGELOG.md b/packages/eql/crates/eql-bindings/CHANGELOG.md index 2133dfa91..4c619584b 100644 --- a/packages/eql/crates/eql-bindings/CHANGELOG.md +++ b/packages/eql/crates/eql-bindings/CHANGELOG.md @@ -11,11 +11,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **`Target::leaf_query_sql_domain`** (`targets` feature, #1350): the domain a document type's ordering at a path binds as, - `eql_v4.query_json_tree_leaf` for `EncryptedJson` and `None` for every + `eql_v4.query_json_leaf` for `JsonSearch` and `None` for every other type, which a resolver hands Stack Encrypt (`TargetDescriptor::with_leaf_query_sql_domain`). -- **`EncryptedJson`, the `json_tree` document, by name** (`targets` +- **`JsonSearch`, the `json_tree` document, by name** (`targets` - **`JsonSearch`, the `json_tree` document, by name** (`targets` feature, #1348, #1349). The target table has a row for it, generated from the catalog's document family (`eql_domains::DOCUMENT_FAMILIES`): diff --git a/packages/eql/crates/eql-bindings/src/encryption/targets.rs b/packages/eql/crates/eql-bindings/src/encryption/targets.rs index e94625cec..dc3edf2e1 100644 --- a/packages/eql/crates/eql-bindings/src/encryption/targets.rs +++ b/packages/eql/crates/eql-bindings/src/encryption/targets.rs @@ -143,8 +143,8 @@ impl Target { } /// The PostgreSQL domain of the type's ordering operand at a path, for - /// a type that stores a JSON document (`EncryptedJson`: - /// `eql_v4.query_json_tree_leaf`); `None` for every other type. Stack + /// a type that stores a JSON document (`JsonSearch`: + /// `eql_v4.query_json_leaf`); `None` for every other type. Stack /// Encrypt binds an ordering at a path as it /// (`TargetDescriptor::with_leaf_query_sql_domain`). pub fn leaf_query_sql_domain(&self) -> Option<&'static str> { diff --git a/packages/eql/tests/encryption/tests/json_tree_data_plan.rs b/packages/eql/tests/encryption/tests/json_tree_data_plan.rs index a393de439..2ea4f5d0e 100644 --- a/packages/eql/tests/encryption/tests/json_tree_data_plan.rs +++ b/packages/eql/tests/encryption/tests/json_tree_data_plan.rs @@ -426,7 +426,7 @@ fn a_document_field_answers_the_json_operators() { ); let unordered = record::plan_with( &dynamic::frame::serde::to_vec(&json!({ - "profile": {"context": ["users", "profile"], "target": "EncryptedJson"} + "profile": {"context": ["users", "profile"], "target": "JsonSearch"} })) .unwrap(), &Eql, @@ -487,7 +487,7 @@ async fn every_operand_is_the_typed_chains() { serde_json::to_value(&typed).unwrap(), "{operator}" ); - assert_eq!(term.domain(), Some("eql_v4.query_json_tree")); + assert_eq!(term.domain(), Some("eql_v4.query_json")); assert_eq!(term.encoding(), "eql"); } @@ -504,7 +504,7 @@ async fn every_operand_is_the_typed_chains() { .await .unwrap(); assert_eq!(one(&eq), serde_json::to_value(&typed_eq).unwrap()); - assert_eq!(eq.domain(), Some("eql_v4.query_json_tree")); + assert_eq!(eq.domain(), Some("eql_v4.query_json")); let lt = data_term( &keyset, @@ -519,7 +519,7 @@ async fn every_operand_is_the_typed_chains() { .await .unwrap(); assert_eq!(one(<), serde_json::to_value(&typed_lt).unwrap()); - assert_eq!(lt.domain(), Some("eql_v4.query_json_tree_leaf")); + assert_eq!(lt.domain(), Some("eql_v4.query_json_leaf")); let between = data_term( &keyset, @@ -552,7 +552,7 @@ async fn every_operand_is_the_typed_chains() { one(<), "the high bound" ); - assert_eq!(between.domain(), Some("eql_v4.query_json_tree_leaf")); + assert_eq!(between.domain(), Some("eql_v4.query_json_leaf")); let path = stack_encrypt::json::JsonPath::parse("$.visits[*].at").unwrap(); let selector = data_term( diff --git a/packages/stack-encrypt/CHANGELOG.md b/packages/stack-encrypt/CHANGELOG.md index f1784b045..7613e190b 100644 --- a/packages/stack-encrypt/CHANGELOG.md +++ b/packages/stack-encrypt/CHANGELOG.md @@ -45,7 +45,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - **A document field's queries and reads through a data plan** (#1350, - #1352). A field naming `EncryptedJson` answers `contains`, + #1352). A field naming `JsonSearch` answers `contains`, `contained_by`, `eq_at`, `selector`, `subtree`, `children` and `child_count`, and, when it orders a path, `lt_at`, `lte_at`, `gt_at`, `gte_at` and `between_at`, each taking an object of named inputs @@ -53,7 +53,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `{"path"}`, `{"path", "wildcard_bound"}`; JSON values as JSON text). `query_term` derives the typed chain's operands, bit for bit: the containment needle and the leaf ordering operand as EQL JSON (bound as - `eql_v4.query_json_tree` and `eql_v4.query_json_tree_leaf`), the + `eql_v4.query_json` and `eql_v4.query_json_leaf`), the selector as `text` and a read's tokens as `text[]`. A position in a path is refused where the operand is a selector (`json_path_position`), a wildcard where the operator names one node (`json_path_wildcard`). diff --git a/packages/stack-encrypt/src/dynamic/json_query.rs b/packages/stack-encrypt/src/dynamic/json_query.rs index 8432e7a56..d0fbccb12 100644 --- a/packages/stack-encrypt/src/dynamic/json_query.rs +++ b/packages/stack-encrypt/src/dynamic/json_query.rs @@ -1,7 +1,7 @@ //! The JSON operators of a document field, for a binding: queries named by //! operator, and reads opened against the request that prepared them. //! -//! A field that names a target storing a document (`EncryptedJson`) runs +//! A field that names a target storing a document (`JsonSearch`) runs //! under the JSON index's options its plan resolved //! ([`TargetOptions::Json`](super::TargetOptions::Json)). Each operator //! takes an object of named inputs and gives a tagged operand @@ -9,9 +9,9 @@ //! //! | operator | input | operand | //! |---|---|---| -//! | `contains`, `contained_by` | `{"json"}`: a JSON needle | EQL: the needle's projection (`eql_v4.query_json_tree`) | +//! | `contains`, `contained_by` | `{"json"}`: a JSON needle | EQL: the needle's projection (`eql_v4.query_json`) | //! | `eq_at` | `{"path", "json"}`: a scalar at a path | EQL: the needle holding the scalar there | -//! | `lt_at`, `lte_at`, `gt_at`, `gte_at` | `{"path", "json"}`: a number or string at an ordered path | EQL: the path's selector and the leaf's ordering term (`eql_v4.query_json_tree_leaf`) | +//! | `lt_at`, `lte_at`, `gt_at`, `gte_at` | `{"path", "json"}`: a number or string at an ordered path | EQL: the path's selector and the leaf's ordering term (`eql_v4.query_json_leaf`) | //! | `between_at` | `{"path", "low", "high"}` | EQL, two of them: the low bound's then the high's | //! | `selector` | `{"path"}` | text: the selector `->` takes | //! | `subtree` | `{"path", "wildcard_bound"?}` | text array: the tokens `json_tree_subtree` takes | @@ -64,9 +64,9 @@ use crate::{KeysetCipher, Label, NonEmpty, Pending}; /// The SQL type a JSON operand binds as. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum Binds { - /// The target's needle domain (`eql_v4.query_json_tree`). + /// The target's needle domain (`eql_v4.query_json`). Needle, - /// The target's leaf operand domain (`eql_v4.query_json_tree_leaf`). + /// The target's leaf operand domain (`eql_v4.query_json_leaf`). Leaf, /// `text`. Text, @@ -410,7 +410,7 @@ struct ReadRequest { } /// The groups of rows a read returned: one list of entries per database -/// row, each entry the `public.eql_v4_json_tree_entry` JSON the SQL +/// row, each entry the `public.eql_v4_json_entry` JSON the SQL /// returned, as text or bytes. #[derive(Deserialize)] #[serde(transparent)] diff --git a/packages/stack-encrypt/src/dynamic/query.rs b/packages/stack-encrypt/src/dynamic/query.rs index d66450905..a8e7c56a9 100644 --- a/packages/stack-encrypt/src/dynamic/query.rs +++ b/packages/stack-encrypt/src/dynamic/query.rs @@ -987,7 +987,7 @@ pub fn check_json_open( /// [`JsonSubtreeKey::open`], for a binding. `request` is the read as the /// binding prepared it, `{"operation": "selector" | "subtree", "path", /// "wildcard_bound"?}`, and `rows` a list per database row of the entries -/// the SQL returned for it (`public.eql_v4_json_tree_entry` JSON, as text +/// the SQL returned for it (`public.eql_v4_json_entry` JSON, as text /// or bytes): for `selector`, the one entry `->` returned or none, and for /// `subtree`, the rows `json_tree_subtree` returned. The engine derives /// the selector or the read key again, under `cipher`'s keyset and the diff --git a/packages/stack-encrypt/tests/heap_hygiene.rs b/packages/stack-encrypt/tests/heap_hygiene.rs index dde1da6d4..afc9af211 100644 --- a/packages/stack-encrypt/tests/heap_hygiene.rs +++ b/packages/stack-encrypt/tests/heap_hygiene.rs @@ -322,25 +322,25 @@ fn verify_leaves_no_plaintext_on_the_heap() { }); } -/// A resolver holding `EncryptedJson` alone, for checking a document +/// A resolver holding `JsonSearch` alone, for checking a document /// field's operators: a check derives nothing, so nothing here runs. struct Documents; impl stack_encrypt::dynamic::TargetResolver for Documents { fn targets(&self) -> Vec { vec![stack_encrypt::dynamic::TargetDescriptor::new( - "EncryptedJson", + "JsonSearch", "json", "", Some(stack_encrypt::dynamic::ValueKind::Json), - "public.eql_v4_json_tree", + "public.eql_v4_json_search", vec!["json".to_string()], - Some("EncryptedJsonQuery".to_string()), - Some("eql_v4.query_json_tree".to_string()), + Some("JsonSearchQuery".to_string()), + Some("eql_v4.query_json".to_string()), true, None, ) - .with_leaf_query_sql_domain("eql_v4.query_json_tree_leaf")] + .with_leaf_query_sql_domain("eql_v4.query_json_leaf")] } fn encrypt<'a, K: 'static>( &self, @@ -381,7 +381,7 @@ impl stack_encrypt::dynamic::TargetResolver for Documents { fn a_json_operators_input_leaves_no_plaintext_on_the_heap() { let wire = serde_json::json!({ "profile": {"context": ["users", "profile"], "type": "json", "null": "sql", - "target": {"name": "EncryptedJson", "options": {"ordered": ["$.age"]}}} + "target": {"name": "JsonSearch", "options": {"ordered": ["$.age"]}}} }); let plan = record::plan_with( &stack_encrypt::dynamic::frame::serde::to_vec(&wire).unwrap(), diff --git a/packages/stack-encrypt/tests/json_operators.rs b/packages/stack-encrypt/tests/json_operators.rs index 7dc6aecc9..a16c8cbd6 100644 --- a/packages/stack-encrypt/tests/json_operators.rs +++ b/packages/stack-encrypt/tests/json_operators.rs @@ -28,7 +28,7 @@ use stack_kms::FakeDataKeySource; type Keyset<'c> = KeysetCipher<'c, FakeDataKeySource>; -/// A resolver holding one type, `EncryptedJson`, sealed with the engine's +/// A resolver holding one type, `JsonSearch`, sealed with the engine's /// JSON index under the options the plan resolved. It stores the /// `JsonDocument` as JSON and needs no query twin: the JSON operators are /// derived by the engine, not the resolver. @@ -37,18 +37,18 @@ struct Documents; impl TargetResolver for Documents { fn targets(&self) -> Vec { vec![TargetDescriptor::new( - "EncryptedJson", + "JsonSearch", "json", "", Some(ValueKind::Json), - "public.eql_v4_json_tree", + "public.eql_v4_json_search", vec!["json".to_string()], - Some("EncryptedJsonQuery".to_string()), - Some("eql_v4.query_json_tree".to_string()), + Some("JsonSearchQuery".to_string()), + Some("eql_v4.query_json".to_string()), true, None, ) - .with_leaf_query_sql_domain("eql_v4.query_json_tree_leaf")] + .with_leaf_query_sql_domain("eql_v4.query_json_leaf")] } fn encrypt<'a, K: 'static>( @@ -102,7 +102,7 @@ impl TargetResolver for Documents { /// `at`, as a binding writes it. fn plan() -> record::Plan { plan_of(json!({ - "name": "EncryptedJson", + "name": "JsonSearch", "options": {"ordered": ["$.age", "$.visits[*].at"]} })) } @@ -272,7 +272,7 @@ async fn every_operator_derives_its_tagged_operand() { ] { let term = term(&keyset, &plan, operator, input).await; assert_eq!(term.encoding(), "eql", "{operator}"); - assert_eq!(term.domain(), Some("eql_v4.query_json_tree"), "{operator}"); + assert_eq!(term.domain(), Some("eql_v4.query_json"), "{operator}"); assert!(needle(&term).is_object(), "{operator}"); } // Equality at a path is a containment needle, the shape `contains` @@ -300,7 +300,7 @@ async fn every_operator_derives_its_tagged_operand() { json!({"path": "$.age", "json": "40"}), ) .await; - assert_eq!(term.domain(), Some("eql_v4.query_json_tree_leaf")); + assert_eq!(term.domain(), Some("eql_v4.query_json_leaf")); assert!(needle(&term).is_object(), "{operator}"); } let between = term( @@ -311,7 +311,7 @@ async fn every_operator_derives_its_tagged_operand() { ) .await; assert!(matches!(between.operand(), Operand::Range { .. })); - assert_eq!(between.domain(), Some("eql_v4.query_json_tree_leaf")); + assert_eq!(between.domain(), Some("eql_v4.query_json_leaf")); let selector = term(&keyset, &plan, "selector", json!({"path": "$.name"})).await; assert_eq!( @@ -371,7 +371,7 @@ fn the_orderings_follow_the_resolved_options() { .collect() }; assert!(names(&plan()).contains(&"lt_at")); - let unordered = plan_of(json!("EncryptedJson")); + let unordered = plan_of(json!("JsonSearch")); assert_eq!( names(&unordered), [ From c502848a7a3f2c619eaba8754809080fd8a13dcf Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 11:29:15 +1100 Subject: [PATCH 09/26] test(go): sort rows by an ordered JSON path in PostgreSQL The PostgreSQL test compared the ordered paths $.age and $.visits[*].at only through <, <=, >= and >, so nothing showed that rows sort by them. Ordering by the entry `->` returns sorts by its ciphertext, which the blockers cannot stop, so the form EQL documents is ORDER BY eql_v4.ord_term(col -> sel), and that form had no Go test. The test now inserts two more rows, giving each path a tie broken by id and a document whose first visit (2) is neither its least nor its greatest element, then sorts by both paths ascending and descending and compares the ids with the order computed from the plaintexts. A row with no value at the path (no age, or no visits) has a NULL term and sorts last ascending and first descending, as PostgreSQL sorts NULL by default. At the wildcard path the term is the first element's, which is what `->` returns. Sorting by the entry itself instead fails all four orderings. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- .../golang/encrypt/json_postgres_test.go | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/languages/golang/encrypt/json_postgres_test.go b/languages/golang/encrypt/json_postgres_test.go index ba9864ee2..60a607efa 100644 --- a/languages/golang/encrypt/json_postgres_test.go +++ b/languages/golang/encrypt/json_postgres_test.go @@ -1,6 +1,7 @@ package encrypt_test import ( + "cmp" "context" "database/sql" "encoding/json" @@ -306,6 +307,97 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { t.Fatal(err) } want("children rows", ids(`SELECT id FROM customers, eql_v4.json_tree_children(profile, $1::text) AS e ORDER BY id`, children), 1, 1, 2) + + // Sorting by an ordered path, in the form EQL documents: + // ORDER BY eql_v4.ord_term(col -> sel). Ordering by the entry itself + // would sort by its ciphertext. Two more rows give each path a tie, + // broken by id, and Eve's visits (first 2, last 9) tell the first match + // `->` takes at the wildcard path from the least or greatest element. + more := []testusers.Customer{ + {ID: 5, Profile: json.RawMessage(`{"name":"Eve","age":34,"visits":[{"at":2},{"at":9}]}`)}, + {ID: 6, Profile: json.RawMessage(`{"name":"Fay","age":7,"visits":[{"at":3}]}`)}, + } + extra, err := testusers.EncryptCustomer(ctx, cipher, more) + if err != nil { + t.Fatal(err) + } + for _, e := range extra { + exec(`INSERT INTO customers VALUES ($1, $2, $3)`, e.ID, e.Profile, e.Prefs) + } + type plain struct { + Age *float64 `json:"age"` + Visits []struct { + At *float64 `json:"at"` + } `json:"visits"` + } + ageOf := func(p plain) *float64 { return p.Age } + firstAtOf := func(p plain) *float64 { + for _, v := range p.Visits { + if v.At != nil { + return v.At + } + } + return nil + } + // The ids in the order PostgreSQL's defaults put the plaintexts in: a + // row with no value at the path has a NULL term, which sorts as the + // greatest value, so last ascending and first descending; ties by id. + plaintextOrder := func(key func(plain) *float64, desc bool) []int64 { + type row struct { + id int64 + v *float64 + } + var rows []row + for _, cust := range append(slices.Clone(jsonCustomers), more...) { + var p plain + if err := json.Unmarshal(cust.Profile, &p); err != nil { + t.Fatal(err) + } + rows = append(rows, row{cust.ID, key(p)}) + } + slices.SortFunc(rows, func(a, b row) int { + var c int + switch { + case a.v == nil && b.v == nil: + case a.v == nil: + c = 1 + case b.v == nil: + c = -1 + default: + c = cmp.Compare(*a.v, *b.v) + } + if desc { + c = -c + } + if c == 0 { + c = cmp.Compare(a.id, b.id) + } + return c + }) + out := make([]int64, len(rows)) + for i, r := range rows { + out[i] = r.id + } + return out + } + for _, tc := range []struct { + name string + selector encrypt.JSONSelector + key func(plain) *float64 + asc, desc []int64 // what plaintextOrder gives, written out + }{ + // Bob and Fay tie at 7, Alice and Eve at 34; Dan has no age. + {"$.age", age.Selector(), ageOf, []int64{2, 6, 1, 5, 3, 4}, []int64{4, 3, 1, 5, 2, 6}}, + // First visits: Eve 2, Alice and Fay 3, Bob 5; Carol's visits are + // empty and Dan has none. + {"$.visits[*].at", at.Selector(), firstAtOf, []int64{5, 1, 6, 2, 3, 4}, []int64{3, 4, 2, 1, 6, 5}}, + } { + asc, desc := plaintextOrder(tc.key, false), plaintextOrder(tc.key, true) + want("plaintext order by "+tc.name, asc, tc.asc...) + want("plaintext order by "+tc.name+" DESC", desc, tc.desc...) + want("ORDER BY "+tc.name, ids(`SELECT id FROM customers ORDER BY eql_v4.ord_term(profile -> $1::text), id`, tc.selector), asc...) + want("ORDER BY "+tc.name+" DESC", ids(`SELECT id FROM customers ORDER BY eql_v4.ord_term(profile -> $1::text) DESC, id`, tc.selector), desc...) + } } func slicesEqual2(a, b [][]string) bool { From e7310dac965658a0f20f1ead0b5bcf3aafbe7357 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 00:33:50 +0000 Subject: [PATCH 10/26] feat(go): tag a JSON document field encrypt_into=JsonSearch With the base's rename the EQL type a document field names is `JsonSearch` (Go `eql.JSONSearch`, over `public.eql_v4_json_search`). The tag is now `encrypt_into=JsonSearch(ordered=$.a;$.b[*].c,sealed_root=false)`; the Go spelling `JSONSearch` in a tag is refused with the same hint as before, naming the engine's spelling. The generator named a document's companion types by appending to the stored type's Go name, `LeafQuery` and `Entry`. Under the v3 names the entry is not `JSONSearchEntry` but `JSONEntry`, named for the plaintext as `public.eql_v4_json_entry` is, so the leaf operand is now derived from the needle (`JSONSearchQuery` to `JSONSearchLeafQuery`) and the entry is `eql.JSONEntry`. The test model, its regenerated customer_stash.go, the READMEs, the CHANGELOG and the PostgreSQL test's SQL (`eql_v4.query_json`, `eql_v4.query_json_leaf`) follow. Refs #1351 Claude-Session: https://claude.ai/code/session_01HLuZxys1NDRmAkv3uhnuoJ --- languages/golang/CHANGELOG.md | 4 +- languages/golang/cmd/stashgen/README.md | 16 ++-- languages/golang/cmd/stashgen/main_test.go | 46 +++++----- languages/golang/encrypt/README.md | 6 +- .../golang/encrypt/gensupport/declaration.go | 6 +- languages/golang/encrypt/gensupport/json.go | 2 +- .../internal/testusers/customer_stash.go | 84 +++++++++---------- .../encrypt/internal/testusers/customers.go | 8 +- languages/golang/encrypt/json.go | 10 +-- languages/golang/encrypt/json_fields_test.go | 4 +- .../golang/encrypt/json_postgres_test.go | 42 +++++----- languages/golang/encrypt/term.go | 6 +- languages/golang/stashgen/declaration.go | 4 +- languages/golang/stashgen/emit.go | 14 ++-- languages/golang/stashgen/engine.go | 2 +- .../golang/stashgen/enginetest/enginetest.go | 2 +- languages/golang/stashgen/refusal_test.go | 2 +- languages/golang/stashgen/tag.go | 2 +- .../testdata/stubsdk/encrypt/encrypt.go | 4 +- 19 files changed, 134 insertions(+), 130 deletions(-) diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index 2db63af13..38d1c0782 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -13,9 +13,9 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. ### Added - **Encrypted JSON documents with `stashgen`** (#1351, #1352). A - `json.RawMessage` field tagged `encrypt_into=EncryptedJson` (options: + `json.RawMessage` field tagged `encrypt_into=JsonSearch` (options: `ordered=$.a;$.b[*].c`, `sealed_root=false`) is stored as an - `eql.EncryptedJSON` in a `public.eql_v4_json_tree` column; a + `eql.JSONSearch` in a `public.eql_v4_json_search` column; a `*json.RawMessage` field takes `null=sql`. Its `Fields` entry has `Contains`, `ContainedBy`, `EqAt`, `Selector`, `Subtree`, `Children` and `ChildCount`, and `At.` per ordered path with `Lt`, `Lte`, `Gt`, diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index 17ca31938..079a37722 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -74,8 +74,8 @@ The first part of a tag is the field's name, which is the column name in a datab | `stash:"notes,encrypt"` | seal the field, with no index | | `stash:"email,encrypt,index=equality;match"` | seal the field, and derive each index beside it | | `stash:"score,index=ore"` | derive the index alone; no ciphertext is stored, so `Decrypt` leaves the field at its zero value | -| `stash:"profile,encrypt_into=EncryptedJson(ordered=$.age)"` | on a `json.RawMessage`: store the field as an encrypted JSON document; see "Encrypted JSON documents" | -| `stash:"attrs,index=json"` | refused: a JSON document is stored with `encrypt_into=EncryptedJson`, not as an index beside a ciphertext | +| `stash:"profile,encrypt_into=JsonSearch(ordered=$.age)"` | on a `json.RawMessage`: store the field as an encrypted JSON document; see "Encrypted JSON documents" | +| `stash:"attrs,index=json"` | refused: a JSON document is stored with `encrypt_into=JsonSearch`, not as an index beside a ciphertext | | `stash:"id,passthrough"` | store the field as it is | | `stash:"tenant,context_field"` | the field's value is the context of every other field in the struct, in place of a `context=` tag; stored as it is | | `stash:"-"` | leave the field out | @@ -109,27 +109,27 @@ An index takes its options in parentheses after its name, separated by commas: ` This build refuses an index with options: a query term is derived with the default options only, so a stored term with other options would never match one. These words are the same as the Rust API's words for the same behaviour, with two that only Go has. `opaque` seals the whole struct as one `bytes` field holding a storage blob (see "What crosses the binding"). -`json` names the JSON index, which a field reaches only through `encrypt_into=EncryptedJson`. +`json` names the JSON index, which a field reaches only through `encrypt_into=JsonSearch`. ### Encrypted JSON documents -A `json.RawMessage` field with `encrypt_into=EncryptedJson` is stored as one encrypted JSON document, an `eql.EncryptedJSON` in a `public.eql_v4_json_tree` column. +A `json.RawMessage` field with `encrypt_into=JsonSearch` is stored as one encrypted JSON document, an `eql.JSONSearch` in a `public.eql_v4_json_search` column. The field holds the document's JSON text; the engine parses it, so an object that names a member twice, an integer outside the 64-bit range and text that is not JSON are refused with `encrypt.ErrEncoding`. Values come back as JSON text holding the same values; key order and the spelling of numbers are not kept. A `*json.RawMessage` field is optional and must say `null=sql`: a document has no fixed length to hide a missing one behind. A non-nil pointer to `null` is a stored JSON `null`. A `json.RawMessage` sealed any other way (`encrypt`, `index=equality`) is bytes, as any type defined over `[]byte` is. The type takes options in parentheses: `ordered=` names the paths whose numbers and strings carry an ordering term, separated by `;`, and `sealed_root=false` turns off the sealed root (on by default). -`encrypt_into=EncryptedJson(ordered=$.age;$.visits[*].at)` orders two paths. +`encrypt_into=JsonSearch(ordered=$.age;$.visits[*].at)` orders two paths. A path in a tag is plain JSONPath: `$`, then `.key` steps of letters, digits and underscores, and `[n]` or `[*]` steps; an ordered path names no position. The engine checks every path when `stashgen` runs. The field's `Fields` entry has a method per operation: | Method | Returns | SQL | |---|---|---| -| `Contains(ctx, c, needle)`, `ContainedBy(ctx, c, needle)` | `eql.EncryptedJSONQuery` | `col @> $1::eql_v4.query_json_tree`, `col <@ $1::eql_v4.query_json_tree` | -| `EqAt(ctx, c, path, v)` | `eql.EncryptedJSONQuery` | `col @> $1::eql_v4.query_json_tree` | -| `At.Age.Lt(ctx, c, v)`, `Lte`, `Gt`, `Gte`, `Between` | `eql.EncryptedJSONLeafQuery` | `(col -> $sel::text) < $1::eql_v4.query_json_tree_leaf`, `$sel` from `At.Age.Selector(ctx, c)` | +| `Contains(ctx, c, needle)`, `ContainedBy(ctx, c, needle)` | `eql.JSONSearchQuery` | `col @> $1::eql_v4.query_json`, `col <@ $1::eql_v4.query_json` | +| `EqAt(ctx, c, path, v)` | `eql.JSONSearchQuery` | `col @> $1::eql_v4.query_json` | +| `At.Age.Lt(ctx, c, v)`, `Lte`, `Gt`, `Gte`, `Between` | `eql.JSONSearchLeafQuery` | `(col -> $sel::text) < $1::eql_v4.query_json_leaf`, `$sel` from `At.Age.Selector(ctx, c)` | | `Selector(ctx, c, path)` | `encrypt.JSONSelection`: `Selector()` and `Open` | `col -> $1::text` | | `Subtree(ctx, c, path, opts)` | `encrypt.JSONRead`: `Tokens()`, `Open` and `OpenOne` | `eql_v4.json_tree_subtree(col, $1::text[], 64)` | | `Children(ctx, c, path)`, `ChildCount(ctx, c, path)` | `encrypt.JSONToken` | `eql_v4.json_tree_children(col, $1::text)`, `eql_v4.json_tree_child_count(col, $1::text)` | diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index f2f27d73c..c83f34fbd 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -216,13 +216,13 @@ func TestRunAsksTheEmbeddedEngine(t *testing.T) { } // A document type over another Go type is refused by naming the Go // types that carry one, not by a kind mismatch. - document := strings.Replace(userSource, "Email string `stash:\"email,encrypt_into=TextEq\"`", "Profile []byte `stash:\"profile,encrypt_into=EncryptedJson\"`", 1) + document := strings.Replace(userSource, "Email string `stash:\"email,encrypt_into=TextEq\"`", "Profile []byte `stash:\"profile,encrypt_into=JsonSearch\"`", 1) dir = writeModule(t, map[string]string{"model.go": document}) stderr.Reset() if code := run([]string{"-type", "User"}, dir, &stdout, &stderr, stashgen.GuestEngine); code != 1 { t.Fatalf("exit %d, want 1\n%s", code, stderr.String()) } - if !strings.Contains(stderr.String(), "User.Profile") || !strings.Contains(stderr.String(), "EncryptedJson seals a json document: declare the field json.RawMessage") { + if !strings.Contains(stderr.String(), "User.Profile") || !strings.Contains(stderr.String(), "JsonSearch seals a json document: declare the field json.RawMessage") { t.Fatalf("stderr = %q", stderr.String()) } // A context of two segments leaves an EQL field no column: refused by @@ -271,7 +271,7 @@ func TestEQLGoNameAgreesWithTheGeneratedPackage(t *testing.T) { t.Fatal("the rule renamed nothing: the Json family is spelled JSON") } if stashgen.EQLGoName("Json") != "JSON" || stashgen.EQLGoName("TextEq") != "TextEq" || stashgen.EQLGoName("SteVecQuery") != "SteVecQuery" || - stashgen.EQLGoName("EncryptedJsonLeafQuery") != "EncryptedJSONLeafQuery" || stashgen.EQLGoName("Jsonify") != "Jsonify" || + stashgen.EQLGoName("JsonSearchLeafQuery") != "JSONSearchLeafQuery" || stashgen.EQLGoName("Jsonify") != "Jsonify" || // A letter after the word is read as a character, as Rust reads it, // not as its first byte. stashgen.EQLGoName("Jsonécrit") != "Jsonécrit" || stashgen.EQLGoName("JsonÉcrit") != "JSONÉcrit" { @@ -295,7 +295,7 @@ func TestModelFlagsRepeat(t *testing.T) { // embedded engine with the EQL types: the generated entry has the // document's methods and an accessor per ordered path, and every refusal // names the field before anything is written. -func TestRunGeneratesEncryptedJSONFields(t *testing.T) { +func TestRunGeneratesJSONSearchFields(t *testing.T) { checker, err := encrypt.NewChecker(context.Background()) if err != nil { t.Skip(err) @@ -311,20 +311,20 @@ func TestRunGeneratesEncryptedJSONFields(t *testing.T) { written, _ := os.ReadFile(filepath.Join(dir, "customer_stash.go")) return string(written), stderr.String(), code } - written, stderr, code := generate("\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$.age;$.visits[*].at,sealed_root=false)\"`\n\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=EncryptedJson,null=sql\"`") + written, stderr, code := generate("\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.age;$.visits[*].at,sealed_root=false)\"`\n\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=JsonSearch,null=sql\"`") if code != 0 { t.Fatalf("exit %d\n%s", code, stderr) } for _, want := range []string{ - "Profile eql.EncryptedJSON", - "Prefs *eql.EncryptedJSON", - `EncryptInto("profile", gensupport.JSON, "EncryptedJson").`, + "Profile eql.JSONSearch", + "Prefs *eql.JSONSearch", + `EncryptInto("profile", gensupport.JSON, "JsonSearch").`, `Options("profile", gensupport.TargetOptions{SealedRoot: gensupport.Ptr(false), Ordered: []string{"$.age", "$.visits[*].at"}})`, - "func (f ProfileField) Contains(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.EncryptedJSONQuery, error)", - "func (f ProfileField) Subtree(ctx context.Context, c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[eql.EncryptedJSONEntry], error)", + "func (f ProfileField) Contains(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.JSONSearchQuery, error)", + "func (f ProfileField) Subtree(ctx context.Context, c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[eql.JSONEntry], error)", "Age ProfileFieldPath // $.age", "VisitsAt ProfileFieldPath // $.visits[*].at", - "func (p ProfileFieldPath) Between(ctx context.Context, c *encrypt.Cipher, lo, hi json.RawMessage) (encrypt.JSONRange[eql.EncryptedJSONLeafQuery], error)", + "func (p ProfileFieldPath) Between(ctx context.Context, c *encrypt.Cipher, lo, hi json.RawMessage) (encrypt.JSONRange[eql.JSONSearchLeafQuery], error)", } { if !strings.Contains(written, want) { t.Errorf("the generated file lacks %q", want) @@ -333,12 +333,12 @@ func TestRunGeneratesEncryptedJSONFields(t *testing.T) { if strings.Contains(written, "PrefsFieldAt") { t.Error("Prefs orders no path, and got an At") } - if !strings.Contains(written, "func (p ProfileFieldPath) Selector(ctx context.Context, c *encrypt.Cipher) (encrypt.JSONSelection[eql.EncryptedJSONEntry], error)") { + if !strings.Contains(written, "func (p ProfileFieldPath) Selector(ctx context.Context, c *encrypt.Cipher) (encrypt.JSONSelection[eql.JSONEntry], error)") { t.Error("an ordered path has no Selector of its own") } // A json.RawMessage sealed any other way is bytes, as it was before // documents; an object key of digits names its accessor. - written, stderr, code = generate("\tData json.RawMessage `stash:\"data,encrypt,index=equality\"`\n\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$.a.1)\"`") + written, stderr, code = generate("\tData json.RawMessage `stash:\"data,encrypt,index=equality\"`\n\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.a.1)\"`") if code != 0 { t.Fatalf("exit %d\n%s", code, stderr) } @@ -348,16 +348,16 @@ func TestRunGeneratesEncryptedJSONFields(t *testing.T) { } } for _, c := range []struct{ name, fields, want string }{ - {"a sealed null", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=EncryptedJson,null=sealed\"`", "Customer.Prefs"}, - {"a nil document that does not say null=sql", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=EncryptedJson\"`", "null=sql"}, - {"a position in an ordered path", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$.scores[2])\"`", "the engine refuses the declaration"}, - {"a quoted key in a tag", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$[\\\"a b\\\"])\"`", "not a plain JSONPath"}, - {"an unknown option", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(bound=3)\"`", "no option \"bound\""}, - {"the Go name in the tag", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJSON\"`", "as the engine does, EncryptedJson"}, - {"two paths, one accessor", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$.a_b;$.aB)\"`", "both name the accessor At.AB"}, - {"the same ordered path twice", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$.a;$.a)\"`", "the ordered path $.a is given twice"}, - {"a path with no accessor name", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=EncryptedJson(ordered=$.__)\"`", "no Go accessor name"}, - {"a string into a document", "\tProfile string `stash:\"profile,encrypt_into=EncryptedJson\"`", "EncryptedJson seals a json"}, + {"a sealed null", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=JsonSearch,null=sealed\"`", "Customer.Prefs"}, + {"a nil document that does not say null=sql", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=JsonSearch\"`", "null=sql"}, + {"a position in an ordered path", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.scores[2])\"`", "the engine refuses the declaration"}, + {"a quoted key in a tag", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$[\\\"a b\\\"])\"`", "not a plain JSONPath"}, + {"an unknown option", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(bound=3)\"`", "no option \"bound\""}, + {"the Go name in the tag", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JSONSearch\"`", "as the engine does, JsonSearch"}, + {"two paths, one accessor", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.a_b;$.aB)\"`", "both name the accessor At.AB"}, + {"the same ordered path twice", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.a;$.a)\"`", "the ordered path $.a is given twice"}, + {"a path with no accessor name", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.__)\"`", "no Go accessor name"}, + {"a string into a document", "\tProfile string `stash:\"profile,encrypt_into=JsonSearch\"`", "JsonSearch seals a json"}, {"options on a type that takes none", "\tEmail string `stash:\"email,encrypt_into=TextEq(ordered=$.a)\"`", "the engine refuses the declaration"}, } { t.Run(c.name, func(t *testing.T) { diff --git a/languages/golang/encrypt/README.md b/languages/golang/encrypt/README.md index 5ba5516a5..72ec0be90 100644 --- a/languages/golang/encrypt/README.md +++ b/languages/golang/encrypt/README.md @@ -593,15 +593,15 @@ column. ### Encrypted JSON documents -`EncryptedJson` stores a whole JSON document in one `public.eql_v4_json_tree` -column, from a `json.RawMessage` field (`stash:"profile,encrypt_into=EncryptedJson(ordered=$.age)"`; +`JsonSearch` stores a whole JSON document in one `public.eql_v4_json_search` +column, from a `json.RawMessage` field (`stash:"profile,encrypt_into=JsonSearch(ordered=$.age)"`; see the stashgen README). One entry per node is encrypted, so the database can test containment, equality and ordering at a path, return one node with `->`, and walk a subtree, without decrypting: ```go q, err := users.Fields.Profile.Contains(ctx, cipher, json.RawMessage(`{"plan":"pro"}`)) -rows, err := db.QueryContext(ctx, `SELECT id FROM users WHERE profile @> $1::eql_v4.query_json_tree`, q) +rows, err := db.QueryContext(ctx, `SELECT id FROM users WHERE profile @> $1::eql_v4.query_json`, q) read, err := users.Fields.Profile.Subtree(ctx, cipher, "$.visits[*].at", encrypt.JSONReadOptions{}) // SELECT e FROM users, eql_v4.json_tree_subtree(profile, $1::text[], 64) AS e WHERE id = $2 diff --git a/languages/golang/encrypt/gensupport/declaration.go b/languages/golang/encrypt/gensupport/declaration.go index f74c054cf..9ac0b6a1d 100644 --- a/languages/golang/encrypt/gensupport/declaration.go +++ b/languages/golang/encrypt/gensupport/declaration.go @@ -48,7 +48,7 @@ const ( Timestamp Kind = "timestamp" String Kind = "string" Bytes Kind = "bytes" - // JSON is a json.RawMessage field stored as an EncryptedJSON document: + // JSON is a json.RawMessage field stored as a JSONSearch document: // it crosses as its JSON text, which the engine parses. JSON Kind = "json" ) @@ -85,7 +85,7 @@ type field struct { } // TargetOptions are the options an EncryptInto field gives its EQL type: -// for EncryptedJson, the JSON index's. The engine checks them when the +// for JsonSearch, the JSON index's. The engine checks them when the // declaration is first used (stashgen asked it when the file was written). type TargetOptions struct { // SealedRoot says whether the document's root is sealed; nil is the @@ -187,7 +187,7 @@ func (d Declaration) EncryptInto(name string, kind Kind, eqlType string) Declara } // Options gives the named EncryptInto field's EQL type its options: the -// `encrypt_into=EncryptedJson(ordered=...)` tag. +// `encrypt_into=JsonSearch(ordered=...)` tag. func (d Declaration) Options(name string, options TargetOptions) Declaration { if d.err != nil { return d diff --git a/languages/golang/encrypt/gensupport/json.go b/languages/golang/encrypt/gensupport/json.go index 717e1821b..3212138ba 100644 --- a/languages/golang/encrypt/gensupport/json.go +++ b/languages/golang/encrypt/gensupport/json.go @@ -10,7 +10,7 @@ import ( "github.com/cipherstash/vitaminc/bindings/go/vcvalue" ) -// The operators of a field stored as an EncryptedJSON document, for the +// The operators of a field stored as a JSONSearch document, for the // generated methods of its Fields entry. Each sends the operator's named // inputs, JSON values as their JSON text ([]byte of the json.RawMessage), // and checks the engine answered in the encoding and with the number of diff --git a/languages/golang/encrypt/internal/testusers/customer_stash.go b/languages/golang/encrypt/internal/testusers/customer_stash.go index 50b266aa1..86bb5579d 100644 --- a/languages/golang/encrypt/internal/testusers/customer_stash.go +++ b/languages/golang/encrypt/internal/testusers/customer_stash.go @@ -20,8 +20,8 @@ const _ = gensupport.GeneratedVersion2 type EncryptedCustomer struct { ID int64 - Profile eql.EncryptedJSON - Prefs *eql.EncryptedJSON + Profile eql.JSONSearch + Prefs *eql.JSONSearch } func (e EncryptedCustomer) String() string { @@ -44,9 +44,9 @@ type customerShape struct { var customerDeclaration = gensupport.Declare("customers"). Passthrough("id"). - EncryptInto("profile", gensupport.JSON, "EncryptedJson"). + EncryptInto("profile", gensupport.JSON, "JsonSearch"). Options("profile", gensupport.TargetOptions{Ordered: []string{"$.age", "$.visits[*].at"}}). - EncryptInto("prefs", gensupport.JSON, "EncryptedJson"). + EncryptInto("prefs", gensupport.JSON, "JsonSearch"). Null("prefs", gensupport.NullSQL) var customerCodec = gensupport.New(gensupport.Generated[Customer, EncryptedCustomer]{ @@ -66,9 +66,9 @@ var customerCodec = gensupport.New(gensupport.Generated[Customer, EncryptedCusto if e.ID, err = gensupport.Passthrough[int64](rec, "id"); err != nil { return EncryptedCustomer{}, err } - e.Profile = eql.EncryptedJSON(rec["profile"].EQL) + e.Profile = eql.JSONSearch(rec["profile"].EQL) if o, ok := rec["prefs"]; ok { - v := eql.EncryptedJSON(o.EQL) + v := eql.JSONSearch(o.EQL) e.Prefs = &v } return e, nil @@ -134,43 +134,43 @@ func newCustomerProfileField(f gensupport.Field[json.RawMessage]) CustomerProfil }} } -func (f CustomerProfileField) Encrypt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.EncryptedJSON, error) { +func (f CustomerProfileField) Encrypt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.JSONSearch, error) { out, err := f.field.Encrypt(ctx, c, v) - return eql.EncryptedJSON(out.EQL), err + return eql.JSONSearch(out.EQL), err } // Contains derives the query for documents that contain needle, a JSON value: // col @> query. Each array element of the needle must sit inside one element of // the document's array. -func (f CustomerProfileField) Contains(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.EncryptedJSONQuery, error) { - return gensupport.JSONQuery[eql.EncryptedJSONQuery](ctx, f.field, c, "contains", needle) +func (f CustomerProfileField) Contains(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.JSONSearchQuery, error) { + return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contains", needle) } // ContainedBy derives the query for documents contained by needle, a JSON // value: col <@ query. -func (f CustomerProfileField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.EncryptedJSONQuery, error) { - return gensupport.JSONQuery[eql.EncryptedJSONQuery](ctx, f.field, c, "contained_by", needle) +func (f CustomerProfileField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.JSONSearchQuery, error) { + return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contained_by", needle) } // EqAt derives the query for documents holding the scalar v at path: col @> // query. A wildcard step matches when any element holds it; a position ([2]) is // refused, since the stored selector reads every position as the wildcard. -func (f CustomerProfileField) EqAt(ctx context.Context, c *encrypt.Cipher, path string, v json.RawMessage) (eql.EncryptedJSONQuery, error) { - return gensupport.JSONQueryAt[eql.EncryptedJSONQuery](ctx, f.field, c, "eq_at", path, v) +func (f CustomerProfileField) EqAt(ctx context.Context, c *encrypt.Cipher, path string, v json.RawMessage) (eql.JSONSearchQuery, error) { + return gensupport.JSONQueryAt[eql.JSONSearchQuery](ctx, f.field, c, "eq_at", path, v) } // Selector prepares `->` at path: bind Selector() as col -> $1::text, and open // the entries it returns with Open. A position ([2]) is refused; read one with // Subtree. -func (f CustomerProfileField) Selector(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONSelection[eql.EncryptedJSONEntry], error) { - return gensupport.JSONSelect[eql.EncryptedJSONEntry](ctx, f.field, c, path) +func (f CustomerProfileField) Selector(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONSelection[eql.JSONEntry], error) { + return gensupport.JSONSelect[eql.JSONEntry](ctx, f.field, c, path) } // Subtree prepares a read of the value at path: bind Tokens() as // eql_v4.json_tree_subtree(col, $1::text[], budget), and open the rows it // returns with Open or OpenOne. -func (f CustomerProfileField) Subtree(ctx context.Context, c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[eql.EncryptedJSONEntry], error) { - return gensupport.JSONSubtree[eql.EncryptedJSONEntry](ctx, f.field, c, path, opts) +func (f CustomerProfileField) Subtree(ctx context.Context, c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[eql.JSONEntry], error) { + return gensupport.JSONSubtree[eql.JSONEntry](ctx, f.field, c, path, opts) } // Children derives the token of the node at path, a path with no wildcard, @@ -201,31 +201,31 @@ type CustomerProfileFieldPath struct { path string } -func (p CustomerProfileFieldPath) Lt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.EncryptedJSONLeafQuery, error) { - return gensupport.JSONQueryAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, "lt_at", p.path, v) +func (p CustomerProfileFieldPath) Lt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.JSONSearchLeafQuery, error) { + return gensupport.JSONQueryAt[eql.JSONSearchLeafQuery](ctx, p.field, c, "lt_at", p.path, v) } -func (p CustomerProfileFieldPath) Lte(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.EncryptedJSONLeafQuery, error) { - return gensupport.JSONQueryAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, "lte_at", p.path, v) +func (p CustomerProfileFieldPath) Lte(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.JSONSearchLeafQuery, error) { + return gensupport.JSONQueryAt[eql.JSONSearchLeafQuery](ctx, p.field, c, "lte_at", p.path, v) } -func (p CustomerProfileFieldPath) Gt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.EncryptedJSONLeafQuery, error) { - return gensupport.JSONQueryAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, "gt_at", p.path, v) +func (p CustomerProfileFieldPath) Gt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.JSONSearchLeafQuery, error) { + return gensupport.JSONQueryAt[eql.JSONSearchLeafQuery](ctx, p.field, c, "gt_at", p.path, v) } -func (p CustomerProfileFieldPath) Gte(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.EncryptedJSONLeafQuery, error) { - return gensupport.JSONQueryAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, "gte_at", p.path, v) +func (p CustomerProfileFieldPath) Gte(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.JSONSearchLeafQuery, error) { + return gensupport.JSONQueryAt[eql.JSONSearchLeafQuery](ctx, p.field, c, "gte_at", p.path, v) } -func (p CustomerProfileFieldPath) Between(ctx context.Context, c *encrypt.Cipher, lo, hi json.RawMessage) (encrypt.JSONRange[eql.EncryptedJSONLeafQuery], error) { - return gensupport.JSONBetweenAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, p.path, lo, hi) +func (p CustomerProfileFieldPath) Between(ctx context.Context, c *encrypt.Cipher, lo, hi json.RawMessage) (encrypt.JSONRange[eql.JSONSearchLeafQuery], error) { + return gensupport.JSONBetweenAt[eql.JSONSearchLeafQuery](ctx, p.field, c, p.path, lo, hi) } // Selector prepares `->` at this path: the value an ordering here compares is // col -> Selector(), so bind it beside the ordering's operand, from the same // path. -func (p CustomerProfileFieldPath) Selector(ctx context.Context, c *encrypt.Cipher) (encrypt.JSONSelection[eql.EncryptedJSONEntry], error) { - return gensupport.JSONSelect[eql.EncryptedJSONEntry](ctx, p.field, c, p.path) +func (p CustomerProfileFieldPath) Selector(ctx context.Context, c *encrypt.Cipher) (encrypt.JSONSelection[eql.JSONEntry], error) { + return gensupport.JSONSelect[eql.JSONEntry](ctx, p.field, c, p.path) } type CustomerPrefsField struct { @@ -236,43 +236,43 @@ func newCustomerPrefsField(f gensupport.Field[json.RawMessage]) CustomerPrefsFie return CustomerPrefsField{field: f} } -func (f CustomerPrefsField) Encrypt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.EncryptedJSON, error) { +func (f CustomerPrefsField) Encrypt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.JSONSearch, error) { out, err := f.field.Encrypt(ctx, c, v) - return eql.EncryptedJSON(out.EQL), err + return eql.JSONSearch(out.EQL), err } // Contains derives the query for documents that contain needle, a JSON value: // col @> query. Each array element of the needle must sit inside one element of // the document's array. -func (f CustomerPrefsField) Contains(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.EncryptedJSONQuery, error) { - return gensupport.JSONQuery[eql.EncryptedJSONQuery](ctx, f.field, c, "contains", needle) +func (f CustomerPrefsField) Contains(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.JSONSearchQuery, error) { + return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contains", needle) } // ContainedBy derives the query for documents contained by needle, a JSON // value: col <@ query. -func (f CustomerPrefsField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.EncryptedJSONQuery, error) { - return gensupport.JSONQuery[eql.EncryptedJSONQuery](ctx, f.field, c, "contained_by", needle) +func (f CustomerPrefsField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.JSONSearchQuery, error) { + return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contained_by", needle) } // EqAt derives the query for documents holding the scalar v at path: col @> // query. A wildcard step matches when any element holds it; a position ([2]) is // refused, since the stored selector reads every position as the wildcard. -func (f CustomerPrefsField) EqAt(ctx context.Context, c *encrypt.Cipher, path string, v json.RawMessage) (eql.EncryptedJSONQuery, error) { - return gensupport.JSONQueryAt[eql.EncryptedJSONQuery](ctx, f.field, c, "eq_at", path, v) +func (f CustomerPrefsField) EqAt(ctx context.Context, c *encrypt.Cipher, path string, v json.RawMessage) (eql.JSONSearchQuery, error) { + return gensupport.JSONQueryAt[eql.JSONSearchQuery](ctx, f.field, c, "eq_at", path, v) } // Selector prepares `->` at path: bind Selector() as col -> $1::text, and open // the entries it returns with Open. A position ([2]) is refused; read one with // Subtree. -func (f CustomerPrefsField) Selector(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONSelection[eql.EncryptedJSONEntry], error) { - return gensupport.JSONSelect[eql.EncryptedJSONEntry](ctx, f.field, c, path) +func (f CustomerPrefsField) Selector(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONSelection[eql.JSONEntry], error) { + return gensupport.JSONSelect[eql.JSONEntry](ctx, f.field, c, path) } // Subtree prepares a read of the value at path: bind Tokens() as // eql_v4.json_tree_subtree(col, $1::text[], budget), and open the rows it // returns with Open or OpenOne. -func (f CustomerPrefsField) Subtree(ctx context.Context, c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[eql.EncryptedJSONEntry], error) { - return gensupport.JSONSubtree[eql.EncryptedJSONEntry](ctx, f.field, c, path, opts) +func (f CustomerPrefsField) Subtree(ctx context.Context, c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[eql.JSONEntry], error) { + return gensupport.JSONSubtree[eql.JSONEntry](ctx, f.field, c, path, opts) } // Children derives the token of the node at path, a path with no wildcard, diff --git a/languages/golang/encrypt/internal/testusers/customers.go b/languages/golang/encrypt/internal/testusers/customers.go index f6a330798..e32245de8 100644 --- a/languages/golang/encrypt/internal/testusers/customers.go +++ b/languages/golang/encrypt/internal/testusers/customers.go @@ -4,12 +4,12 @@ import "encoding/json" //go:generate go run github.com/cipherstash/stack/languages/golang/cmd/stashgen -type Customer -name Customer -// Customer holds two encrypted JSON documents (the EQL type EncryptedJson, -// stored in public.eql_v4_json_tree): a profile whose $.age and every +// Customer holds two encrypted JSON documents (the EQL type JsonSearch, +// stored in public.eql_v4_json_search): a profile whose $.age and every // visit's $.at order, and optional preferences, stored as SQL NULL when nil. type Customer struct { _ struct{} `stash:"context=customers"` ID int64 `stash:"id,passthrough"` - Profile json.RawMessage `stash:"profile,encrypt_into=EncryptedJson(ordered=$.age;$.visits[*].at)"` - Prefs *json.RawMessage `stash:"prefs,encrypt_into=EncryptedJson,null=sql"` + Profile json.RawMessage `stash:"profile,encrypt_into=JsonSearch(ordered=$.age;$.visits[*].at)"` + Prefs *json.RawMessage `stash:"prefs,encrypt_into=JsonSearch,null=sql"` } diff --git a/languages/golang/encrypt/json.go b/languages/golang/encrypt/json.go index 3e13bee40..50b9de025 100644 --- a/languages/golang/encrypt/json.go +++ b/languages/golang/encrypt/json.go @@ -96,7 +96,7 @@ func (cph *Cipher) JSONOpen(ctx context.Context, plan *record.Plan, field string } // JSONSelector is a JSON path's selector: what `->`, `->>` and the path -// functions of an EncryptedJSON column take. It binds as SQL text; cast it, +// functions of a JSONSearch column take. It binds as SQL text; cast it, // `col -> $1::text`, so PostgreSQL picks EQL's operator and not jsonb's. // It is column-wide: the same path has the same selector in every row. type JSONSelector string @@ -174,12 +174,12 @@ type JSONRange[Q ~[]byte] struct { High Q } -// JSONSelection is a prepared `->` on an EncryptedJSON field: the selector +// JSONSelection is a prepared `->` on a JSONSearch field: the selector // to bind, and what opens the entries the database returns for it. It keeps // the request it was prepared from (the field's declaration, the path and // the cipher's keyset), so the entries open only as answers to it: the // engine derives the selector again and refuses an entry that does not -// carry it. E is the column's entry type, eql.EncryptedJSONEntry. +// carry it. E is the column's entry type, eql.JSONEntry. type JSONSelection[E ~[]byte] struct { cipher *Cipher plan *record.Plan @@ -236,12 +236,12 @@ func (s JSONSelection[E]) Open(ctx context.Context, rows []*E) ([]JSONValue, err return out, nil } -// JSONRead is a prepared subtree read of an EncryptedJSON field: the tokens +// JSONRead is a prepared subtree read of a JSONSearch field: the tokens // to bind, and what opens the rows eql_v4.json_tree_subtree returns for // them. It keeps the request it was prepared from (the field's declaration, // the path, the bound and the cipher's keyset); the engine derives the read // again from it, and checks every row against it. E is the column's entry -// type, eql.EncryptedJSONEntry. +// type, eql.JSONEntry. type JSONRead[E ~[]byte] struct { cipher *Cipher plan *record.Plan diff --git a/languages/golang/encrypt/json_fields_test.go b/languages/golang/encrypt/json_fields_test.go index 976ad14c5..3dde05150 100644 --- a/languages/golang/encrypt/json_fields_test.go +++ b/languages/golang/encrypt/json_fields_test.go @@ -14,7 +14,7 @@ import ( "github.com/cipherstash/stack/languages/golang/encrypt/internal/testusers" ) -// Generated code for json.RawMessage fields stored as EncryptedJson: +// Generated code for json.RawMessage fields stored as JsonSearch: // testusers.Customer's Profile, ordered at $.age and $.visits[*].at, and its // optional Prefs. @@ -134,7 +134,7 @@ func TestACustomersFieldsDeriveEachQuery(t *testing.T) { } // A selection over rows with no entry opens to no value without a // request. - none, err := sel.Open(ctx, make([]*eql.EncryptedJSONEntry, 3)) + none, err := sel.Open(ctx, make([]*eql.JSONEntry, 3)) if err != nil || len(none) != 3 || none[0].Found || none[2].Found { t.Fatalf("an empty page: %+v %v", none, err) } diff --git a/languages/golang/encrypt/json_postgres_test.go b/languages/golang/encrypt/json_postgres_test.go index ba9864ee2..ad3704c63 100644 --- a/languages/golang/encrypt/json_postgres_test.go +++ b/languages/golang/encrypt/json_postgres_test.go @@ -13,7 +13,7 @@ import ( "github.com/cipherstash/stack/languages/golang/encrypt/internal/testusers" ) -// Encrypted JSON documents (EncryptedJson, public.eql_v4_json_tree) the Go +// Encrypted JSON documents (JsonSearch, public.eql_v4_json_search) the Go // SDK stores, found by the queries its generated methods derive and read by // its selections and reads, inside PostgreSQL with EQL v4 installed. Each // query's rows are the ones the plaintexts say. Skipped unless DATABASE_URL @@ -41,7 +41,7 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { t.Fatalf("%s: %v", q, err) } } - exec(`CREATE TEMP TABLE customers (id bigint PRIMARY KEY, profile public.eql_v4_json_tree NOT NULL, prefs public.eql_v4_json_tree)`) + exec(`CREATE TEMP TABLE customers (id bigint PRIMARY KEY, profile public.eql_v4_json_search NOT NULL, prefs public.eql_v4_json_search)`) t.Cleanup(func() { _, _ = conn.ExecContext(context.Background(), `DROP TABLE IF EXISTS customers`) }) encrypted, err := testusers.EncryptCustomer(ctx, cipher, jsonCustomers) if err != nil { @@ -114,21 +114,21 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { } // Containment, both ways. - q := eql.EncryptedJSONQuery(must(f.Contains(ctx, cipher, json.RawMessage(`{"name":"Alice"}`)))) - want("contains name", ids(`SELECT id FROM customers WHERE profile @> $1::eql_v4.query_json_tree ORDER BY id`, q), 1) - q = eql.EncryptedJSONQuery(must(f.Contains(ctx, cipher, json.RawMessage(`{"visits":[{"at":5}]}`)))) - want("contains an element", ids(`SELECT id FROM customers WHERE profile @> $1::eql_v4.query_json_tree ORDER BY id`, q), 2) - q = eql.EncryptedJSONQuery(must(f.ContainedBy(ctx, cipher, json.RawMessage(`{"name":"Dan","x":1}`)))) - want("contained by", ids(`SELECT id FROM customers WHERE profile <@ $1::eql_v4.query_json_tree ORDER BY id`, q), 4) + q := eql.JSONSearchQuery(must(f.Contains(ctx, cipher, json.RawMessage(`{"name":"Alice"}`)))) + want("contains name", ids(`SELECT id FROM customers WHERE profile @> $1::eql_v4.query_json ORDER BY id`, q), 1) + q = eql.JSONSearchQuery(must(f.Contains(ctx, cipher, json.RawMessage(`{"visits":[{"at":5}]}`)))) + want("contains an element", ids(`SELECT id FROM customers WHERE profile @> $1::eql_v4.query_json ORDER BY id`, q), 2) + q = eql.JSONSearchQuery(must(f.ContainedBy(ctx, cipher, json.RawMessage(`{"name":"Dan","x":1}`)))) + want("contained by", ids(`SELECT id FROM customers WHERE profile <@ $1::eql_v4.query_json ORDER BY id`, q), 4) // Equality at a wildcard path matches any element. - q = eql.EncryptedJSONQuery(must(f.EqAt(ctx, cipher, "$.visits[*].at", json.RawMessage(`1`)))) - want("eq_at", ids(`SELECT id FROM customers WHERE profile @> $1::eql_v4.query_json_tree ORDER BY id`, q), 1) + q = eql.JSONSearchQuery(must(f.EqAt(ctx, cipher, "$.visits[*].at", json.RawMessage(`1`)))) + want("eq_at", ids(`SELECT id FROM customers WHERE profile @> $1::eql_v4.query_json ORDER BY id`, q), 1) // Containment on the optional column: SQL NULL rows match nothing. pq, err := testusers.CustomerFields.Prefs.Contains(ctx, cipher, json.RawMessage(`{"theme":"dark"}`)) if err != nil { t.Fatal(err) } - want("prefs contains", ids(`SELECT id FROM customers WHERE prefs @> $1::eql_v4.query_json_tree ORDER BY id`, pq), 1) + want("prefs contains", ids(`SELECT id FROM customers WHERE prefs @> $1::eql_v4.query_json ORDER BY id`, pq), 1) // Orderings at an ordered path compare the leaf `->` extracts there, // selected by the accessor's own Selector, so the path is named once. @@ -140,12 +140,12 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { if err != nil { t.Fatal(err) } - want("age < 40", ids(`SELECT id FROM customers WHERE (profile -> $1::text) < $2::eql_v4.query_json_tree_leaf ORDER BY id`, age.Selector(), lt), 1, 2) + want("age < 40", ids(`SELECT id FROM customers WHERE (profile -> $1::text) < $2::eql_v4.query_json_leaf ORDER BY id`, age.Selector(), lt), 1, 2) rng, err := f.At.Age.Between(ctx, cipher, json.RawMessage(`30`), json.RawMessage(`70`)) if err != nil { t.Fatal(err) } - want("age between", ids(`SELECT id FROM customers WHERE (profile -> $1::text) >= $2::eql_v4.query_json_tree_leaf AND (profile -> $1::text) <= $3::eql_v4.query_json_tree_leaf ORDER BY id`, age.Selector(), rng.Low, rng.High), 1, 3) + want("age between", ids(`SELECT id FROM customers WHERE (profile -> $1::text) >= $2::eql_v4.query_json_leaf AND (profile -> $1::text) <= $3::eql_v4.query_json_leaf ORDER BY id`, age.Selector(), rng.Low, rng.High), 1, 3) at, err := f.At.VisitsAt.Selector(ctx, cipher) if err != nil { t.Fatal(err) @@ -154,20 +154,20 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { if err != nil { t.Fatal(err) } - want("first visit > 2", ids(`SELECT id FROM customers WHERE (profile -> $1::text) > $2::eql_v4.query_json_tree_leaf ORDER BY id`, at.Selector(), gt), 1, 2) + want("first visit > 2", ids(`SELECT id FROM customers WHERE (profile -> $1::text) > $2::eql_v4.query_json_leaf ORDER BY id`, at.Selector(), gt), 1, 2) // `->` opened against the selection that asked for it: each row's // value, and no value where the row has none. - selected := func(sel encrypt.JSONSelection[eql.EncryptedJSONEntry]) []encrypt.JSONValue { + selected := func(sel encrypt.JSONSelection[eql.JSONEntry]) []encrypt.JSONValue { t.Helper() rows, err := conn.QueryContext(ctx, `SELECT profile -> $1::text FROM customers ORDER BY id`, sel.Selector()) if err != nil { t.Fatal(err) } defer rows.Close() - var entries []*eql.EncryptedJSONEntry + var entries []*eql.JSONEntry for rows.Next() { - var e *eql.EncryptedJSONEntry + var e *eql.JSONEntry if err := rows.Scan(&e); err != nil { t.Fatal(err) } @@ -206,14 +206,14 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { } walk := func(budget int) [][]json.RawMessage { t.Helper() - groups := make([][]eql.EncryptedJSONEntry, len(jsonCustomers)) + groups := make([][]eql.JSONEntry, len(jsonCustomers)) for i, cust := range jsonCustomers { rows, err := conn.QueryContext(ctx, `SELECT e FROM customers, eql_v4.json_tree_subtree(profile, $2::text[], $3) AS e WHERE id = $1`, cust.ID, read.Tokens(), budget) if err != nil { t.Fatal(err) } for rows.Next() { - var e eql.EncryptedJSONEntry + var e eql.JSONEntry if err := rows.Scan(&e); err != nil { t.Fatal(err) } @@ -252,13 +252,13 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { if err != nil { t.Fatal(err) } - var entries []eql.EncryptedJSONEntry + var entries []eql.JSONEntry rs, err := conn.QueryContext(ctx, `SELECT e FROM customers, eql_v4.json_tree_subtree(profile, $1::text[]) AS e WHERE id = 1`, visits.Tokens()) if err != nil { t.Fatal(err) } for rs.Next() { - var e eql.EncryptedJSONEntry + var e eql.JSONEntry if err := rs.Scan(&e); err != nil { t.Fatal(err) } diff --git a/languages/golang/encrypt/term.go b/languages/golang/encrypt/term.go index e30d1c97d..608f7004b 100644 --- a/languages/golang/encrypt/term.go +++ b/languages/golang/encrypt/term.go @@ -75,7 +75,7 @@ type JSONOption interface { // by stashgen, and at run time by the engine. // // Deprecated: store a JSON document as a json.RawMessage field with -// `encrypt_into=EncryptedJson`, whose EQL type carries the JSON index. +// `encrypt_into=JsonSearch`, whose EQL type carries the JSON index. func JSON(options ...JSONOption) Index { _ = options return index("json") @@ -158,8 +158,8 @@ func (t OpeTerm) Less(other OpeTerm) bool { return t.Compare(other) < 0 } // derive it; the type exists so a generated declaration that names [JSON] // has a Go type to fail into. // -// Deprecated: an EncryptedJson field's queries are eql.EncryptedJSONQuery -// and eql.EncryptedJSONLeafQuery. +// Deprecated: a JsonSearch field's queries are eql.JSONSearchQuery +// and eql.JSONSearchLeafQuery. type JSONTerm []byte // compareCLLW is cllw-ore's compare_lex: compare_slice over the common diff --git a/languages/golang/stashgen/declaration.go b/languages/golang/stashgen/declaration.go index de10e50eb..9b8070767 100644 --- a/languages/golang/stashgen/declaration.go +++ b/languages/golang/stashgen/declaration.go @@ -46,7 +46,7 @@ type Field struct { // EQLType is the EQL type an EncryptInto field seals into. EQLType string // TargetOptions are the EQL type's options, for a type that takes them - // (EncryptedJson); nil for its defaults. + // (JsonSearch); nil for its defaults. TargetOptions *record.TargetOptions // Identity is the field's part of its context when it differs from Name: // a column that was renamed keeps the identity it was first written @@ -276,7 +276,7 @@ const ( // KindDecimal is encrypt.Decimal. KindDecimal // KindJSON is json.RawMessage: a JSON document, stored as an - // EncryptedJSON value. + // JSONSearch value. KindJSON ) diff --git a/languages/golang/stashgen/emit.go b/languages/golang/stashgen/emit.go index 314781003..080966b31 100644 --- a/languages/golang/stashgen/emit.go +++ b/languages/golang/stashgen/emit.go @@ -977,7 +977,7 @@ func joinNames(names []string) string { return strings.Join(names[:len(names)-1], ", ") + " and " + names[len(names)-1] } -// isJSON reports whether the field is stored as an EncryptedJSON document: +// isJSON reports whether the field is stored as a JSONSearch document: // a json.RawMessage sealed into an EQL type. func (g genField) isJSON() bool { return g.Verb == VerbEncryptInto && g.GoType.Kind == KindJSON @@ -1004,7 +1004,7 @@ func planJSON(f *genFile, g *genField) error { // The engine answers selector for a field whose type stores a json_tree // document; the leaf and entry types below are that family's. if !document { - return fieldErr(f.typeName, g.GoName, "the engine answers no document operator on %s, and the generator writes json.RawMessage fields of EncryptedJson only", g.EQLType) + return fieldErr(f.typeName, g.GoName, "the engine answers no document operator on %s, and the generator writes json.RawMessage fields of JsonSearch only", g.EQLType) } g.jsonPaths = nil seen := map[string]string{} @@ -1070,10 +1070,14 @@ func (w *writer) jsonEntry(g genField) { for _, op := range g.operators { answered[op.Name] = true } - eqlName := strings.TrimPrefix(g.outputType, "eql.") needle := g.queryType - leaf := "eql." + eqlName + "LeafQuery" - entry := "eql." + eqlName + "Entry" + // The document's companion types carry the names EQL v3 gives its own + // (eql_domains::JSON_TREE): the leaf operand puts Leaf before the + // needle's Query (JSONSearchQuery, JSONSearchLeafQuery), and the entry + // is named for the plaintext it holds one node of (JSONEntry, over + // public.eql_v4_json_entry), not for the stored type. + leaf := strings.TrimSuffix(needle, "Query") + "LeafQuery" + entry := "eql." + EQLGoName("JsonEntry") ordered := len(g.jsonPaths) > 0 && answered["lt_at"] field := "gensupport.Field[" + g.valueExpr + "]" diff --git a/languages/golang/stashgen/engine.go b/languages/golang/stashgen/engine.go index 08275aaa1..0368d876f 100644 --- a/languages/golang/stashgen/engine.go +++ b/languages/golang/stashgen/engine.go @@ -271,7 +271,7 @@ func lowerDeclaration(d Declaration, eqlTypes []EQLType) (*record.Plan, error) { if !ok { reason := fmt.Sprintf("the engine does not derive the %s index yet", idx.Name) if idx.Name == IndexJSON { - reason = "the json index is not an output beside a ciphertext: store a JSON document as a json.RawMessage field with encrypt_into=EncryptedJson, whose EQL type carries the index" + reason = "the json index is not an output beside a ciphertext: store a JSON document as a json.RawMessage field with encrypt_into=JsonSearch, whose EQL type carries the index" } return nil, &FieldError{Type: d.Type, Field: f.GoName, Reason: reason} } diff --git a/languages/golang/stashgen/enginetest/enginetest.go b/languages/golang/stashgen/enginetest/enginetest.go index 2f91c56d3..e3a582f49 100644 --- a/languages/golang/stashgen/enginetest/enginetest.go +++ b/languages/golang/stashgen/enginetest/enginetest.go @@ -278,7 +278,7 @@ func indexApplies(name stashgen.IndexName, t stashgen.GoType) error { return nil } case stashgen.IndexJSON: - return fmt.Errorf("the json index is not an output beside a ciphertext: store a JSON document with encrypt_into=EncryptedJson") + return fmt.Errorf("the json index is not an output beside a ciphertext: store a JSON document with encrypt_into=JsonSearch") } return fmt.Errorf("%s does not apply to %s", name, t) } diff --git a/languages/golang/stashgen/refusal_test.go b/languages/golang/stashgen/refusal_test.go index 3297fdc9c..3d4855ff5 100644 --- a/languages/golang/stashgen/refusal_test.go +++ b/languages/golang/stashgen/refusal_test.go @@ -113,7 +113,7 @@ func TestRefusals(t *testing.T) { {"an opaque struct with a map keyed by a struct", user("\t_ struct{} `stash:\"context=users,opaque\"`\n\tM map[hidden]int") + "\ntype hidden struct{ A int }\n", stashgen.Request{Type: "User"}, "M", "map key that is not a string or an integer"}, {"an EQL type the engine cannot produce yet", user(ctx + "\tAge int64 `stash:\"age,encrypt_into=BigintOrdOre\"`"), stashgen.Request{Type: "User"}, "Age", "has no EQL type BigintOrdOre"}, {"an index on a composite", user(ctx + "\tAttrs map[string]string `stash:\"attrs,encrypt,index=equality\"`"), stashgen.Request{Type: "User"}, "Attrs", "seals only as part of an opaque struct"}, - {"the json index as an output", user(ctx + "\tAttrs string `stash:\"attrs,index=json\"`"), stashgen.Request{Type: "User"}, "Attrs", "encrypt_into=EncryptedJson"}, + {"the json index as an output", user(ctx + "\tAttrs string `stash:\"attrs,index=json\"`"), stashgen.Request{Type: "User"}, "Attrs", "encrypt_into=JsonSearch"}, {"an index option a query term cannot use", user(ctx + "\tEmail string `stash:\"email,encrypt,index=match(k=3)\"`"), stashgen.Request{Type: "User"}, "Email", "a query term uses only the default index options"}, {"a passthrough field that has an index", user(ctx + "\tID int64 `stash:\"id,passthrough,index=equality\"`"), stashgen.Request{Type: "User"}, "ID", "passthrough field has no index"}, {"an embedded struct from another package with no tag", user(ctx + "\tgorm.Model\n\tEmail string `stash:\"email,encrypt\"`"), stashgen.Request{Type: "User"}, "Model", "cannot carry tags; tag the field"}, diff --git a/languages/golang/stashgen/tag.go b/languages/golang/stashgen/tag.go index 925b5a093..0e7584c94 100644 --- a/languages/golang/stashgen/tag.go +++ b/languages/golang/stashgen/tag.go @@ -26,7 +26,7 @@ const tagKey = "stash" // stash:"tenant,context_field" Name, VerbContextField // stash:",passthrough" VerbPassthrough with no Name, on an embedded struct // stash:"dob,encrypt_into=Date,null=sql" Null, on a sealed field declared over a pointer -// stash:"profile,encrypt_into=EncryptedJson(ordered=$.age;$.visits[*].at,sealed_root=false)" +// stash:"profile,encrypt_into=JsonSearch(ordered=$.age;$.visits[*].at,sealed_root=false)" // EQLType and EQLOptions type tag struct { Omit bool diff --git a/languages/golang/stashgen/testdata/stubsdk/encrypt/encrypt.go b/languages/golang/stashgen/testdata/stubsdk/encrypt/encrypt.go index 6ddfbe7bc..aa1117201 100644 --- a/languages/golang/stashgen/testdata/stubsdk/encrypt/encrypt.go +++ b/languages/golang/stashgen/testdata/stubsdk/encrypt/encrypt.go @@ -141,13 +141,13 @@ type JSONReadOptions struct{ WildcardBound uint32 } // JSONRange is both bounds of a between at a path. type JSONRange[Q ~[]byte] struct{ Low, High Q } -// JSONSelection is a prepared `->` on an EncryptedJSON field. +// JSONSelection is a prepared `->` on a JSONSearch field. type JSONSelection[E ~[]byte] struct{ _ E } func (JSONSelection[E]) Selector() JSONSelector { return "" } func (JSONSelection[E]) Open(context.Context, []*E) ([]JSONValue, error) { return nil, nil } -// JSONRead is a prepared subtree read on an EncryptedJSON field. +// JSONRead is a prepared subtree read on a JSONSearch field. type JSONRead[E ~[]byte] struct{ _ E } func (JSONRead[E]) Tokens() JSONTokens { return nil } From ed858fbbc171cb51e9b30533ed4d9bd46ad03ae1 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:10:15 +1100 Subject: [PATCH 11/26] fix(go): decide OpenOne's wildcard from the engine's token count JSONRead.OpenOne refused a wildcard read by looking for "[*]" in the path text. That copies a rule of the engine's path parser into Go, and gets it wrong for a quoted key: $["a[*]"] is one member key, and OpenOne refused it. The engine already answers the question. It derives exactly one token for a path that names one node, and for a wildcard one token per element below the bound plus a probe at the bound, which is two or more at any bound of 1 or more. Generated code never sends a bound of 0. OpenOne now refuses a read with more or fewer than one token. A read prepared by hand with a bound of 0 passes the check, but it reaches no value: a row with an element there is refused as truncated, and any other opens to none. The refusal no longer prints the path, which can hold a customer's key. The parameter is named entries, for what it holds: one database row's json_tree_subtree entries. JSONRead no longer keeps the path, which nothing reads now, so NewJSONRead stops taking it. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/encrypt/gensupport/json.go | 2 +- languages/golang/encrypt/json.go | 34 ++++++++++++-------- languages/golang/encrypt/json_fields_test.go | 14 ++++++-- 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/languages/golang/encrypt/gensupport/json.go b/languages/golang/encrypt/gensupport/json.go index 3212138ba..04c034f60 100644 --- a/languages/golang/encrypt/gensupport/json.go +++ b/languages/golang/encrypt/gensupport/json.go @@ -94,7 +94,7 @@ func JSONSubtree[E ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.C if err != nil { return encrypt.JSONRead[E]{}, err } - return encrypt.NewJSONRead[E](c, f.plan, f.name, path, input, term.Text), nil + return encrypt.NewJSONRead[E](c, f.plan, f.name, input, term.Text), nil } // JSONNodeToken derives the token of the node at a path with no wildcard diff --git a/languages/golang/encrypt/json.go b/languages/golang/encrypt/json.go index 50b9de025..800f700ec 100644 --- a/languages/golang/encrypt/json.go +++ b/languages/golang/encrypt/json.go @@ -246,7 +246,6 @@ type JSONRead[E ~[]byte] struct { cipher *Cipher plan *record.Plan field string - path string // input is the subtree operator's input the tokens were derived from; // opening sends it back as the read's request, unchanged. input vcvalue.Object @@ -255,8 +254,8 @@ type JSONRead[E ~[]byte] struct { // NewJSONRead is a read prepared by [Cipher.QueryTerm]'s subtree operator // from input, its {"path", "wildcard_bound"?} object. For generated code. -func NewJSONRead[E ~[]byte](c *Cipher, plan *record.Plan, field, path string, input vcvalue.Object, tokens []string) JSONRead[E] { - return JSONRead[E]{cipher: c, plan: plan, field: field, path: path, input: input, tokens: JSONTokens(tokens)} +func NewJSONRead[E ~[]byte](c *Cipher, plan *record.Plan, field string, input vcvalue.Object, tokens []string) JSONRead[E] { + return JSONRead[E]{cipher: c, plan: plan, field: field, input: input, tokens: JSONTokens(tokens)} } // Tokens are the tokens to bind: @@ -301,17 +300,26 @@ func (r JSONRead[E]) Open(ctx context.Context, rows [][]E) ([][]json.RawMessage, return out, nil } -// OpenOne opens one row's rows to the one value at the read's path: Found -// with its JSON, or not Found where the row has no node there. A read that -// reaches more than one value (a wildcard) is [ErrEncoding]; open it with -// [JSONRead.Open]. -func (r JSONRead[E]) OpenOne(ctx context.Context, rows []E) (JSONValue, error) { - // A wildcard reaches a value per element: refused before anything is - // opened, not only for the rows that happen to hold two. - if strings.Contains(r.path, "[*]") { - return JSONValue{}, fmt.Errorf("%w: OpenOne reads one value, and %s has a wildcard; open it with Open", ErrEncoding, r.path) +// OpenOne opens one database row's entries to the one value at the read's +// path: Found with its JSON, or not Found where the row has no node there. +// A read that reaches more than one value (a wildcard) is [ErrEncoding]; +// open it with [JSONRead.Open]. +func (r JSONRead[E]) OpenOne(ctx context.Context, entries []E) (JSONValue, error) { + // The engine derives one token for a path that names one node, and for + // a wildcard one per element below the bound plus a probe at it: two + // or more at any bound of 1 or more. The generated Subtree never sends + // a bound of 0 (WildcardBound 0 sends none, and the engine's default + // is 64). So the count is the engine's own answer to whether the path + // names one node: a quoted key that spells [*] is one member. A + // wildcard is refused here, before anything is opened, not only for + // the rows that happen to hold two values. A read prepared with a + // bound of 0 expands a wildcard to the probe alone and passes this + // check, but it reaches no value: a row with an element there is + // refused as truncated, and any other opens to none. + if r.cipher != nil && len(r.tokens) != 1 { + return JSONValue{}, fmt.Errorf("%w: OpenOne reads one value, and this read's path has a wildcard; open it with Open", ErrEncoding) } - opened, err := r.Open(ctx, [][]E{rows}) + opened, err := r.Open(ctx, [][]E{entries}) if err != nil { return JSONValue{}, err } diff --git a/languages/golang/encrypt/json_fields_test.go b/languages/golang/encrypt/json_fields_test.go index 3dde05150..e0609e0c5 100644 --- a/languages/golang/encrypt/json_fields_test.go +++ b/languages/golang/encrypt/json_fields_test.go @@ -122,10 +122,20 @@ func TestACustomersFieldsDeriveEachQuery(t *testing.T) { t.Fatal("the accessor's selector is another path's") } // OpenOne refuses a wildcard read before anything is opened, whatever - // the rows hold; a zero selector or token binds nothing. - if _, err := read.OpenOne(ctx, nil); !errors.Is(err, encrypt.ErrEncoding) { + // the rows hold, and its error does not name the path, which can hold + // a customer's key. A quoted key that spells [*] names one member: the + // engine derives one token for it, and OpenOne takes it. A zero + // selector or token binds nothing. + if _, err := read.OpenOne(ctx, nil); !errors.Is(err, encrypt.ErrEncoding) || strings.Contains(err.Error(), "visits") { t.Fatalf("OpenOne on a wildcard read: %v", err) } + quoted, err := f.Subtree(ctx, cipher, `$["a[*]"]`, encrypt.JSONReadOptions{}) + if err != nil || len(quoted.Tokens()) != 1 { + t.Fatalf("Subtree at a quoted key: %v %v", quoted.Tokens(), err) + } + if v, err := quoted.OpenOne(ctx, nil); err != nil || v.Found { + t.Fatalf("OpenOne at a quoted key that spells [*]: %+v %v", v, err) + } if _, err := (encrypt.JSONSelector("")).Value(); !errors.Is(err, encrypt.ErrEncoding) { t.Fatalf("a zero selector binds: %v", err) } From 247bd3239dc1e90f4c52b10330bb21c7d4e80422 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:10:46 +1100 Subject: [PATCH 12/26] fix(go): refuse empty JSONTokens rather than bind them as {} A zero JSONSelector or JSONToken is refused with ErrEncoding when bound, because the empty value matches no row without an error. JSONTokens did not follow the rule: a nil or empty list bound as the text[] literal {}, so a zero JSONRead's tokens read nothing from every row and said nothing. Every read the engine prepares has at least one token, so an empty list is always a read that was never prepared. It is now refused the same way, and the changelog's sentence covers the token list too. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/CHANGELOG.md | 4 ++-- languages/golang/encrypt/json.go | 7 ++++++- languages/golang/encrypt/json_fields_test.go | 8 +++++++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index 55a41957f..a2e9ffb1c 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -23,8 +23,8 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. and `encrypt.JSONRead` open the rows `->` and `eql_v4.json_tree_subtree` return against the request that prepared them (`OpenOne` refuses a wildcard read up front); `encrypt.JSONSelector`, `JSONToken` and - `JSONTokens` bind as `text` and `text[]`, and a zero selector or token is - refused rather than bound. A `json.RawMessage` sealed any other way stays + `JSONTokens` bind as `text` and `text[]`, and a zero selector, token or + token list is refused rather than bound. A `json.RawMessage` sealed any other way stays bytes. `gensupport.JSON`, `TargetOptions`, `Declaration.Options` and the `JSON*` helpers are the generated code's. diff --git a/languages/golang/encrypt/json.go b/languages/golang/encrypt/json.go index 800f700ec..20f072aac 100644 --- a/languages/golang/encrypt/json.go +++ b/languages/golang/encrypt/json.go @@ -131,8 +131,13 @@ func (t JSONToken) Value() (driver.Value, error) { type JSONTokens []string // Value binds the tokens as a text[] literal. A token is base64, so it -// needs no escape inside the quotes. +// needs no escape inside the quotes. Empty JSONTokens (a zero JSONRead's) +// are refused, as a zero [JSONToken] is: bound as {}, they would read +// nothing from every row. func (t JSONTokens) Value() (driver.Value, error) { + if len(t) == 0 { + return nil, fmt.Errorf("%w: empty JSONTokens: prepare a read with the field's Subtree", ErrEncoding) + } var b strings.Builder b.WriteByte('{') for i, token := range t { diff --git a/languages/golang/encrypt/json_fields_test.go b/languages/golang/encrypt/json_fields_test.go index e0609e0c5..eb38d31c8 100644 --- a/languages/golang/encrypt/json_fields_test.go +++ b/languages/golang/encrypt/json_fields_test.go @@ -125,7 +125,7 @@ func TestACustomersFieldsDeriveEachQuery(t *testing.T) { // the rows hold, and its error does not name the path, which can hold // a customer's key. A quoted key that spells [*] names one member: the // engine derives one token for it, and OpenOne takes it. A zero - // selector or token binds nothing. + // selector, token or tokens binds nothing. if _, err := read.OpenOne(ctx, nil); !errors.Is(err, encrypt.ErrEncoding) || strings.Contains(err.Error(), "visits") { t.Fatalf("OpenOne on a wildcard read: %v", err) } @@ -142,6 +142,12 @@ func TestACustomersFieldsDeriveEachQuery(t *testing.T) { if _, err := (encrypt.JSONToken("")).Value(); !errors.Is(err, encrypt.ErrEncoding) { t.Fatalf("a zero token binds: %v", err) } + if _, err := (encrypt.JSONTokens(nil)).Value(); !errors.Is(err, encrypt.ErrEncoding) { + t.Fatalf("no tokens bind: %v", err) + } + if _, err := (encrypt.JSONRead[eql.JSONEntry]{}).Tokens().Value(); !errors.Is(err, encrypt.ErrEncoding) { + t.Fatalf("a zero read's tokens bind: %v", err) + } // A selection over rows with no entry opens to no value without a // request. none, err := sel.Open(ctx, make([]*eql.JSONEntry, 3)) From 2424891309cef3d030ba6ccb788019fc3b67c8c6 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:12:21 +1100 Subject: [PATCH 13/26] fix(go): print and log the JSON read types redacted JSONToken and JSONTokens are documented as capabilities: a token opens the structure of its node's subtree in every row of the column. They still printed in the clear with %v and through slog. JSONSelection and JSONRead printed their selector, tokens and path with %+v, and a path's keys can be a customer's data (Go principle 12). Each now has String, GoString, Format and LogValue, as the query types in query.go do, so every fmt verb and slog record shows the type, the field and a token count, with [redacted] for the tokens, the selector and the path. The values still bind through driver.Valuer, which pgx and database/sql consult before a Stringer. JSONValue's doc now says it holds plaintext JSON that prints in the clear, as the generated plaintext types' docs do. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/CHANGELOG.md | 5 +- languages/golang/encrypt/json.go | 97 +++++++++++++++++++- languages/golang/encrypt/json_fields_test.go | 48 ++++++++++ 3 files changed, 146 insertions(+), 4 deletions(-) diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index a2e9ffb1c..22e73c11d 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -24,7 +24,10 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. return against the request that prepared them (`OpenOne` refuses a wildcard read up front); `encrypt.JSONSelector`, `JSONToken` and `JSONTokens` bind as `text` and `text[]`, and a zero selector, token or - token list is refused rather than bound. A `json.RawMessage` sealed any other way stays + token list is refused rather than bound. `JSONToken`, `JSONTokens`, + `JSONSelection` and `JSONRead` print and log with the tokens, the + selector and the path redacted; `encrypt.JSONValue` holds plaintext and + prints it. A `json.RawMessage` sealed any other way stays bytes. `gensupport.JSON`, `TargetOptions`, `Declaration.Options` and the `JSON*` helpers are the generated code's. diff --git a/languages/golang/encrypt/json.go b/languages/golang/encrypt/json.go index 20f072aac..648ec3a37 100644 --- a/languages/golang/encrypt/json.go +++ b/languages/golang/encrypt/json.go @@ -5,6 +5,8 @@ import ( "database/sql/driver" "encoding/json" "fmt" + "io" + "log/slog" "strings" "github.com/cipherstash/stack/languages/golang/internal/record" @@ -113,7 +115,8 @@ func (s JSONSelector) Value() (driver.Value, error) { // JSONToken is one node's token: what eql_v4.json_tree_children and // eql_v4.json_tree_child_count take. It binds as SQL text. A token is a // capability over the node's subtree in every row of the column: send it -// to the database for a query, and do not log it. +// to the database for a query, and nowhere else. It prints and logs +// redacted. type JSONToken string // Value binds the token as text. A zero JSONToken is refused, as a zero @@ -125,9 +128,31 @@ func (t JSONToken) Value() (driver.Value, error) { return string(t), nil } +// redacted stands in for a token, a selector or a path in what the JSON +// read types print and log. +const redacted = "[redacted]" + +// String hides the token: it is a capability, so it does not print. +func (t JSONToken) String() string { + if t == "" { + return "encrypt.JSONToken{}" + } + return "encrypt.JSONToken{" + redacted + "}" +} + +// GoString is what %#v prints: [JSONToken.String]. +func (t JSONToken) GoString() string { return t.String() } + +// Format prints [JSONToken.String] for every verb fmt hands it. +func (t JSONToken) Format(f fmt.State, _ rune) { _, _ = io.WriteString(f, t.String()) } + +// LogValue is what slog records: [redacted], never the token. +func (t JSONToken) LogValue() slog.Value { return slog.StringValue(redacted) } + // JSONTokens are a subtree read's tokens: what eql_v4.json_tree_subtree // takes. They bind as a PostgreSQL text[] literal; cast the parameter, -// `$2::text[]`. Like a [JSONToken], each is a capability: do not log them. +// `$2::text[]`. Like a [JSONToken], each is a capability, and they print +// and log redacted. type JSONTokens []string // Value binds the tokens as a text[] literal. A token is base64, so it @@ -152,10 +177,27 @@ func (t JSONTokens) Value() (driver.Value, error) { return b.String(), nil } +// String counts the tokens and hides them: each is a capability. +func (t JSONTokens) String() string { + return fmt.Sprintf("encrypt.JSONTokens{%d tokens: %s}", len(t), redacted) +} + +// GoString is what %#v prints: [JSONTokens.String]. +func (t JSONTokens) GoString() string { return t.String() } + +// Format prints [JSONTokens.String] for every verb fmt hands it. +func (t JSONTokens) Format(f fmt.State, _ rune) { _, _ = io.WriteString(f, t.String()) } + +// LogValue is what slog records: how many tokens, never the tokens. +func (t JSONTokens) LogValue() slog.Value { + return slog.GroupValue(slog.Int("tokens", len(t))) +} + // JSONValue is one value a read reached, or none. Found is false for no // value at the path (the row has no node there, or SQL NULL); a JSON null // that is there is Found, with JSON "null". JSON is the value's JSON text: -// plaintext, which the caller owns. +// plaintext, which the caller owns. JSONValue prints its JSON in the +// clear: it has no String or LogValue method, so do not log one. type JSONValue struct { Found bool JSON json.RawMessage @@ -202,6 +244,30 @@ func NewJSONSelection[E ~[]byte](c *Cipher, plan *record.Plan, field, path, sele // Selector is the selector to bind: `col -> $1::text`. func (s JSONSelection[E]) Selector() JSONSelector { return s.selector } +// String names the field and hides the path and the selector: a path's +// keys can be a customer's data. +func (s JSONSelection[E]) String() string { + if s.cipher == nil { + return "encrypt.JSONSelection{}" + } + return fmt.Sprintf("encrypt.JSONSelection{field: %q, path: %s, selector: %s}", s.field, redacted, redacted) +} + +// GoString is what %#v prints: [JSONSelection.String]. +func (s JSONSelection[E]) GoString() string { return s.String() } + +// Format prints [JSONSelection.String] for every verb fmt hands it. +func (s JSONSelection[E]) Format(f fmt.State, _ rune) { _, _ = io.WriteString(f, s.String()) } + +// LogValue is what slog records: the field, and [redacted] for the path +// and the selector. +func (s JSONSelection[E]) LogValue() slog.Value { + if s.cipher == nil { + return slog.GroupValue() + } + return slog.GroupValue(slog.String("field", s.field), slog.String("path", redacted), slog.String("selector", redacted)) +} + // Open opens the entries `col -> selector` returned, one per database row, // nil for a row where it returned SQL NULL, with one ZeroKMS request. Each // opens to the scalar it holds (Found), or for the sealed root at "$" the @@ -267,6 +333,31 @@ func NewJSONRead[E ~[]byte](c *Cipher, plan *record.Plan, field string, input vc // `eql_v4.json_tree_subtree(col, $1::text[], 64)`. func (r JSONRead[E]) Tokens() JSONTokens { return r.tokens } +// String names the field and counts the tokens, and hides the path and +// the tokens: a path's keys can be a customer's data, and a token is a +// capability. +func (r JSONRead[E]) String() string { + if r.cipher == nil { + return "encrypt.JSONRead{}" + } + return fmt.Sprintf("encrypt.JSONRead{field: %q, path: %s, tokens: %d}", r.field, redacted, len(r.tokens)) +} + +// GoString is what %#v prints: [JSONRead.String]. +func (r JSONRead[E]) GoString() string { return r.String() } + +// Format prints [JSONRead.String] for every verb fmt hands it. +func (r JSONRead[E]) Format(f fmt.State, _ rune) { _, _ = io.WriteString(f, r.String()) } + +// LogValue is what slog records: the field, [redacted] for the path, and +// how many tokens. +func (r JSONRead[E]) LogValue() slog.Value { + if r.cipher == nil { + return slog.GroupValue() + } + return slog.GroupValue(slog.String("field", r.field), slog.String("path", redacted), slog.Int("tokens", len(r.tokens))) +} + // Open opens the rows the read returned, one group per database row, with // one ZeroKMS request: per row, the value at each concrete path the read // named that the row has, in document order. A row with no node there diff --git a/languages/golang/encrypt/json_fields_test.go b/languages/golang/encrypt/json_fields_test.go index eb38d31c8..6384c92d7 100644 --- a/languages/golang/encrypt/json_fields_test.go +++ b/languages/golang/encrypt/json_fields_test.go @@ -1,10 +1,12 @@ package encrypt_test import ( + "bytes" "context" "encoding/json" "errors" "fmt" + "log/slog" "reflect" "strings" "testing" @@ -174,3 +176,49 @@ func TestACustomersFieldsDeriveEachQuery(t *testing.T) { t.Fatal("Prefs has an At") } } + +// A token is a capability and a path's keys can be a customer's data: the +// read types print and log neither, under any verb or through slog. +func TestJSONReadTypesPrintAndLogRedacted(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + cipher := c.DefaultKeyset() + f := testusers.CustomerFields.Profile + sel, err := f.Selector(ctx, cipher, "$.secretkey") + if err != nil { + t.Fatal(err) + } + read, err := f.Subtree(ctx, cipher, "$.secretkey[*]", encrypt.JSONReadOptions{WildcardBound: 2}) + if err != nil { + t.Fatal(err) + } + token, err := f.Children(ctx, cipher, "$.secretkey") + if err != nil { + t.Fatal(err) + } + hidden := append([]string{"secretkey", string(sel.Selector()), string(token)}, read.Tokens()...) + var logged bytes.Buffer + logger := slog.New(slog.NewJSONHandler(&logged, nil)) + for name, v := range map[string]any{"selection": sel, "read": read, "token": token, "tokens": read.Tokens()} { + logger.Info(name, "v", v) + for _, verb := range []string{"%v", "%+v", "%#v", "%s", "%q", "%x", "%d"} { + out := fmt.Sprintf(verb, v) + for _, h := range hidden { + if strings.Contains(out, h) { + t.Errorf("%s printed with %s shows %q: %s", name, verb, h, out) + } + } + if !strings.Contains(out, "encrypt.JSON") { + t.Errorf("%s printed with %s does not name its type: %s", name, verb, out) + } + } + } + for _, h := range hidden { + if strings.Contains(logged.String(), h) { + t.Errorf("slog shows %q: %s", h, logged.String()) + } + } + if !strings.Contains(logged.String(), `"field":"profile"`) || !strings.Contains(logged.String(), `"tokens":3`) { + t.Errorf("slog records no field or token count: %s", logged.String()) + } +} From e33cf3e9f786f58d618408a0c30d318495cebe64 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:12:44 +1100 Subject: [PATCH 14/26] docs(go): name the JSON read constructors' declaration parameter NewJSONSelection and NewJSONRead are exported, so their parameter names are part of what godoc shows. The parameter was named plan, a word for the engine's internal form that a user never sees (Go principle 7). It is now declaration, for what it is: the declaration of the record type the field belongs to, and the docs say so. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/encrypt/json.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/languages/golang/encrypt/json.go b/languages/golang/encrypt/json.go index 648ec3a37..77c3de098 100644 --- a/languages/golang/encrypt/json.go +++ b/languages/golang/encrypt/json.go @@ -236,9 +236,10 @@ type JSONSelection[E ~[]byte] struct { } // NewJSONSelection is a selection prepared by [Cipher.QueryTerm]'s -// selector operator. For generated code. -func NewJSONSelection[E ~[]byte](c *Cipher, plan *record.Plan, field, path, selector string) JSONSelection[E] { - return JSONSelection[E]{cipher: c, plan: plan, field: field, path: path, selector: JSONSelector(selector)} +// selector operator on field of the record type that declaration +// declares, at path. For generated code. +func NewJSONSelection[E ~[]byte](c *Cipher, declaration *record.Plan, field, path, selector string) JSONSelection[E] { + return JSONSelection[E]{cipher: c, plan: declaration, field: field, path: path, selector: JSONSelector(selector)} } // Selector is the selector to bind: `col -> $1::text`. @@ -324,9 +325,10 @@ type JSONRead[E ~[]byte] struct { } // NewJSONRead is a read prepared by [Cipher.QueryTerm]'s subtree operator -// from input, its {"path", "wildcard_bound"?} object. For generated code. -func NewJSONRead[E ~[]byte](c *Cipher, plan *record.Plan, field string, input vcvalue.Object, tokens []string) JSONRead[E] { - return JSONRead[E]{cipher: c, plan: plan, field: field, input: input, tokens: JSONTokens(tokens)} +// on field of the record type that declaration declares, from input, its +// {"path", "wildcard_bound"?} object. For generated code. +func NewJSONRead[E ~[]byte](c *Cipher, declaration *record.Plan, field string, input vcvalue.Object, tokens []string) JSONRead[E] { + return JSONRead[E]{cipher: c, plan: declaration, field: field, input: input, tokens: JSONTokens(tokens)} } // Tokens are the tokens to bind: From 8736697e2da1abc0536beac006a85376059df448 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:14:10 +1100 Subject: [PATCH 15/26] fix(go): refuse ordered paths the engine answers no ordering on stashgen wrote a document field's At accessors only when the engine answered lt_at on it. A tag that names ordered paths on a field the engine answers no ordering on means the generator and the engine it embeds disagree, and the generator then wrote the field with no At and said nothing: the paths the user asked to order had no methods, and the build still passed. planJSON now refuses that field by name, so the drift fails generation instead of producing a quieter API. jsonEntry writes At whenever the field has ordered paths, since planJSON has already checked the engine answers on them. The test runs the embedded engine with lt_at removed from its answer, which is the one way to reach the refusal: the real engine always answers the orderings on an ordered field. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/cmd/stashgen/main_test.go | 42 ++++++++++++++++++++++ languages/golang/stashgen/emit.go | 19 +++++++++- 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index c83f34fbd..df224efc5 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -347,6 +347,20 @@ func TestRunGeneratesJSONSearchFields(t *testing.T) { t.Errorf("the generated file lacks %q", want) } } + // An engine that answers no ordering on a field whose tag orders paths + // disagrees with the generator: refused by name, not written as a field + // with no At. + dir := writeModule(t, map[string]string{"model.go": source("\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.age)\"`")}) + stderrBuf := bytes.Buffer{} + if code := run([]string{"-type", "Customer"}, dir, &bytes.Buffer{}, &stderrBuf, withoutOperator("lt_at")); code != 1 { + t.Fatalf("exit %d, want 1\n%s", code, stderrBuf.String()) + } + if got := stderrBuf.String(); !strings.Contains(got, "Customer.Profile") || !strings.Contains(got, "the engine answers no ordering (lt_at)") { + t.Fatalf("stderr = %q", got) + } + if _, err := os.Stat(filepath.Join(dir, "customer_stash.go")); !os.IsNotExist(err) { + t.Fatal("a file was written after the refusal") + } for _, c := range []struct{ name, fields, want string }{ {"a sealed null", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=JsonSearch,null=sealed\"`", "Customer.Prefs"}, {"a nil document that does not say null=sql", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=JsonSearch\"`", "null=sql"}, @@ -371,3 +385,31 @@ func TestRunGeneratesJSONSearchFields(t *testing.T) { }) } } + +// withoutOperator is the embedded engine with op removed from every field's +// answer: an engine that disagrees with the generator. +func withoutOperator(op string) func(context.Context) (stashgen.Engine, error) { + return func(ctx context.Context) (stashgen.Engine, error) { + engine, err := stashgen.GuestEngine(ctx) + return droppingEngine{Engine: engine, op: op}, err + } +} + +type droppingEngine struct { + stashgen.Engine + op string +} + +func (e droppingEngine) Check(ctx context.Context, d stashgen.Declaration) (map[string][]stashgen.Operator, error) { + answered, err := e.Engine.Check(ctx, d) + for field, ops := range answered { + kept := ops[:0] + for _, o := range ops { + if o.Name != e.op { + kept = append(kept, o) + } + } + answered[field] = kept + } + return answered, err +} diff --git a/languages/golang/stashgen/emit.go b/languages/golang/stashgen/emit.go index 080966b31..f31fcdb42 100644 --- a/languages/golang/stashgen/emit.go +++ b/languages/golang/stashgen/emit.go @@ -1026,10 +1026,26 @@ func planJSON(f *genFile, g *genField) error { g.jsonPaths = append(g.jsonPaths, jsonPath{name: name, path: path}) } } + // The engine answers the orderings on a field that orders a path. A + // tag that orders one and gets none is the engine and the generator + // disagreeing: refused, rather than written as a field with no At. + if len(g.jsonPaths) > 0 && !answers(g, "lt_at") { + return fieldErr(f.typeName, g.GoName, "the tag orders %d paths, and the engine answers no ordering (lt_at) on %s: the generator and its engine disagree, which is a bug in this stashgen", len(g.jsonPaths), g.EQLType) + } f.imports.add("encoding/json", "json") return nil } +// answers reports whether the engine answers op on the field. +func answers(g *genField, op string) bool { + for _, o := range g.operators { + if o.Name == op { + return true + } + } + return false +} + // jsonPathName is an ordered path's accessor name: its keys in Go case, // wildcard steps dropped. $.age is Age, $.visits[*].at is VisitsAt, $.a.1 // is A1, and $ is Root. An ordered path names no position (the engine @@ -1078,7 +1094,8 @@ func (w *writer) jsonEntry(g genField) { // public.eql_v4_json_entry), not for the stored type. leaf := strings.TrimSuffix(needle, "Query") + "LeafQuery" entry := "eql." + EQLGoName("JsonEntry") - ordered := len(g.jsonPaths) > 0 && answered["lt_at"] + // planJSON refuses ordered paths the engine answers no ordering on. + ordered := len(g.jsonPaths) > 0 field := "gensupport.Field[" + g.valueExpr + "]" w.p("type %s struct {", g.fieldType) From ef1bbd41685f5a0a5b29305aaaad9304a1b3543a Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:14:51 +1100 Subject: [PATCH 16/26] test(go): assert the reason stashgen gives for a sealed null document The "a sealed null" case asserted only that stderr named Customer.Prefs, so any refusal of that field passed it, including one for the wrong reason. It now asserts the field and the reason together: a document field carries an index other than equality, a nil has only an equality term, so a sealed nil would show which documents are absent, and the field must say null=sql. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/cmd/stashgen/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index df224efc5..df5410e40 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -362,7 +362,7 @@ func TestRunGeneratesJSONSearchFields(t *testing.T) { t.Fatal("a file was written after the refusal") } for _, c := range []struct{ name, fields, want string }{ - {"a sealed null", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=JsonSearch,null=sealed\"`", "Customer.Prefs"}, + {"a sealed null", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=JsonSearch,null=sealed\"`", "Customer.Prefs: a nil has an equality term and no other, so another index's column would show which values are absent; write null=sql"}, {"a nil document that does not say null=sql", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=JsonSearch\"`", "null=sql"}, {"a position in an ordered path", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.scores[2])\"`", "the engine refuses the declaration"}, {"a quoted key in a tag", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$[\\\"a b\\\"])\"`", "not a plain JSONPath"}, From 66a81a1bf8b2d8b5bc890f9853993d9ed44a549b Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:15:50 +1100 Subject: [PATCH 17/26] docs(go): say JsonSearch refuses a type defined over json.RawMessage stashgen accepts a scalar field declared with any type defined over the scalar (type Email string), so a reader would expect type Doc json.RawMessage to work for JsonSearch too. It does not: stashgen knows a document by the json.RawMessage type itself, so a defined type is bytes, and JsonSearch refuses it with "declare the field json.RawMessage". An alias (type Doc = json.RawMessage) is the type itself and is accepted. Both READMEs now say so, and the stashgen test pins both: the alias generates an eql.JSONSearch field, and the defined type is refused by name with that reason. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/cmd/stashgen/README.md | 1 + languages/golang/cmd/stashgen/main_test.go | 9 ++++++++- languages/golang/encrypt/README.md | 4 +++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index a72f31923..d5585c9b6 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -118,6 +118,7 @@ The field holds the document's JSON text; the engine parses it, so an object tha Values come back as JSON text holding the same values; key order and the spelling of numbers are not kept. A `*json.RawMessage` field is optional and must say `null=sql`: a document has no fixed length to hide a missing one behind. A non-nil pointer to `null` is a stored JSON `null`. A `json.RawMessage` sealed any other way (`encrypt`, `index=equality`) is bytes, as any type defined over `[]byte` is. +`JsonSearch` takes `json.RawMessage` itself, or an alias of it (`type Doc = json.RawMessage`), and refuses a type defined over it (`type Doc json.RawMessage`): `stashgen` knows a document by the `json.RawMessage` type, where a scalar type is accepted with any type defined over it. The type takes options in parentheses: `ordered=` names the paths whose numbers and strings carry an ordering term, separated by `;`, and `sealed_root=false` turns off the sealed root (on by default). `encrypt_into=JsonSearch(ordered=$.age;$.visits[*].at)` orders two paths. diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index df5410e40..860e8d309 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -302,7 +302,7 @@ func TestRunGeneratesJSONSearchFields(t *testing.T) { } _ = checker.Close() source := func(fields string) string { - return "package users\n\nimport \"encoding/json\"\n\nvar _ json.RawMessage\n\ntype Customer struct {\n\t_ struct{} `stash:\"context=customers\"`\n\tID int64 `stash:\"id,passthrough\"`\n" + fields + "\n}\n" + return "package users\n\nimport \"encoding/json\"\n\ntype Doc json.RawMessage\n\ntype DocAlias = json.RawMessage\n\ntype Customer struct {\n\t_ struct{} `stash:\"context=customers\"`\n\tID int64 `stash:\"id,passthrough\"`\n" + fields + "\n}\n" } generate := func(fields string) (string, string, int) { dir := writeModule(t, map[string]string{"model.go": source(fields)}) @@ -347,6 +347,12 @@ func TestRunGeneratesJSONSearchFields(t *testing.T) { t.Errorf("the generated file lacks %q", want) } } + // An alias of json.RawMessage is the type itself, and is a document; a + // type defined over it is refused below. + written, stderr, code = generate("\tProfile DocAlias `stash:\"profile,encrypt_into=JsonSearch\"`") + if code != 0 || !strings.Contains(written, "Profile eql.JSONSearch") { + t.Fatalf("an alias of json.RawMessage: exit %d\n%s", code, stderr) + } // An engine that answers no ordering on a field whose tag orders paths // disagrees with the generator: refused by name, not written as a field // with no At. @@ -371,6 +377,7 @@ func TestRunGeneratesJSONSearchFields(t *testing.T) { {"two paths, one accessor", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.a_b;$.aB)\"`", "both name the accessor At.AB"}, {"the same ordered path twice", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.a;$.a)\"`", "the ordered path $.a is given twice"}, {"a path with no accessor name", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.__)\"`", "no Go accessor name"}, + {"a type defined over json.RawMessage", "\tProfile Doc `stash:\"profile,encrypt_into=JsonSearch\"`", "Customer.Profile: JsonSearch seals a json document: declare the field json.RawMessage"}, {"a string into a document", "\tProfile string `stash:\"profile,encrypt_into=JsonSearch\"`", "JsonSearch seals a json"}, {"options on a type that takes none", "\tEmail string `stash:\"email,encrypt_into=TextEq(ordered=$.a)\"`", "the engine refuses the declaration"}, } { diff --git a/languages/golang/encrypt/README.md b/languages/golang/encrypt/README.md index 72ec0be90..96343c0ca 100644 --- a/languages/golang/encrypt/README.md +++ b/languages/golang/encrypt/README.md @@ -595,7 +595,9 @@ column. `JsonSearch` stores a whole JSON document in one `public.eql_v4_json_search` column, from a `json.RawMessage` field (`stash:"profile,encrypt_into=JsonSearch(ordered=$.age)"`; -see the stashgen README). One entry per node is encrypted, so the database +see the stashgen README). Unlike the types above, a type defined over +`json.RawMessage` (`type Doc json.RawMessage`) is refused; an alias of it +(`type Doc = json.RawMessage`) is accepted. One entry per node is encrypted, so the database can test containment, equality and ordering at a path, return one node with `->`, and walk a subtree, without decrypting: From 726e3a3b6eda4aa07036c88bccf5fe93d916c8c7 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:16:11 +1100 Subject: [PATCH 18/26] docs(go): say what the JSON read types print and refuse to bind The encrypt README's JSON section said how the selector, token and token list bind, but not that a zero one is refused, nor that the read types print and log redacted while JSONValue prints the plaintext it holds. A reader deciding what is safe to log needs both, and the README is where they look before the godoc. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/encrypt/README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/languages/golang/encrypt/README.md b/languages/golang/encrypt/README.md index 96343c0ca..2376a2db5 100644 --- a/languages/golang/encrypt/README.md +++ b/languages/golang/encrypt/README.md @@ -617,7 +617,12 @@ path and the cipher's keyset, and refuses a row that is not one it asked for. It does not prove the database returned every row, or rows of the database row you meant. `encrypt.JSONTokens` binds as a `text[]` literal, and a selector or token as `text`; cast each parameter, as above, so -PostgreSQL picks EQL's operator and not `jsonb`'s. Numbers order as `float64`, +PostgreSQL picks EQL's operator and not `jsonb`'s; a zero one is refused +rather than bound. A token is a capability over its node's subtree in every +row of the column, and a path's keys can be your customer's data, so +`JSONToken`, `JSONTokens`, `JSONSelection` and `JSONRead` print and log with +the tokens, the selector and the path redacted. A `JSONValue` holds the +plaintext a read opened, and prints it. Numbers order as `float64`, so integers past 2^53 can tie at an ordered path; containment and `EqAt` compare them exactly. From 0d38691be5449ea52014ff222b64586077126e99 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:27:17 +1100 Subject: [PATCH 19/26] fix(stack-encrypt,go): follow the random entry order and the cascade in the JSON operators #1379 stores a document's entries in a random order, shuffles every containment query's selector set, and makes an entry's ciphertext commit to its children's. Three things here assumed otherwise: - the data plan's operand parity test compared a containment or equality operand with the typed chain's byte for byte; it now reads both back as JsonQuery, which compares as sets; - `->` on a container with members now fails as JsonNotALeaf (Go: ErrEncoding, code stack_encrypt::json_not_a_leaf), so a selector read of `$.visits` is pinned to that refusal in the data plan test and through the Go guest, and json_open's and Cipher.JSONOpen's docs say it; - the docs said an ordering or `->` through a wildcard path compares the first element; it sees an unspecified one, and an ordering over every element is written over all the matches (json_tree_path_query). The same wording is corrected in JsonOptions::ordered, JsonOrderingAtPosition and the typed orderings' position check. The data plan's identifiers follow #1379's naming: a containment operand is a document's selector set, not a needle (DocumentInput, Input::Document, Binds::Query). Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/encrypt/guest/src/ops.rs | 2 +- languages/golang/encrypt/json.go | 6 ++- languages/golang/encrypt/json_tree_test.go | 12 ++++- .../encryption/tests/json_tree_data_plan.rs | 53 +++++++++++++++---- .../stack-encrypt/src/dynamic/json_query.rs | 38 +++++++------ packages/stack-encrypt/src/dynamic/query.rs | 8 ++- packages/stack-encrypt/src/dynamic/target.rs | 2 +- packages/stack-encrypt/src/json/mod.rs | 7 +-- packages/stack-encrypt/src/json/query.rs | 4 +- packages/stack-encrypt/src/sem/mod.rs | 8 +-- 10 files changed, 99 insertions(+), 41 deletions(-) diff --git a/languages/golang/encrypt/guest/src/ops.rs b/languages/golang/encrypt/guest/src/ops.rs index 548c013e6..b152e312e 100644 --- a/languages/golang/encrypt/guest/src/ops.rs +++ b/languages/golang/encrypt/guest/src/ops.rs @@ -249,7 +249,7 @@ where /// that names an EQL type runs the type's query twin through this build's /// resolver, and `term` holds the twin's JSON bytes and `domain` its /// PostgreSQL domain. A document's operator gives `encoding` `eql` (a -/// needle or leaf operand), `text` (a selector or node token) or +/// containment query or leaf operand), `text` (a selector or node token) or /// `text_array` (a subtree read's tokens). No data key is minted. pub async fn query_term( cipher: &KeysetCipher<'_, K>, diff --git a/languages/golang/encrypt/json.go b/languages/golang/encrypt/json.go index e2e4752ab..313514d59 100644 --- a/languages/golang/encrypt/json.go +++ b/languages/golang/encrypt/json.go @@ -21,7 +21,11 @@ import ( // the field's declaration and this cipher's keyset, and checks every row // against it; the rows never say which path they are for. A row that is not // one the read asked for is [ErrEncoding]; a wildcard read of an array -// longer than its bound is too. What comes back is, per group, the JSON +// longer than its bound is too, and so is a selector entry that does not +// open alone: a container with members, whose entry commits to children +// col -> does not return, or an altered entry, which the client cannot +// tell apart (code stack_encrypt::json_not_a_leaf). Read a container with a +// subtree read. What comes back is, per group, the JSON // text of each value the read reached, in document order: plaintext, which // the caller owns. One ZeroKMS request for every group. For generated code. // diff --git a/languages/golang/encrypt/json_tree_test.go b/languages/golang/encrypt/json_tree_test.go index 3c4027576..057b34ff3 100644 --- a/languages/golang/encrypt/json_tree_test.go +++ b/languages/golang/encrypt/json_tree_test.go @@ -201,7 +201,7 @@ func TestJSONSelectorReadsOpenAgainstTheirRequest(t *testing.T) { c := deterministicEQLClient(t) ctx := context.Background() cipher := c.DefaultKeyset() - sealed, err := cipher.Seal(ctx, jsonPlan(), []record.Source{{"profile": []byte(`{"name":"Alice","age":34}`)}}) + sealed, err := cipher.Seal(ctx, jsonPlan(), []record.Source{{"profile": []byte(`{"name":"Alice","age":34,"visits":[{"at":3}]}`)}}) if err != nil { t.Fatal(err) } @@ -246,6 +246,16 @@ func TestJSONSelectorReadsOpenAgainstTheirRequest(t *testing.T) { if !errors.Is(err, encrypt.ErrEncoding) { t.Fatalf("another path's entry: %v", err) } + // col -> on a container with members returns its entry alone, which + // commits to children the read does not hold: it does not open, and + // the refusal is ErrEncoding (a container or an altered entry, the + // client cannot tell which), never ErrAuthentication. + visits := extract(selector("$.visits")) + _, err = cipher.JSONOpen(ctx, jsonPlan(), "profile", vcvalue.Object{{Key: "operation", Value: "selector"}, {Key: "path", Value: "$.visits"}}, [][][]byte{{visits}}) + var d *encrypt.Diagnostic + if !errors.Is(err, encrypt.ErrEncoding) || errors.Is(err, encrypt.ErrAuthentication) || !errors.As(err, &d) || d.Code != "stack_encrypt::json_not_a_leaf" { + t.Fatalf("a container through ->: %v", err) + } } // decodeNumbers decodes JSON text with each number kept as its text. diff --git a/packages/eql/tests/encryption/tests/json_tree_data_plan.rs b/packages/eql/tests/encryption/tests/json_tree_data_plan.rs index 6f1d1f86a..77fcb1265 100644 --- a/packages/eql/tests/encryption/tests/json_tree_data_plan.rs +++ b/packages/eql/tests/encryption/tests/json_tree_data_plan.rs @@ -27,7 +27,7 @@ use serde_json::{json, Value}; use stack_encrypt::dynamic::{ self, record, Scope, TargetDescriptor, TargetError, TargetResolver, ValueFrame, }; -use stack_encrypt::json::{JsonDocument, JsonOptions}; +use stack_encrypt::json::{JsonDocument, JsonOptions, JsonQuery}; use stack_encrypt::target::{ExpectedContext, Layout}; use stack_encrypt::{CipherText, Encrypt, KeysetCipher, Label, NonEmpty, Pending, StackCipher}; @@ -527,6 +527,13 @@ fn one(term: &dynamic::QueryTerm) -> Value { serde_json::from_slice(bytes).unwrap() } +/// A containment or equality operand, read back as the engine's +/// `JsonQuery`, which compares as sets: each derivation sends its selector +/// set in a random order of its own. +fn containment(operand: Value) -> JsonQuery { + serde_json::from_value(operand).unwrap() +} + /// The operators a document field answers, with the input each takes: /// the orderings at a path only when the field orders a path. #[test] @@ -585,7 +592,8 @@ fn a_document_field_answers_the_json_operators() { } /// Every operand the data path derives is the typed chain's, in the EQL -/// wire form, bound as the domain the target names for it. +/// wire form, bound as the domain the target names for it. A containment +/// or equality operand is the same selector set, in its own random order. #[tokio::test] async fn every_operand_is_the_typed_chains() { use stack_encrypt::op; @@ -593,32 +601,32 @@ async fn every_operand_is_the_typed_chains() { let keyset = cipher.default_keyset(); let plan = plan(); let typed = typed_plan(); - let needle = json!({"visits": [{"at": 3}], "name": "Alice"}); + let document = json!({"visits": [{"at": 3}], "name": "Alice"}); for (operator, typed_op) in [("contains", true), ("contained_by", false)] { let term = data_term( &keyset, &plan, operator, - &[("json", json!(needle.to_string()))], + &[("json", json!(document.to_string()))], ) .await; let typed = if typed_op { cipher - .query(op::Contains, &needle) + .query(op::Contains, &document) .on(&typed) .await .unwrap() } else { cipher - .query(op::ContainedBy, &needle) + .query(op::ContainedBy, &document) .on(&typed) .await .unwrap() }; assert_eq!( - one(&term), - serde_json::to_value(&typed).unwrap(), + containment(one(&term)), + containment(serde_json::to_value(&typed).unwrap()), "{operator}" ); assert_eq!(term.domain(), Some("eql_v4.query_json")); @@ -637,7 +645,10 @@ async fn every_operand_is_the_typed_chains() { .on(&typed) .await .unwrap(); - assert_eq!(one(&eq), serde_json::to_value(&typed_eq).unwrap()); + assert_eq!( + containment(one(&eq)), + containment(serde_json::to_value(&typed_eq).unwrap()) + ); assert_eq!(eq.domain(), Some("eql_v4.query_json")); let lt = data_term( @@ -876,7 +887,8 @@ fn subtree_rows(doc: &stack_encrypt::json::JsonDocument, tokens: &[String]) -> V .collect() } -/// The entry `doc -> selector` returns: the first entry carrying it. +/// The entry `doc -> selector` returns: the first entry carrying it in +/// stored order. fn extracted_row(doc: &stack_encrypt::json::JsonDocument, selector: &str) -> Option { use base64::Engine; let selector: [u8; 16] = base64::engine::general_purpose::STANDARD @@ -990,6 +1002,27 @@ async fn reads_open_against_the_request_that_prepared_them() { "absence is no value, not null" ); + // `->` on a container with members returns its entry alone, which + // commits to children the read does not return: it does not open, and + // the refusal names that cause and an altered entry, claiming neither. + // A container is read with `subtree`. + let visits = data_term(&keyset, &plan, "selector", &[("path", json!("$.visits"))]).await; + let dynamic::Operand::Text(visits) = visits.operand() else { + panic!() + }; + let container = open( + &keyset, + &plan, + json!({"operation": "selector", "path": "$.visits"}), + vec![extracted_row(&stored[0], visits).into_iter().collect()], + ) + .await; + assert!( + matches!(container, Err(stack_encrypt::Error::JsonNotALeaf)), + "{:?}", + container.err() + ); + // A wildcard read: every element's `at` below the bound, per row. let tokens = data_term( &keyset, diff --git a/packages/stack-encrypt/src/dynamic/json_query.rs b/packages/stack-encrypt/src/dynamic/json_query.rs index 86060dcbf..8a59d40be 100644 --- a/packages/stack-encrypt/src/dynamic/json_query.rs +++ b/packages/stack-encrypt/src/dynamic/json_query.rs @@ -9,8 +9,8 @@ //! //! | operator | input | operand | //! |---|---|---| -//! | `contains`, `contained_by` | `{"json"}`: a JSON needle | EQL: the needle's projection (`eql_v4.query_json`) | -//! | `eq_at` | `{"path", "json"}`: a scalar at a path | EQL: the needle holding the scalar there | +//! | `contains`, `contained_by` | `{"json"}`: a JSON document | EQL: the document's selector set (`eql_v4.query_json`) | +//! | `eq_at` | `{"path", "json"}`: a scalar at a path | EQL: the selector set of the document holding the scalar there | //! | `lt_at`, `lte_at`, `gt_at`, `gte_at` | `{"path", "json"}`: a number or string at an ordered path | EQL: the path's selector and the leaf's ordering term (`eql_v4.query_json_leaf`) | //! | `between_at` | `{"path", "low", "high"}` | EQL, two of them: the low bound's then the high's | //! | `selector` | `{"path"}` | text: the selector `->` takes | @@ -24,9 +24,13 @@ //! `selector`): a selector reads every position as the wildcard, so the //! query would compare whichever element the database found first. A //! positional read is a `subtree` read. Equality at a wildcard path matches -//! when any element matches; an ordering or `->` through one compares the -//! first. Two `eq_at` predicates on one row need not hold of the same -//! array element; one containment needle holding both does. +//! when any element matches. `->` at a wildcard path, and an ordering +//! written over it (`eql_v4.ord_term(col -> sel)`), see an unspecified +//! element of the row, since entries are stored in a random order; an +//! ordering over every element is written over all its matches +//! (`json_tree_path_query`). Two `eq_at` predicates on one row need not +//! hold of the same array element; one containment query of a document +//! holding both does. //! //! # Reads //! @@ -64,8 +68,8 @@ use crate::{KeysetCipher, Label, NonEmpty, Pending}; /// The SQL type a JSON operand binds as. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum Binds { - /// The target's needle domain (`eql_v4.query_json`). - Needle, + /// The target's query domain (`eql_v4.query_json`). + Query, /// The target's leaf operand domain (`eql_v4.query_json_leaf`). Leaf, /// `text`. @@ -80,7 +84,7 @@ pub(super) enum Binds { /// must be placed. pub(super) fn binds(op: OpSpec) -> Option { match op { - OpSpec::Contains | OpSpec::ContainedBy | OpSpec::EqAt => Some(Binds::Needle), + OpSpec::Contains | OpSpec::ContainedBy | OpSpec::EqAt => Some(Binds::Query), OpSpec::LtAt | OpSpec::LteAt | OpSpec::GtAt | OpSpec::GteAt | OpSpec::BetweenAt => { Some(Binds::Leaf) } @@ -96,11 +100,11 @@ pub(super) fn binds(op: OpSpec) -> Option { } } -/// The SQL type an operand of `binds` binds as for `target`: its needle +/// The SQL type an operand of `binds` binds as for `target`: its query /// or leaf domain, which a resolver may not name, or text. pub(super) fn domain(target: &super::TargetDescriptor, binds: Binds) -> Option { match binds { - Binds::Needle => target.query_sql_domain.clone(), + Binds::Query => target.query_sql_domain.clone(), Binds::Leaf => target.leaf_query_sql_domain.clone(), Binds::Text => Some("text".to_owned()), Binds::TextArray => Some("text[]".to_owned()), @@ -162,7 +166,7 @@ fn json_reason(error: crate::json::JsonTextError) -> Reason { #[derive(Deserialize)] #[serde(deny_unknown_fields)] -struct NeedleInput { +struct DocumentInput { json: JsonText, } @@ -198,7 +202,7 @@ struct ReadInput { /// An operator's input, read from its frame and checked: everything /// [`derive`] would refuse before it derives anything. enum Input { - Needle(Parsed), + Document(Parsed), At(JsonPath, Parsed), Between(JsonPath, Parsed, Parsed), Path(JsonPath), @@ -243,8 +247,8 @@ fn term(error: TermError) -> Error { /// Read and check an operator's input against the field's options. fn read_input(op: OpSpec, options: &JsonOptions, input: &ValueFrame) -> Result { match op { - OpSpec::Contains | OpSpec::ContainedBy => Ok(Input::Needle( - read_frame::(input)?.json.parse()?, + OpSpec::Contains | OpSpec::ContainedBy => Ok(Input::Document( + read_frame::(input)?.json.parse()?, )), OpSpec::EqAt => read_eq_at(input), OpSpec::LtAt | OpSpec::LteAt | OpSpec::GtAt | OpSpec::GteAt => { @@ -355,7 +359,7 @@ pub(super) fn derive<'a, K: 'static>( ) -> Result, Error> { let column = || NonEmpty::from(label.clone()); Ok(match read_input(op, options, input)? { - Input::Needle(needle) => terms(keyset, options, label, needle.0.clone()) + Input::Document(document) => terms(keyset, options, label, document.0.clone()) .try_map(|terms| eql(&JsonQuery::of_terms(terms)).map(Operand::One)), Input::At(path, value) if op == OpSpec::EqAt => { terms(keyset, options, label, path.holding(value.0.clone())) @@ -490,7 +494,9 @@ enum Opener { /// not parse, a wildcard that expands too far). A row that is not the one /// the read asked for fails the pending /// ([`crate::Error::JsonWrongEntry`]), as does a wildcard read of an array -/// longer than the bound ([`crate::Error::JsonWildcardTruncated`]). +/// longer than the bound ([`crate::Error::JsonWildcardTruncated`]). A +/// `selector` entry that does not open alone, a container with members or +/// an altered entry, is [`crate::Error::JsonNotALeaf`]. pub(super) fn open<'a, K: 'static>( keyset: &'a KeysetCipher<'_, K>, label: &Label, diff --git a/packages/stack-encrypt/src/dynamic/query.rs b/packages/stack-encrypt/src/dynamic/query.rs index 75eabe874..43fe66844 100644 --- a/packages/stack-encrypt/src/dynamic/query.rs +++ b/packages/stack-encrypt/src/dynamic/query.rs @@ -120,13 +120,13 @@ pub enum OpSpec { /// the field's match index fixes. Matches, /// `"contains"`: [`op::Contains`](crate::op::Contains), a stored - /// document containing a JSON needle (`@>`). Answered by the JSON index + /// document containing a JSON document (`@>`). Answered by the JSON index /// of a field that stores a document. The inputs of this and the /// operators below are named by [`OpSpec::input`]; [`query_term`] /// derives their operands and [`json_open`] opens their reads. Contains, /// `"contained_by"`: [`op::ContainedBy`](crate::op::ContainedBy), a - /// stored document contained by a JSON needle (`<@`). + /// stored document contained by a JSON document (`<@`). ContainedBy, /// `"eq_at"`: [`op::AtPath`](crate::op::AtPath)`(op::Eq)`, a scalar at /// a path. @@ -1011,6 +1011,10 @@ pub fn check_json_open( /// ([`Error::JsonWrongEntry`](crate::Error::JsonWrongEntry)), as does a /// wildcard read of an array longer than its bound /// ([`Error::JsonWildcardTruncated`](crate::Error::JsonWildcardTruncated)). +/// A `selector` entry that does not open alone, a container with members +/// or an altered entry, fails it as +/// [`Error::JsonNotALeaf`](crate::Error::JsonNotALeaf); a container is read +/// with `subtree`. pub fn json_open<'a, K>( cipher: &'a KeysetCipher<'_, K>, plan: &Plan, diff --git a/packages/stack-encrypt/src/dynamic/target.rs b/packages/stack-encrypt/src/dynamic/target.rs index 01bd4581e..197c51cc4 100644 --- a/packages/stack-encrypt/src/dynamic/target.rs +++ b/packages/stack-encrypt/src/dynamic/target.rs @@ -106,7 +106,7 @@ pub struct TargetDescriptor { /// The PostgreSQL domain of the type's ordering operand at a path, for /// a type that stores a JSON document /// ([`with_leaf_query_sql_domain`](Self::with_leaf_query_sql_domain)): - /// what an ordering at a path binds as, beside the containment needle + /// what an ordering at a path binds as, beside the containment query /// [`query_sql_domain`](Self::query_sql_domain) names. Not on the /// wire; a generator reads it from the operators' terms. pub leaf_query_sql_domain: Option, diff --git a/packages/stack-encrypt/src/json/mod.rs b/packages/stack-encrypt/src/json/mod.rs index ede81a383..b550df447 100644 --- a/packages/stack-encrypt/src/json/mod.rs +++ b/packages/stack-encrypt/src/json/mod.rs @@ -195,9 +195,10 @@ pub struct JsonOptions { /// (`$.scores[2]`) is refused, here and in an ordering query /// ([`TermError::JsonOrderingAtPosition`](crate::sem::TermError::JsonOrderingAtPosition)): /// an ordering compares the leaf `->` extracts by the path's selector, - /// which reads every position as the wildcard, so it reaches the first - /// element at the path and never the one a position names. An ordering - /// asked at a path no entry here covers is refused + /// which reads every position as the wildcard, so it reaches an + /// unspecified element at the path (entries are stored in a random + /// order) and never the one a position names. An ordering asked at a + /// path no entry here covers is refused /// ([`TermError::JsonPathNotOrdered`](crate::sem::TermError::JsonPathNotOrdered)). pub ordered: BTreeSet, } diff --git a/packages/stack-encrypt/src/json/query.rs b/packages/stack-encrypt/src/json/query.rs index 99246ae85..3f86bd731 100644 --- a/packages/stack-encrypt/src/json/query.rs +++ b/packages/stack-encrypt/src/json/query.rs @@ -785,8 +785,8 @@ fn ordered(value: &Protected) -> Result<(), Error> { /// Whether the index orders the leaves at `path`: some path in its /// `ordered` option covers it. A path that names a position is refused /// first: the comparison extracts by selector, which reads the position as -/// the wildcard, so it would compare the first element at the path rather -/// than the one named. +/// the wildcard, so it would compare an unspecified element at the path +/// rather than the one named. fn orders(index: &Json, path: &JsonPath) -> Result<(), Error> { if path.names_a_position() { Err(TermError::JsonOrderingAtPosition.into()) diff --git a/packages/stack-encrypt/src/sem/mod.rs b/packages/stack-encrypt/src/sem/mod.rs index 35dcfce34..c544e0df4 100644 --- a/packages/stack-encrypt/src/sem/mod.rs +++ b/packages/stack-encrypt/src/sem/mod.rs @@ -341,13 +341,13 @@ pub enum TermError { /// An ordered path, or an ordering asked at a path, names an array /// position. An ordering compares the leaf `->` extracts by the path's /// selector, and a selector reads every position as the wildcard, so - /// `->` returns the first element at the path whatever position was - /// named: the comparison would be against another element's leaf, or - /// against one with no term. + /// `->` returns an unspecified element at the path whatever position + /// was named: the comparison would be against another element's leaf, + /// or against one with no term. #[error("an ordering at a JSON path cannot name an array position")] #[diagnostic( code(stack_encrypt::json_ordering_at_position), - help("Write `[*]` where an ordered path, or the path an ordering asks at, crosses an array: `$.orders[*].total`, not `$.orders[2].total`. The comparison extracts the first element at the path.") + help("Write `[*]` where an ordered path, or the path an ordering asks at, crosses an array: `$.orders[*].total`, not `$.orders[2].total`. The comparison extracts one element at the path, by selector, never the one a position names.") )] JsonOrderingAtPosition, #[error("equality at a JSON path takes a scalar")] From 36898b2ffed9d37e25601d57ac738a39bae7bec5 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:27:33 +1100 Subject: [PATCH 20/26] fix(stack-encrypt): refuse a JSON operator path deeper than a document eq_at and the orderings build the document holding a value at the path they are given, one level per step, before anything checks its depth, and JsonPath::parse caps no length. A path of fifty thousand steps from a binding built a value that deep, whose recursive drop can overflow the stack and abort the host. A path with more steps than a document has levels (MAX_DEPTH) names no node, so every operator's path is now refused as JsonTooDeep when it is read, before anything is built or any keyset resolved. The rest of the operator module's docs: check_query and check_json_open say they refuse what the input alone shows (the limits a read's key hits are derive's and open's, after the keyset), json_open documents its query_context_field refusal (also in the changelog), OpSpec::fidelity says what exact means for the reads and tokens, and comments say why an ordering's position refusal is the index's own error while eq_at's and selector's is the data plan's, and what is_positional means at node_path's call site. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- packages/stack-encrypt/CHANGELOG.md | 5 ++- .../stack-encrypt/src/dynamic/json_query.rs | 32 +++++++++++++++++-- packages/stack-encrypt/src/dynamic/query.rs | 16 +++++++--- packages/stack-encrypt/src/json/mod.rs | 2 +- 4 files changed, 46 insertions(+), 9 deletions(-) diff --git a/packages/stack-encrypt/CHANGELOG.md b/packages/stack-encrypt/CHANGELOG.md index d95f260ed..f84f411d5 100644 --- a/packages/stack-encrypt/CHANGELOG.md +++ b/packages/stack-encrypt/CHANGELOG.md @@ -82,7 +82,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 prepared it: the engine derives the selector or the read key again from the request, the field's options and the keyset, checks every row with the index's own checks, and opens each database row's group on its own, - in one key request. `check_json_open` is its cipher-free check. + in one key request. `check_json_open` is its cipher-free check. A plan + whose context comes from a field of each record (`context_field`) is + refused by both (`query_context_field`), as it is by `query_term`: a read + through the data plan names no record to take a context from. - **A JSON document through a data plan** (#1349). A plan field may declare the new `json` value kind and name a target that stores a diff --git a/packages/stack-encrypt/src/dynamic/json_query.rs b/packages/stack-encrypt/src/dynamic/json_query.rs index 8a59d40be..ea5e261f6 100644 --- a/packages/stack-encrypt/src/dynamic/json_query.rs +++ b/packages/stack-encrypt/src/dynamic/json_query.rs @@ -216,8 +216,18 @@ fn read_frame Deserialize<'de>>(input: &ValueFrame) -> Result Result { - JsonPath::parse(text).map_err(|error| Error::Cipher(TermError::JsonPath(error).into())) + let path = + JsonPath::parse(text).map_err(|error| Error::Cipher(TermError::JsonPath(error).into()))?; + if path.segments().len() > crate::json::MAX_DEPTH { + return Err(term(TermError::JsonTooDeep)); + } + Ok(path) } /// A path whose operand is a selector, which reads a position as the @@ -233,6 +243,8 @@ fn selector_path(text: &str) -> Result { /// A path that names one node: no wildcard. fn node_path(text: &str) -> Result { let path = path(text)?; + // `is_positional`: every array step names a position, so the path + // has no wildcard and names at most one node. if path.is_positional() { Ok(path) } else { @@ -303,6 +315,14 @@ fn read_between(options: &JsonOptions, input: &ValueFrame) -> Result Result { let path = path(text)?; if path.names_a_position() { @@ -323,7 +343,11 @@ fn leaf(value: &Value) -> Result<(), Error> { } /// Check a JSON operator's input without a cipher: what [`derive`] -/// refuses before it derives anything. +/// refuses from the input alone (its shape, its paths and values, a path +/// deeper than a document). The limits a subtree read's key hits as it is +/// built, a wildcard expanding too wide or a path too deep for its +/// tokens (`JsonWildcardTooWide`, `JsonTooWide`, `JsonTooDeep`), are +/// [`derive`]'s, after the keyset is resolved. pub(super) fn check(op: OpSpec, options: &JsonOptions, input: &ValueFrame) -> Result<(), Error> { read_input(op, options, input).map(drop) } @@ -462,7 +486,9 @@ fn read_open(request: &ValueFrame, rows: &ValueFrame) -> Result<(Read, Vec Result<(), Error> { read_open(request, rows).map(drop) } diff --git a/packages/stack-encrypt/src/dynamic/query.rs b/packages/stack-encrypt/src/dynamic/query.rs index 43fe66844..84aaa0b4c 100644 --- a/packages/stack-encrypt/src/dynamic/query.rs +++ b/packages/stack-encrypt/src/dynamic/query.rs @@ -241,7 +241,13 @@ impl OpSpec { } /// How far the server's answer to this operator can be trusted: what - /// the typed path says by the trait the term implements. + /// the typed path says by the trait the term implements. For the JSON + /// reads and tokens (`selector`, `subtree`, `children`, `child_count`), + /// which are not predicates, `exact` says only that the server finds + /// the nodes the operand names, not that it returns them all (a read + /// opens what the rows hold; see [`json_open`]). Equality at a wildcard + /// path matches when any element does, and an ordering compared through + /// `col -> sel` at one sees an unspecified element of each row. pub fn fidelity(self) -> Fidelity { match self { OpSpec::Matches => Fidelity::Probabilistic, @@ -1005,9 +1011,11 @@ pub fn check_json_open( /// # Errors /// /// As [`query_term`] refuses a field that does not answer the read's -/// operator, and [`Error::Source`] naming the field for a request or rows -/// that do not parse ([`Reason::JsonInput`]). A row that is not one the -/// read asked for fails the pending +/// operator, and a plan whose context comes from a field of each record +/// ([`Reason::QueryContextField`]): a read names no record, so it has no +/// context to derive under. [`Error::Source`] names the field for a +/// request or rows that do not parse ([`Reason::JsonInput`]). A row that +/// is not one the read asked for fails the pending /// ([`Error::JsonWrongEntry`](crate::Error::JsonWrongEntry)), as does a /// wildcard read of an array longer than its bound /// ([`Error::JsonWildcardTruncated`](crate::Error::JsonWildcardTruncated)). diff --git a/packages/stack-encrypt/src/json/mod.rs b/packages/stack-encrypt/src/json/mod.rs index b550df447..5587c04c0 100644 --- a/packages/stack-encrypt/src/json/mod.rs +++ b/packages/stack-encrypt/src/json/mod.rs @@ -150,7 +150,7 @@ pub fn wipe_value(value: &mut serde_json::Value) { plain::wipe(value); } -pub(crate) use derive::{seal_pending, terms_pending}; +pub(crate) use derive::{seal_pending, terms_pending, MAX_DEPTH}; pub(crate) use plain::PlainJson; pub(crate) use shuffle::Randomness; From 8afa98f1db131ab23f596cd3f410ec9f380a858e Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:27:33 +1100 Subject: [PATCH 21/26] test(stack-encrypt): pin the operators' refusals, rows that are not a read's, and their heap The malformed-input test mapped every refusal without a data plan reason to "engine", so a refusal of another kind passed; each row now names the index's term error (JsonOrderingAtPosition, JsonPathNotOrdered, ...), and gains the over-deep path rows. The orderings test's doc promised a check it does not make; it now says where options on a target that takes none are refused. A new test refuses rows that are not the read's: a document sealed under another keyset (JsonWrongEntry for a selector, ForeignKeyset for a subtree, both before any key is retrieved), an entry of another field of the plan, and a subtree read's rows handed over as a selector read's; a leaf's selector row read as a subtree of the same path opens to the same value, which is the node that read names. heap_hygiene gains a watched run of query_term (Contains, EqAt, GtAt, BetweenAt) and of json_open (selector and subtree) through the data plan, with the marker in the operands and the stored values. It fails when the JSON terms operation drops its owned document unwiped (checked by making that release a plain drop). Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- packages/stack-encrypt/tests/heap_hygiene.rs | 167 ++++++++++++ .../stack-encrypt/tests/json_operators.rs | 250 ++++++++++++++++-- 2 files changed, 390 insertions(+), 27 deletions(-) diff --git a/packages/stack-encrypt/tests/heap_hygiene.rs b/packages/stack-encrypt/tests/heap_hygiene.rs index 5afe4182b..72a533386 100644 --- a/packages/stack-encrypt/tests/heap_hygiene.rs +++ b/packages/stack-encrypt/tests/heap_hygiene.rs @@ -800,6 +800,173 @@ mod json_index { drop(query); } + /// A document field's operators through the data plan + /// (`dynamic::query_term`) and its reads opened against their request + /// (`dynamic::json_open`) free no block holding plaintext or a data + /// key: the operator's parsed input and the document built from it, + /// and each opened value once its text is written. + #[test] + fn a_document_fields_operators_and_reads_leave_no_plaintext_or_key_on_the_heap() { + use base64ct::{Base64, Encoding}; + use stack_encrypt::dynamic::json_open; + + let rt = runtime(); + let cipher = rt.block_on(cipher()); + let keyset = cipher.default_keyset(); + let wire = json!({ + "profile": {"context": ["users", "profile"], "type": "json", "null": "sql", + "target": {"name": "JsonSearch", "options": {"ordered": ["$.age"]}}} + }); + let plan = record::plan_with( + &stack_encrypt::dynamic::frame::serde::to_vec(&wire).unwrap(), + &Documents, + ) + .unwrap(); + + let mark = marked("operand"); + let quoted = format!("{mark:?}"); + let document = format!(r#"{{{quoted}: [{quoted}], "name": {quoted}}}"#); + let inputs = [ + ( + OpSpec::Contains, + object(vec![("json", frame(document.as_str()))]), + ), + ( + OpSpec::EqAt, + object(vec![ + ("path", frame("$.name")), + ("json", frame(quoted.as_str())), + ]), + ), + ( + OpSpec::GtAt, + object(vec![ + ("path", frame("$.age")), + ("json", frame(quoted.as_str())), + ]), + ), + ( + OpSpec::BetweenAt, + object(vec![ + ("path", frame("$.age")), + ("low", frame(quoted.as_str())), + ("high", frame(quoted.as_str())), + ]), + ), + ]; + for (operator, input) in &inputs { + let term = watched(&format!("query_term ({operator:?})"), || { + rt.block_on(async { + query_term(&keyset, &plan, "profile", *operator, input, &Documents) + .expect("a document operator") + .await + .expect("its operand") + }) + }); + drop(term); + } + + // A stored document, sealed and laid out as the SQL returns its + // rows, before the watch. + let value = json!({ + "name": marked("name"), + "age": 34, + "visits": [{"at": 3, "place": marked("place")}, {"at": 1}], + }); + let doc = rt + .block_on(async { + keyset + .run( + json_document::<_, Borrowed>( + JsonOptions::new().ordering(["$.age"]).unwrap(), + ), + &value, + context(), + ) + .await + }) + .unwrap(); + let operand = |operator: OpSpec, path: &str| { + rt.block_on(async { + query_term( + &keyset, + &plan, + "profile", + operator, + object(vec![("path", frame(path))]), + &Documents, + ) + .unwrap() + .await + .unwrap() + }) + }; + let tag = + |text: &str| -> [u8; 16] { Base64::decode_vec(text).unwrap().try_into().unwrap() }; + let row = |j: usize, d: Option| { + let mut wire = serde_json::to_value(doc.node(j).unwrap()).unwrap(); + if let Some(d) = d { + wire["d"] = json!(d); + } + wire.to_string() + }; + let stack_encrypt::dynamic::Operand::Text(name) = + operand(OpSpec::Selector, "$.name").operand().clone() + else { + panic!("a selector is text") + }; + let at = doc + .entries() + .iter() + .position(|entry| entry.selector() == &tag(&name)) + .unwrap(); + let selected = vec![vec![row(at, None)]]; + let stack_encrypt::dynamic::Operand::TextList(tokens) = + operand(OpSpec::Subtree, "$.visits").operand().clone() + else { + panic!("a subtree read's tokens are a text array") + }; + let tokens: Vec<[u8; 16]> = tokens.iter().map(|t| tag(t)).collect(); + let read = vec![doc + .subtree(&tokens) + .into_iter() + .map(|(j, _, d)| row(j, Some(u64::from(d)))) + .collect::>()]; + for (what, request, rows) in [ + ( + "json_open (selector)", + object(vec![ + ("operation", frame("selector")), + ("path", frame("$.name")), + ]), + selected, + ), + ( + "json_open (subtree)", + object(vec![ + ("operation", frame("subtree")), + ("path", frame("$.visits")), + ]), + read, + ), + ] { + let rows = frame(rows); + let opened = watched(what, || { + rt.block_on(async { + json_open(&keyset, &plan, "profile", &request, &rows, &Documents) + .expect("a read") + .await + .expect("its values") + }) + }); + assert!(opened + .risky_ref() + .windows(MARK.len()) + .any(|window| window == MARK)); + } + *KEY.lock().unwrap() = None; + } + /// The spy sees the data key too: a copy of the key a seal was handed, /// freed while watched, fails the watch. #[test] diff --git a/packages/stack-encrypt/tests/json_operators.rs b/packages/stack-encrypt/tests/json_operators.rs index a16c8cbd6..fe67dcf36 100644 --- a/packages/stack-encrypt/tests/json_operators.rs +++ b/packages/stack-encrypt/tests/json_operators.rs @@ -24,9 +24,12 @@ use stack_encrypt::dynamic::{ use stack_encrypt::json::{JsonDocument, Tag}; use stack_encrypt::target::{Borrowed, CallerContext, IndexSpec}; use stack_encrypt::{CipherText, Encrypt, KeysetCipher, Label, NonEmpty, Pending, StackCipher}; -use stack_kms::FakeDataKeySource; +use stack_kms::{DataKeySource, IndexKeySource}; -type Keyset<'c> = KeysetCipher<'c, FakeDataKeySource>; +/// What the helpers below run over: any key source a test builds its +/// cipher on. +trait Source: DataKeySource + IndexKeySource + Send + Sync + 'static {} +impl Source for K {} /// A resolver holding one type, `JsonSearch`, sealed with the engine's /// JSON index under the options the plan resolved. It stores the @@ -123,7 +126,12 @@ fn json_frame(value: &Value) -> ValueFrame { ValueFrame::from_bytes(&dynamic::frame::serde::to_vec(value).unwrap()).unwrap() } -async fn term(keyset: &Keyset<'_>, plan: &record::Plan, operator: &str, input: Value) -> QueryTerm { +async fn term( + keyset: &KeysetCipher<'_, K>, + plan: &record::Plan, + operator: &str, + input: Value, +) -> QueryTerm { dynamic::query_term( keyset, plan, @@ -138,7 +146,11 @@ async fn term(keyset: &Keyset<'_>, plan: &record::Plan, operator: &str, input: V } /// A document sealed through the plan, as the engine's `JsonDocument`. -async fn seal(keyset: &Keyset<'_>, plan: &record::Plan, document: &Value) -> JsonDocument { +async fn seal( + keyset: &KeysetCipher<'_, K>, + plan: &record::Plan, + document: &Value, +) -> JsonDocument { let text = document.to_string(); let row = record::encrypt_with( keyset, @@ -149,16 +161,20 @@ async fn seal(keyset: &Keyset<'_>, plan: &record::Plan, document: &Value) -> Jso .unwrap() .await .unwrap(); - let CipherText::Map(fields) = &row else { + stored(&row, "profile") +} + +/// The document a sealed row stores for `field`. +fn stored(row: &stack_encrypt::StackCipherText, field: &str) -> JsonDocument { + let CipherText::Map(fields) = row else { panic!("a row is a map") }; - let Some((_, CipherText::Map(outputs))) = fields.iter().find(|(name, _)| name == "profile") - else { - panic!("profile is a map of outputs") + let Some((_, CipherText::Map(outputs))) = fields.iter().find(|(name, _)| name == field) else { + panic!("{field} is a map of outputs") }; let Some((_, CipherText::Passthrough(payload))) = outputs.iter().find(|(key, _)| key == "eql") else { - panic!("profile's stored value is a passthrough") + panic!("{field}'s stored value is a passthrough") }; let bytes: bytes::Bytes = dynamic::frame::serde::from_slice(&payload.0).unwrap(); serde_json::from_slice(&bytes.0).unwrap() @@ -197,8 +213,8 @@ fn subtree(doc: &JsonDocument, tokens: &[String]) -> Vec { .collect() } -async fn open( - keyset: &Keyset<'_>, +async fn open( + keyset: &KeysetCipher<'_, K>, plan: &record::Plan, request: Value, groups: Vec>, @@ -359,8 +375,10 @@ async fn every_operator_derives_its_tagged_operand() { assert!(matches!(children.operand(), Operand::Text(_))); } -/// A field with no ordered path answers no ordering; a target given -/// options it does not take is refused when the plan is built. +/// A field with no ordered path lists no ordering, and an ordering asked +/// of it is not answered. (Options on a target that takes none are +/// refused when the plan is built, `Reason::TargetOptions`: the record +/// module's plan tests.) #[test] fn the_orderings_follow_the_resolved_options() { let names = |plan: &record::Plan| -> Vec<&'static str> { @@ -404,10 +422,22 @@ fn malformed_inputs_are_refused_with_their_reason() { let check = |operator: OpSpec, input: Value| { dynamic::check_query(&plan, "profile", operator, json_frame(&input), &Documents) }; - let reason = |operator: OpSpec, input: Value| { + // The data plan's reason, or for the engine's own refusals the index's + // term error by name, so a refusal of another kind fails the row. + let reason = |operator: OpSpec, input: Value| -> String { let error = check(operator, input).unwrap_err(); assert!(!error.to_string().contains("secret"), "{error}"); - error.reason().map(Reason::as_str).unwrap_or("engine") + match (&error, error.reason()) { + (_, Some(reason)) => reason.as_str().to_owned(), + (dynamic::Error::Cipher(stack_encrypt::Error::Term(term)), None) => { + let name = format!("{term:?}"); + name.split(['(', ' ', '{']) + .next() + .unwrap_or_default() + .to_owned() + } + (other, None) => panic!("neither a reason nor a term error: {other:?}"), + } }; for (operator, input, want) in [ (OpSpec::Contains, json!({"needle": "{}"}), "json_input"), @@ -447,9 +477,9 @@ fn malformed_inputs_are_refused_with_their_reason() { ( OpSpec::LtAt, json!({"path": "$.visits[0].at", "json": "1"}), - // A position is not the ordered wildcard path: the engine - // refuses the path as one the field does not order. - "engine", + // A position on an ordering is the index's own refusal, the + // one the typed ordering gives. + "JsonOrderingAtPosition", ), ( OpSpec::Selector, @@ -471,25 +501,33 @@ fn malformed_inputs_are_refused_with_their_reason() { ( OpSpec::EqAt, json!({"path": "$.a", "json": r#"{"secret": 1}"#}), - "engine", + "JsonValueNotScalar", ), ( OpSpec::LtAt, json!({"path": "$.name", "json": "1"}), - "engine", + "JsonPathNotOrdered", ), ( OpSpec::GtAt, json!({"path": "$.age", "json": "true"}), - "engine", + "JsonLeafNotOrdered", + ), + (OpSpec::Selector, json!({"path": "secret"}), "JsonPath"), + // A path with more steps than a document has levels names no node, + // and is refused before a document as deep is built. + ( + OpSpec::EqAt, + json!({"path": format!("${}", ".a".repeat(50_000)), "json": "1"}), + "JsonTooDeep", + ), + ( + OpSpec::Selector, + json!({"path": format!("${}", ".a".repeat(128))}), + "JsonTooDeep", ), - (OpSpec::Selector, json!({"path": "secret"}), "engine"), ] { - assert_eq!( - reason(operator, input.clone()), - want, - "{operator:?} {input}" - ); + assert_eq!(reason(operator, input.clone()), want, "{operator:?}"); } for (operator, input) in [ (OpSpec::Contains, json!({"json": "[1, {\"a\": null}]"})), @@ -622,6 +660,164 @@ async fn reads_open_against_the_request_that_prepared_them() { ); } +/// Rows that are not the read's are refused whichever way they differ: +/// rows of a document sealed under another keyset (refused before any key +/// is retrieved, since the read opens through the keyset that derived it), +/// an entry of another field of the plan, and rows of a selector read +/// handed over as a subtree read's, or the reverse. +#[tokio::test] +async fn rows_that_are_not_the_reads_are_refused() { + let (cipher, _, retrieves) = common::counting_cipher().await; + let keyset = cipher.default_keyset(); + let other_keyset = cipher + .keyset(stack_kms::IdentifiedBy::Name("other".to_string().into())) + .await + .unwrap(); + let plan = plan(); + let document = json!({"name": "Alice", "age": 34, "visits": [{"at": 3}]}); + let mine = seal(&keyset, &plan, &document).await; + let theirs = seal(&other_keyset, &plan, &document).await; + let text = |t: QueryTerm| match t.operand() { + Operand::Text(text) => text.clone(), + other => panic!("{other:?}"), + }; + let list = |t: QueryTerm| match t.operand() { + Operand::TextList(list) => list.clone(), + other => panic!("{other:?}"), + }; + + // Another keyset's document: its selectors are not this keyset's, so + // no row is the one asked for, and nothing is retrieved. + let name = text(term(&keyset, &plan, "selector", json!({"path": "$.name"})).await); + let their_name = text(term(&other_keyset, &plan, "selector", json!({"path": "$.name"})).await); + let before = retrieves.load(std::sync::atomic::Ordering::SeqCst); + let refused = open( + &keyset, + &plan, + json!({"operation": "selector", "path": "$.name"}), + vec![extracted(&theirs, &their_name)], + ) + .await; + assert!( + matches!(refused, Err(stack_encrypt::Error::JsonWrongEntry { .. })), + "{refused:?}" + ); + let visits = list(term(&keyset, &plan, "subtree", json!({"path": "$.visits"})).await); + let their_visits = + list(term(&other_keyset, &plan, "subtree", json!({"path": "$.visits"})).await); + let refused = open( + &keyset, + &plan, + json!({"operation": "subtree", "path": "$.visits"}), + vec![subtree(&theirs, &their_visits)], + ) + .await; + // Rows the key's tokens do match, but under a header naming another + // keyset: the read opens through its own keyset alone. + assert!( + matches!(refused, Err(stack_encrypt::Error::ForeignKeyset { .. })), + "{refused:?}" + ); + assert_eq!( + retrieves.load(std::sync::atomic::Ordering::SeqCst), + before, + "another keyset's rows are refused before any key is retrieved" + ); + + // An entry of another field of the plan, under another column: the + // selector is derived under this field's column, so it is not the + // one asked for. + let two = record::plan_with( + &dynamic::frame::serde::to_vec(&json!({ + "profile": {"context": ["users", "profile"], "type": "json", "null": "sql", + "target": {"name": "JsonSearch", "options": {"ordered": ["$.age"]}}}, + "prefs": {"context": ["users", "prefs"], "type": "json", "null": "sql", + "target": "JsonSearch"} + })) + .unwrap(), + &Documents, + ) + .unwrap(); + let text_of = document.to_string(); + let row = record::encrypt_with( + &keyset, + frame(BTreeMap::from([("prefs", text_of.as_str())])), + &two, + &Documents, + ) + .unwrap() + .await + .unwrap(); + let prefs = stored(&row, "prefs"); + let prefs_name = text( + dynamic::query_term( + &keyset, + &two, + "prefs", + OpSpec::Selector, + json_frame(&json!({"path": "$.name"})), + &Documents, + ) + .unwrap() + .await + .unwrap(), + ); + assert_ne!(prefs_name, name, "a selector is the column's"); + let refused = open( + &keyset, + &plan, + json!({"operation": "selector", "path": "$.name"}), + vec![extracted(&prefs, &prefs_name)], + ) + .await; + assert!( + matches!(refused, Err(stack_encrypt::Error::JsonWrongEntry { .. })), + "{refused:?}" + ); + + // A selector read's row handed over as a subtree read's: the entry + // `->` returns for a leaf is the node a subtree read of the same path + // names, so it opens to the same value, and under another path's + // subtree read it is refused. A subtree read's rows of a container + // handed over as a selector read's are more than the one entry `->` + // returns, and are refused before anything is derived. + let same = open( + &keyset, + &plan, + json!({"operation": "subtree", "path": "$.name"}), + vec![extracted(&mine, &name)], + ) + .await + .unwrap(); + assert_eq!(same, [vec![json!("Alice")]]); + let refused = open( + &keyset, + &plan, + json!({"operation": "subtree", "path": "$.age"}), + vec![extracted(&mine, &name)], + ) + .await; + assert!( + matches!(refused, Err(stack_encrypt::Error::JsonWrongEntry { .. })), + "{refused:?}" + ); + let rows = subtree(&mine, &visits); + assert!(rows.len() > 1, "a container and its members"); + let refused = dynamic::json_open( + &keyset, + &plan, + "profile", + json_frame(&json!({"operation": "selector", "path": "$.visits"})), + frame(&vec![rows]), + &Documents, + ); + assert_eq!( + refused.err().and_then(|error| error.reason()), + Some(Reason::JsonInput), + "a subtree read's rows as a selector read's" + ); +} + /// A read's request and rows are checked before anything is derived. #[test] fn a_malformed_read_is_refused_before_any_key() { From a9a4da73307bf9045e8af2f730e64cd5ff63b10a Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:27:34 +1100 Subject: [PATCH 22/26] test(go): read a bounded subtree request through Cipher.JSONOpen A subtree read's request carries its wildcard bound as Go writes an integer; the guest test now opens one (a row with no node opens to no values) and refuses a bound that is not an integer as ErrEncoding. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/encrypt/json_tree_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/languages/golang/encrypt/json_tree_test.go b/languages/golang/encrypt/json_tree_test.go index 057b34ff3..b55a6302f 100644 --- a/languages/golang/encrypt/json_tree_test.go +++ b/languages/golang/encrypt/json_tree_test.go @@ -256,6 +256,21 @@ func TestJSONSelectorReadsOpenAgainstTheirRequest(t *testing.T) { if !errors.Is(err, encrypt.ErrEncoding) || errors.Is(err, encrypt.ErrAuthentication) || !errors.As(err, &d) || d.Code != "stack_encrypt::json_not_a_leaf" { t.Fatalf("a container through ->: %v", err) } + // A subtree read's request carries its wildcard bound as Go writes an + // integer, which the engine reads as its bound: a row with no node at + // the path opens to no values, and a bound the engine cannot read would + // be refused as ErrEncoding. (A read past the bound is refused as + // ErrEncoding too; TestJSONDocumentsQueryAndReadInPostgres reads one + // from PostgreSQL's own rows.) + bounded := vcvalue.Object{{Key: "operation", Value: "subtree"}, {Key: "path", Value: "$.visits[*].at"}, {Key: "wildcard_bound", Value: uint32(1)}} + opened, err = cipher.JSONOpen(ctx, jsonPlan(), "profile", bounded, [][][]byte{{}}) + if err != nil || len(opened) != 1 || len(opened[0]) != 0 { + t.Fatalf("a bounded subtree read of no rows: %q %v", opened, err) + } + unreadable := vcvalue.Object{{Key: "operation", Value: "subtree"}, {Key: "path", Value: "$.visits[*].at"}, {Key: "wildcard_bound", Value: "1"}} + if _, err := cipher.JSONOpen(ctx, jsonPlan(), "profile", unreadable, [][][]byte{{}}); !errors.Is(err, encrypt.ErrEncoding) { + t.Fatalf("a bound that is not an integer: %v", err) + } } // decodeNumbers decodes JSON text with each number kept as its text. From 9b01531f42e94ee6c78a5529858ddca9bd8f1e7a Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:27:34 +1100 Subject: [PATCH 23/26] docs(eql-bindings): drop a duplicated changelog line A merge left the JsonSearch entry's first line twice. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- packages/eql/crates/eql-bindings/CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/eql/crates/eql-bindings/CHANGELOG.md b/packages/eql/crates/eql-bindings/CHANGELOG.md index 4c619584b..d7ba01796 100644 --- a/packages/eql/crates/eql-bindings/CHANGELOG.md +++ b/packages/eql/crates/eql-bindings/CHANGELOG.md @@ -15,7 +15,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 other type, which a resolver hands Stack Encrypt (`TargetDescriptor::with_leaf_query_sql_domain`). -- **`JsonSearch`, the `json_tree` document, by name** (`targets` - **`JsonSearch`, the `json_tree` document, by name** (`targets` feature, #1348, #1349). The target table has a row for it, generated from the catalog's document family (`eql_domains::DOCUMENT_FAMILIES`): From 83bee5734e7b6f0d3c6d4421f9fc65d475da5e2e Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:43:08 +1100 Subject: [PATCH 24/26] fix(stack-encrypt): export MAX_DEPTH to the data plan alone The re-export the operator path cap reads is unused without the dynamic feature, which warnings-as-errors refuses; it is now gated on it. The operator test passes its rows by value, as clippy asks. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- packages/stack-encrypt/src/json/mod.rs | 6 +++++- packages/stack-encrypt/tests/json_operators.rs | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/stack-encrypt/src/json/mod.rs b/packages/stack-encrypt/src/json/mod.rs index 5587c04c0..34857bca5 100644 --- a/packages/stack-encrypt/src/json/mod.rs +++ b/packages/stack-encrypt/src/json/mod.rs @@ -150,7 +150,11 @@ pub fn wipe_value(value: &mut serde_json::Value) { plain::wipe(value); } -pub(crate) use derive::{seal_pending, terms_pending, MAX_DEPTH}; +/// The deepest a document nests: the data plan refuses a path with more +/// steps, which names no node. +#[cfg(feature = "dynamic")] +pub(crate) use derive::MAX_DEPTH; +pub(crate) use derive::{seal_pending, terms_pending}; pub(crate) use plain::PlainJson; pub(crate) use shuffle::Randomness; diff --git a/packages/stack-encrypt/tests/json_operators.rs b/packages/stack-encrypt/tests/json_operators.rs index fe67dcf36..174fc59cf 100644 --- a/packages/stack-encrypt/tests/json_operators.rs +++ b/packages/stack-encrypt/tests/json_operators.rs @@ -808,7 +808,7 @@ async fn rows_that_are_not_the_reads_are_refused() { &plan, "profile", json_frame(&json!({"operation": "selector", "path": "$.visits"})), - frame(&vec![rows]), + frame(vec![rows]), &Documents, ); assert_eq!( From 955b9a1f5f0f953d6e86020a0c003e1daac105a7 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:51:59 +1100 Subject: [PATCH 25/26] fix(go): write orderings at a wildcard path over every match A JSON document's entries are now stored in a random order (#1379), so `col -> sel` at a wildcard path, and an ordering through it, sees an unspecified element of each row rather than the first. The At accessors stay: their operands are what a comparison over every match takes, and the engine stores ordering terms at a wildcard path when asked to, so refusing ordered= there would leave those terms unreachable from Go. What changes is the SQL they are documented and tested with. - The generated XPath comment gives both forms: col -> Selector() at a path with no array step, and at a wildcard path EXISTS over eql_v4.json_tree_path_query for a comparison and the least or greatest match's ordering term for ORDER BY. Lt, Lte, Gt, Gte and Between get doc comments of their own, which they lacked, and Selector says which form it binds into. The READMEs and the CHANGELOG say the same. - TestJSONDocumentsQueryAndReadInPostgres compares at $.visits[*].at with EXISTS (some visit > 2, and some visit < 2, which only Alice's second visit meets), orders rows by their least visit ascending and greatest descending against the plaintexts' order, and reads a wildcard past its bound from PostgreSQL's own rows, which Open refuses as ErrEncoding. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/CHANGELOG.md | 4 +- languages/golang/cmd/stashgen/README.md | 12 +++ languages/golang/encrypt/README.md | 5 +- .../internal/testusers/customer_stash.go | 38 +++++++-- .../golang/encrypt/json_postgres_test.go | 80 ++++++++++++++----- languages/golang/stashgen/emit.go | 8 +- 6 files changed, 117 insertions(+), 30 deletions(-) diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index 22e73c11d..cdcbb0763 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -19,7 +19,9 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. `*json.RawMessage` field takes `null=sql`. Its `Fields` entry has `Contains`, `ContainedBy`, `EqAt`, `Selector`, `Subtree`, `Children` and `ChildCount`, and `At.` per ordered path with `Lt`, `Lte`, `Gt`, - `Gte`, `Between` and the path's own `Selector`. `encrypt.JSONSelection` + `Gte`, `Between` and the path's own `Selector`; at a wildcard path their + operands are compared over every match (`eql_v4.json_tree_path_query`), + since `->` there returns an unspecified element. `encrypt.JSONSelection` and `encrypt.JSONRead` open the rows `->` and `eql_v4.json_tree_subtree` return against the request that prepared them (`OpenOne` refuses a wildcard read up front); `encrypt.JSONSelector`, `JSONToken` and diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index d5585c9b6..45d473b53 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -137,6 +137,18 @@ The field's `Fields` entry has a method per operation: `At` has one accessor per ordered path, named for its keys (`$.visits[*].at` is `At.VisitsAt`, `$.a.1` is `At.A1`); two paths with one name, or one path given twice, stop `stashgen`. An ordering compares the value `->` extracts at its path, and EQL's `<` compares ordering terms alone, so take the selector from the same accessor (`At.Age.Selector`) rather than spelling the path again. +At a wildcard path (`$.visits[*].at`), `->` returns an unspecified element of each row (the entries are stored in a random order), so write the comparison over every match and order by the least or greatest one: + +```sql +-- rows with some visit after $2 +SELECT id FROM customers WHERE EXISTS ( + SELECT 1 FROM eql_v4.json_tree_path_query(profile, $1::text) AS e WHERE e > $2::eql_v4.query_json_leaf) +-- rows by their earliest visit +SELECT id FROM customers ORDER BY ( + SELECT eql_v4.ord_term(e) FROM eql_v4.json_tree_path_query(profile, $1::text) AS e ORDER BY 1 LIMIT 1) +``` + +with `$1` from `At.VisitsAt.Selector(ctx, c)` and `$2` from `At.VisitsAt.Gt(ctx, c, v)`; for the latest visit, order by `ORDER BY 1 DESC NULLS LAST` inside and `DESC` outside. `OpenOne` reads one value and refuses a read whose path has a wildcard before anything is opened; open a wildcard read with `Open`. The needle and the values are `json.RawMessage`: write the JSON you mean, such as `json.RawMessage(`+"`"+`{"active":true}`+"`"+`)`. A position in a path is refused for `EqAt` and `Selector`, whose selector reads every position as the wildcard; read a position with `Subtree`. diff --git a/languages/golang/encrypt/README.md b/languages/golang/encrypt/README.md index 2376a2db5..80261ab83 100644 --- a/languages/golang/encrypt/README.md +++ b/languages/golang/encrypt/README.md @@ -624,7 +624,10 @@ row of the column, and a path's keys can be your customer's data, so the tokens, the selector and the path redacted. A `JSONValue` holds the plaintext a read opened, and prints it. Numbers order as `float64`, so integers past 2^53 can tie at an ordered path; containment and `EqAt` -compare them exactly. +compare them exactly. At a wildcard path, `->` returns an unspecified +element of each row, so an ordering over every element is written over +`eql_v4.json_tree_path_query(col, $sel)`, its matches (the stashgen README +shows the SQL). ## Building diff --git a/languages/golang/encrypt/internal/testusers/customer_stash.go b/languages/golang/encrypt/internal/testusers/customer_stash.go index 86bb5579d..20acbf6f9 100644 --- a/languages/golang/encrypt/internal/testusers/customer_stash.go +++ b/languages/golang/encrypt/internal/testusers/customer_stash.go @@ -192,38 +192,62 @@ type CustomerProfileFieldAt struct { VisitsAt CustomerProfileFieldPath // $.visits[*].at } -// CustomerProfileFieldPath is one ordered path of Profile: its orderings -// compare the first value `->` extracts there, by the leaf's ordering term. -// Numbers order as float64, so integers past 2^53 can tie; text orders case- -// and accent-folded. +// CustomerProfileFieldPath is one ordered path of Profile. Its orderings derive +// the leaf operand (eql_v4.query_json_leaf) a stored leaf's ordering term is +// compared with, and Selector the path's selector. At a path with no array +// step, compare col -> Selector() with the operand, and order by +// eql_v4.ord_term(col -> Selector()). At a wildcard path ([*]) col -> +// Selector() is an unspecified element of each row, so write the comparison +// over every match, EXISTS (SELECT 1 FROM eql_v4.json_tree_path_query(col, +// $sel) e WHERE e < $1::eql_v4.query_json_leaf), and order by the least or +// greatest match's term: (SELECT eql_v4.ord_term(e) FROM +// eql_v4.json_tree_path_query(col, $sel) e ORDER BY 1 LIMIT 1). Numbers order +// as float64, so integers past 2^53 can tie; text orders case- and +// accent-folded. type CustomerProfileFieldPath struct { field gensupport.Field[json.RawMessage] path string } +// Lt derives the operand that finds a leaf at this path less than v: (col -> +// $sel) < $1::eql_v4.query_json_leaf, or at a wildcard path the same comparison +// over every match (see CustomerProfileFieldPath). func (p CustomerProfileFieldPath) Lt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.JSONSearchLeafQuery, error) { return gensupport.JSONQueryAt[eql.JSONSearchLeafQuery](ctx, p.field, c, "lt_at", p.path, v) } +// Lte derives the operand that finds a leaf at this path at most v: (col -> +// $sel) <= $1::eql_v4.query_json_leaf, or at a wildcard path the same +// comparison over every match (see CustomerProfileFieldPath). func (p CustomerProfileFieldPath) Lte(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.JSONSearchLeafQuery, error) { return gensupport.JSONQueryAt[eql.JSONSearchLeafQuery](ctx, p.field, c, "lte_at", p.path, v) } +// Gt derives the operand that finds a leaf at this path greater than v: (col -> +// $sel) > $1::eql_v4.query_json_leaf, or at a wildcard path the same comparison +// over every match (see CustomerProfileFieldPath). func (p CustomerProfileFieldPath) Gt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.JSONSearchLeafQuery, error) { return gensupport.JSONQueryAt[eql.JSONSearchLeafQuery](ctx, p.field, c, "gt_at", p.path, v) } +// Gte derives the operand that finds a leaf at this path at least v: (col -> +// $sel) >= $1::eql_v4.query_json_leaf, or at a wildcard path the same +// comparison over every match (see CustomerProfileFieldPath). func (p CustomerProfileFieldPath) Gte(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.JSONSearchLeafQuery, error) { return gensupport.JSONQueryAt[eql.JSONSearchLeafQuery](ctx, p.field, c, "gte_at", p.path, v) } +// Between derives the two operands that find a leaf at this path from lo to hi, +// inclusive: (col -> $sel) >= Low AND (col -> $sel) <= High, or at a wildcard +// path the same over every match, with both bounds on one element (see +// CustomerProfileFieldPath). func (p CustomerProfileFieldPath) Between(ctx context.Context, c *encrypt.Cipher, lo, hi json.RawMessage) (encrypt.JSONRange[eql.JSONSearchLeafQuery], error) { return gensupport.JSONBetweenAt[eql.JSONSearchLeafQuery](ctx, p.field, c, p.path, lo, hi) } -// Selector prepares `->` at this path: the value an ordering here compares is -// col -> Selector(), so bind it beside the ordering's operand, from the same -// path. +// Selector prepares `->` at this path: bind Selector() beside the ordering's +// operand, from the same path, as col -> $sel, or at a wildcard path as +// eql_v4.json_tree_path_query(col, $sel), whose matches are every element. func (p CustomerProfileFieldPath) Selector(ctx context.Context, c *encrypt.Cipher) (encrypt.JSONSelection[eql.JSONEntry], error) { return gensupport.JSONSelect[eql.JSONEntry](ctx, p.field, c, p.path) } diff --git a/languages/golang/encrypt/json_postgres_test.go b/languages/golang/encrypt/json_postgres_test.go index 25cdb0370..f9e2a130a 100644 --- a/languages/golang/encrypt/json_postgres_test.go +++ b/languages/golang/encrypt/json_postgres_test.go @@ -5,6 +5,8 @@ import ( "context" "database/sql" "encoding/json" + "errors" + "fmt" "os" "slices" "testing" @@ -155,7 +157,16 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { if err != nil { t.Fatal(err) } - want("first visit > 2", ids(`SELECT id FROM customers WHERE (profile -> $1::text) > $2::eql_v4.query_json_leaf ORDER BY id`, at.Selector(), gt), 1, 2) + // At a wildcard path `->` sees an unspecified element of each row, so a + // comparison over the elements is written over every match. + some := `SELECT id FROM customers WHERE EXISTS (SELECT 1 FROM eql_v4.json_tree_path_query(profile, $1::text) AS e WHERE e %s $2::eql_v4.query_json_leaf) ORDER BY id` + want("some visit > 2", ids(fmt.Sprintf(some, ">"), at.Selector(), gt), 1, 2) + lt2, err := f.At.VisitsAt.Lt(ctx, cipher, json.RawMessage(`2`)) + if err != nil { + t.Fatal(err) + } + // Alice's second visit, at 1, is the only one before 2. + want("some visit < 2", ids(fmt.Sprintf(some, "<"), at.Selector(), lt2), 1) // `->` opened against the selection that asked for it: each row's // value, and no value where the row has none. @@ -248,6 +259,29 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { } } + // A wildcard read of an array longer than its bound is refused, not + // opened to the elements below the bound: Alice has two visits. + short, err := f.Subtree(ctx, cipher, "$.visits[*].at", encrypt.JSONReadOptions{WildcardBound: 1}) + if err != nil { + t.Fatal(err) + } + var shortRows []eql.JSONEntry + sr, err := conn.QueryContext(ctx, `SELECT e FROM customers, eql_v4.json_tree_subtree(profile, $1::text[]) AS e WHERE id = 1`, short.Tokens()) + if err != nil { + t.Fatal(err) + } + for sr.Next() { + var e eql.JSONEntry + if err := sr.Scan(&e); err != nil { + t.Fatal(err) + } + shortRows = append(shortRows, e) + } + _ = sr.Close() + if _, err := short.Open(ctx, [][]eql.JSONEntry{shortRows}); !errors.Is(err, encrypt.ErrEncoding) { + t.Errorf("a wildcard read past its bound: %v, want ErrEncoding", err) + } + // A concrete read of a container, opened to one value or none. visits, err := f.Subtree(ctx, cipher, "$.visits", encrypt.JSONReadOptions{}) if err != nil { @@ -309,10 +343,12 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { want("children rows", ids(`SELECT id FROM customers, eql_v4.json_tree_children(profile, $1::text) AS e ORDER BY id`, children), 1, 1, 2) // Sorting by an ordered path, in the form EQL documents: - // ORDER BY eql_v4.ord_term(col -> sel). Ordering by the entry itself - // would sort by its ciphertext. Two more rows give each path a tie, - // broken by id, and Eve's visits (first 2, last 9) tell the first match - // `->` takes at the wildcard path from the least or greatest element. + // ORDER BY eql_v4.ord_term(col -> sel) at a path with no array step. + // Ordering by the entry itself would sort by its ciphertext. At the + // wildcard path `->` sees an unspecified element, so a row is ordered by + // its least match's term ascending and its greatest descending, over + // json_tree_path_query. Two more rows give each path a tie, broken by + // id, and Eve's visits (2 and 9) tell the least from the greatest. more := []testusers.Customer{ {ID: 5, Profile: json.RawMessage(`{"name":"Eve","age":34,"visits":[{"at":2},{"at":9}]}`)}, {ID: 6, Profile: json.RawMessage(`{"name":"Fay","age":7,"visits":[{"at":3}]}`)}, @@ -331,13 +367,17 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { } `json:"visits"` } ageOf := func(p plain) *float64 { return p.Age } - firstAtOf := func(p plain) *float64 { - for _, v := range p.Visits { - if v.At != nil { - return v.At + // The least or greatest `at` of a row's visits, nil for none. + atOf := func(greatest bool) func(plain) *float64 { + return func(p plain) *float64 { + var out *float64 + for _, v := range p.Visits { + if v.At != nil && (out == nil || (*v.At > *out) == greatest) { + out = v.At + } } + return out } - return nil } // The ids in the order PostgreSQL's defaults put the plaintexts in: a // row with no value at the path has a NULL term, which sorts as the @@ -380,23 +420,27 @@ func TestJSONDocumentsQueryAndReadInPostgres(t *testing.T) { } return out } + byTerm := `SELECT id FROM customers ORDER BY eql_v4.ord_term(profile -> $1::text) %[1]s, id` + byMatch := `SELECT id FROM customers ORDER BY (SELECT eql_v4.ord_term(e) FROM eql_v4.json_tree_path_query(profile, $1::text) AS e ORDER BY 1 %[1]s NULLS LAST LIMIT 1) %[1]s, id` for _, tc := range []struct { name string + sql string selector encrypt.JSONSelector - key func(plain) *float64 + ascKey func(plain) *float64 + descKey func(plain) *float64 asc, desc []int64 // what plaintextOrder gives, written out }{ // Bob and Fay tie at 7, Alice and Eve at 34; Dan has no age. - {"$.age", age.Selector(), ageOf, []int64{2, 6, 1, 5, 3, 4}, []int64{4, 3, 1, 5, 2, 6}}, - // First visits: Eve 2, Alice and Fay 3, Bob 5; Carol's visits are - // empty and Dan has none. - {"$.visits[*].at", at.Selector(), firstAtOf, []int64{5, 1, 6, 2, 3, 4}, []int64{3, 4, 2, 1, 6, 5}}, + {"$.age", byTerm, age.Selector(), ageOf, ageOf, []int64{2, 6, 1, 5, 3, 4}, []int64{4, 3, 1, 5, 2, 6}}, + // Least visits: Alice 1, Eve 2, Fay 3, Bob 5; greatest: Eve 9, + // Bob 5, Alice and Fay 3. Carol's visits are empty and Dan has none. + {"$.visits[*].at", byMatch, at.Selector(), atOf(false), atOf(true), []int64{1, 5, 6, 2, 3, 4}, []int64{3, 4, 5, 2, 1, 6}}, } { - asc, desc := plaintextOrder(tc.key, false), plaintextOrder(tc.key, true) + asc, desc := plaintextOrder(tc.ascKey, false), plaintextOrder(tc.descKey, true) want("plaintext order by "+tc.name, asc, tc.asc...) want("plaintext order by "+tc.name+" DESC", desc, tc.desc...) - want("ORDER BY "+tc.name, ids(`SELECT id FROM customers ORDER BY eql_v4.ord_term(profile -> $1::text), id`, tc.selector), asc...) - want("ORDER BY "+tc.name+" DESC", ids(`SELECT id FROM customers ORDER BY eql_v4.ord_term(profile -> $1::text) DESC, id`, tc.selector), desc...) + want("ORDER BY "+tc.name, ids(fmt.Sprintf(tc.sql, "ASC"), tc.selector), asc...) + want("ORDER BY "+tc.name+" DESC", ids(fmt.Sprintf(tc.sql, "DESC"), tc.selector), desc...) } } diff --git a/languages/golang/stashgen/emit.go b/languages/golang/stashgen/emit.go index f31fcdb42..ce34e5ca6 100644 --- a/languages/golang/stashgen/emit.go +++ b/languages/golang/stashgen/emit.go @@ -1182,29 +1182,31 @@ func (w *writer) jsonEntry(g genField) { } w.p("}") w.nl() - w.b.WriteString(wrapComment(fmt.Sprintf("%sPath is one ordered path of %s: its orderings compare the first value `->` extracts there, by the leaf's ordering term. Numbers order as float64, so integers past 2^53 can tie; text orders case- and accent-folded.", g.fieldType, g.GoName), 80)) + w.b.WriteString(wrapComment(fmt.Sprintf("%sPath is one ordered path of %s. Its orderings derive the leaf operand (eql_v4.query_json_leaf) a stored leaf's ordering term is compared with, and Selector the path's selector. At a path with no array step, compare col -> Selector() with the operand, and order by eql_v4.ord_term(col -> Selector()). At a wildcard path ([*]) col -> Selector() is an unspecified element of each row, so write the comparison over every match, EXISTS (SELECT 1 FROM eql_v4.json_tree_path_query(col, $sel) e WHERE e < $1::eql_v4.query_json_leaf), and order by the least or greatest match's term: (SELECT eql_v4.ord_term(e) FROM eql_v4.json_tree_path_query(col, $sel) e ORDER BY 1 LIMIT 1). Numbers order as float64, so integers past 2^53 can tie; text orders case- and accent-folded.", g.fieldType, g.GoName), 80)) w.p("type %sPath struct {", g.fieldType) w.p("\tfield %s", field) w.p("\tpath string") w.p("}") - for _, op := range []struct{ name, method string }{{"lt_at", "Lt"}, {"lte_at", "Lte"}, {"gt_at", "Gt"}, {"gte_at", "Gte"}} { + for _, op := range []struct{ name, method, sign, words string }{{"lt_at", "Lt", "<", "less than"}, {"lte_at", "Lte", "<=", "at most"}, {"gt_at", "Gt", ">", "greater than"}, {"gte_at", "Gte", ">=", "at least"}} { if !answered[op.name] { continue } w.nl() + w.b.WriteString(wrapComment(fmt.Sprintf("%s derives the operand that finds a leaf at this path %s v: (col -> $sel) %s $1::eql_v4.query_json_leaf, or at a wildcard path the same comparison over every match (see %sPath).", op.method, op.words, op.sign, g.fieldType), 80)) w.p("func (p %sPath) %s(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (%s, error) {", g.fieldType, op.method, leaf) w.p("\treturn gensupport.JSONQueryAt[%s](ctx, p.field, c, %q, p.path, v)", leaf, op.name) w.p("}") } if answered["between_at"] { w.nl() + w.b.WriteString(wrapComment(fmt.Sprintf("Between derives the two operands that find a leaf at this path from lo to hi, inclusive: (col -> $sel) >= Low AND (col -> $sel) <= High, or at a wildcard path the same over every match, with both bounds on one element (see %sPath).", g.fieldType), 80)) w.p("func (p %sPath) Between(ctx context.Context, c *encrypt.Cipher, lo, hi json.RawMessage) (encrypt.JSONRange[%s], error) {", g.fieldType, leaf) w.p("\treturn gensupport.JSONBetweenAt[%s](ctx, p.field, c, p.path, lo, hi)", leaf) w.p("}") } if answered["selector"] { w.nl() - w.b.WriteString(wrapComment("Selector prepares `->` at this path: the value an ordering here compares is col -> Selector(), so bind it beside the ordering's operand, from the same path.", 80)) + w.b.WriteString(wrapComment("Selector prepares `->` at this path: bind Selector() beside the ordering's operand, from the same path, as col -> $sel, or at a wildcard path as eql_v4.json_tree_path_query(col, $sel), whose matches are every element.", 80)) w.p("func (p %sPath) Selector(ctx context.Context, c *encrypt.Cipher) (encrypt.JSONSelection[%s], error) {", g.fieldType, entry) w.p("\treturn gensupport.JSONSelect[%s](ctx, p.field, c, p.path)", entry) w.p("}") From be251066afd2451438b96f9d12fbfe5c277cc5bb Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 22:28:30 +1100 Subject: [PATCH 26/26] test(stack-encrypt): accept a JSON path exactly as deep as a document The mutants gate found the depth cap's comparison unpinned: refusing a path of MAX_DEPTH steps instead of more than MAX_DEPTH passed every test. A selector path of 127 steps, the deepest that can name a node, is now among the inputs the check accepts. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- packages/stack-encrypt/tests/json_operators.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/stack-encrypt/tests/json_operators.rs b/packages/stack-encrypt/tests/json_operators.rs index 174fc59cf..d9e1b7ac6 100644 --- a/packages/stack-encrypt/tests/json_operators.rs +++ b/packages/stack-encrypt/tests/json_operators.rs @@ -538,6 +538,12 @@ fn malformed_inputs_are_refused_with_their_reason() { ), (OpSpec::Subtree, json!({"path": "$.visits[1]"})), (OpSpec::ChildCount, json!({"path": "$.visits"})), + // As many steps as a document has levels: the deepest path that + // can name a node is not refused as too deep. + ( + OpSpec::Selector, + json!({"path": format!("${}", ".a".repeat(127))}), + ), ] { assert!( check(operator, input.clone()).is_ok(),