feat(stack-encrypt, golang): JSON operators and data-plan reads, with generated Go JSON document fields (#1374)

.cargo/mutants.toml

@@ -116,6 +116,9 @@ exclude_re = [
   # A document always has a root, so its terms are never empty: `false` is
   # the value `is_empty` returns. The `true` replacement is still caught.
   'stack-encrypt/src/json/derive\.rs:\d+:\d+: replace JsonTerms::is_empty -> bool with false$',
+  # A JSON operator's input: the `JsonText` visitor's `expecting` writes a
+  # message `read_frame` discards for its fixed `Reason::JsonInput`.
+  'stack-encrypt/src/dynamic/json_query\.rs:\d+:\d+: replace <impl Deserialize<\x27de> for JsonText>::deserialize::<impl Visitor<\x27_> for V>::expecting -> std::fmt::Result with Ok\(Default::default\(\)\)$',
 ]
 
 # Headroom over the measured baseline before a slow-but-correct mutant is

.changeset/eql-encrypted-json-operations.md

@@ -0,0 +1,5 @@
+---
+"@cipherstash/eql": minor
+---
+
+**`JsonSearch` answers its JSON operators by name.** `eql-bindings`' target table row now tells Stack Encrypt the domain an ordering at a path binds as (`Target::leaf_query_sql_domain`, `eql_v4.query_json_leaf`), so a data plan's `lt_at`, `lte_at`, `gt_at`, `gte_at` and `between_at` bind their operand as that domain, beside `contains`, `contained_by` and `eq_at`, which bind the needle as `eql_v4.query_json`. No SQL changes.

languages/golang/CHANGELOG.md

@@ -12,6 +12,35 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`.
 
 ### Added
 
+- **Encrypted JSON documents with `stashgen`** (#1351, #1352). A
+  `json.RawMessage` field tagged `encrypt_into=JsonSearch` (options:
+  `ordered=$.a;$.b[*].c`, `sealed_root=false`) is stored as an
+  `eql.JSONSearch` in a `public.eql_v4_json_search` column; a
+  `*json.RawMessage` field takes `null=sql`. Its `Fields` entry has
+  `Contains`, `ContainedBy`, `EqAt`, `Selector`, `Subtree`, `Children` and
+  `ChildCount`, and `At.<Path>` per ordered path with `Lt`, `Lte`, `Gt`,
+  `Gte`, `Between` and the path's own `Selector`; at a wildcard path their
+  operands are compared over every match (`eql_v4.json_tree_path_query`),
+  since `->` there returns an unspecified element. `encrypt.JSONSelection`
+  and `encrypt.JSONRead` open the rows `->` and `eql_v4.json_tree_subtree`
+  return against the request that prepared them (`OpenOne` refuses a
+  wildcard read up front); `encrypt.JSONSelector`, `JSONToken` and
+  `JSONTokens` bind as `text` and `text[]`, and a zero selector, token or
+  token list is refused rather than bound. `JSONToken`, `JSONTokens`,
+  `JSONSelection` and `JSONRead` print and log with the tokens, the
+  selector and the path redacted; `encrypt.JSONValue` holds plaintext and
+  prints it. A `json.RawMessage` sealed any other way stays
+  bytes. `gensupport.JSON`, `TargetOptions`,
+  `Declaration.Options` and the `JSON*` helpers are the generated code's.
+
+- **Query and read an encrypted JSON document, at the record level**
+  (#1350, #1352). `Cipher.QueryTerm` derives a document field's operands:
+  `QueryTerm.Encoding` says what the term holds, and `QueryTerm.Text`
+  carries a selector, a node's token or a read's tokens.
+  `Cipher.JSONOpen` opens the rows a selector or subtree read returned
+  against the request that prepared it. `record.Operator.Input` names an
+  operator's input. Generated methods follow with `stashgen`.
+
 - **The EQL types for an encrypted JSON document** (#1348):
   `eql.JSONSearch` (`public.eql_v4_json_search`), its containment needle
   `eql.JSONSearchQuery`, its ordering operand

languages/golang/cmd/stashgen/README.md

@@ -74,7 +74,8 @@ The first part of a tag is the field's name, which is the column name in a datab
 | `stash:"notes,encrypt"` | seal the field, with no index |
 | `stash:"email,encrypt,index=equality;match"` | seal the field, and derive each index beside it |
 | `stash:"score,index=ore"` | derive the index alone; no ciphertext is stored, so `Decrypt` leaves the field at its zero value |
-| `stash:"attrs,index=json"` | refused in this build: the engine does not derive the `json` index yet |
+| `stash:"profile,encrypt_into=JsonSearch(ordered=$.age)"` | on a `json.RawMessage`: store the field as an encrypted JSON document; see "Encrypted JSON documents" |
+| `stash:"attrs,index=json"` | refused: a JSON document is stored with `encrypt_into=JsonSearch`, not as an index beside a ciphertext |
 | `stash:"id,passthrough"` | store the field as it is |
 | `stash:"tenant,context_field"` | the field's value is the context of every other field in the struct, in place of a `context=` tag; stored as it is |
 | `stash:"-"` | leave the field out |
@@ -108,7 +109,49 @@ An index takes its options in parentheses after its name, separated by commas: `
 This build refuses an index with options: a query term is derived with the default options only, so a stored term with other options would never match one.
 These words are the same as the Rust API's words for the same behaviour, with two that only Go has.
 `opaque` seals the whole struct as one `bytes` field holding a storage blob (see "What crosses the binding").
-`json` names the JSON index, which the Rust API does not declare yet; this build refuses it.
+`json` names the JSON index, which a field reaches only through `encrypt_into=JsonSearch`.
+
+### Encrypted JSON documents
+
+A `json.RawMessage` field with `encrypt_into=JsonSearch` is stored as one encrypted JSON document, an `eql.JSONSearch` in a `public.eql_v4_json_search` column.
+The field holds the document's JSON text; the engine parses it, so an object that names a member twice, an integer outside the 64-bit range and text that is not JSON are refused with `encrypt.ErrEncoding`.
+Values come back as JSON text holding the same values; key order and the spelling of numbers are not kept.
+A `*json.RawMessage` field is optional and must say `null=sql`: a document has no fixed length to hide a missing one behind. A non-nil pointer to `null` is a stored JSON `null`.
+A `json.RawMessage` sealed any other way (`encrypt`, `index=equality`) is bytes, as any type defined over `[]byte` is.
+`JsonSearch` takes `json.RawMessage` itself, or an alias of it (`type Doc = json.RawMessage`), and refuses a type defined over it (`type Doc json.RawMessage`): `stashgen` knows a document by the `json.RawMessage` type, where a scalar type is accepted with any type defined over it.
+
+The type takes options in parentheses: `ordered=` names the paths whose numbers and strings carry an ordering term, separated by `;`, and `sealed_root=false` turns off the sealed root (on by default).
+`encrypt_into=JsonSearch(ordered=$.age;$.visits[*].at)` orders two paths.
+A path in a tag is plain JSONPath: `$`, then `.key` steps of letters, digits and underscores, and `[n]` or `[*]` steps; an ordered path names no position. The engine checks every path when `stashgen` runs.
+
+The field's `Fields` entry has a method per operation:
+
+| Method | Returns | SQL |
+|---|---|---|
+| `Contains(ctx, c, needle)`, `ContainedBy(ctx, c, needle)` | `eql.JSONSearchQuery` | `col @> $1::eql_v4.query_json`, `col <@ $1::eql_v4.query_json` |
+| `EqAt(ctx, c, path, v)` | `eql.JSONSearchQuery` | `col @> $1::eql_v4.query_json` |
+| `At.Age.Lt(ctx, c, v)`, `Lte`, `Gt`, `Gte`, `Between` | `eql.JSONSearchLeafQuery` | `(col -> $sel::text) < $1::eql_v4.query_json_leaf`, `$sel` from `At.Age.Selector(ctx, c)` |
+| `Selector(ctx, c, path)` | `encrypt.JSONSelection`: `Selector()` and `Open` | `col -> $1::text` |
+| `Subtree(ctx, c, path, opts)` | `encrypt.JSONRead`: `Tokens()`, `Open` and `OpenOne` | `eql_v4.json_tree_subtree(col, $1::text[], 64)` |
+| `Children(ctx, c, path)`, `ChildCount(ctx, c, path)` | `encrypt.JSONToken` | `eql_v4.json_tree_children(col, $1::text)`, `eql_v4.json_tree_child_count(col, $1::text)` |
+
+`At` has one accessor per ordered path, named for its keys (`$.visits[*].at` is `At.VisitsAt`, `$.a.1` is `At.A1`); two paths with one name, or one path given twice, stop `stashgen`.
+An ordering compares the value `->` extracts at its path, and EQL's `<` compares ordering terms alone, so take the selector from the same accessor (`At.Age.Selector`) rather than spelling the path again.
+At a wildcard path (`$.visits[*].at`), `->` returns an unspecified element of each row (the entries are stored in a random order), so write the comparison over every match and order by the least or greatest one:
+
+```sql
+-- rows with some visit after $2
+SELECT id FROM customers WHERE EXISTS (
+  SELECT 1 FROM eql_v4.json_tree_path_query(profile, $1::text) AS e WHERE e > $2::eql_v4.query_json_leaf)
+-- rows by their earliest visit
+SELECT id FROM customers ORDER BY (
+  SELECT eql_v4.ord_term(e) FROM eql_v4.json_tree_path_query(profile, $1::text) AS e ORDER BY 1 LIMIT 1)
+```
+
+with `$1` from `At.VisitsAt.Selector(ctx, c)` and `$2` from `At.VisitsAt.Gt(ctx, c, v)`; for the latest visit, order by `ORDER BY 1 DESC NULLS LAST` inside and `DESC` outside.
+`OpenOne` reads one value and refuses a read whose path has a wildcard before anything is opened; open a wildcard read with `Open`.
+The needle and the values are `json.RawMessage`: write the JSON you mean, such as `json.RawMessage(`+"`"+`{"active":true}`+"`"+`)`.
+A position in a path is refused for `EqAt` and `Selector`, whose selector reads every position as the wildcard; read a position with `Subtree`.
 
 ### Optional fields
 

languages/golang/cmd/stashgen/main_test.go

@@ -214,15 +214,15 @@ func TestRunAsksTheEmbeddedEngine(t *testing.T) {
 	if !strings.Contains(stderr.String(), "User.Age") || !strings.Contains(stderr.String(), "TextEq seals a string, and int64 is int") {
 		t.Fatalf("stderr = %q", stderr.String())
 	}
-	// A document type is refused by what is missing, not by a kind
-	// mismatch no Go type could fix.
+	// A document type over another Go type is refused by naming the Go
+	// types that carry one, not by a kind mismatch.
 	document := strings.Replace(userSource, "Email string   `stash:\"email,encrypt_into=TextEq\"`", "Profile []byte `stash:\"profile,encrypt_into=JsonSearch\"`", 1)
 	dir = writeModule(t, map[string]string{"model.go": document})
 	stderr.Reset()
 	if code := run([]string{"-type", "User"}, dir, &stdout, &stderr, stashgen.GuestEngine); code != 1 {
 		t.Fatalf("exit %d, want 1\n%s", code, stderr.String())
 	}
-	if !strings.Contains(stderr.String(), "User.Profile") || !strings.Contains(stderr.String(), "JsonSearch seals a json document, and stashgen does not generate a field of it yet") {
+	if !strings.Contains(stderr.String(), "User.Profile") || !strings.Contains(stderr.String(), "JsonSearch seals a json document: declare the field json.RawMessage") {
 		t.Fatalf("stderr = %q", stderr.String())
 	}
 	// A context of two segments leaves an EQL field no column: refused by
@@ -290,3 +290,133 @@ func TestModelFlagsRepeat(t *testing.T) {
 		t.Fatalf("String = %q", got)
 	}
 }
+
+// A json.RawMessage field stored as an encrypted JSON document, against the
+// embedded engine with the EQL types: the generated entry has the
+// document's methods and an accessor per ordered path, and every refusal
+// names the field before anything is written.
+func TestRunGeneratesJSONSearchFields(t *testing.T) {
+	checker, err := encrypt.NewChecker(context.Background())
+	if err != nil {
+		t.Skip(err)
+	}
+	_ = checker.Close()
+	source := func(fields string) string {
+		return "package users\n\nimport \"encoding/json\"\n\ntype Doc json.RawMessage\n\ntype DocAlias = json.RawMessage\n\ntype Customer struct {\n\t_ struct{} `stash:\"context=customers\"`\n\tID int64 `stash:\"id,passthrough\"`\n" + fields + "\n}\n"
+	}
+	generate := func(fields string) (string, string, int) {
+		dir := writeModule(t, map[string]string{"model.go": source(fields)})
+		var stdout, stderr bytes.Buffer
+		code := run([]string{"-type", "Customer"}, dir, &stdout, &stderr, stashgen.GuestEngine)
+		written, _ := os.ReadFile(filepath.Join(dir, "customer_stash.go"))
+		return string(written), stderr.String(), code
+	}
+	written, stderr, code := generate("\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.age;$.visits[*].at,sealed_root=false)\"`\n\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=JsonSearch,null=sql\"`")
+	if code != 0 {
+		t.Fatalf("exit %d\n%s", code, stderr)
+	}
+	for _, want := range []string{
+		"Profile eql.JSONSearch",
+		"Prefs   *eql.JSONSearch",
+		`EncryptInto("profile", gensupport.JSON, "JsonSearch").`,
+		`Options("profile", gensupport.TargetOptions{SealedRoot: gensupport.Ptr(false), Ordered: []string{"$.age", "$.visits[*].at"}})`,
+		"func (f ProfileField) Contains(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.JSONSearchQuery, error)",
+		"func (f ProfileField) Subtree(ctx context.Context, c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[eql.JSONEntry], error)",
+		"Age      ProfileFieldPath // $.age",
+		"VisitsAt ProfileFieldPath // $.visits[*].at",
+		"func (p ProfileFieldPath) Between(ctx context.Context, c *encrypt.Cipher, lo, hi json.RawMessage) (encrypt.JSONRange[eql.JSONSearchLeafQuery], error)",
+	} {
+		if !strings.Contains(written, want) {
+			t.Errorf("the generated file lacks %q", want)
+		}
+	}
+	if strings.Contains(written, "PrefsFieldAt") {
+		t.Error("Prefs orders no path, and got an At")
+	}
+	if !strings.Contains(written, "func (p ProfileFieldPath) Selector(ctx context.Context, c *encrypt.Cipher) (encrypt.JSONSelection[eql.JSONEntry], error)") {
+		t.Error("an ordered path has no Selector of its own")
+	}
+	// A json.RawMessage sealed any other way is bytes, as it was before
+	// documents; an object key of digits names its accessor.
+	written, stderr, code = generate("\tData json.RawMessage `stash:\"data,encrypt,index=equality\"`\n\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.a.1)\"`")
+	if code != 0 {
+		t.Fatalf("exit %d\n%s", code, stderr)
+	}
+	for _, want := range []string{`EncryptIndex("data", gensupport.Bytes`, "A1 ProfileFieldPath // $.a.1"} {
+		if !strings.Contains(written, want) {
+			t.Errorf("the generated file lacks %q", want)
+		}
+	}
+	// An alias of json.RawMessage is the type itself, and is a document; a
+	// type defined over it is refused below.
+	written, stderr, code = generate("\tProfile DocAlias `stash:\"profile,encrypt_into=JsonSearch\"`")
+	if code != 0 || !strings.Contains(written, "Profile eql.JSONSearch") {
+		t.Fatalf("an alias of json.RawMessage: exit %d\n%s", code, stderr)
+	}
+	// An engine that answers no ordering on a field whose tag orders paths
+	// disagrees with the generator: refused by name, not written as a field
+	// with no At.
+	dir := writeModule(t, map[string]string{"model.go": source("\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.age)\"`")})
+	stderrBuf := bytes.Buffer{}
+	if code := run([]string{"-type", "Customer"}, dir, &bytes.Buffer{}, &stderrBuf, withoutOperator("lt_at")); code != 1 {
+		t.Fatalf("exit %d, want 1\n%s", code, stderrBuf.String())
+	}
+	if got := stderrBuf.String(); !strings.Contains(got, "Customer.Profile") || !strings.Contains(got, "the engine answers no ordering (lt_at)") {
+		t.Fatalf("stderr = %q", got)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "customer_stash.go")); !os.IsNotExist(err) {
+		t.Fatal("a file was written after the refusal")
+	}
+	for _, c := range []struct{ name, fields, want string }{
+		{"a sealed null", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=JsonSearch,null=sealed\"`", "Customer.Prefs: a nil has an equality term and no other, so another index's column would show which values are absent; write null=sql"},
+		{"a nil document that does not say null=sql", "\tPrefs *json.RawMessage `stash:\"prefs,encrypt_into=JsonSearch\"`", "null=sql"},
+		{"a position in an ordered path", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.scores[2])\"`", "the engine refuses the declaration"},
+		{"a quoted key in a tag", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$[\\\"a b\\\"])\"`", "not a plain JSONPath"},
+		{"an unknown option", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(bound=3)\"`", "no option \"bound\""},
+		{"the Go name in the tag", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JSONSearch\"`", "as the engine does, JsonSearch"},
+		{"two paths, one accessor", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.a_b;$.aB)\"`", "both name the accessor At.AB"},
+		{"the same ordered path twice", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.a;$.a)\"`", "the ordered path $.a is given twice"},
+		{"a path with no accessor name", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.__)\"`", "no Go accessor name"},
+		{"a type defined over json.RawMessage", "\tProfile Doc `stash:\"profile,encrypt_into=JsonSearch\"`", "Customer.Profile: JsonSearch seals a json document: declare the field json.RawMessage"},
+		{"a string into a document", "\tProfile string `stash:\"profile,encrypt_into=JsonSearch\"`", "JsonSearch seals a json"},
+		{"options on a type that takes none", "\tEmail string `stash:\"email,encrypt_into=TextEq(ordered=$.a)\"`", "the engine refuses the declaration"},
+	} {
+		t.Run(c.name, func(t *testing.T) {
+			written, stderr, code := generate(c.fields)
+			if code != 1 || written != "" {
+				t.Fatalf("exit %d, wrote %d bytes\n%s", code, len(written), stderr)
+			}
+			if !strings.Contains(stderr, c.want) {
+				t.Fatalf("stderr = %q, want %q", stderr, c.want)
+			}
+		})
+	}
+}
+
+// withoutOperator is the embedded engine with op removed from every field's
+// answer: an engine that disagrees with the generator.
+func withoutOperator(op string) func(context.Context) (stashgen.Engine, error) {
+	return func(ctx context.Context) (stashgen.Engine, error) {
+		engine, err := stashgen.GuestEngine(ctx)
+		return droppingEngine{Engine: engine, op: op}, err
+	}
+}
+
+type droppingEngine struct {
+	stashgen.Engine
+	op string
+}
+
+func (e droppingEngine) Check(ctx context.Context, d stashgen.Declaration) (map[string][]stashgen.Operator, error) {
+	answered, err := e.Engine.Check(ctx, d)
+	for field, ops := range answered {
+		kept := ops[:0]
+		for _, o := range ops {
+			if o.Name != e.op {
+				kept = append(kept, o)
+			}
+		}
+		answered[field] = kept
+	}
+	return answered, err
+}