diff --git a/.github/workflows/tests-golang.yml b/.github/workflows/tests-golang.yml index 6ef27e601..7494b4dcb 100644 --- a/.github/workflows/tests-golang.yml +++ b/.github/workflows/tests-golang.yml @@ -17,6 +17,11 @@ name: Tests (Go) # go-binding-cross # the same Go tests on macOS and Windows, against the guests # Linux built, after checking their sha256. +# eql-postgres +# the Go module's PostgreSQL tests, against PostgreSQL 17 with +# the EQL v4 bundle built from this tree installed: rows the Go +# SDK stores in every EQL type it produces, found by the queries +# it derives, through EQL's operators. # live the Go live tests and the stack-encrypt examples, against real # ZeroKMS with the repository's CS_* credentials. # @@ -36,6 +41,13 @@ on: # The eql guest build links eql-bindings. - packages/eql/crates/eql-bindings/** - packages/eql/crates/eql-domains/** + # eql-postgres installs the EQL v4 bundle `mise run build` writes from + # these, so a SQL change starts the Go test that runs against it. + - packages/eql/src/** + - packages/eql/tasks/** + - packages/eql/crates/eql-codegen/** + - packages/eql/Cargo.toml + - packages/eql/mise.toml # The Go tests read the cross-language fixtures the EQL suite writes, # which are derived under EQL's lockfile. - packages/eql/tests/encryption/** @@ -65,6 +77,13 @@ on: # The eql guest build links eql-bindings. - packages/eql/crates/eql-bindings/** - packages/eql/crates/eql-domains/** + # eql-postgres installs the EQL v4 bundle `mise run build` writes from + # these, so a SQL change starts the Go test that runs against it. + - packages/eql/src/** + - packages/eql/tasks/** + - packages/eql/crates/eql-codegen/** + - packages/eql/Cargo.toml + - packages/eql/mise.toml # The Go tests read the cross-language fixtures the EQL suite writes, # which are derived under EQL's lockfile. - packages/eql/tests/encryption/** @@ -111,24 +130,6 @@ jobs: # caches) the whole test toolset, which `mise run` would otherwise # install uncached. MISE_ENV: test - # The Go module's PostgreSQL test (encrypt/postgres_test.go) stores - # generated rows and runs the query methods against it through - # database/sql; it skips where DATABASE_URL is unset, as on the macOS - # and Windows jobs. - services: - postgres: - image: postgres:17 - env: - POSTGRES_PASSWORD: postgres - # Container port only: Docker picks the host port, and the Go step - # reads it from `job.services`, so the number is written once. - ports: - - 5432 - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: @@ -242,7 +243,6 @@ jobs: - name: Go binding env: STACK_ENCRYPT_TESTS_REQUIRE_LOCK: "1" - DATABASE_URL: "postgres://postgres:postgres@localhost:${{ job.services.postgres.ports['5432'] }}/postgres?sslmode=disable" run: | ulimit -l "$(ulimit -H -l)" echo "RLIMIT_MEMLOCK: $(ulimit -l) KiB" @@ -370,6 +370,83 @@ jobs: - name: Go binding run: scripts/go-binding-test.sh languages/golang + # The Go SDK's rows and queries inside PostgreSQL with EQL installed: the + # path a program using the SDK runs. Its own job because it builds the EQL + # v4 bundle under packages/eql's mise config, which pins its own Rust; the + # guests are the ones wasi-check built and checked. + eql-postgres: + name: Go rows and queries in PostgreSQL with EQL v4 + needs: wasi-check + runs-on: blacksmith-16vcpu-ubuntu-2204 + timeout-minutes: 30 + services: + postgres: + image: postgres:17 + env: + POSTGRES_PASSWORD: postgres + # Container port only: Docker picks the host port, and the test step + # reads it from `job.services`, so the number is written once. + ports: + - 5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + # The root config for Go, and EQL's for the build of its bundle. + - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4 + with: + version: 2026.4.0 + install: true + working_directory: . + cache: true + # Must name runs-on's image; see scripts/__tests__/workflow-mise-cache-image.test.mjs + cache_key_prefix: mise-v1-ubuntu-2204 + + - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4 + with: + version: 2026.4.0 + install: true + working_directory: packages/eql + cache: true + # Must name runs-on's image; see scripts/__tests__/workflow-mise-cache-image.test.mjs + cache_key_prefix: mise-v1-ubuntu-2204 + + # `mise run build` compiles eql-codegen to write the bundle. + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + with: + workspaces: packages/eql + + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: wasm-guests + path: languages/golang + + - name: The guests are the ones wasi-check built and checked + run: | + cd languages/golang + for guest in encrypt/testdata/stack_encrypt_guest_deterministic.wasm encrypt/testdata/stack_encrypt_guest_eql_deterministic.wasm; do + want=$(cat "$guest.sha256") + got=$(openssl dgst -sha256 "$guest" | awk '{print $NF}') + if [ "$want" != "$got" ]; then + echo "$guest checksum mismatch: artifact says $want, file is $got" >&2 + exit 1 + fi + echo "$guest sha256 $got" + done + + # Builds the bundle from the tree, installs it, and runs the tests; a + # skip fails the task, so a missing guest or database cannot pass. + - name: Go rows and queries through EQL v4 + env: + EQL_TEST_DATABASE_URL: "host=localhost port=${{ job.services.postgres.ports['5432'] }} user=postgres password=postgres dbname=postgres sslmode=disable" + run: mise run go:test:postgres + # Round trips through real ZeroKMS. The Go live tests skip without their # STACK_ENCRYPT_TEST_* variables, so the secrets pre-flight runs first and # turns a missing credential into a failure rather than a silent skip. The diff --git a/AGENTS.md b/AGENTS.md index 338df5c22..6850d4d56 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -723,7 +723,9 @@ exact pin would stop the suite sharing one `cts-common` with it. - **CI:** `tests-crates.yml` (fmt, clippy, nextest including the trybuild `tests/ui` binary, doctests, rustdoc, the node bindings and `stack-auth-wasm`), `tests-golang.yml` (WASI checks, the guests, the Go - module on Linux, macOS and Windows, and the live tests), `crap-crates.yml`, + module on Linux, macOS and Windows, its PostgreSQL tests against the EQL v4 + bundle built from the tree — `mise run go:test:postgres` locally — and the + live tests), `crap-crates.yml`, `mutants.yml`, `fuzz.yml`, `miri.yml` and `udeps.yml`. `scripts/__tests__/crates-ci.test.mjs` fails when a mise task in the root `mise.toml` or the five crate `tasks.toml` files loses its last CI caller. diff --git a/languages/golang/encrypt/eql_postgres_test.go b/languages/golang/encrypt/eql_postgres_test.go new file mode 100644 index 000000000..bee34bb0f --- /dev/null +++ b/languages/golang/encrypt/eql_postgres_test.go @@ -0,0 +1,348 @@ +package encrypt_test + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "slices" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgconn" + _ "github.com/jackc/pgx/v5/stdlib" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/eql" + "github.com/cipherstash/stack/languages/golang/encrypt/gensupport" +) + +// Rows the Go SDK stores, found by queries the Go SDK derives, inside +// PostgreSQL with EQL v4 installed: the path a program using the SDK runs. +// Every EQL type the engine produces gets a temporary table with one column +// of its domain and four rows, low, middle, middle, high. Each query the +// type answers is derived through the SDK and run through EQL's operators, +// and must find exactly the rows its plaintexts say. The values are also +// read back from the database and opened, since PostgreSQL stores jsonb +// re-serialised. +// +// The list of types is eql.Types, generated from the EQL catalog, so a type +// the engine starts producing is covered here the day it lands. A type whose +// plaintext or index this file has no case for fails rather than being +// skipped. +// +// Skipped unless DATABASE_URL is set. With it set, EQL v4 must be installed +// there: a skip would read the same as a pass. CI installs the bundle built +// from the tree (tests-golang.yml); locally, `mise run go:test:postgres`. + +// The four rows' plaintexts for each wire kind, in ascending order with the +// middle two equal. Text is chosen so that only the middle rows share a +// trigram with the match query below. +var ( + eqlInt16s = [4]int16{-5, 7, 7, 40} + eqlInt32s = [4]int32{-5, 7, 7, 40} + eqlInt64s = [4]int64{-5_000_000_000, 7, 7, 40_000_000_000} + eqlFloat32s = [4]float32{-1.5, 2.25, 2.25, 10} + eqlFloat64s = [4]float64{-1.5, 2.25, 2.25, 10} + eqlBools = [4]bool{false, true, true, true} + eqlStrings = [4]string{"apple pie", "banana split", "banana split", "cherry tart"} + eqlMatch = "banana" + eqlDates = [4]encrypt.Date{ + encrypt.DateOf(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)), + encrypt.DateOf(time.Date(2021, 6, 15, 0, 0, 0, 0, time.UTC)), + encrypt.DateOf(time.Date(2021, 6, 15, 0, 0, 0, 0, time.UTC)), + encrypt.DateOf(time.Date(2023, 12, 31, 0, 0, 0, 0, time.UTC)), + } + eqlTimestamps = [4]time.Time{ + time.Date(2020, 1, 1, 8, 0, 0, 0, time.UTC), + time.Date(2021, 6, 15, 12, 30, 0, 0, time.UTC), + time.Date(2021, 6, 15, 12, 30, 0, 0, time.UTC), + time.Date(2023, 12, 31, 23, 59, 59, 0, time.UTC), + } + eqlDecimals = [4]string{"-1.50", "12.25", "12.25", "400"} +) + +// The rows each query must find, by the operator the SDK names it with. +var eqlExpected = map[string][]int{ + "eq": {2, 3}, + "lt": {1}, + "lte": {1, 2, 3}, + "gt": {4}, + "gte": {2, 3, 4}, +} + +// The SQL operator each SDK query operator is run through. The query +// operand is always on the right; the Rust suite covers it on the left. +var eqlOperators = map[string]string{ + "eq": "=", + "lt": "<", + "lte": "<=", + "gt": ">", + "gte": ">=", +} + +// The queries each index answers, by SDK operator. "match" is not here: it +// is a match query, derived differently, and handled by its own case. +var eqlIndexOperators = map[string][]string{ + "eq": {"eq"}, + "ope": {"lt", "lte", "gt", "gte"}, + "ore": {"lt", "lte", "gt", "gte"}, +} + +func TestEQLTypesRoundTripAndQueryInPostgres(t *testing.T) { + url := os.Getenv("DATABASE_URL") + if url == "" { + t.Skip("DATABASE_URL is not set") + } + ctx := context.Background() + c := deterministicEQLClient(t) + cipher := c.DefaultKeyset() + + db, err := sql.Open("pgx", url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + // One connection: the tables are temporary and live in this session. + conn, err := db.Conn(ctx) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + + var installed bool + if err := conn.QueryRowContext(ctx, `SELECT to_regnamespace('eql_v4') IS NOT NULL`).Scan(&installed); err != nil { + t.Fatal(err) + } + if !installed { + t.Fatal("EQL v4 is not installed in DATABASE_URL: install the bundle built from the tree (`mise run build` in packages/eql writes release/cipherstash-encrypt-v4.sql), or run `mise run go:test:postgres`") + } + + covered := 0 + for _, typ := range eql.Types { + if !typ.Producible { + continue + } + covered++ + t.Run(typ.Name, func(t *testing.T) { + h := eqlHarness{t: t, ctx: ctx, cipher: cipher, conn: conn, typ: typ} + switch typ.Plaintext { + case "int16": + runEQLType(h, gensupport.Int16, eqlInt16s, nil) + case "int32": + runEQLType(h, gensupport.Int32, eqlInt32s, nil) + case "int64": + runEQLType(h, gensupport.Int64, eqlInt64s, nil) + case "float32": + runEQLType(h, gensupport.Float32, eqlFloat32s, nil) + case "float64": + runEQLType(h, gensupport.Float64, eqlFloat64s, nil) + case "bool": + runEQLType(h, gensupport.Bool, eqlBools, nil) + case "string": + runEQLType(h, gensupport.String, eqlStrings, func(f gensupport.Field[string]) ([]byte, error) { + q, err := gensupport.EQLMatches[[]byte](ctx, f, cipher, eqlMatch) + return q.Query(), err + }) + case "date": + runEQLType(h, gensupport.Date, eqlDates, nil) + case "timestamp": + runEQLType(h, gensupport.Timestamp, eqlTimestamps, nil) + case "decimal": + var ds [4]encrypt.Decimal + for i, s := range eqlDecimals { + d, err := encrypt.ParseDecimal(s) + if err != nil { + t.Fatal(err) + } + ds[i] = d + } + runEQLType(h, gensupport.Decimal, ds, nil) + default: + t.Fatalf("no plaintexts for wire kind %q: add a case so %s is tested", typ.Plaintext, typ.Name) + } + }) + } + if covered == 0 { + t.Fatal("eql.Types lists no producible type, so this test checked nothing") + } +} + +type eqlHarness struct { + t *testing.T + ctx context.Context + cipher *encrypt.Cipher + conn *sql.Conn + typ eql.Type +} + +// runEQLType stores the four values in a column of typ's domain, runs every +// query typ's indexes answer, and opens what the database hands back. +// match derives the match query for a type with the match index. +func runEQLType[T comparable](h eqlHarness, kind gensupport.Kind, values [4]T, match func(gensupport.Field[string]) ([]byte, error)) { + t, ctx, typ := h.t, h.ctx, h.typ + t.Helper() + decl := gensupport.Declare("go_eql_postgres").Passthrough("id").EncryptInto("v", kind, typ.Name) + if err := decl.Err(); err != nil { + t.Fatal(err) + } + field := gensupport.NewField[T](decl, "v") + table := "go_eql_" + strings.ToLower(typ.Name) + + exec := func(stmt string, args ...any) { + t.Helper() + if _, err := h.conn.ExecContext(ctx, stmt, args...); err != nil { + t.Fatalf("%s: %v", stmt, err) + } + } + exec(fmt.Sprintf(`CREATE TEMPORARY TABLE %s (id int PRIMARY KEY, v %s NOT NULL)`, table, typ.SQLDomain)) + t.Cleanup(func() { _, _ = h.conn.ExecContext(context.Background(), `DROP TABLE IF EXISTS `+table) }) + + for i, v := range values { + out, err := field.Encrypt(ctx, h.cipher, v) + if err != nil { + t.Fatal(err) + } + exec(fmt.Sprintf(`INSERT INTO %s VALUES ($1, $2::text::jsonb::%s)`, table, typ.SQLDomain), i+1, string(out.EQL)) + } + + ids := func(where string, operand []byte) ([]int, error) { + rows, err := h.conn.QueryContext(ctx, fmt.Sprintf(`SELECT id FROM %s WHERE %s ORDER BY id`, table, where), string(operand)) + if err != nil { + return nil, err + } + defer rows.Close() + var found []int + for rows.Next() { + var id int + if err := rows.Scan(&id); err != nil { + return nil, err + } + found = append(found, id) + } + return found, rows.Err() + } + operand := "$1::text::jsonb::" + typ.QuerySQLDomain + + answered := map[string]bool{} + for _, index := range typ.Indexes { + if index == "match" { + if match == nil { + t.Fatalf("%s has the match index but %s is not text", typ.Name, typ.Plaintext) + } + sf := gensupport.NewField[string](decl, "v") + q, err := match(sf) + if err != nil { + t.Fatal(err) + } + found, err := ids("v @@ "+operand, q) + if err != nil { + t.Fatalf("match: %v", err) + } + if !slices.Equal(found, []int{2, 3}) { + t.Errorf("v @@ match(%q) found rows %v, want [2 3]", eqlMatch, found) + } + continue + } + operators, ok := eqlIndexOperators[index] + if !ok { + t.Fatalf("no query for the %q index: add one so %s is tested", index, typ.Name) + } + for _, op := range operators { + if answered[op] { + continue + } + answered[op] = true + term, err := field.Term(ctx, h.cipher, op, values[1]) + if err != nil { + t.Fatalf("%s: %v", op, err) + } + found, err := ids(fmt.Sprintf("v %s %s", eqlOperators[op], operand), term) + if err != nil { + t.Fatalf("%s: %v", op, err) + } + if want := eqlExpected[op]; !slices.Equal(found, want) { + t.Errorf("v %s middle found rows %v, want %v", eqlOperators[op], found, want) + } + } + } + + // What the type does not answer, PostgreSQL refuses rather than + // comparing the stored jsonb. A storage-only type has no query domain, + // so it is compared with itself. + switch { + case len(typ.Indexes) == 0: + expectRefused(t, h.conn, ctx, fmt.Sprintf(`SELECT id FROM %s WHERE v = v`, table), "=", typ.SQLDomain) + case !answered["lt"]: + expectRefused(t, h.conn, ctx, fmt.Sprintf(`SELECT id FROM %s WHERE v < v`, table), "<", typ.SQLDomain) + } + + // Read back what PostgreSQL stored, and open it. + rows, err := h.conn.QueryContext(ctx, fmt.Sprintf(`SELECT v::text FROM %s ORDER BY id`, table)) + if err != nil { + t.Fatal(err) + } + var stored []gensupport.Record + for rows.Next() { + var text string + if err := rows.Scan(&text); err != nil { + t.Fatal(err) + } + stored = append(stored, gensupport.Record{"id": {Value: int32(len(stored) + 1)}, "v": {EQL: []byte(text)}}) + } + if err := rows.Close(); err != nil { + t.Fatal(err) + } + opened, err := eqlCodec(decl).Decrypt(ctx, h.cipher, stored) + if err != nil { + t.Fatal(err) + } + for i, vals := range opened { + got, err := gensupport.Get[T](vals, "v") + if err != nil { + t.Fatal(err) + } + if !eqlSame(got, values[i]) { + t.Errorf("row %d opened as %v, want %v", i+1, got, values[i]) + } + } +} + +// eqlCodec opens records of one declaration without a generated type: the +// record is the encrypted form and the values map the plaintext one. +func eqlCodec(decl gensupport.Declaration) *gensupport.Codec[gensupport.Values, gensupport.Record] { + return gensupport.New(gensupport.Generated[gensupport.Values, gensupport.Record]{ + TypeName: "GoEQLPostgres", + Declaration: decl, + Source: func(v gensupport.Values) gensupport.Values { return v }, + Seal: func(r gensupport.Record) (gensupport.Record, error) { return r, nil }, + Open: func(r gensupport.Record) gensupport.Record { return r }, + Value: func(_ gensupport.Record, v gensupport.Values) (gensupport.Values, error) { return v, nil }, + }) +} + +// eqlSame compares two plaintexts, decimals by value rather than by their +// internal representation. +func eqlSame[T comparable](a, b T) bool { + if da, ok := any(a).(encrypt.Decimal); ok { + return da.String() == any(b).(encrypt.Decimal).String() + } + return a == b +} + +// expectRefused runs query and wants EQL's own refusal of operator on +// domain, not some other error that happens to stop the query. +func expectRefused(t *testing.T, conn *sql.Conn, ctx context.Context, query, operator, domain string) { + t.Helper() + rows, err := conn.QueryContext(ctx, query) + if err == nil { + err = rows.Close() + } + want := fmt.Sprintf("operator %s is not supported for %s", operator, domain) + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || !strings.Contains(pgErr.Message, want) { + t.Errorf("%s: want PostgreSQL to refuse it with %q, got %v", query, want, err) + } +} diff --git a/languages/golang/go.mod b/languages/golang/go.mod index a32bfab2a..fa53e7c3a 100644 --- a/languages/golang/go.mod +++ b/languages/golang/go.mod @@ -10,7 +10,7 @@ require ( ) require ( - github.com/jackc/pgx/v5 v5.11.0 // test only: encrypt/postgres_test.go + github.com/jackc/pgx/v5 v5.11.0 // test only: encrypt/postgres_test.go, encrypt/eql_postgres_test.go golang.org/x/sys v0.49.0 golang.org/x/tools v0.51.0 google.golang.org/protobuf v1.36.12 diff --git a/mise.toml b/mise.toml index e9c094ce9..3da1f794b 100644 --- a/mise.toml +++ b/mise.toml @@ -364,6 +364,31 @@ description = "Format check, vet and test the Go module (languages/golang: encry # embeds is named relative to the module root. run = "scripts/go-binding-test.sh languages/golang" +[tasks."go:test:postgres"] +description = "Install the EQL v4 bundle built from the tree into the database EQL_TEST_DATABASE_URL names and run the Go module's PostgreSQL tests against it: rows the Go SDK stores in every EQL type it produces, found by the queries it derives; needs `wasm:guest:build:deterministic` and `wasm:guest:build:eql:deterministic` first" +# The bundle is built from packages/eql, as test:encryption:postgres builds +# it: the copy under crates/eql-bindings/sql is the RELEASED bundle and moves +# only at the lockstep bump, so it cannot show a SQL change in the same pull +# request. One connection string serves psql and the Go tests; pgx reads a +# libpq keyword/value string as well as a URL. +run = """ +#!/usr/bin/env bash +set -euo pipefail +: "${EQL_TEST_DATABASE_URL:?set EQL_TEST_DATABASE_URL to a libpq connection string for a disposable PostgreSQL}" +mise -C packages/eql run build +psql "$EQL_TEST_DATABASE_URL" -v ON_ERROR_STOP=1 -q -f packages/eql/release/cipherstash-encrypt-v4.sql +cd languages/golang +# The tests skip without DATABASE_URL and without their guest builds; -v and +# the grep turn either skip into a failure here, where both are provided. +DATABASE_URL="$EQL_TEST_DATABASE_URL" CGO_ENABLED=0 go test -count=1 -v \ + -run '^(TestEQLTypesRoundTripAndQueryInPostgres|TestQueriesRunAgainstPostgres)$' ./encrypt \ + | tee "${TMPDIR:-/tmp}/go-test-postgres.log" +if grep -q -- '--- SKIP' "${TMPDIR:-/tmp}/go-test-postgres.log"; then + echo "error: a PostgreSQL test skipped; build the deterministic guests first" >&2 + exit 1 +fi +""" + [tasks."go:lint"] description = "Lint and format-check the Go module (languages/golang) with golangci-lint; config in languages/golang/.golangci.yaml" dir = "languages/golang"