diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index ebc0cad8..fdffea5c 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -235,3 +235,69 @@ jobs: cargo fmt --manifest-path test/m1-realdata-e2e/Cargo.toml -p m1-realdata-e2e -- --check - name: Run deterministic M1 contract tests run: cargo test --locked --manifest-path test/m1-realdata-e2e/Cargo.toml + + # TC-887 S0: a real node on native PostgreSQL and native MinIO (no service + # containers). MinIO/mc are pinned community releases whose sha256 is pinned + # in harness.sh; PostgreSQL is the runner image's own native install. The + # green checks prove KV puts land in MinIO and SQL writes land in + # database_artifact (checkpoint and WAL delta). The RED checks assert + # encryption at rest; until TC-887 S3/S4 land, the step passes only if both + # fail with their "TC-887 RED:" findings (`--expect red`), and fails loudly + # if they error or unexpectedly pass. Switch to `--expect green` (with the + # at-rest mode set) when S3/S4 merge. + at-rest-harness: + name: at-rest harness (native PostgreSQL + MinIO) + runs-on: ubuntu-latest + timeout-minutes: 45 + permissions: + contents: read + steps: + - name: Checkout TinyCloud repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Install reviewed Rust toolchain + uses: dtolnay/rust-toolchain@1.97.1 + with: + components: clippy, rustfmt + - name: Cache Rust dependencies + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 + with: + shared-key: rust-1.97.1-at-rest-harness + workspaces: . -> target + save-if: ${{ github.ref == 'refs/heads/main' }} + + - name: Harness paths + run: | + { + echo "TC887_SCRATCH_ROOT=$RUNNER_TEMP/tc887" + echo "TC887_RUN_DIR=$RUNNER_TEMP/tc887/harness-run-ci" + echo "TC887_STATE=$RUNNER_TEMP/tc887/harness-run-ci/state.json" + } >> "$GITHUB_ENV" + + - name: Fetch pinned MinIO and mc (sha256-verified) + run: bash scripts/at-rest-harness/harness.sh bins + + - name: Start the harness (native PostgreSQL, MinIO, tinycloud-node) + run: bash scripts/at-rest-harness/harness.sh up + + - name: Harness checks (writes reach MinIO and PostgreSQL) + run: bash scripts/at-rest-harness/harness.sh test green "$TC887_STATE" + + - name: TC-887 RED checks (must be RED until S3/S4) + run: bash scripts/at-rest-harness/harness.sh test red --expect red "$TC887_STATE" + + - name: Harness logs + if: failure() + run: | + if [ -f "$TC887_STATE" ]; then + bash scripts/at-rest-harness/harness.sh logs "$TC887_STATE" + fi + + - name: Stop the harness + if: always() + run: | + if [ -f "$TC887_STATE" ]; then + bash scripts/at-rest-harness/harness.sh down --purge "$TC887_STATE" + fi diff --git a/Cargo.lock b/Cargo.lock index 2f23023d..bcf67a7c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9711,6 +9711,7 @@ dependencies = [ "core-foundation", "core-foundation-sys", "ed25519-dalek", + "flate2", "futures", "hex", "hmac", diff --git a/agent.changelog.md b/agent.changelog.md index 7943d4fc..a905bf2d 100644 --- a/agent.changelog.md +++ b/agent.changelog.md @@ -3,6 +3,11 @@ Track changes to agent-facing development guidance for TinyCloud Node. Add a concise entry when `agent.dev.md` or related agent workflow expectations change. +## 2026-10-17 + +- Added the TC-887 native at-rest harness (`scripts/at-rest-harness/`): a real node on native + PostgreSQL + MinIO, green storage-reach checks, and RED checks for plaintext at rest. + ## 2026-05-17 - Added initial agent development notes covering project context, related repositories, build and diff --git a/agent.dev.md b/agent.dev.md index 0485a479..ad326af8 100644 --- a/agent.dev.md +++ b/agent.dev.md @@ -90,6 +90,13 @@ For persistence-sensitive work, add a restart/cold-cache check: 5. Restart against the same durable database. 6. Query through the SDK and confirm the data is still present. +For work on what reaches storage (S3 blocks, `database_artifact`, encryption at rest), use the +native harness in `scripts/at-rest-harness/` (README there): `harness.sh up` runs the real node on +native PostgreSQL and MinIO with no containers and prints its state path, `harness.sh test +green|red STATE` runs the checks in `tinycloud-node-server/tests/at_rest_harness.rs`, and +`harness.sh down --purge STATE` stops only the process groups that run started. Commands never +default to another run's state. + ## Debugging Health and feature discovery: diff --git a/scripts/at-rest-harness/README.md b/scripts/at-rest-harness/README.md new file mode 100644 index 00000000..1639d362 --- /dev/null +++ b/scripts/at-rest-harness/README.md @@ -0,0 +1,118 @@ +# TC-887 at-rest harness (S0) + +A real `tinycloud` node on native PostgreSQL and native MinIO, plus the checks +that show what reaches storage. No Docker or containers. Everything binds +loopback on free ports. Each run lives in its own directory +(`$TC887_SCRATCH_ROOT/harness-run-*`, default `~/scratch/tc887`, mode 0700). +Every process the harness starts leads its own process group (`setsid`; the +postmaster calls `setsid` itself). The state file records each PID with its +kernel start time, and `down` signals only those groups. + +| Piece | What runs | +|---|---| +| PostgreSQL | native `initdb`/`pg_ctl` (newest `/usr/lib/postgresql/*/bin`, or `PG_BIN`), scram password auth, TCP only. Holds the node's capability DB, including `database_artifact` | +| MinIO | `minio` `RELEASE.2025-09-07T16-13-09Z`; bucket `tinycloud-blocks` is created with `mc` `RELEASE.2025-08-13T08-35-41Z` | +| Node | a copy of `target/debug/tinycloud` with `TINYCLOUD_STORAGE__DATABASE` = PG, `TINYCLOUD_STORAGE__BLOCKS__TYPE=S3` (bucket, endpoint, `AWS_*` creds, region `us-east-1`), `TINYCLOUD_STORAGE__SQL__MEMORY_THRESHOLD=0` (file-backed SQL, so writes after the first checkpoint are saved as WAL deltas), and `TINYCLOUD_KEYS__TYPE=Static` with a per-run secret | + +Secrets (PG password, MinIO root credentials, node Static root) are generated +per run into `secrets/` and the 0600 files `harness.env` (read by the checks) +and `node.env` (the node's entire environment). They are never put on argv or +in logs. `state.json` (ports, PIDs and start times, binaries, log paths, +`env_file`, at-rest settings) holds no secrets. + +## Prerequisites + +- Linux x86_64 (the pinned MinIO/mc are linux-amd64; other hosts are refused), `bash`, `curl`, `python3`, and the Rust toolchain from `rust-toolchain.toml`. +- Native PostgreSQL server binaries (`initdb`, `pg_ctl`, `postgres`, `psql`). + Ubuntu: `postgresql-14` or newer. The version is not pinned: the harness uses + the newest one installed, or `PG_BIN`. +- MinIO and mc: `harness.sh bins` downloads the pinned community releases from + their (archived) GitHub releases into `~/.cache/tc887-harness/bin` and checks + the sha256 pinned in the script. dl.min.io no longer serves community builds + (HTTP 410). To work offline, place binaries with the pinned sha256 there. + +## Use + +```sh +STATE=$(scripts/at-rest-harness/harness.sh up) # builds the node (nice -n 10, -j 8); prints the state path last +scripts/at-rest-harness/harness.sh test green "$STATE" # writes reach MinIO and PG (passes) +scripts/at-rest-harness/harness.sh test red --expect red "$STATE" # exit 0 only if both RED checks fail as RED +scripts/at-rest-harness/harness.sh down --purge "$STATE" # stop this run's processes, remove its dir +``` + +Every command after `up` needs the explicit `STATE` (the state.json path or its +run dir). Nothing defaults to a "latest" run, so concurrent runs on a shared +host can't act on each other. + +- `up --no-build` reuses an already-built binary (or set `TC887_NODE_BIN`). +- If `up` fails, times out or is interrupted, it stops every process it + started. It keeps the run dir with its logs; remove it with `down --purge`. +- `status STATE` prints the state file and `logs STATE` tails the logs. +- Overrides: `TC887_SCRATCH_ROOT`, `TC887_RUN_DIR` (must be named + `harness-run-*`), `TC887_{PG,MINIO,MINIO_CONSOLE,NODE}_PORT`, `TC887_BIN_DIR`, + `TC887_CARGO_JOBS`, `TC887_NODE_READY_SECS`. +- Ports are picked by binding port 0 and releasing it, so another process can + take one before the harness does. A collision makes `up` fail cleanly; it + can't produce a false green, because the checks talk to MinIO and PG directly. + +`test` exits 0 only on the expected outcome: + +| Exit | Meaning | +|---|---| +| 0 | as expected (`GREEN as expected` / `RED as expected`) | +| 3 | unexpected result: RED checks passed (`UNEXPECTEDLY PASSED for KV/SQL`), or `--expect green` while still RED | +| 4 | errored: compile failure, harness failure, a precondition (`harness no-op`/`harness gap`) or a node error, i.e. not a RED finding | + +`--expect` defaults to `green` for `test green`. For `test red` it defaults to +`red`, or to `green` if the run was started with `TC887_AT_REST_MODE=encrypt` +or `strict`. + +## At-rest mode + +The config names belong to the S1 key-continuity slice and may change. They +are defined in one marked block at the top of `harness.sh`: + +| Harness env (at `up`) | Node env written to `node.env` | +|---|---| +| `TC887_AT_REST_MODE` (`off`/`encrypt`/`strict`) | `TINYCLOUD_AT_REST__MODE` | +| `TC887_AT_REST_ESCROW` (e.g. `none`, the Static self-host acknowledgement, plan §5) | `TINYCLOUD_AT_REST__ESCROW` | + +If either is unset, the harness sets nothing and the node runs its default +(today: no encryption at rest). + +## Checks + +`tinycloud-node-server/tests/at_rest_harness.rs`. The harness checks are +`#[ignore]`d, so `cargo test --workspace` stays green. `harness.sh test` runs +them with `--ignored` and checks the count and the outcome. Each check signs in +like the SDK (host SIWE delegation, then a session delegation) on two fresh +random wallets and spaces, and writes the same plaintext into both through +`/invoke`: +- a short value carrying a unique marker; +- a 64 KiB value of the marker repeated, which compresses well. + +SQL runs the same two batches in both spaces: create + insert (a checkpoint), +then insert (a WAL delta). + +| Test | Asserts | Today | +|---|---|---| +| `green_kv_put_reaches_minio` | every put round-trips through the node and adds a non-empty object to the bucket | pass | +| `green_sql_write_reaches_postgres` | the writes round-trip, and `database_artifact` has a `payload` and a non-NULL `delta_payload` | pass | +| `red_kv_blocks_are_sealed_in_minio` | object keys contain no space id, wallet address or blake3 fingerprint of the value (b64url padded/unpadded, base64 padded/unpadded, hex; raw digest and multihash); raw bytes hold no marker; the 64 KiB object doesn't compress (zlib ratio > 0.95); the two spaces' objects for the same value share no 32-byte run | **fail** | +| `red_sql_artifacts_are_sealed_in_postgres` | `payload` doesn't start with `SQLite format 3\0` and `delta_payload` doesn't start with the WAL magic; neither holds a marker or compresses (ratio > 0.95); the two spaces' payloads and deltas share no 32-byte run | **fail** | + +The RED checks re-run the green preconditions first, so a RED result is never +a harness no-op. The 32-byte limit: two sealed blobs of the same plaintext +share only fixed TCE1 header fields (≤ 20 contiguous bytes), while any +deterministic transform (encoding, XOR, compression, an unsalted cipher) keeps +runs as long as the data. The reasoning is in the test's `MAX_SHARED_RUN` doc. +`selftest_predicates_flag_deterministic_transforms_not_ciphertext` runs in plain +`cargo test` and checks the predicates offline: base64, XOR and zlib are +flagged, and sealed-like random blobs are not. + +## Not covered + +- The dstack simulator (`DSTACK_SIMULATOR_ENDPOINT`). No native simulator is + installed here, so the node runs with a per-run Static root; S1 adds a + Dstack-root variant. +- DuckDB artifacts (`service=duckdb`, an opt-in feature): S4 follow-up. diff --git a/scripts/at-rest-harness/harness.sh b/scripts/at-rest-harness/harness.sh new file mode 100755 index 00000000..0a51e8a9 --- /dev/null +++ b/scripts/at-rest-harness/harness.sh @@ -0,0 +1,651 @@ +#!/usr/bin/env bash +# TC-887 S0: native at-rest harness. A real tinycloud-node binary on native +# PostgreSQL (capability DB + SQL artifacts) and native MinIO (S3 block store). +# No containers. Everything binds 127.0.0.1 on free ports; all state lives in +# one per-run directory; `down` stops only the process groups this run started. +# +# harness.sh bins fetch + sha256-verify the pinned MinIO/mc binaries +# harness.sh up [--no-build] start PG, MinIO, the node; prints the STATE path last +# harness.sh test green|red [--expect green|red] STATE +# run the checks; exit 0 only on the expected outcome +# harness.sh status STATE print the state file +# harness.sh logs STATE tail the harness logs (no secrets are written to them) +# harness.sh down [--purge] STATE stop the recorded processes; --purge removes the run dir +# +# STATE is the state.json path `up` printed (or its run directory). Commands +# never guess it. See README.md next to this script. +set -Eeuo pipefail + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +SCRATCH_ROOT=${TC887_SCRATCH_ROOT:-$HOME/scratch/tc887} +BIN_DIR=${TC887_BIN_DIR:-${XDG_CACHE_HOME:-$HOME/.cache}/tc887-harness/bin} +STATE_SCHEMA=tc887-at-rest-harness/v2 + +# Pinned MinIO community releases (linux-amd64). dl.min.io no longer serves +# community binaries (HTTP 410), so these come from the archived GitHub +# releases. The sha256 values are pinned here, not trusted from the download. +MINIO_RELEASE=RELEASE.2025-09-07T16-13-09Z +MINIO_SHA256=7c5bd8512c6e966455b1d198209358b2d191c77a83ab377c4073281065fb855f +MC_RELEASE=RELEASE.2025-08-13T08-35-41Z +MC_SHA256=01f866e9c5f9b87c2b09116fa5d7c06695b106242d829a8bb32990c00312e891 +MINIO_URL=https://github.com/minio/minio/releases/download/$MINIO_RELEASE/minio.linux-amd64.$MINIO_RELEASE +MC_URL=https://github.com/minio/mc/releases/download/$MC_RELEASE/mc.linux-amd64.$MC_RELEASE + +# --------------------------------------------------------------------------- +# TC-887 at-rest node config. These two names belong to the S1 key-continuity +# slice and may change; this block is the only place the harness defines them +# (README "At-rest mode" documents the same mapping). +# TC887_AT_REST_MODE -> TINYCLOUD_AT_REST__MODE (off|encrypt|strict) +# TC887_AT_REST_ESCROW -> TINYCLOUD_AT_REST__ESCROW (Static self-host +# `none` acknowledgement, plan §5) +# Unset: the harness sets nothing and the node runs its default (today: no +# encryption at rest). With mode encrypt/strict, `test red` expects green. +AT_REST_MODE_VAR=TINYCLOUD_AT_REST__MODE +AT_REST_ESCROW_VAR=TINYCLOUD_AT_REST__ESCROW +# --------------------------------------------------------------------------- + +BUCKET=tinycloud-blocks +REGION=us-east-1 +PG_USER=tc887 +PG_DB=tinycloud + +die() { printf 'at-rest-harness: %s\n' "$*" >&2; exit 2; } +log() { printf 'at-rest-harness: %s\n' "$*" >&2; } + +sha256_of() { sha256sum "$1" | awk '{print $1}'; } + +# Bind-and-release; another process can take the port before we use it. +free_port() { + python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()' +} + +random_hex() { python3 -c 'import secrets; print(secrets.token_hex(24))'; } + +# Write stdin to a new 0600 file. +secret_file() { (umask 077 && cat >"$1"); } + +# Kernel start time (clock ticks since boot) of PID: with the PID, it names +# one process for its whole life, across exec and binary rebuilds. +proc_start() { + local stat + stat=$(cat "/proc/$1/stat" 2>/dev/null) || return 1 + stat=${stat##*) } + # shellcheck disable=SC2086 + set -- $stat + printf '%s\n' "${20}" +} + +# ---------------------------------------------------------------- binaries -- + +fetch_one() { + local name=$1 url=$2 want=$3 dest=$BIN_DIR/$1 + if [[ -x "$dest" && "$(sha256_of "$dest")" == "$want" ]]; then + return 0 + fi + mkdir -p "$BIN_DIR" + local tmp got + tmp=$(mktemp "$BIN_DIR/.$name.XXXXXX") + log "downloading $name from $url" + if ! curl -fsSL --retry 3 --max-time 600 -o "$tmp" "$url"; then + rm -f "$tmp" + die "downloading $name failed" + fi + got=$(sha256_of "$tmp") + if [[ "$got" != "$want" ]]; then + rm -f "$tmp" + die "$name sha256 mismatch: got $got, pinned $want" + fi + chmod 0755 "$tmp" + mv -f "$tmp" "$dest" +} + +cmd_bins() { + local os arch + os=$(uname -s) arch=$(uname -m) + [[ "$os" == Linux && "$arch" == x86_64 ]] \ + || die "the pinned MinIO/mc binaries are linux-amd64 only; this host is $os $arch" + fetch_one minio "$MINIO_URL" "$MINIO_SHA256" + fetch_one mc "$MC_URL" "$MC_SHA256" + log "minio: $("$BIN_DIR/minio" --version | head -1)" + log "mc: $("$BIN_DIR/mc" --version | head -1)" +} + +detect_pg_bin() { + if [[ -n "${PG_BIN:-}" ]]; then + [[ -x "$PG_BIN/initdb" ]] || die "PG_BIN=$PG_BIN has no initdb" + printf '%s\n' "$PG_BIN" + return + fi + local best + best=$(find /usr/lib/postgresql -mindepth 3 -maxdepth 3 -path '*/bin/initdb' 2>/dev/null | sort -V | tail -1) + if [[ -n "$best" ]]; then + dirname "$best" + return + fi + if command -v pg_config >/dev/null && [[ -x "$(pg_config --bindir)/initdb" ]]; then + pg_config --bindir + return + fi + die "no native PostgreSQL found; set PG_BIN to a directory with initdb/pg_ctl/postgres" +} + +node_target_dir() { + cargo metadata --format-version 1 --no-deps --manifest-path "$ROOT/Cargo.toml" \ + | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])' +} + +# ------------------------------------------------------------ processes --- + +# Is the process group led by PID (or PID itself) still alive? +group_alive() { kill -0 -- "-$1" 2>/dev/null || kill -0 "$1" 2>/dev/null; } + +signal_group() { kill "-$1" -- "-$2" 2>/dev/null || kill "-$1" "$2" 2>/dev/null || true; } + +# Stop the process group led by a recorded PID. Each harness process is a +# session/group leader (setsid; pg_ctl's postmaster calls setsid), so the +# group holds it and the children it forks. (PostgreSQL moves its backends +# into their own groups; the postmaster stops them on SIGTERM, and they exit +# by themselves if it dies.) The leader is signalled only if its kernel start +# time still matches the recorded one. A group whose leader is gone still +# holds our leftovers: Linux never reuses a PID while it names a live process +# group. Returns 1 if a live PID was left alone. +stop_recorded() { + local what=$1 pid=$2 start=$3 + [[ -n "$pid" ]] || return 0 + if kill -0 "$pid" 2>/dev/null; then + local actual + actual=$(proc_start "$pid" || true) + if [[ -z "$start" || "$actual" != "$start" ]]; then + log "pid $pid is not the recorded $what (start time ${actual:-?} != ${start:-unrecorded}); leaving it alone" + return 1 + fi + elif ! group_alive "$pid"; then + log "$what (pid $pid) is not running" + return 0 + fi + signal_group TERM "$pid" + local i + for ((i = 0; i < 100; i++)); do + group_alive "$pid" || { log "$what (process group $pid) stopped"; return 0; } + sleep 0.1 + done + signal_group KILL "$pid" + for ((i = 0; i < 50; i++)); do + group_alive "$pid" || { log "$what (process group $pid) killed"; return 0; } + sleep 0.1 + done + log "$what (process group $pid) survived SIGKILL" + return 1 +} + +# pg_ctl stop (a clean shutdown of our data dir), then the group fallback. +stop_postgres() { + local run=$1 pid=$2 start=$3 pg_bin=$4 + if [[ -n "$pid" && -n "$pg_bin" && -f "$run/pgdata/postmaster.pid" ]] \ + && kill -0 "$pid" 2>/dev/null && [[ "$(proc_start "$pid" || true)" == "$start" ]]; then + if "$pg_bin/pg_ctl" -D "$run/pgdata" -m fast -w -t 30 stop >/dev/null 2>&1 && ! group_alive "$pid"; then + log "postgres (process group $pid) stopped by pg_ctl" + return 0 + fi + fi + stop_recorded postgres "$pid" "$start" +} + +# --------------------------------------------------------------------- up --- + +UP_RUN="" UP_PG_PID="" UP_PG_START="" UP_MINIO_PID="" UP_MINIO_START="" UP_NODE_PID="" UP_NODE_START="" + +wait_http() { + local url=$1 pid=$2 what=$3 seconds=$4 i + for ((i = 0; i < seconds * 5; i++)); do + kill -0 "$pid" 2>/dev/null || die "$what (pid $pid) exited during startup" + if curl -fsS --max-time 2 -o /dev/null "$url" 2>/dev/null; then + return 0 + fi + sleep 0.2 + done + die "$what did not become ready at $url within ${seconds}s" +} + +# EXIT trap of `up`: a failed, interrupted or timed-out `up` stops every +# process group it started and keeps the run dir (logs) for inspection. +up_abort() { + local status=$? + trap '' INT TERM + trap - EXIT + set +e + if [[ -n "$UP_RUN" && -d "$UP_RUN" ]]; then + log "up failed (exit $status); stopping every process this run started" + if [[ -z "$UP_PG_PID" && -f "$UP_RUN/pgdata/postmaster.pid" ]]; then + UP_PG_PID=$(head -1 "$UP_RUN/pgdata/postmaster.pid") + UP_PG_START=$(proc_start "$UP_PG_PID") + fi + write_state "$UP_RUN" failed + stop_recorded tinycloud-node "$UP_NODE_PID" "$UP_NODE_START" + stop_recorded minio "$UP_MINIO_PID" "$UP_MINIO_START" + stop_postgres "$UP_RUN" "$UP_PG_PID" "$UP_PG_START" "${STATE_PG_BIN:-}" + log "logs kept in $UP_RUN/logs; remove the run with: $0 down --purge $UP_RUN/state.json" + fi + exit "$status" +} + +valid_setting() { [[ "$1" =~ ^[A-Za-z0-9_.:-]+$ ]]; } + +cmd_up() { + local build=1 + for arg in "$@"; do + case $arg in + --no-build) build=0 ;; + *) die "up: unknown argument $arg" ;; + esac + done + trap up_abort EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + command -v python3 >/dev/null || die "python3 is required" + command -v curl >/dev/null || die "curl is required" + local at_rest_mode=${TC887_AT_REST_MODE:-} at_rest_escrow=${TC887_AT_REST_ESCROW:-} + [[ -z "$at_rest_mode" ]] || valid_setting "$at_rest_mode" || die "invalid TC887_AT_REST_MODE" + [[ -z "$at_rest_escrow" ]] || valid_setting "$at_rest_escrow" || die "invalid TC887_AT_REST_ESCROW" + cmd_bins + + local pg_bin + pg_bin=$(detect_pg_bin) + + local node_bin=${TC887_NODE_BIN:-} + if [[ -z "$node_bin" ]]; then + if ((build)); then + log "building tinycloud-node (nice -n 10, -j ${TC887_CARGO_JOBS:-8})" + (cd "$ROOT" && nice -n 10 cargo build -p tinycloud-node --bin tinycloud -j "${TC887_CARGO_JOBS:-8}") + fi + node_bin=$(node_target_dir)/debug/tinycloud + fi + [[ -x "$node_bin" ]] || die "node binary $node_bin is missing (build it or set TC887_NODE_BIN)" + node_bin=$(readlink -f "$node_bin") + + mkdir -p "$SCRATCH_ROOT" + local run + run=${TC887_RUN_DIR:-$SCRATCH_ROOT/harness-run-$(date -u +%Y%m%dT%H%M%SZ)-$$} + [[ "$(basename "$run")" == harness-run-* ]] || die "run dir must be named harness-run-*: $run" + [[ ! -e "$run/state.json" ]] || die "$run already holds a harness" + mkdir -p "$run" + chmod 0700 "$run" + run=$(cd "$run" && pwd) + UP_RUN=$run + mkdir -p "$run"/{bin,secrets,logs,home,tmp,node-data,minio-data,mc} + chmod 0700 "$run/secrets" + # Run a private copy, so a rebuild of target/ can't swap the binary under + # the node. + log "node binary: $node_bin" + install -m 0755 "$node_bin" "$run/bin/tinycloud" + node_bin=$run/bin/tinycloud + + local pg_port=${TC887_PG_PORT:-$(free_port)} + local minio_port=${TC887_MINIO_PORT:-$(free_port)} + local console_port=${TC887_MINIO_CONSOLE_PORT:-$(free_port)} + local node_port=${TC887_NODE_PORT:-$(free_port)} + # Recorded by every write_state call, including a failed up's. + export STATE_PG_BIN=$pg_bin STATE_NODE_BIN=$node_bin STATE_PG_PORT=$pg_port \ + STATE_MINIO_PORT=$minio_port STATE_CONSOLE_PORT=$console_port STATE_NODE_PORT=$node_port \ + STATE_AT_REST_MODE=$at_rest_mode STATE_AT_REST_ESCROW=$at_rest_escrow + + # Per-run secrets: generated here, kept in 0600 files, never on argv. + random_hex | secret_file "$run/secrets/pg-password" + printf 'tc887%s\n' "$(python3 -c 'import secrets; print(secrets.token_hex(8))')" | secret_file "$run/secrets/minio-user" + random_hex | secret_file "$run/secrets/minio-password" + python3 -c 'import secrets; print(secrets.token_urlsafe(32))' | secret_file "$run/secrets/node-static-secret" + local pg_password minio_user minio_password node_secret + pg_password=$(<"$run/secrets/pg-password") + minio_user=$(<"$run/secrets/minio-user") + minio_password=$(<"$run/secrets/minio-password") + node_secret=$(<"$run/secrets/node-static-secret") + printf '127.0.0.1:%s:*:%s:%s\n' "$pg_port" "$PG_USER" "$pg_password" | secret_file "$run/secrets/pgpass" + + local pg_url="postgres://$PG_USER:$pg_password@127.0.0.1:$pg_port/$PG_DB" + local s3_endpoint="http://127.0.0.1:$minio_port" + local node_url="http://127.0.0.1:$node_port" + + # Test-side credentials (read by the Rust harness checks). + secret_file "$run/harness.env" <>"$run/node.env" + log "at-rest: $AT_REST_MODE_VAR=$at_rest_mode" + fi + if [[ -n "$at_rest_escrow" ]]; then + printf '%s=%s\n' "$AT_REST_ESCROW_VAR" "$at_rest_escrow" >>"$run/node.env" + log "at-rest: $AT_REST_ESCROW_VAR=$at_rest_escrow" + fi + printf '[global]\nlog_level = "normal"\n' >"$run/tinycloud.toml" + write_state "$run" starting + + # ---- PostgreSQL + log "PostgreSQL: $("$pg_bin/postgres" --version) on 127.0.0.1:$pg_port" + "$pg_bin/initdb" -D "$run/pgdata" -U "$PG_USER" --auth=scram-sha-256 \ + --pwfile="$run/secrets/pg-password" -E UTF8 --locale=C >"$run/logs/initdb.log" 2>&1 + cat >>"$run/pgdata/postgresql.conf" </dev/null + UP_PG_PID=$(head -1 "$run/pgdata/postmaster.pid") + UP_PG_START=$(proc_start "$UP_PG_PID") + write_state "$run" starting + PGPASSFILE=$run/secrets/pgpass "$pg_bin/psql" -X -q -h 127.0.0.1 -p "$pg_port" -U "$PG_USER" \ + -d postgres -v ON_ERROR_STOP=1 -c "CREATE DATABASE $PG_DB" + + # ---- MinIO + log "MinIO: $MINIO_RELEASE on $s3_endpoint" + ( + cd "$run" + export MINIO_ROOT_USER=$minio_user MINIO_ROOT_PASSWORD=$minio_password + export MINIO_BROWSER=off MINIO_UPDATE=off HOME=$run/home + exec setsid "$BIN_DIR/minio" server "$run/minio-data" --quiet \ + --address "127.0.0.1:$minio_port" --console-address "127.0.0.1:$console_port" \ + "$run/logs/minio.log" 2>&1 + ) & + UP_MINIO_PID=$! + UP_MINIO_START=$(proc_start "$UP_MINIO_PID" || true) + write_state "$run" starting + wait_http "$s3_endpoint/minio/health/live" "$UP_MINIO_PID" MinIO 60 + # mc reads the alias (with credentials) from the environment, not argv. + MC_CONFIG_DIR=$run/mc MC_HOST_tc887="http://$minio_user:$minio_password@127.0.0.1:$minio_port" \ + "$BIN_DIR/mc" --quiet mb "tc887/$BUCKET" >"$run/logs/mc.log" 2>&1 + + # ---- tinycloud-node + log "tinycloud-node: $node_bin on $node_url" + ( + cd "$run" + exec env -i NODE_ENV_FILE="$run/node.env" setsid bash -c \ + 'set -a; . "$NODE_ENV_FILE"; set +a; unset NODE_ENV_FILE; exec "$0"' "$node_bin" \ + "$run/logs/node.log" 2>&1 + ) & + UP_NODE_PID=$! + UP_NODE_START=$(proc_start "$UP_NODE_PID" || true) + write_state "$run" starting + wait_http "$node_url/healthz" "$UP_NODE_PID" tinycloud-node "${TC887_NODE_READY_SECS:-180}" + + write_state "$run" up + trap - EXIT INT TERM + log "up: state file $run/state.json" + printf '%s\n' "$run/state.json" +} + +# write_state RUN STATUS; PIDs come from UP_*, the rest from STATE_* env. +write_state() { + RUN=$1 STATUS=$2 SCHEMA=$STATE_SCHEMA \ + PG_PID=$UP_PG_PID PG_START=$UP_PG_START MINIO_PID=$UP_MINIO_PID MINIO_START=$UP_MINIO_START \ + NODE_PID=$UP_NODE_PID NODE_START=$UP_NODE_START \ + BIN_DIR=$BIN_DIR MINIO_RELEASE=$MINIO_RELEASE MC_RELEASE=$MC_RELEASE \ + BUCKET=$BUCKET REGION=$REGION PG_USER=$PG_USER PG_DB=$PG_DB \ + AT_REST_MODE_VAR=$AT_REST_MODE_VAR AT_REST_ESCROW_VAR=$AT_REST_ESCROW_VAR python3 - <<'PY' +import json, os, datetime +e = os.environ +run = e["RUN"] +def num(name): + v = e.get(name, "") + return int(v) if v else None +def text(name): + return e.get(name) or None +pg_bin = text("STATE_PG_BIN") +node_port = num("STATE_NODE_PORT") +minio_port = num("STATE_MINIO_PORT") +state = { + "schema": e["SCHEMA"], + "status": e["STATUS"], + "run_dir": run, + "updated_at": datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds"), + "env_file": os.path.join(run, "harness.env"), + "at_rest": { + "mode": text("STATE_AT_REST_MODE"), + "escrow": text("STATE_AT_REST_ESCROW"), + "mode_var": e["AT_REST_MODE_VAR"], + "escrow_var": e["AT_REST_ESCROW_VAR"], + }, + "node": { + "pid": num("NODE_PID"), + "pid_start": num("NODE_START"), + "port": node_port, + "url": f"http://127.0.0.1:{node_port}" if node_port else None, + "bin": text("STATE_NODE_BIN"), + "data_dir": os.path.join(run, "node-data"), + "log": os.path.join(run, "logs", "node.log"), + "keys": "Static (per-run, secrets/node-static-secret)", + "sql_memory_threshold": 0, + }, + "postgres": { + "pid": num("PG_PID"), + "pid_start": num("PG_START"), + "port": num("STATE_PG_PORT"), + "bin_dir": pg_bin, + "data_dir": os.path.join(run, "pgdata"), + "user": e["PG_USER"], + "database": e["PG_DB"], + "log": os.path.join(run, "logs", "postgres.log"), + }, + "minio": { + "pid": num("MINIO_PID"), + "pid_start": num("MINIO_START"), + "port": minio_port, + "console_port": num("STATE_CONSOLE_PORT"), + "endpoint": f"http://127.0.0.1:{minio_port}" if minio_port else None, + "bin": os.path.join(e["BIN_DIR"], "minio"), + "release": e["MINIO_RELEASE"], + "mc_release": e["MC_RELEASE"], + "bucket": e["BUCKET"], + "region": e["REGION"], + "data_dir": os.path.join(run, "minio-data"), + "log": os.path.join(run, "logs", "minio.log"), + }, +} +path = os.path.join(run, "state.json") +with open(path + ".tmp", "w") as f: + json.dump(state, f, indent=2) + f.write("\n") +os.replace(path + ".tmp", path) +PY +} + +# ------------------------------------------------------------ state file --- + +# The explicit STATE argument (state.json or its run dir), checked. +resolve_state() { + local target=${1:-} + [[ -n "$target" ]] || die "STATE is required: pass the state.json path \`up\` printed (no default; refusing to guess)" + [[ -d "$target" ]] && target=$target/state.json + [[ -f "$target" ]] || die "no harness state at $target" + target=$(readlink -f "$target") + [[ "$(state_get "$target" schema)" == "$STATE_SCHEMA" ]] || die "$target is not a $STATE_SCHEMA state file" + [[ "$(state_get "$target" run_dir)" == "$(dirname "$target")" ]] || die "$target does not live in its recorded run dir" + printf '%s\n' "$target" +} + +state_get() { + python3 - "$1" "$2" <<'PY' +import json, sys +value = json.load(open(sys.argv[1])) +for key in sys.argv[2].split("."): + value = value.get(key) if isinstance(value, dict) else None +print("" if value is None else value) +PY +} + +# ------------------------------------------------------------------- down --- + +cmd_down() { + local purge=0 target="" + for arg in "$@"; do + case $arg in + --purge) purge=1 ;; + -*) die "down: unknown option $arg" ;; + *) [[ -z "$target" ]] || die "down: one STATE only"; target=$arg ;; + esac + done + local state run survivors=0 + state=$(resolve_state "$target") + run=$(state_get "$state" run_dir) + stop_recorded tinycloud-node "$(state_get "$state" node.pid)" "$(state_get "$state" node.pid_start)" || survivors=1 + stop_recorded minio "$(state_get "$state" minio.pid)" "$(state_get "$state" minio.pid_start)" || survivors=1 + stop_postgres "$run" "$(state_get "$state" postgres.pid)" "$(state_get "$state" postgres.pid_start)" \ + "$(state_get "$state" postgres.bin_dir)" || survivors=1 + if ((purge)); then + ((survivors == 0)) || die "not purging $run: a recorded process is still running" + [[ "$(basename "$run")" == harness-run-* ]] || die "not purging $run: not a harness-run-* dir" + rm -rf -- "$run" + log "removed $run" + elif ((survivors == 0)); then + python3 - "$state" <<'PY' +import json, os, sys +path = sys.argv[1] +state = json.load(open(path)) +state["status"] = "down" +with open(path + ".tmp", "w") as f: + json.dump(state, f, indent=2) + f.write("\n") +os.replace(path + ".tmp", path) +PY + fi + return "$survivors" +} + +# ------------------------------------------------------------------- test --- + +annotate() { + local level=$1 + shift + if [[ "${GITHUB_ACTIONS:-}" == true ]]; then + printf '::%s::%s\n' "$level" "$*" + fi + log "$*" +} + +cmd_test() { + local kind="" expect="" target="" + while (($#)); do + case $1 in + green|red) [[ -z "$kind" ]] || die "test: one kind only"; kind=$1 ;; + --expect) (($# >= 2)) || die "--expect needs green or red"; shift; expect=$1 ;; + --expect=*) expect=${1#--expect=} ;; + -*) die "test: unknown option $1" ;; + *) [[ -z "$target" ]] || die "test: one STATE only"; target=$1 ;; + esac + shift + done + [[ -n "$kind" ]] || die "usage: harness.sh test green|red [--expect green|red] STATE" + local state + state=$(resolve_state "$target") + [[ "$(state_get "$state" status)" == up ]] || die "harness at $state is not up" + if [[ -z "$expect" ]]; then + expect=green + if [[ "$kind" == red ]]; then + case $(state_get "$state" at_rest.mode) in + encrypt|strict) expect=green ;; + *) expect=red ;; + esac + fi + fi + [[ "$expect" == green || "$expect" == red ]] || die "--expect must be green or red" + + local out status=0 + out=$(mktemp) + (cd "$ROOT" && TC887_HARNESS_STATE=$state nice -n 10 cargo test -p tinycloud-node \ + -j "${TC887_CARGO_JOBS:-8}" --test at_rest_harness -- --ignored --nocapture --test-threads=1 "${kind}_") \ + 2>&1 | tee "$out" || status=$? + local passed failed kv_red=0 sql_red=0 + read -r passed failed < <(grep -Eo 'test result: [A-Za-z]+\. [0-9]+ passed; [0-9]+ failed' "$out" \ + | awk '{p += $4; f += $6} END {print p + 0, f + 0}') + if [[ "$kind" == red ]]; then + grep -q 'TC-887 RED: KV data is not sealed at rest' "$out" && kv_red=1 + grep -q 'TC-887 RED: SQL artifacts are not sealed at rest' "$out" && sql_red=1 + fi + rm -f "$out" + local reds=$((kv_red + sql_red)) ran=$((passed + failed)) + + # libtest exits 0 when a filter matches nothing; never let that pass (TC-381). + if ((ran != 2)); then + annotate error "$kind checks ERRORED: expected 2 to run, $ran ran (cargo exit $status: compile or harness failure)" + return 4 + fi + if ((failed != reds)); then + annotate error "$kind checks ERRORED: $failed failed but only $reds with a TC-887 RED finding (harness, precondition or node error; see output above)" + return 4 + fi + if [[ "$expect" == green ]]; then + if ((status == 0 && failed == 0)); then + annotate notice "$kind checks GREEN as expected ($passed passed)" + return 0 + fi + annotate error "$kind checks still RED ($reds of 2 report plaintext at rest) but expected green" + return 3 + fi + if ((status == 101 && reds == 2)); then + annotate notice "RED as expected: both TC-887 RED checks report plaintext at rest (KV and SQL)" + return 0 + fi + local flipped=() + ((kv_red)) || flipped+=(KV) + ((sql_red)) || flipped+=(SQL) + annotate error "$kind checks UNEXPECTEDLY PASSED under --expect red (no TC-887 RED finding for ${flipped[*]}): the at-rest property holds there; run with --expect green (or set TC887_AT_REST_MODE) and update CI" + return 3 +} + +cmd_status() { + local state + state=$(resolve_state "${1:-}") + cat "$state" +} + +cmd_logs() { + local state run f + state=$(resolve_state "${1:-}") + run=$(state_get "$state" run_dir) + for f in "$run"/logs/*.log; do + printf '==> %s <==\n' "$f" + tail -n "${TC887_LOG_LINES:-40}" "$f" + done +} + +case ${1:-} in + bins) shift; cmd_bins "$@" ;; + up) shift; cmd_up "$@" ;; + down) shift; cmd_down "$@" ;; + test) shift; cmd_test "$@" ;; + status) shift; cmd_status "$@" ;; + logs) shift; cmd_logs "$@" ;; + *) die "usage: harness.sh bins | up [--no-build] | test green|red [--expect green|red] STATE | status STATE | logs STATE | down [--purge] STATE" ;; +esac diff --git a/tinycloud-node-server/Cargo.toml b/tinycloud-node-server/Cargo.toml index 67c23011..5f9d96c9 100644 --- a/tinycloud-node-server/Cargo.toml +++ b/tinycloud-node-server/Cargo.toml @@ -124,6 +124,9 @@ tokio = { version = "1", features = ["test-util"] } axum = { version = "0.7", features = ["ws"] } bs58 = "0.5" chrono = { version = "0.4", default-features = false, features = ["clock", "serde", "std"] } +# zlib compressibility of raw stored bytes in tests/at_rest_harness.rs (TC-887). +# Already in the lock through ssi-status; adds no new crate. +flate2 = "1" k256 = { version = "0.13.4", features = ["ecdsa"] } sha3 = "0.10" # The Rust half of the JS SDK: builds the same host and session SIWE diff --git a/tinycloud-node-server/tests/at_rest_harness.rs b/tinycloud-node-server/tests/at_rest_harness.rs new file mode 100644 index 00000000..27eb7b86 --- /dev/null +++ b/tinycloud-node-server/tests/at_rest_harness.rs @@ -0,0 +1,985 @@ +//! TC-887 S0: at-rest checks against the native harness in +//! `scripts/at-rest-harness/` (a real `tinycloud` binary on native PostgreSQL +//! and native MinIO). Every test is `#[ignore]`d: they need the harness up and +//! are run through `scripts/at-rest-harness/harness.sh test green|red STATE`, +//! which sets `TC887_HARNESS_STATE` and checks the expected outcome. +//! +//! Each check writes the same plaintext into two fresh spaces through the +//! node's public API (SDK-style sign-in, `/invoke`): a short value carrying a +//! unique marker and a 64 KiB value of the marker repeated (highly +//! compressible). For SQL, the second write lands as a WAL delta +//! (`delta_payload`), because the harness runs the node with +//! `storage.sql.memory_threshold = 0`. +//! +//! - `green_*` prove the harness is live: the writes round-trip through the +//! node and land in MinIO / `database_artifact` (including a delta). They +//! stay green before and after encryption, and the RED checks re-run the +//! same preconditions, so a RED result is never a harness no-op. +//! - `red_*` assert the TC-887 at-rest property on the raw stored bytes. They +//! FAIL today (plaintext at rest) and turn green with slices S3 (KV) and S4 +//! (SQL). Besides the marker, the space id/address and SQLite magic, they +//! reject the plaintext fingerprint in object names and any deterministic, +//! non-encrypting transform (encoding, XOR, compression): the two spaces' +//! stored bytes must share no [`MAX_SHARED_RUN`]-byte run, and stored bytes +//! must not compress below [`MIN_INCOMPRESSIBLE_RATIO`]. + +use std::collections::{BTreeSet, HashMap}; +use std::io::Write; +use std::time::Duration; + +use anyhow::{anyhow, bail, ensure, Context, Result}; +use k256::ecdsa::SigningKey; +use reqwest::{Client, StatusCode}; +use serde_json::{json, Value}; +use sha3::{Digest, Keccak256}; +use tinycloud_auth::{ + cacaos::{ + siwe::{encode_eip55, Message}, + siwe_cacao::Signature, + }, + resource::{Path, Service, SpaceId}, + siwe_recap::Ability, + ssi::dids::DIDBuf, +}; +use tinycloud_core::models::database_artifact; +use tinycloud_core::sea_orm::{ + ColumnTrait, Database, DatabaseConnection, EntityTrait, QueryFilter, +}; +use tinycloud_sdk_wasm::{ + host::{generate_host_siwe_message, siwe_to_delegation_headers, HostConfig, SignedMessage}, + session::{complete_session_setup, prepare_session, Session, SessionConfig, SignedSession}, +}; + +const SQLITE_MAGIC: &[u8] = b"SQLite format 3\0"; +const SQLITE_WAL_MAGIC: [[u8; 4]; 2] = [[0x37, 0x7f, 0x06, 0x82], [0x37, 0x7f, 0x06, 0x83]]; +const SQL_DATABASE: &str = "tc887"; +const KV_PUT: &str = "tinycloud.kv/put"; +const KV_GET: &str = "tinycloud.kv/get"; +const SQL_ADMIN: &str = "tinycloud.sql/admin"; +/// Size of the compressible probe value (the marker repeated). +const BIG_VALUE_LEN: usize = 64 * 1024; + +/// Stored bytes of two independent writes of the same plaintext (in two +/// spaces) must not share a run of this many bytes. +/// +/// A sealed blob (plan §1.2-1.3: a 64-byte TCE1 header, then AES-256-GCM +/// segments under a per-blob key derived from a random 32-byte salt) repeats +/// only fixed header fields between two writes of the same plaintext: magic, +/// version, algorithm, flags, reserved, `dek_id` and `plaintext_len` (20 +/// bytes), then the random salt, then `seg_log2` and padding (12 bytes). So +/// honest ciphertexts share runs of at most ~20 bytes, while a chance 32-byte +/// match between pseudorandom strings of n and m bytes has probability about +/// n·m/2^256. A deterministic transform of identical plaintext (encoding, +/// XOR, compression, an unsalted cipher keyed the same in both spaces) keeps +/// runs as long as the data itself. +const MAX_SHARED_RUN: usize = 32; + +/// zlib must not shrink stored bytes below this fraction of their length. +/// Ciphertext is incompressible (zlib ratio ≈ 1.0); the 64 KiB repeated-marker +/// plaintext compresses to ≈ 0.01, and base64 or XOR of it to ≈ 0.03. +const MIN_INCOMPRESSIBLE_RATIO: f64 = 0.95; + +/// The running harness, from the state file `harness.sh up` wrote. +struct Harness { + node: String, + http: Client, + s3: aws_sdk_s3::Client, + bucket: String, + pg: DatabaseConnection, +} + +impl Harness { + async fn connect() -> Result { + let path = std::env::var("TC887_HARNESS_STATE").context( + "TC887_HARNESS_STATE is unset: start the harness with \ + `scripts/at-rest-harness/harness.sh up` and run these checks with \ + `scripts/at-rest-harness/harness.sh test green|red`", + )?; + let state: Value = serde_json::from_slice( + &std::fs::read(&path).with_context(|| format!("read harness state {path}"))?, + )?; + ensure!(state["status"] == "up", "harness at {path} is not up"); + let env_file = state["env_file"] + .as_str() + .context("state file has no env_file")?; + let env: HashMap = std::fs::read_to_string(env_file) + .with_context(|| format!("read harness env file {env_file}"))? + .lines() + .filter_map(|line| line.split_once('=')) + .map(|(key, value)| (key.to_owned(), value.to_owned())) + .collect(); + let var = |key: &str| { + env.get(key) + .cloned() + .with_context(|| format!("{key} missing from harness env file")) + }; + + let sdk = aws_config::from_env() + .region(aws_sdk_s3::Region::new(var("TC887_S3_REGION")?)) + .credentials_provider(aws_sdk_s3::Credentials::new( + var("TC887_S3_ACCESS_KEY_ID")?, + var("TC887_S3_SECRET_ACCESS_KEY")?, + None, + None, + "tc887-harness", + )) + .load() + .await; + let endpoint: rocket::http::hyper::Uri = var("TC887_S3_ENDPOINT")?.parse()?; + let s3 = aws_sdk_s3::Client::from_conf( + aws_sdk_s3::config::Builder::from(&sdk) + .endpoint_resolver(aws_sdk_s3::Endpoint::immutable(endpoint)) + .build(), + ); + let pg = Database::connect(var("TC887_PG_URL")?) + .await + .context("connect to the harness PostgreSQL")?; + Ok(Self { + node: var("TC887_NODE_URL")?, + http: Client::builder() + .no_proxy() + .timeout(Duration::from_secs(60)) + .build()?, + s3, + bucket: var("TC887_S3_BUCKET")?, + pg, + }) + } + + /// Every object key in the harness bucket. + async fn keys(&self) -> Result> { + let mut keys = BTreeSet::new(); + let mut token = None; + loop { + let page = self + .s3 + .list_objects_v2() + .bucket(&self.bucket) + .set_continuation_token(token) + .send() + .await + .context("list the harness bucket")?; + for object in page.contents().unwrap_or_default() { + keys.insert(object.key().context("object without key")?.to_owned()); + } + if !page.is_truncated() { + return Ok(keys); + } + token = page.next_continuation_token().map(str::to_owned); + } + } + + /// One object's raw bytes, as MinIO stores them. + async fn object(&self, key: &str) -> Result> { + Ok(self + .s3 + .get_object() + .bucket(&self.bucket) + .key(key) + .send() + .await + .with_context(|| format!("GET raw object {key}"))? + .body + .collect() + .await? + .into_bytes() + .to_vec()) + } + + async fn delegate(&self, authorization: String) -> Result<()> { + let response = self + .http + .post(format!("{}/delegate", self.node)) + .header("Authorization", authorization) + .send() + .await?; + let status = response.status(); + let body = response.text().await?; + ensure!(status == StatusCode::OK, "/delegate: {status} {body}"); + Ok(()) + } + + /// Host a fresh space and open an SDK session on it, the way the SDK's + /// `signIn` does: host SIWE delegation to the node's peer, then a session + /// delegation to an ephemeral Ed25519 key. + async fn sign_in(&self) -> Result<(Account, Session)> { + let account = Account::random()?; + let response = self + .http + .get(format!("{}/peer/generate/{}", self.node, account.space)) + .send() + .await?; + ensure!( + response.status() == StatusCode::OK, + "peer generation failed" + ); + let host = generate_host_siwe_message(HostConfig { + address: account.address, + chain_id: 1, + domain: "localhost".parse()?, + issued_at: time::OffsetDateTime::now_utc().into(), + space_id: account.space.clone(), + peer_id: response.text().await?, + })?; + let signature = account.sign(&host)?; + self.delegate(authorization(&siwe_to_delegation_headers(SignedMessage { + siwe: host, + signature, + }))?) + .await?; + + let now = time::OffsetDateTime::now_utc(); + let session_key = ed25519_dalek::SigningKey::from_bytes(&uuid_seed()); + let jwk = serde_json::from_value(json!({ + "kty": "OKP", "crv": "Ed25519", + "d": base64::encode_config(session_key.to_bytes(), base64::URL_SAFE_NO_PAD), + "x": base64::encode_config(session_key.verifying_key().to_bytes(), base64::URL_SAFE_NO_PAD) + }))?; + let prepared = prepare_session(SessionConfig { + abilities: HashMap::from([ + ( + "kv".parse()?, + HashMap::from([("".parse()?, vec![KV_PUT.parse()?, KV_GET.parse()?])]), + ), + ( + "sql".parse()?, + HashMap::from([(SQL_DATABASE.parse()?, vec![SQL_ADMIN.parse()?])]), + ), + ]), + space_abilities: None, + raw_abilities: HashMap::new(), + address: account.address, + chain_id: 1, + domain: "localhost".parse()?, + issued_at: now.into(), + space_id: account.space.clone(), + additional_spaces: None, + not_before: None, + expiration_time: (now + time::Duration::hours(1)).into(), + parents: None, + jwk: Some(jwk), + delegate_uri: None, + nonce: None, + invocation_ttl_secs: None, + })?; + let signature = account.sign(&prepared.siwe)?; + let session = complete_session_setup(SignedSession { + session: prepared, + signature, + })?; + self.delegate(authorization(&session.delegation_header)?) + .await?; + Ok((account, session)) + } + + async fn invoke( + &self, + session: &Session, + service: &str, + path: &str, + ability: &str, + body: reqwest::Body, + content_type: &str, + ) -> Result<(StatusCode, Vec)> { + let invocation = session + .invoke( + [( + service.parse::()?, + path.parse::()?, + None, + None, + [ability.parse::()?], + )], + None, + )? + .encode()?; + let response = self + .http + .post(format!("{}/invoke", self.node)) + .header("Authorization", invocation) + .header("Content-Type", content_type) + .body(body) + .send() + .await?; + Ok((response.status(), response.bytes().await?.to_vec())) + } + + async fn sql(&self, session: &Session, request: Value) -> Result { + let (status, body) = self + .invoke( + session, + "sql", + SQL_DATABASE, + SQL_ADMIN, + serde_json::to_vec(&request)?.into(), + "application/json", + ) + .await?; + ensure!( + status == StatusCode::OK, + "sql {}: {status} {}", + request["action"], + String::from_utf8_lossy(&body) + ); + Ok(serde_json::from_slice(&body)?) + } +} + +fn authorization(headers: &impl serde::Serialize) -> Result { + serde_json::to_value(headers)?["Authorization"] + .as_str() + .map(str::to_owned) + .context("SDK delegation must contain Authorization") +} + +/// 32 fresh random bytes (two v4 UUIDs: 244 bits from the OS RNG). +fn uuid_seed() -> [u8; 32] { + let mut seed = [0; 32]; + seed[..16].copy_from_slice(uuid::Uuid::new_v4().as_bytes()); + seed[16..].copy_from_slice(uuid::Uuid::new_v4().as_bytes()); + seed +} + +fn marker(kind: &str) -> String { + format!("TC887-{kind}-MARKER-{}", uuid::Uuid::new_v4().simple()) +} + +/// A synthetic per-run wallet and its space; never a real credential. +struct Account { + key: SigningKey, + address: [u8; 20], + space: SpaceId, +} + +impl Account { + fn random() -> Result { + let key = SigningKey::from_bytes(&uuid_seed().into())?; + let public = key.verifying_key().to_encoded_point(false); + let digest = Keccak256::digest(&public.as_bytes()[1..]); + let address: [u8; 20] = digest[12..].try_into()?; + let did = format!("did:pkh:eip155:1:0x{}", encode_eip55(&address)); + let name = format!("tc887{}", &uuid::Uuid::new_v4().simple().to_string()[..12]); + let space = SpaceId::new(did.parse::()?, name.parse()?); + Ok(Self { + key, + address, + space, + }) + } + + fn sign(&self, message: &Message) -> Result { + let (signature, recovery) = self + .key + .sign_prehash_recoverable(&message.eip191_hash().map_err(|e| anyhow!("{e}"))?)?; + let mut bytes = [0; 65]; + bytes[..64].copy_from_slice(signature.to_bytes().as_ref()); + bytes[64] = u8::from(recovery) + 27; + Ok(Signature::from(bytes)) + } + + /// Lower-case hex wallet address, as it can appear in any casing. + fn address_hex(&self) -> String { + hex::encode(self.address) + } +} + +fn find(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|window| window == needle) +} + +/// `marker` repeated (space-separated) to at least `len` bytes. +fn repeated(marker: &str, len: usize) -> String { + let mut value = String::with_capacity(len + marker.len() + 1); + while value.len() < len { + value.push_str(marker); + value.push(' '); + } + value +} + +/// zlib-compressed size over input size. +fn zlib_ratio(bytes: &[u8]) -> Result { + let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(bytes)?; + Ok(encoder.finish()?.len() as f64 / bytes.len().max(1) as f64) +} + +/// The longest run starting at the first [`MAX_SHARED_RUN`]-byte window of +/// `a` that also occurs in `b`: (offset in a, offset in b, run length). +fn shared_run(a: &[u8], b: &[u8]) -> Option<(usize, usize, usize)> { + if a.len() < MAX_SHARED_RUN || b.len() < MAX_SHARED_RUN { + return None; + } + let mut windows = HashMap::new(); + for (offset, window) in b.windows(MAX_SHARED_RUN).enumerate() { + windows.entry(window).or_insert(offset); + } + let (i, j) = a + .windows(MAX_SHARED_RUN) + .enumerate() + .find_map(|(i, window)| windows.get(window).map(|&j| (i, j)))?; + let len = a[i..] + .iter() + .zip(&b[j..]) + .take_while(|(x, y)| x == y) + .count(); + Some((i, j, len)) +} + +/// Findings for stored bytes `a` and `b` of the same plaintext written into +/// two different spaces. +fn compare_writes(what: &str, a: &[u8], b: &[u8], findings: &mut Vec) { + if a == b { + findings.push(format!( + "{what}: the two spaces stored byte-identical {}-byte blobs for the same plaintext \ + (deterministic, no per-blob randomness)", + a.len() + )); + } else if let Some((i, j, len)) = shared_run(a, b) { + findings.push(format!( + "{what}: the two spaces' blobs ({} and {} bytes) for the same plaintext share a \ + {len}-byte run (space A byte {i}, space B byte {j}; limit {})", + a.len(), + b.len(), + MAX_SHARED_RUN - 1 + )); + } +} + +/// Finding if zlib shrinks `bytes` below [`MIN_INCOMPRESSIBLE_RATIO`]. +fn check_incompressible(what: &str, bytes: &[u8], findings: &mut Vec) -> Result<()> { + let ratio = zlib_ratio(bytes)?; + if ratio <= MIN_INCOMPRESSIBLE_RATIO { + findings.push(format!( + "{what} ({} bytes) compresses to {ratio:.3} of its size with zlib \ + (ciphertext stays above {MIN_INCOMPRESSIBLE_RATIO})", + bytes.len() + )); + } + Ok(()) +} + +/// A text encoding of the plaintext's blake3 fingerprint. +struct Fingerprint { + label: String, + text: String, + case_insensitive: bool, +} + +/// blake3(value), as the raw digest and as the multihash the node stores, +/// in b64url (padded and unpadded), standard base64 (padded and unpadded) +/// and hex. +fn fingerprints(value: &[u8]) -> Vec { + let hash = tinycloud_core::hash::hash(value); + let digest = hash.as_ref().to_vec(); + let multihash: Vec = hash.into(); + let mut out = Vec::new(); + for (form, bytes) in [("blake3", &digest), ("blake3 multihash", &multihash)] { + for (encoding, config) in [ + ("b64url", base64::URL_SAFE), + ("b64url-nopad", base64::URL_SAFE_NO_PAD), + ("base64", base64::STANDARD), + ("base64-nopad", base64::STANDARD_NO_PAD), + ] { + out.push(Fingerprint { + label: format!("{form} {encoding}"), + text: base64::encode_config(bytes, config), + case_insensitive: false, + }); + } + out.push(Fingerprint { + label: format!("{form} hex"), + text: hex::encode(bytes), + case_insensitive: true, + }); + } + out +} + +/// A raw object in MinIO. +struct StoredObject { + key: String, + body: Vec, +} + +/// KV probes written into one space, with the objects each put created. +struct KvSpace { + account: Account, + /// Per probe (same order as [`KvRun::probes`]). + objects: Vec>, +} + +/// The same KV probes written into two fresh spaces. +struct KvRun { + marker: String, + probes: Vec<(&'static str, String)>, + spaces: Vec, +} + +/// One KV put through the node, round-tripped through a get; returns the +/// objects it added to the bucket. Fails if it added none (a harness no-op). +async fn kv_put( + harness: &Harness, + session: &Session, + account: &Account, + path: &str, + value: &str, +) -> Result> { + let before = harness.keys().await?; + let (status, body) = harness + .invoke( + session, + "kv", + path, + KV_PUT, + value.to_owned().into(), + "text/plain", + ) + .await?; + ensure!( + status == StatusCode::OK, + "kv put {path}: {status} {}", + String::from_utf8_lossy(&body) + ); + let (status, body) = harness + .invoke(session, "kv", path, KV_GET, Vec::new().into(), "text/plain") + .await?; + ensure!( + status == StatusCode::OK && body == value.as_bytes(), + "kv get {path} through the node: {status}, {} bytes back for a {}-byte value", + body.len(), + value.len() + ); + let mut objects = Vec::new(); + for key in harness.keys().await?.difference(&before) { + let body = harness.object(key).await?; + ensure!(!body.is_empty(), "new object {key} is empty"); + objects.push(StoredObject { + key: key.clone(), + body, + }); + } + if objects.is_empty() { + bail!( + "harness no-op: the {}-byte KV put to {}/{path} created no object in the MinIO bucket", + value.len(), + account.space + ); + } + Ok(objects) +} + +async fn kv_run(harness: &Harness) -> Result { + let marker = marker("KV"); + let probes = vec![ + ("short", format!("at-rest probe {marker}")), + ("compressible", repeated(&marker, BIG_VALUE_LEN)), + ]; + let mut spaces = Vec::new(); + for _ in 0..2 { + let (account, session) = harness.sign_in().await?; + let mut objects = Vec::new(); + for (label, value) in &probes { + objects.push( + kv_put( + harness, + &session, + &account, + &format!("tc887/{label}"), + value, + ) + .await?, + ); + } + spaces.push(KvSpace { account, objects }); + } + Ok(KvRun { + marker, + probes, + spaces, + }) +} + +/// SQL probes written into one space, with its `database_artifact` rows. +struct SqlSpace { + account: Account, + rows: Vec, +} + +/// The same SQL statements run in two fresh spaces. +struct SqlRun { + markers: [String; 2], + spaces: Vec, +} + +async fn sql_run(harness: &Harness) -> Result { + let markers = [marker("SQL-WRITE1"), marker("SQL-WRITE2")]; + let big = [ + repeated(&markers[0], BIG_VALUE_LEN), + repeated(&markers[1], BIG_VALUE_LEN), + ]; + let insert = "INSERT INTO probe (secret) VALUES (?)"; + let writes = [ + // No checkpoint exists yet: the node saves a full checkpoint into + // `database_artifact.payload` before replying. + json!({"action": "batch", "statements": [ + {"sql": "CREATE TABLE probe (id INTEGER PRIMARY KEY, secret TEXT NOT NULL)", "params": []}, + {"sql": insert, "params": [markers[0]]}, + {"sql": insert, "params": [big[0]]}, + ]}), + // The database is file-backed (memory_threshold = 0), so this write + // is saved as a WAL delta into `delta_payload` before replying. + json!({"action": "batch", "statements": [ + {"sql": insert, "params": [markers[1]]}, + {"sql": insert, "params": [big[1]]}, + ]}), + ]; + let expected = json!([[markers[0]], [big[0]], [markers[1]], [big[1]]]); + let mut spaces = Vec::new(); + for _ in 0..2 { + let (account, session) = harness.sign_in().await?; + for write in &writes { + harness.sql(&session, write.clone()).await?; + } + let read = harness + .sql( + &session, + json!({"action": "query", "sql": "SELECT secret FROM probe ORDER BY id", "params": []}), + ) + .await?; + ensure!( + read["rows"] == expected, + "sql read through the node for {} returned other rows ({} row(s))", + account.space, + read["rows"].as_array().map_or(0, Vec::len) + ); + let rows = database_artifact::Entity::find() + .filter(database_artifact::Column::Service.eq("sql")) + .filter(database_artifact::Column::Space.eq(account.space.to_string())) + .all(&harness.pg) + .await + .context("read database_artifact")?; + let space = SqlSpace { account, rows }; + sql_reached_postgres(&space)?; + spaces.push(space); + } + Ok(SqlRun { markers, spaces }) +} + +/// The writes reached PostgreSQL as a checkpoint and a WAL delta. +fn sql_reached_postgres(space: &SqlSpace) -> Result<()> { + if space.rows.is_empty() { + bail!( + "harness no-op: no database_artifact row for service=sql space={}", + space.account.space + ); + } + ensure!( + space + .rows + .iter() + .any(|row| row.revision >= 1 && !row.payload.is_empty()), + "harness no-op: database_artifact rows for {} carry no payload", + space.account.space + ); + ensure!( + space.rows.iter().any(|row| row + .delta_payload + .as_ref() + .is_some_and(|delta| !delta.is_empty())), + "harness gap: database_artifact rows for {} have no delta_payload, so the WAL-delta \ + path did not run (the harness must start the node with \ + TINYCLOUD_STORAGE__SQL__MEMORY_THRESHOLD=0)", + space.account.space + ); + Ok(()) +} + +fn row_id(row: &database_artifact::Model) -> String { + format!( + "database_artifact({}, {}, {})", + row.service, row.space, row.name + ) +} + +#[tokio::test] +#[ignore = "TC-887 harness: run via scripts/at-rest-harness/harness.sh test green STATE"] +async fn green_kv_put_reaches_minio() -> Result<()> { + let harness = Harness::connect().await?; + let run = kv_run(&harness).await?; + for space in &run.spaces { + for ((label, value), objects) in run.probes.iter().zip(&space.objects) { + println!( + "KV {label} put ({} bytes) for {} reached MinIO: {} new object(s), {} bytes", + value.len(), + space.account.space, + objects.len(), + objects + .iter() + .map(|object| object.body.len()) + .sum::() + ); + } + } + Ok(()) +} + +#[tokio::test] +#[ignore = "TC-887 harness: run via scripts/at-rest-harness/harness.sh test green STATE"] +async fn green_sql_write_reaches_postgres() -> Result<()> { + let harness = Harness::connect().await?; + let run = sql_run(&harness).await?; + for row in run.spaces.iter().flat_map(|space| &space.rows) { + println!( + "{} revision={} payload={}B delta_payload={}", + row_id(row), + row.revision, + row.payload.len(), + row.delta_payload + .as_ref() + .map_or("NULL".to_owned(), |delta| format!("{}B", delta.len())) + ); + } + Ok(()) +} + +#[tokio::test] +#[ignore = "TC-887 RED: plaintext at rest until S3/S4"] +async fn red_kv_blocks_are_sealed_in_minio() -> Result<()> { + let harness = Harness::connect().await?; + let run = kv_run(&harness).await?; + let mut findings = Vec::new(); + for space in &run.spaces { + let space_id = space.account.space.to_string(); + let address = space.account.address_hex(); + for ((label, value), objects) in run.probes.iter().zip(&space.objects) { + let fingerprints = fingerprints(value.as_bytes()); + for StoredObject { key, body } in objects { + if key.contains(&space_id) { + findings.push(format!( + "object key {key:?} contains the space id {space_id}" + )); + } + let lower = key.to_ascii_lowercase(); + if lower.contains(&address) { + findings.push(format!( + "object key {key:?} contains the wallet address 0x{address} (case-insensitive)" + )); + } + let leaked = fingerprints + .iter() + .filter(|fp| { + if fp.case_insensitive { + lower.contains(&fp.text) + } else { + key.contains(&fp.text) + } + }) + .map(|fp| format!("{} {}", fp.label, fp.text)) + .collect::>(); + if !leaked.is_empty() { + findings.push(format!( + "object key {key:?} contains the plaintext fingerprint of the {label} value: {}", + leaked.join(", ") + )); + } + if let Some(offset) = find(body, run.marker.as_bytes()) { + findings.push(format!( + "raw object {key:?} ({} bytes, {label} value) contains the plaintext marker {:?} at byte {offset}", + body.len(), + run.marker + )); + } + if *label == "compressible" { + check_incompressible( + &format!("raw object {key:?} [{label} value]"), + body, + &mut findings, + )?; + } + } + } + } + for (index, (label, _)) in run.probes.iter().enumerate() { + let a = &run.spaces[0].objects[index]; + let b = &run.spaces[1].objects[index]; + for (a, b) in a.iter().zip(b) { + compare_writes( + &format!("KV {label} value ({:?} vs {:?})", a.key, b.key), + &a.body, + &b.body, + &mut findings, + ); + } + } + ensure!( + findings.is_empty(), + "TC-887 RED: KV data is not sealed at rest in MinIO ({} finding(s)):\n {}", + findings.len(), + findings.join("\n ") + ); + Ok(()) +} + +#[tokio::test] +#[ignore = "TC-887 RED: plaintext at rest until S3/S4"] +async fn red_sql_artifacts_are_sealed_in_postgres() -> Result<()> { + let harness = Harness::connect().await?; + let run = sql_run(&harness).await?; + let mut findings = Vec::new(); + for row in run.spaces.iter().flat_map(|space| &space.rows) { + let id = row_id(row); + if row.payload.starts_with(SQLITE_MAGIC) { + findings.push(format!( + "{id}.payload ({} bytes) starts with the SQLite magic {:?}", + row.payload.len(), + String::from_utf8_lossy(&row.payload[..SQLITE_MAGIC.len()]) + )); + } + if let Some(delta) = &row.delta_payload { + if SQLITE_WAL_MAGIC + .iter() + .any(|magic| delta.starts_with(magic)) + { + findings.push(format!( + "{id}.delta_payload ({} bytes) starts with the SQLite WAL magic {}", + delta.len(), + hex::encode(&delta[..4]) + )); + } + } + for (column, bytes) in [ + ("payload", Some(&row.payload)), + ("delta_payload", row.delta_payload.as_ref()), + ] { + let Some(bytes) = bytes else { continue }; + for marker in &run.markers { + if let Some(offset) = find(bytes, marker.as_bytes()) { + findings.push(format!( + "{id}.{column} contains the plaintext marker {marker:?} at byte {offset}" + )); + } + } + check_incompressible(&format!("{id}.{column}"), bytes, &mut findings)?; + } + } + for a in &run.spaces[0].rows { + let b = run.spaces[1] + .rows + .iter() + .find(|b| b.name == a.name) + .with_context(|| { + format!( + "no database_artifact row named {} for the second space {}", + a.name, run.spaces[1].account.space + ) + })?; + compare_writes( + &format!("database_artifact payload of {}", a.name), + &a.payload, + &b.payload, + &mut findings, + ); + if let (Some(da), Some(db)) = (&a.delta_payload, &b.delta_payload) { + compare_writes( + &format!("database_artifact delta_payload of {}", a.name), + da, + db, + &mut findings, + ); + } + } + ensure!( + findings.is_empty(), + "TC-887 RED: SQL artifacts are not sealed at rest in PostgreSQL ({} finding(s)):\n {}", + findings.len(), + findings.join("\n ") + ); + Ok(()) +} + +fn random_bytes(len: usize) -> Vec { + (0..len.div_ceil(16)) + .flat_map(|_| *uuid::Uuid::new_v4().as_bytes()) + .take(len) + .collect() +} + +fn zlib(bytes: &[u8]) -> Vec { + let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(bytes).expect("zlib into memory"); + encoder.finish().expect("zlib into memory") +} + +/// A stand-in for one sealed blob of `len` plaintext bytes: the fixed TCE1 +/// header fields two writes of the same plaintext share (plan §1.3), a random +/// salt, and random ciphertext. +fn sealed_like(len: usize) -> Vec { + let mut blob = b"TCE1\x01\x01\x00\x00".to_vec(); + blob.extend(1u32.to_be_bytes()); // dek_id + blob.extend((len as u64).to_be_bytes()); // plaintext_len + blob.extend(random_bytes(32)); // salt + blob.push(16); // seg_log2 + blob.extend([0; 11]); // pad + blob.extend(random_bytes(len + 16 * len.div_ceil(1 << 16))); + blob +} + +/// Offline check of the RED predicates themselves (runs in plain `cargo +/// test`, no harness; its name stays clear of the `green_`/`red_` filters +/// `harness.sh test` uses): deterministic transforms of identical plaintext +/// are flagged even with the marker and magic gone, and two sealed-like +/// blobs of the same plaintext are not. +#[test] +fn selftest_predicates_flag_deterministic_transforms_not_ciphertext() -> Result<()> { + let marker = marker("SELFTEST"); + let plaintext = repeated(&marker, BIG_VALUE_LEN).into_bytes(); + type Transform = fn(&[u8]) -> Vec; + let transforms: [(&str, Transform); 4] = [ + ("identity", |bytes| bytes.to_vec()), + ("base64", |bytes| base64::encode(bytes).into_bytes()), + ("xor 0x5a", |bytes| { + bytes.iter().map(|byte| byte ^ 0x5a).collect() + }), + ("zlib", zlib), + ]; + for (name, transform) in transforms { + let (a, b) = (transform(&plaintext), transform(&plaintext)); + let mut findings = Vec::new(); + compare_writes(name, &a, &b, &mut findings); + ensure!( + findings.len() == 1, + "{name}: two-space comparison missed it" + ); + if name != "zlib" { + check_incompressible(name, &a, &mut findings)?; + ensure!(findings.len() == 2, "{name}: compressibility missed it"); + } + // A per-write random prefix doesn't hide a deterministic body. + let (a, b) = ( + [random_bytes(16), a].concat(), + [random_bytes(16), b].concat(), + ); + let mut findings = Vec::new(); + compare_writes(name, &a, &b, &mut findings); + ensure!( + findings.len() == 1 && findings[0].contains("run"), + "{name} behind a random prefix: shared-run check missed it" + ); + } + for len in [62, BIG_VALUE_LEN] { + let (a, b) = (sealed_like(len), sealed_like(len)); + let mut findings = Vec::new(); + compare_writes("sealed", &a, &b, &mut findings); + if len == BIG_VALUE_LEN { + check_incompressible("sealed", &a, &mut findings)?; + } + ensure!( + find(&a, marker.as_bytes()).is_none() && findings.is_empty(), + "sealed-like blobs of {len} bytes were flagged: {findings:?}" + ); + } + Ok(()) +}