From 4a8aab8a50ce2e1fa75ed11967c89a67485d5b53 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 8 Oct 2026 15:24:44 -0400 Subject: [PATCH] feat(otdf-local): export benchmark manifest pins from otdf-local env --- otdf-local/src/otdf_local/cli.py | 58 +++++++++++++ otdf-local/tests/test_bench_env.py | 112 ++++++++++++++++++++++++ xtest/perf/README.md | 135 +++++++++++++++++++++++++++++ 3 files changed, 305 insertions(+) create mode 100644 otdf-local/tests/test_bench_env.py diff --git a/otdf-local/src/otdf_local/cli.py b/otdf-local/src/otdf_local/cli.py index 76403667c..6cfd24566 100644 --- a/otdf-local/src/otdf_local/cli.py +++ b/otdf-local/src/otdf_local/cli.py @@ -4,6 +4,7 @@ import shutil import sys import time +from pathlib import Path from typing import Annotated import httpx @@ -565,6 +566,55 @@ def restart( raise typer.Exit(1) +def bench_manifest_path(xtest_root: Path) -> Path: + """Where `otdf-sdk-mgr` writes the benchmark installation manifest. + + Mirrors `otdf_sdk_mgr.bench_prepare.default_manifest_path`. The two packages + do not depend on each other, so the path and the JSON shape are a written + convention rather than a shared import; change one and you must change both. + """ + return xtest_root / "sdk" / "benchmark.installed.json" + + +def bench_manifest_env(manifest_path: Path) -> dict[str, str]: + """Environment for a prepared benchmark run, or empty if none was prepared. + + A missing manifest is not an error: most `otdf-local env` invocations have + nothing to do with benchmarking. + """ + if not manifest_path.exists(): + return {} + + env_vars = {"BENCH_INSTALLATION_MANIFEST": str(manifest_path.resolve())} + + # OTDFCTL_HEADS names the *provisioning* CLI, and only that. The measured + # SDK arms are deliberately excluded: letting a benchmark arm supply otdfctl + # is what made "otdfctl" mean "whatever the go arm happens to be", so the + # build under measurement would also be provisioning the fixtures it is + # measured against. + # + # The value is a JSON array, not a comma-separated list: `conftest.py` + # reads it with `json.loads` (see `load_otdfctl`), and CI passes the same + # shape through `fromJson` in xtest.yml. A bare string is invalid JSON + # there and gets silently discarded, dropping the run back onto the + # implicit `sdk/go/dist/main` fallback. + # + # A release pin is exported the same as a source pin. `installed_tag` is + # the `dist/` directory name either way, and the point of pinning at all + # is that the pin is what runs -- silently ignoring released pins would + # reinstate the fallback for exactly the runs that asked not to have it. + try: + manifest = json.loads(manifest_path.read_text()) + otdfctl = manifest.get("otdfctl") or {} + tag = otdfctl.get("installed_tag", "") + if tag: + env_vars["OTDFCTL_HEADS"] = json.dumps([tag]) + except (OSError, ValueError, AttributeError) as e: + print_warning(f"Could not read OTDFCTL_HEADS from {manifest_path}: {e}") + + return env_vars + + @app.command() def env( format: Annotated[ @@ -646,6 +696,14 @@ def env( except Exception as e: # noqa: BLE001 - best-effort, as above print_warning(f"Could not get platform version: {e}", err_console) + # Benchmark preparation: XT_TMP_DIR for generated fixtures + tmp_dir = settings.xtest_root / "tmp" + tmp_dir.mkdir(parents=True, exist_ok=True) + env_vars["XT_TMP_DIR"] = str(tmp_dir.resolve()) + + # Benchmark preparation: point to installation manifest if it exists + env_vars.update(bench_manifest_env(bench_manifest_path(settings.xtest_root))) + # Output in requested format if format == "json": console.print_json(json.dumps(env_vars, indent=2)) diff --git a/otdf-local/tests/test_bench_env.py b/otdf-local/tests/test_bench_env.py new file mode 100644 index 000000000..83c72ac63 --- /dev/null +++ b/otdf-local/tests/test_bench_env.py @@ -0,0 +1,112 @@ +"""Tests for the benchmark environment `otdf-local env` exports. + +The contract under test is cross-package: `otdf-sdk-mgr` writes the manifest, +`otdf-local` turns it into environment variables, and `xtest/conftest.py` +consumes them. Nothing in the type system connects those three, so the shape is +asserted here explicitly -- in particular that `OTDFCTL_HEADS` survives the +`json.loads` in `conftest.load_otdfctl`, which is the failure mode that hides: +an unparseable value is discarded silently and the run falls back to an +unpinned CLI while still reporting success. +""" + +import json +from pathlib import Path + +from otdf_local.cli import bench_manifest_env, bench_manifest_path + + +def _write_manifest( + path: Path, otdfctl: dict[str, object] | None, **rest: object +) -> Path: + manifest: dict[str, object] = { + "schema_version": "v1alpha1", + "platform": None, + "otdfctl": otdfctl, + "sdk_arms": [], + "status": "success", + "status_message": "", + **rest, + } + path.write_text(json.dumps(manifest)) + return path + + +def _identity(alias: str, tag: str, method: str) -> dict[str, object]: + return { + "requested_alias": alias, + "repository": "https://github.com/opentdf/otdfctl.git", + "commit_sha": "0" * 40, + "installed_tag": tag, + "installed_path": f"/dist/{tag}", + "installation_method": method, + "build_settings": {}, + } + + +class TestBenchManifestPath: + def test_matches_the_sdk_manager_default(self, tmp_path: Path): + # Kept in step with otdf_sdk_mgr.bench_prepare.default_manifest_path, + # which is get_sdk_dir() / "benchmark.installed.json" where get_sdk_dir + # is /sdk. + assert ( + bench_manifest_path(tmp_path) + == tmp_path / "sdk" / "benchmark.installed.json" + ) + + +class TestBenchManifestEnv: + def test_no_manifest_exports_nothing(self, tmp_path: Path): + assert bench_manifest_env(tmp_path / "absent.json") == {} + + def test_manifest_pointer_is_absolute(self, tmp_path: Path): + path = _write_manifest(tmp_path / "m.json", None) + env = bench_manifest_env(path) + assert Path(env["BENCH_INSTALLATION_MANIFEST"]).is_absolute() + + def test_otdfctl_heads_is_json_conftest_can_parse(self, tmp_path: Path): + path = _write_manifest( + tmp_path / "m.json", _identity("pr:123", "refs--pull--123--head", "source") + ) + env = bench_manifest_env(path) + # The exact parse conftest.load_otdfctl performs. + assert json.loads(env["OTDFCTL_HEADS"]) == ["refs--pull--123--head"] + + def test_released_pin_is_exported_too(self, tmp_path: Path): + # A release pin that is not exported silently reinstates the + # dist/main fallback for a run that explicitly asked for a pin. + path = _write_manifest( + tmp_path / "m.json", _identity("v0.24.0", "v0.24.0", "release") + ) + env = bench_manifest_env(path) + assert json.loads(env["OTDFCTL_HEADS"]) == ["v0.24.0"] + + def test_measured_arms_never_supply_otdfctl(self, tmp_path: Path): + # The provisioning CLI is pinned separately from the arms under + # measurement; an arm must not become the tool that provisions it. + path = _write_manifest( + tmp_path / "m.json", + None, + sdk_arms=[ + _identity("main", "main", "source"), + _identity("v0.29.0", "v0.29.0", "release"), + ], + ) + env = bench_manifest_env(path) + assert "OTDFCTL_HEADS" not in env + assert "BENCH_INSTALLATION_MANIFEST" in env + + def test_unreadable_manifest_warns_without_exporting_heads( + self, tmp_path: Path, capsys + ): + path = tmp_path / "m.json" + path.write_text("{not json") + env = bench_manifest_env(path) + assert "OTDFCTL_HEADS" not in env + # Still points at the manifest so the failure is diagnosable. + assert "BENCH_INSTALLATION_MANIFEST" in env + assert "OTDFCTL_HEADS" in capsys.readouterr().out + + def test_pin_without_a_tag_exports_nothing(self, tmp_path: Path): + # An empty tag would build the path sdk/go/dist//otdfctl.sh. + path = _write_manifest(tmp_path / "m.json", _identity("main", "", "source")) + assert "OTDFCTL_HEADS" not in bench_manifest_env(path) diff --git a/xtest/perf/README.md b/xtest/perf/README.md index cc826dce8..fe769fa27 100644 --- a/xtest/perf/README.md +++ b/xtest/perf/README.md @@ -552,3 +552,138 @@ two builds doing different amounts of work) is invisible in the output. Every one of these fails *silently* and *plausibly* when broken: the numbers still look like numbers. That is why they are written down. + +--- + +## 3. Local preparation and execution + +A benchmark run locally is the same run CI performs — same resolution, same +installs, same manifest, same environment variables. That is the point of this +section: a local sequence that diverges from the workflow measures something +else, and the difference shows up as an unreproducible result rather than as +an error. + +### Prerequisites + +- `otdf-sdk-mgr` installed: `cd otdf-sdk-mgr && uv tool install --editable .` +- Nothing else listening on port 8080. A benchmark starts the platform and its + default KAS only — no alpha/beta/gamma/delta, no km instances. That is + deliberate: every extra Go service is a process competing for the CPU being + measured. + +### Sequence + +```bash +# 1. Resolve and install every arm in one step, recording what was installed. +# The first ref is the reference; the rest are candidates. +cd otdf-sdk-mgr +uv run otdf-sdk-mgr install benchmark go v0.29.0 main --otdfctl v0.30.0 + +# 2. Start the platform and its default KAS. +cd ../otdf-local +uv run otdf-local up --services docker,platform + +# 3. Export the environment: log paths for the audit assertions, XT_TMP_DIR, +# and the pins recorded in step 1. +eval $(uv run otdf-local env) + +# 4. Measure. +cd ../xtest +uv run pytest --bench --sdks go \ + --bench-baseline go@v0.29.0 \ + --bench-candidate go@main \ + -v test_benchmarks.py + +# 5. Read the result. The JSON holds every sample and every interval; the +# terminal summary is a view of it, not a separate source. +cat test-results/benchmarks/go.json + +# 6. Stop the services, whether or not the measurement passed. Logs stay +# under otdf-local's log directory afterwards. +cd ../otdf-local +uv run otdf-local down +``` + +`install benchmark` is what writes the manifest. `install stable` and +`install tip` still work for ordinary test runs, but they install without +recording provenance, so step 3 has no pins to export and the run falls back +to whatever happens to be in `dist/`. + +### Environment variables + +`otdf-local env` exports, in addition to the platform settings: + +- `XT_TMP_DIR` — root for generated fixtures and ciphertexts +- `BENCH_INSTALLATION_MANIFEST` — path to the manifest written in step 1 +- `OTDFCTL_HEADS` — a **JSON array** naming the `dist/` directory of the + provisioning CLI, e.g. `["v0.30.0"]`. `conftest.load_otdfctl` parses it with + `json.loads`; any other shape is discarded silently and the run falls back + to `sdk/go/dist/main/otdfctl.sh` or a system binary. It names the + provisioning CLI only — never a measured arm, which would make a build under + measurement responsible for provisioning its own fixtures. + +Only set when the corresponding file exists: `PLATFORM_LOG_FILE`, +`KAS_*_LOG_FILE` (the audit assertions need these), `SCHEMA_FILE`, +`OT_ROOT_KEY`, `PLATFORM_VERSION`. + +### Installation manifest + +`install benchmark` writes `xtest/sdk/benchmark.installed.json`, a versioned +record of what was installed. Per artifact: requested alias, repository, +immutable commit SHA, installed tag and path, installation method (release +artifact or source build), and build settings. Request order is preserved, so +the first `sdk_arms` entry is the reference. + +The platform pin and the provisioning `otdfctl` pin are recorded in their own +fields, separate from the measured arms. + +The manifest is written on **every** exit path, including the failing ones, so +a partial or failed preparation still says what it managed to install. Its +`status` is one of `success`, `neutral`, `partial`, or `failed`, and +`status_message` explains whichever it is — `neutral` is not an error, so the +field is not named for one. + +A source build records its commit in `dist//.version` alongside the +build. A later run reuses that directory only if the recorded commit matches; +an unrecorded or mismatched one is discarded and rebuilt. Adopting a stale +build costs nothing at install time and mislabels every measurement that +follows. + +### Neutral preparation + +When two distinct requested aliases resolve to one commit — `main` and +`latest` both at `abc123` — preparation stops before any build and reports +**neutral**. There is nothing to compare, which is not the same as something +having gone wrong; the manifest still records the commit everything collapsed +onto. Local and CI share this path, so the outcome reads the same in both. + +### Troubleshooting + +**Services won't start**: +```bash +cd otdf-local +uv run otdf-local status +uv run otdf-local logs -f +``` + +**Missing SDK build**: +```bash +ls -la xtest/sdk/go/dist/ # what is installed +cat xtest/sdk/benchmark.installed.json # and what preparation believes it installed +``` +A build with no `.version` beside it was not installed by +`install benchmark`, so its provenance is unknown and the next preparation +will rebuild it. + +**Audit log assertions fail** — the log paths are only exported if the files +already exist, so re-export after the services have started: +```bash +echo $PLATFORM_LOG_FILE +eval $(cd otdf-local && uv run otdf-local env) +``` + +**Platform not reachable**: +```bash +curl http://localhost:8080/healthz +cd otdf-local && uv run otdf-local restart platform +```