feat(xtest): export benchmark manifest pins from otdf-local env

otdf-local/src/otdf_local/cli.py

@@ -4,6 +4,7 @@
 import shutil
 import sys
 import time
+from pathlib import Path
 from typing import Annotated
 
 import httpx
@@ -565,6 +566,55 @@ def restart(
     raise typer.Exit(1)
 
 
+def bench_manifest_path(xtest_root: Path) -> Path:
+    """Where `otdf-sdk-mgr` writes the benchmark installation manifest.
+
+    Mirrors `otdf_sdk_mgr.bench_prepare.default_manifest_path`. The two packages
+    do not depend on each other, so the path and the JSON shape are a written
+    convention rather than a shared import; change one and you must change both.
+    """
+    return xtest_root / "sdk" / "benchmark.installed.json"
+
+
+def bench_manifest_env(manifest_path: Path) -> dict[str, str]:
+    """Environment for a prepared benchmark run, or empty if none was prepared.
+
+    A missing manifest is not an error: most `otdf-local env` invocations have
+    nothing to do with benchmarking.
+    """
+    if not manifest_path.exists():
+        return {}
+
+    env_vars = {"BENCH_INSTALLATION_MANIFEST": str(manifest_path.resolve())}
+
+    # OTDFCTL_HEADS names the *provisioning* CLI, and only that. The measured
+    # SDK arms are deliberately excluded: letting a benchmark arm supply otdfctl
+    # is what made "otdfctl" mean "whatever the go arm happens to be", so the
+    # build under measurement would also be provisioning the fixtures it is
+    # measured against.
+    #
+    # The value is a JSON array, not a comma-separated list: `conftest.py`
+    # reads it with `json.loads` (see `load_otdfctl`), and CI passes the same
+    # shape through `fromJson` in xtest.yml. A bare string is invalid JSON
+    # there and gets silently discarded, dropping the run back onto the
+    # implicit `sdk/go/dist/main` fallback.
+    #
+    # A release pin is exported the same as a source pin. `installed_tag` is
+    # the `dist/` directory name either way, and the point of pinning at all
+    # is that the pin is what runs -- silently ignoring released pins would
+    # reinstate the fallback for exactly the runs that asked not to have it.
+    try:
+        manifest = json.loads(manifest_path.read_text())
+        otdfctl = manifest.get("otdfctl") or {}
+        tag = otdfctl.get("installed_tag", "")
+        if tag:
+            env_vars["OTDFCTL_HEADS"] = json.dumps([tag])
+    except (OSError, ValueError, AttributeError) as e:
+        print_warning(f"Could not read OTDFCTL_HEADS from {manifest_path}: {e}")
+
+    return env_vars
+
+
 @app.command()
 def env(
     format: Annotated[
@@ -646,6 +696,14 @@ def env(
     except Exception as e:  # noqa: BLE001 - best-effort, as above
         print_warning(f"Could not get platform version: {e}", err_console)
 
+    # Benchmark preparation: XT_TMP_DIR for generated fixtures
+    tmp_dir = settings.xtest_root / "tmp"
+    tmp_dir.mkdir(parents=True, exist_ok=True)
+    env_vars["XT_TMP_DIR"] = str(tmp_dir.resolve())
+
+    # Benchmark preparation: point to installation manifest if it exists
+    env_vars.update(bench_manifest_env(bench_manifest_path(settings.xtest_root)))
+
     # Output in requested format
     if format == "json":
         console.print_json(json.dumps(env_vars, indent=2))

otdf-local/tests/test_bench_env.py

@@ -0,0 +1,112 @@
+"""Tests for the benchmark environment `otdf-local env` exports.
+
+The contract under test is cross-package: `otdf-sdk-mgr` writes the manifest,
+`otdf-local` turns it into environment variables, and `xtest/conftest.py`
+consumes them. Nothing in the type system connects those three, so the shape is
+asserted here explicitly -- in particular that `OTDFCTL_HEADS` survives the
+`json.loads` in `conftest.load_otdfctl`, which is the failure mode that hides:
+an unparseable value is discarded silently and the run falls back to an
+unpinned CLI while still reporting success.
+"""
+
+import json
+from pathlib import Path
+
+from otdf_local.cli import bench_manifest_env, bench_manifest_path
+
+
+def _write_manifest(
+    path: Path, otdfctl: dict[str, object] | None, **rest: object
+) -> Path:
+    manifest: dict[str, object] = {
+        "schema_version": "v1alpha1",
+        "platform": None,
+        "otdfctl": otdfctl,
+        "sdk_arms": [],
+        "status": "success",
+        "status_message": "",
+        **rest,
+    }
+    path.write_text(json.dumps(manifest))
+    return path
+
+
+def _identity(alias: str, tag: str, method: str) -> dict[str, object]:
+    return {
+        "requested_alias": alias,
+        "repository": "https://github.com/opentdf/otdfctl.git",
+        "commit_sha": "0" * 40,
+        "installed_tag": tag,
+        "installed_path": f"/dist/{tag}",
+        "installation_method": method,
+        "build_settings": {},
+    }
+
+
+class TestBenchManifestPath:
+    def test_matches_the_sdk_manager_default(self, tmp_path: Path):
+        # Kept in step with otdf_sdk_mgr.bench_prepare.default_manifest_path,
+        # which is get_sdk_dir() / "benchmark.installed.json" where get_sdk_dir
+        # is <xtest>/sdk.
+        assert (
+            bench_manifest_path(tmp_path)
+            == tmp_path / "sdk" / "benchmark.installed.json"
+        )
+
+
+class TestBenchManifestEnv:
+    def test_no_manifest_exports_nothing(self, tmp_path: Path):
+        assert bench_manifest_env(tmp_path / "absent.json") == {}
+
+    def test_manifest_pointer_is_absolute(self, tmp_path: Path):
+        path = _write_manifest(tmp_path / "m.json", None)
+        env = bench_manifest_env(path)
+        assert Path(env["BENCH_INSTALLATION_MANIFEST"]).is_absolute()
+
+    def test_otdfctl_heads_is_json_conftest_can_parse(self, tmp_path: Path):
+        path = _write_manifest(
+            tmp_path / "m.json", _identity("pr:123", "refs--pull--123--head", "source")
+        )
+        env = bench_manifest_env(path)
+        # The exact parse conftest.load_otdfctl performs.
+        assert json.loads(env["OTDFCTL_HEADS"]) == ["refs--pull--123--head"]
+
+    def test_released_pin_is_exported_too(self, tmp_path: Path):
+        # A release pin that is not exported silently reinstates the
+        # dist/main fallback for a run that explicitly asked for a pin.
+        path = _write_manifest(
+            tmp_path / "m.json", _identity("v0.24.0", "v0.24.0", "release")
+        )
+        env = bench_manifest_env(path)
+        assert json.loads(env["OTDFCTL_HEADS"]) == ["v0.24.0"]
+
+    def test_measured_arms_never_supply_otdfctl(self, tmp_path: Path):
+        # The provisioning CLI is pinned separately from the arms under
+        # measurement; an arm must not become the tool that provisions it.
+        path = _write_manifest(
+            tmp_path / "m.json",
+            None,
+            sdk_arms=[
+                _identity("main", "main", "source"),
+                _identity("v0.29.0", "v0.29.0", "release"),
+            ],
+        )
+        env = bench_manifest_env(path)
+        assert "OTDFCTL_HEADS" not in env
+        assert "BENCH_INSTALLATION_MANIFEST" in env
+
+    def test_unreadable_manifest_warns_without_exporting_heads(
+        self, tmp_path: Path, capsys
+    ):
+        path = tmp_path / "m.json"
+        path.write_text("{not json")
+        env = bench_manifest_env(path)
+        assert "OTDFCTL_HEADS" not in env
+        # Still points at the manifest so the failure is diagnosable.
+        assert "BENCH_INSTALLATION_MANIFEST" in env
+        assert "OTDFCTL_HEADS" in capsys.readouterr().out
+
+    def test_pin_without_a_tag_exports_nothing(self, tmp_path: Path):
+        # An empty tag would build the path sdk/go/dist//otdfctl.sh.
+        path = _write_manifest(tmp_path / "m.json", _identity("main", "", "source"))
+        assert "OTDFCTL_HEADS" not in bench_manifest_env(path)

xtest/perf/README.md

@@ -552,3 +552,138 @@ two builds doing different amounts of work) is invisible in the output.
 
 Every one of these fails *silently* and *plausibly* when broken: the numbers
 still look like numbers. That is why they are written down.
+
+---
+
+## 3. Local preparation and execution
+
+A benchmark run locally is the same run CI performs — same resolution, same
+installs, same manifest, same environment variables. That is the point of this
+section: a local sequence that diverges from the workflow measures something
+else, and the difference shows up as an unreproducible result rather than as
+an error.
+
+### Prerequisites
+
+- `otdf-sdk-mgr` installed: `cd otdf-sdk-mgr && uv tool install --editable .`
+- Nothing else listening on port 8080. A benchmark starts the platform and its
+  default KAS only — no alpha/beta/gamma/delta, no km instances. That is
+  deliberate: every extra Go service is a process competing for the CPU being
+  measured.
+
+### Sequence
+
+```bash
+# 1. Resolve and install every arm in one step, recording what was installed.
+#    The first ref is the reference; the rest are candidates.
+cd otdf-sdk-mgr
+uv run otdf-sdk-mgr install benchmark go v0.29.0 main --otdfctl v0.30.0
+
+# 2. Start the platform and its default KAS.
+cd ../otdf-local
+uv run otdf-local up --services docker,platform
+
+# 3. Export the environment: log paths for the audit assertions, XT_TMP_DIR,
+#    and the pins recorded in step 1.
+eval $(uv run otdf-local env)
+
+# 4. Measure.
+cd ../xtest
+uv run pytest --bench --sdks go \
+  --bench-baseline go@v0.29.0 \
+  --bench-candidate go@main \
+  -v test_benchmarks.py
+
+# 5. Read the result. The JSON holds every sample and every interval; the
+#    terminal summary is a view of it, not a separate source.
+cat test-results/benchmarks/go.json
+
+# 6. Stop the services, whether or not the measurement passed. Logs stay
+#    under otdf-local's log directory afterwards.
+cd ../otdf-local
+uv run otdf-local down
+```
+
+`install benchmark` is what writes the manifest. `install stable` and
+`install tip` still work for ordinary test runs, but they install without
+recording provenance, so step 3 has no pins to export and the run falls back
+to whatever happens to be in `dist/`.
+
+### Environment variables
+
+`otdf-local env` exports, in addition to the platform settings:
+
+- `XT_TMP_DIR` — root for generated fixtures and ciphertexts
+- `BENCH_INSTALLATION_MANIFEST` — path to the manifest written in step 1
+- `OTDFCTL_HEADS` — a **JSON array** naming the `dist/` directory of the
+  provisioning CLI, e.g. `["v0.30.0"]`. `conftest.load_otdfctl` parses it with
+  `json.loads`; any other shape is discarded silently and the run falls back
+  to `sdk/go/dist/main/otdfctl.sh` or a system binary. It names the
+  provisioning CLI only — never a measured arm, which would make a build under
+  measurement responsible for provisioning its own fixtures.
+
+Only set when the corresponding file exists: `PLATFORM_LOG_FILE`,
+`KAS_*_LOG_FILE` (the audit assertions need these), `SCHEMA_FILE`,
+`OT_ROOT_KEY`, `PLATFORM_VERSION`.
+
+### Installation manifest
+
+`install benchmark` writes `xtest/sdk/benchmark.installed.json`, a versioned
+record of what was installed. Per artifact: requested alias, repository,
+immutable commit SHA, installed tag and path, installation method (release
+artifact or source build), and build settings. Request order is preserved, so
+the first `sdk_arms` entry is the reference.
+
+The platform pin and the provisioning `otdfctl` pin are recorded in their own
+fields, separate from the measured arms.
+
+The manifest is written on **every** exit path, including the failing ones, so
+a partial or failed preparation still says what it managed to install. Its
+`status` is one of `success`, `neutral`, `partial`, or `failed`, and
+`status_message` explains whichever it is — `neutral` is not an error, so the
+field is not named for one.
+
+A source build records its commit in `dist/<slug>/.version` alongside the
+build. A later run reuses that directory only if the recorded commit matches;
+an unrecorded or mismatched one is discarded and rebuilt. Adopting a stale
+build costs nothing at install time and mislabels every measurement that
+follows.
+
+### Neutral preparation
+
+When two distinct requested aliases resolve to one commit — `main` and
+`latest` both at `abc123` — preparation stops before any build and reports
+**neutral**. There is nothing to compare, which is not the same as something
+having gone wrong; the manifest still records the commit everything collapsed
+onto. Local and CI share this path, so the outcome reads the same in both.
+
+### Troubleshooting
+
+**Services won't start**:
+```bash
+cd otdf-local
+uv run otdf-local status
+uv run otdf-local logs -f
+```
+
+**Missing SDK build**:
+```bash
+ls -la xtest/sdk/go/dist/            # what is installed
+cat xtest/sdk/benchmark.installed.json  # and what preparation believes it installed
+```
+A build with no `.version` beside it was not installed by
+`install benchmark`, so its provenance is unknown and the next preparation
+will rebuild it.
+
+**Audit log assertions fail** — the log paths are only exported if the files
+already exist, so re-export after the services have started:
+```bash
+echo $PLATFORM_LOG_FILE
+eval $(cd otdf-local && uv run otdf-local env)
+```
+
+**Platform not reachable**:
+```bash
+curl http://localhost:8080/healthz
+cd otdf-local && uv run otdf-local restart platform
+```