From 1a36848eb380673fddd3b33e005db025b744a246 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Mon, 7 Sep 2026 17:11:17 +0200 Subject: [PATCH 1/8] feat(ci): require freshly executed lifecycle contract evidence Extend the canonical lifecycle ledger with conservative candidate-bound obligations, exact pytest witnesses and observed subprocess/model trajectories. Enforce PR and merge-queue lanes through existing CI authorities. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/CODEOWNERS | 13 + .github/workflows/ci-integration.yml | 19 + .github/workflows/ci.yml | 18 + scripts/check_lifecycle_evidence.py | 184 +++++++ scripts/lifecycle_contracts.py | 314 ++++++++++++ tests/fixtures/lifecycle_bug_ledger.json | 7 + tests/quality/test_ci_topology.py | 24 + tests/quality/test_lifecycle_bug_ledger.py | 72 +-- tests/unit/scripts/test_lifecycle_evidence.py | 479 ++++++++++++++++++ tests/utils/isolated_apm_environment.py | 5 +- tests/utils/lifecycle_evidence.py | 145 ++++++ 11 files changed, 1249 insertions(+), 31 deletions(-) create mode 100644 scripts/check_lifecycle_evidence.py create mode 100644 scripts/lifecycle_contracts.py create mode 100644 tests/unit/scripts/test_lifecycle_evidence.py create mode 100644 tests/utils/lifecycle_evidence.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2f2ca7c4e6..8a3409c5b1 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -6,3 +6,16 @@ README.md @danielmeppiel # Workflow files - Lead Maintainer review required for any change to CI # definitions, especially anything touching `pull_request_target` or secrets. .github/workflows/** @danielmeppiel + +# Lifecycle proof obligations and their executing provider share the CI trust boundary. +scripts/check_lifecycle_evidence.py @danielmeppiel +scripts/lifecycle_contracts.py @danielmeppiel +tests/fixtures/lifecycle_bug_ledger.json @danielmeppiel +tests/utils/lifecycle_evidence.py @danielmeppiel +tests/utils/apm_lifecycle_runner.py @danielmeppiel +tests/utils/artifact_snapshot.py @danielmeppiel +tests/utils/lifecycle_state.py @danielmeppiel +tests/utils/isolated_apm_environment.py @danielmeppiel +tests/quality/test_lifecycle_bug_ledger.py @danielmeppiel +tests/unit/scripts/test_lifecycle_evidence.py @danielmeppiel +.github/CODEOWNERS @danielmeppiel diff --git a/.github/workflows/ci-integration.yml b/.github/workflows/ci-integration.yml index 1faf89a27e..40b99e0989 100644 --- a/.github/workflows/ci-integration.yml +++ b/.github/workflows/ci-integration.yml @@ -241,6 +241,25 @@ jobs: # suite so runner variance does not turn into an unbounded job. timeout-minutes: 20 + - name: Execute full lifecycle contracts + if: matrix.shard == 1 + env: + LIFECYCLE_BASE: ${{ github.event.merge_group.base_sha }} + LIFECYCLE_HEAD: ${{ github.event.merge_group.head_sha }} + run: | + git fetch --no-tags origin "$LIFECYCLE_BASE" "$LIFECYCLE_HEAD" + uv run --extra dev python scripts/check_lifecycle_evidence.py \ + --base "$LIFECYCLE_BASE" --head "$LIFECYCLE_HEAD" --lane full \ + --report "$RUNNER_TEMP/lifecycle-evidence-full.json" + timeout-minutes: 6 + + - name: Upload full lifecycle evidence + if: always() && matrix.shard == 1 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: lifecycle-evidence-full + path: ${{ runner.temp }}/lifecycle-evidence-full.json + - name: Rename coverage data for shard if: always() run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46f0b0e7e3..48c58c3806 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -520,6 +520,8 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 @@ -551,3 +553,19 @@ jobs: uv run --extra dev pytest -p no:cacheprovider -q --strict-markers \ -n 2 --dist loadgroup \ -m 'lifecycle_smoke and not lifecycle_merge_group' tests/integration + + - name: Execute changed lifecycle contracts + env: + LIFECYCLE_BASE: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} + LIFECYCLE_HEAD: ${{ github.sha }} + run: | + uv run --extra dev python scripts/check_lifecycle_evidence.py \ + --base "$LIFECYCLE_BASE" --head "$LIFECYCLE_HEAD" --lane pr \ + --report "$RUNNER_TEMP/lifecycle-evidence-pr.json" + + - name: Upload lifecycle evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: lifecycle-evidence-pr + path: ${{ runner.temp }}/lifecycle-evidence-pr.json diff --git a/scripts/check_lifecycle_evidence.py b/scripts/check_lifecycle_evidence.py new file mode 100644 index 0000000000..784317468e --- /dev/null +++ b/scripts/check_lifecycle_evidence.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""Execute lifecycle obligations for an exact candidate; never import receipts.""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +from pathlib import Path +from typing import Any + +# Direct script execution must resolve first-party helpers from this checkout. +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from scripts.lifecycle_contracts import ( # noqa: E402 + LEDGER, + EvidenceError, + command_inventory, + git, + load_ledger, + select_contracts, + validate_execution, +) + +LIMITATIONS = [ + "Applicability, assertions and model semantics require code-owner review.", + "PR code and its verifier are not tamper-proof; protected review is the trust boundary.", + "Bounded source-Python trajectories do not establish packaged or exhaustive parity.", + "A pr-lane pending report is not shipping evidence; execute lane full independently.", +] + + +def source_profile(root: Path) -> tuple[Path, dict[str, str]]: + """Identify the installed editable source and its exact interpreter/launcher.""" + import apm_cli + from tests.utils.lifecycle_evidence import fingerprint + + if Path(apm_cli.__file__).resolve().parent != root / "src/apm_cli": + raise EvidenceError("Installed source does not resolve to the candidate checkout") + executable = Path(sys.executable).parent / ("apm.exe" if os.name == "nt" else "apm") + if os.name == "nt" or not executable.is_file(): + raise EvidenceError("source-python profile requires the Unix installed apm script") + script = executable.read_text(encoding="utf-8") + if "apm_cli.cli" not in script or not script.startswith(f"#!{sys.executable}\n"): + raise EvidenceError("Installed apm entrypoint is not this Python environment") + return executable, { + "kind": "source-python", + "executable": str(executable), + "executable_sha256": fingerprint(executable), + "python": str(Path(sys.executable).resolve()), + "python_sha256": fingerprint(Path(sys.executable).resolve()), + } + + +def candidate(root: Path, base: str, head: str) -> dict[str, str]: + """Require explicit existing revisions and a pristine tested source tree.""" + resolved_base = git(root, "rev-parse", "--verify", "--end-of-options", f"{base}^{{commit}}") + resolved_head = git(root, "rev-parse", "--verify", "--end-of-options", f"{head}^{{commit}}") + if resolved_head != git(root, "rev-parse", "HEAD"): + raise EvidenceError("Requested head is not the checked-out candidate") + if resolved_base == resolved_head: + raise EvidenceError("Base and head must describe a candidate change") + git(root, "merge-base", "--is-ancestor", resolved_base, resolved_head) + if git(root, "status", "--porcelain", "--untracked-files=normal"): + raise EvidenceError("Candidate must be clean, including untracked files") + return { + "base": resolved_base, + "head": resolved_head, + "tested_tree": git(root, "rev-parse", "HEAD^{tree}"), + } + + +def execute(args: argparse.Namespace) -> dict[str, Any]: + """Generate native evidence from this process's own fresh pytest execution.""" + identity = candidate(ROOT, args.base, args.head) + report: dict[str, Any] = { + "version": 1, + **identity, + "lane": args.lane, + "status": "blocked", + "profile": {"kind": "source-python"}, + "contracts": [], + "witnesses": {}, + "limitations": LIMITATIONS, + } + try: + inventory = command_inventory() + base_text = git(ROOT, "show", f"{identity['base']}:{LEDGER}") + base = load_ledger(base_text) + head = load_ledger((ROOT / LEDGER).read_text(encoding="utf-8")) + changed = set( + git( + ROOT, + "diff", + "--name-only", + "--no-renames", + "-z", + identity["base"], + identity["head"], + ).split("\0") + ) - {""} + contracts = select_contracts(base, head, changed, inventory) + report["contracts"] = [contract["id"] for contract in contracts] + report["command_inventory"] = sorted(inventory) + report["changed_paths"] = sorted(changed) + if not contracts: + if candidate(ROOT, args.base, args.head) != identity: + raise EvidenceError("Candidate identity changed during assessment") + report["status"] = "not_applicable" + return report + witnesses = [ + witness + for contract in contracts + for witness in contract["witnesses"] + if args.lane == "full" or witness["kind"] == "deterministic" + ] + if not witnesses: + raise EvidenceError("Empty lifecycle witness selection") + + import pytest + + from tests.utils.lifecycle_evidence import LifecycleEvidencePlugin + + executable, report["profile"] = source_profile(ROOT) + nodeids = sorted({witness["nodeid"] for witness in witnesses}) + plugin = LifecycleEvidencePlugin(nodeids, executable) + os.environ["APM_BINARY_PATH"] = str(executable) + os.environ["APM_E2E_TESTS"] = "1" + # No user-provided pytest options, deselection filters or receipt inputs. + os.environ.pop("PYTEST_ADDOPTS", None) + exitcode = pytest.main( + ["-p", "no:cacheprovider", "-o", "addopts=", "--strict-markers", "-q", *nodeids], + plugins=[plugin], + ) + report["witnesses"] = plugin.records + if exitcode != 0: + raise EvidenceError(f"Lifecycle pytest execution failed with exit code {exitcode}") + for witness in witnesses: + validate_execution(witness, plugin.records.get(witness["nodeid"], {})) + if candidate(ROOT, args.base, args.head) != identity: + raise EvidenceError("Candidate identity changed during execution") + if source_profile(ROOT)[1] != report["profile"]: + raise EvidenceError("Source executable identity changed during execution") + report["status"] = "passed" if args.lane == "full" else "pending" + except (EvidenceError, KeyError, TypeError, ValueError) as exc: + report["error"] = str(exc) + return report + + +def main() -> int: + """CLI entrypoint; nonzero means blocked, pending is explicitly lane-scoped.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base", required=True) + parser.add_argument("--head", required=True) + parser.add_argument("--lane", choices=("pr", "full"), required=True) + parser.add_argument("--report", type=Path, required=True) + args = parser.parse_args() + if args.report.resolve().is_relative_to(ROOT): + parser.error("--report must be outside the candidate checkout") + try: + report = execute(args) + except (EvidenceError, subprocess.CalledProcessError) as exc: + report = { + "version": 1, + "base": args.base, + "head": args.head, + "tested_tree": None, + "lane": args.lane, + "status": "blocked", + "error": str(exc), + "witnesses": {}, + "limitations": LIMITATIONS, + } + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report, indent=2, default=str) + "\n", encoding="ascii") + print(f"Lifecycle evidence: {report['status']} ({args.report})") + return 1 if report["status"] == "blocked" else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/lifecycle_contracts.py b/scripts/lifecycle_contracts.py new file mode 100644 index 0000000000..c95d75bc76 --- /dev/null +++ b/scripts/lifecycle_contracts.py @@ -0,0 +1,314 @@ +"""Authored lifecycle obligations in the existing escaped-defect ledger. + +Semantic applicability is reviewed, not inferred from filenames or test names. +This module owns mechanical validation; pytest owns executable evidence. +""" + +from __future__ import annotations + +import json +import shutil +import subprocess +from pathlib import Path +from typing import Any + +import click + +LEDGER = "tests/fixtures/lifecycle_bug_ledger.json" + + +class EvidenceError(ValueError): + """A candidate has not discharged its lifecycle obligations.""" + + +def git(root: Path, *args: str) -> str: + """Run a read-only git query, preserving command failures.""" + executable = shutil.which("git") + if executable is None: + raise EvidenceError("git is required for candidate identity") + result = subprocess.check_output( # noqa: S603 - fixed executable, argument vector, no shell + [executable, *args], cwd=root, text=True + ) + return result if "-z" in args else result.strip() + + +def command_inventory() -> dict[str, click.Command]: + """Walk actual Click registrations, including groups, defaults and aliases.""" + from apm_cli.cli import cli + + result: dict[str, click.Command] = {} + + def walk(command: click.Command, path: str, parent: click.Context | None) -> None: + result[path] = command + if isinstance(command, click.Group): + with click.Context(command, parent=parent, info_name=path) as context: + for name in command.list_commands(context): + child = command.get_command(context, name) + if child is None: + raise EvidenceError(f"Unresolved Click registration: {path} {name}") + walk(child, f"{path} {name}".strip(), context) + + walk(cli, "", None) + return result + + +def command_path(args: list[str], inventory: dict[str, click.Command]) -> str: + """Resolve command tokens with Click's parsers, not a remembered argv list.""" + path = "" + remaining = args[:] + parent = None + while True: + command = inventory[path] + context = command.make_context(path, remaining, parent=parent, resilient_parsing=True) + if not isinstance(command, click.Group): + return path + # Click keeps the first subcommand separately from its remaining args. + remaining = [*context._protected_args, *context.args] + if not remaining: + return path + candidate = f"{path} {remaining[0]}".strip() + if candidate not in inventory: + return path + path, remaining, parent = candidate, remaining[1:], context + + +def load_ledger(text: str) -> dict[str, Any]: + """Decode the canonical ledger, including additive general contracts.""" + value = json.loads(text) + if not isinstance(value, dict) or value.get("schema_version") != 1: + raise EvidenceError("Unsupported lifecycle ledger") + return value + + +def _require(condition: object, message: str) -> None: + if not condition: + raise EvidenceError(message) + + +def _paths(value: object) -> list[str]: + _require(isinstance(value, list) and value, "Expected nonempty exact paths") + for path in value: + _require( + isinstance(path, str) + and path + and not path.startswith("/") + and "\\" not in path + and all(part not in ("", ".", "..") for part in path.split("/")) + and not any(char in path for char in "*?["), + f"Not an exact repository path: {path}", + ) + return value + + +def validate_contracts( + ledger: dict[str, Any], inventory: dict[str, click.Command] +) -> dict[str, dict[str, Any]]: + """Validate complete applicability and exact witness/dimension obligations.""" + properties = {row["id"] for row in ledger["property_catalog"]} + contracts: dict[str, dict[str, Any]] = {} + for row in ledger.get("lifecycle_contracts", []): + identifier = row["id"] + _require(identifier and identifier not in contracts, "Duplicate/empty contract ID") + _require(row.get("assessment"), f"{identifier}: missing reviewed assessment") + _paths(row["paths"]) + _require( + row["properties"] and set(row["properties"]) <= properties, + f"{identifier}: unknown/missing properties", + ) + _require(set(row["commands"]) == set(inventory), f"{identifier}: incomplete command map") + witnesses = {w["id"]: w for w in row["witnesses"]} + _require( + len(witnesses) == len(row["witnesses"]) and witnesses, + f"{identifier}: duplicate/empty witnesses", + ) + _require( + {w["kind"] for w in witnesses.values()} == {"deterministic", "generated"}, + f"{identifier}: both deterministic and generated witnesses required", + ) + for witness in witnesses.values(): + nodeid = witness["nodeid"] + _require( + isinstance(nodeid, str) and "::" in nodeid and nodeid.startswith("tests/"), + f"{identifier}: invalid exact nodeid", + ) + _paths([nodeid.split("::")[0]]) + _require(len(witness["transitions"]) >= 2, f"{nodeid}: missing trajectory") + for transition in witness["transitions"]: + _require(transition["command"] in inventory, f"{nodeid}: unknown command") + _require(type(transition["returncode"]) is int, f"{nodeid}: invalid exit status") + _require( + transition["state"] in {"observed", "unchanged", "changed"}, + f"{nodeid}: missing state observation", + ) + _require( + isinstance(transition["argv_contains"], list) + and all(isinstance(arg, str) for arg in transition["argv_contains"]), + f"{nodeid}: invalid argv constraints", + ) + for param, values in row["dimensions"].items(): + _require(values and isinstance(values, list), f"{identifier}: empty dimension") + for kind in ("deterministic", "generated"): + observed = { + w["dimensions"].get(param) for w in witnesses.values() if w["kind"] == kind + } + _require( + set(values) <= observed, + f"{identifier}: {kind} missing required dimension {param}", + ) + applicable = False + for command, entry in row["commands"].items(): + _require(entry["reason"].strip(), f"{identifier}: missing command rationale") + disposition = entry["disposition"] + _require(disposition in {"applicable", "semantic_na"}, "Unknown applicability") + if disposition == "semantic_na": + continue + applicable = True + for kind in ("deterministic", "generated"): + refs = entry[kind] + _require(refs, f"{identifier}: {command} missing {kind} witness") + for ref in refs: + _require( + ref in witnesses and witnesses[ref]["kind"] == kind, + f"{identifier}: invalid witness reference {ref}", + ) + _require( + any(t["command"] == command for t in witnesses[ref]["transitions"]), + f"{identifier}: {command} not exercised by {ref}", + ) + _require(applicable, f"{identifier}: no applicable commands") + contracts[identifier] = row + return contracts + + +def runtime_candidate(path: str) -> bool: + """Only known repository-only surfaces are automatically non-runtime. + + Unknown roots/extensions, packages, templates, locks and manifests default + to runtime. Renames are processed as deletion plus addition. + """ + return not ( + path.startswith(("tests/", "docs/", ".github/workflows/")) + or path == ".github/CODEOWNERS" + or ("/" not in path and path.endswith(".md")) + ) + + +def select_contracts( + base: dict[str, Any], + head: dict[str, Any], + changed: set[str], + inventory: dict[str, click.Command], +) -> list[dict[str, Any]]: + """Fail closed on uncovered runtime changes or deleted/weakened obligations.""" + current = validate_contracts(head, inventory) + for row in base["property_catalog"]: + _require(row in head["property_catalog"], f"Removed/changed property: {row['id']}") + bugs = {row["issue"]: row for row in head.get("bugs", [])} + for row in base.get("bugs", []): + replacement = bugs.get(row["issue"], {}) + _require( + all( + set(row[key]) <= set(replacement.get(key, [])) + for key in ("properties", "regression_tests") + ), + f"Removed legacy regression obligation: {row['issue']}", + ) + previous = {row["id"]: row for row in base.get("lifecycle_contracts", [])} + selected = set() + for identifier, old in previous.items(): + _require(identifier in current, f"Removed lifecycle contract: {identifier}") + new = current[identifier] + for key in ("paths", "properties"): + _require(set(old[key]) <= set(new[key]), f"Weakened {identifier}: {key}") + for witness in old["witnesses"]: + _require(witness in new["witnesses"], f"Removed/changed witness: {identifier}") + for name, values in old["dimensions"].items(): + _require( + set(values) <= set(new["dimensions"].get(name, [])), + f"Removed dimension: {identifier}/{name}", + ) + for command, entry in old["commands"].items(): + if entry["disposition"] == "applicable": + replacement = new["commands"].get(command, {}) + _require( + replacement.get("disposition") == "applicable" + and all( + set(entry[kind]) <= set(replacement.get(kind, [])) + for kind in ("deterministic", "generated") + ), + f"Removed command obligation: {identifier}/{command}", + ) + if any(w["nodeid"].split("::")[0] in changed for w in old["witnesses"]): + selected.add(identifier) + + excluded: set[str] = set() + for entry in head.get("non_runtime_changes", []): + _require(entry["reason"].strip(), "Non-runtime assessment needs a reason") + paths = _paths(entry["paths"]) + _require( + all( + path.startswith(("scripts/", ".apm/", ".agents/", ".github/", "packages/")) + for path in paths + ), + "Non-runtime exceptions are limited to reviewed repository tooling/agent content", + ) + # A prior exception cannot silently exempt a later change. + if entry not in base.get("non_runtime_changes", []): + excluded.update(paths) + runtime = {path for path in changed if runtime_candidate(path)} + for path in runtime: + matching = { + identifier + for identifier, row in current.items() + if path in row["paths"] and row != previous.get(identifier) + } + if matching: + selected.update(matching) + else: + _require(path in excluded, f"Missing fresh lifecycle assessment for {path}") + # Contract additions/edits must execute even in an otherwise tests-only diff. + selected.update(key for key, row in current.items() if row != previous.get(key)) + return [current[key] for key in sorted(selected)] + + +def validate_execution(witness: dict[str, Any], evidence: dict[str, Any]) -> None: + """Check actual collection, phases, model execution and persistent trajectory.""" + nodeid = witness["nodeid"] + _require(evidence.get("collected"), f"{nodeid}: not collected (exact nodeid required)") + _require( + evidence.get("phases") == {"setup": "passed", "call": "passed", "teardown": "passed"}, + f"{nodeid}: setup/call/teardown must pass without skip or xfail", + ) + for name, value in witness["dimensions"].items(): + _require( + evidence["dimensions"].get(name) == value, + f"{nodeid}: unexecuted parameter dimension {name}={value}", + ) + if witness["kind"] == "generated": + _require(evidence.get("models", 0) > 0, f"{nodeid}: no executed generated model") + events = evidence.get("events", []) + if witness["kind"] == "generated": + events = [ + event + for event in events + if event.get("model") is not None and 0 < event["model"] <= evidence["models"] + ] + # One trajectory cannot be assembled out of independent temporary projects. + workspaces = {event["cwd"] for event in events} + for workspace in workspaces: + cursor = 0 + for event in (event for event in events if event["cwd"] == workspace): + expected = witness["transitions"][cursor] + if ( + event["command"] == expected["command"] + and event["returncode"] == expected["returncode"] + and all(arg in event["args"] for arg in expected["argv_contains"]) + and ( + expected["state"] == "observed" + or (event["before"] == event["after"]) == (expected["state"] == "unchanged") + ) + ): + cursor += 1 + if cursor == len(witness["transitions"]): + return + raise EvidenceError(f"{nodeid}: required ordered command/state trajectory not executed") diff --git a/tests/fixtures/lifecycle_bug_ledger.json b/tests/fixtures/lifecycle_bug_ledger.json index 5ed22d4021..2b64f79740 100644 --- a/tests/fixtures/lifecycle_bug_ledger.json +++ b/tests/fixtures/lifecycle_bug_ledger.json @@ -1,6 +1,13 @@ { "schema_version": 1, "scope": "Representative post-0.26 escaped defects with reviewable regression evidence; this is not an issue-count census.", + "lifecycle_contracts": [], + "non_runtime_changes": [ + { + "paths": ["scripts/check_lifecycle_evidence.py", "scripts/lifecycle_contracts.py"], + "reason": "Repository-only lifecycle evidence provider and authored-contract validator; neither module is shipped in apm_cli or invoked by an APM command. Exercised by tests/unit/scripts/test_lifecycle_evidence.py and the existing quality/CI topology gates." + } + ], "property_catalog": [ { "id": "filesystem.open_world_observation", diff --git a/tests/quality/test_ci_topology.py b/tests/quality/test_ci_topology.py index 123fb3d2a9..54add7b5af 100644 --- a/tests/quality/test_ci_topology.py +++ b/tests/quality/test_ci_topology.py @@ -85,6 +85,30 @@ FORBIDDEN_CREDENTIAL_EXPRESSIONS = (*FORBIDDEN_CREDENTIAL_ENV, "github.token") +def test_lifecycle_evidence_is_executed_in_required_candidate_lanes() -> None: + """A saved receipt cannot replace either required executing provider lane.""" + smoke = workflow_job(load_workflow(REPO_ROOT / ".github/workflows/ci.yml"), LIFECYCLE_SMOKE_JOB) + step = workflow_step(smoke, "Execute changed lifecycle contracts") + assert "scripts/check_lifecycle_evidence.py" in step["run"] + assert '--base "$LIFECYCLE_BASE" --head "$LIFECYCLE_HEAD" --lane pr' in step["run"] + assert "github.event.pull_request.base.sha" in step["env"]["LIFECYCLE_BASE"] + assert "github.event.merge_group.base_sha" in step["env"]["LIFECYCLE_BASE"] + assert step["env"]["LIFECYCLE_HEAD"] == "${{ github.sha }}" + checkout = next(s for s in smoke["steps"] if "actions/checkout@" in s.get("uses", "")) + assert checkout["with"]["fetch-depth"] == 0 + integration = workflow_job( + load_workflow(MERGE_GROUP_INTEGRATION_WORKFLOW), MERGE_GROUP_INTEGRATION_JOB + ) + full = workflow_step(integration, "Execute full lifecycle contracts") + assert full["if"] == "matrix.shard == 1" + assert "scripts/check_lifecycle_evidence.py" in full["run"] + assert '--base "$LIFECYCLE_BASE" --head "$LIFECYCLE_HEAD" --lane full' in full["run"] + assert full["env"]["LIFECYCLE_BASE"] == "${{ github.event.merge_group.base_sha }}" + assert full["env"]["LIFECYCLE_HEAD"] == "${{ github.event.merge_group.head_sha }}" + assert 'git fetch --no-tags origin "$LIFECYCLE_BASE" "$LIFECYCLE_HEAD"' in full["run"] + assert "continue-on-error" not in step and "continue-on-error" not in full + + def _pytest_ini_markers() -> list[str]: """The registered marker declarations from [tool.pytest.ini_options].markers in pyproject.toml -- the single diff --git a/tests/quality/test_lifecycle_bug_ledger.py b/tests/quality/test_lifecycle_bug_ledger.py index 86c276d4a0..262d71f8a8 100644 --- a/tests/quality/test_lifecycle_bug_ledger.py +++ b/tests/quality/test_lifecycle_bug_ledger.py @@ -2,10 +2,14 @@ from __future__ import annotations -import ast import json +import os +import subprocess +import sys from pathlib import Path +from scripts.lifecycle_contracts import command_inventory, validate_contracts + _REPOSITORY_ROOT = Path(__file__).resolve().parents[2] _LEDGER_PATH = _REPOSITORY_ROOT / "tests/fixtures/lifecycle_bug_ledger.json" _FAILURE_MODES = frozenset( @@ -32,17 +36,6 @@ def _load_ledger() -> dict[str, object]: return payload -def _defined_test_names(path: Path) -> set[str]: - tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - names: set[str] = set() - for node in ast.walk(tree): - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name.startswith( - "test_" - ): - names.add(node.name) - return names - - def test_lifecycle_bug_ledger_has_valid_taxonomy_and_unique_references() -> None: ledger = _load_ledger() @@ -79,6 +72,9 @@ def test_lifecycle_bug_ledger_has_valid_taxonomy_and_unique_references() -> None assert row["regression_tests"] referenced_properties.update(row["properties"]) + contracts = validate_contracts(ledger, command_inventory()) + for contract in contracts.values(): + referenced_properties.update(contract["properties"]) assert referenced_properties == set(property_ids) gap_ids = [gap["id"] for gap in known_gaps] assert gap_ids @@ -90,22 +86,38 @@ def test_lifecycle_bug_ledger_has_valid_taxonomy_and_unique_references() -> None def test_lifecycle_bug_ledger_regression_nodeids_exist() -> None: - ledger = _load_ledger() - defined_by_path: dict[Path, set[str]] = {} + """Legacy bug rows may name parameter families; pytest must collect each one. - for row in ledger["bugs"]: - for nodeid in row["regression_tests"]: - path_text, separator, test_name = nodeid.partition("::") - leaf_name = test_name.split("::")[-1] - assert separator and leaf_name.startswith("test_"), f"Invalid test nodeid: {nodeid}" - path = _REPOSITORY_ROOT / path_text - assert path.is_file(), f"Missing regression test file: {path_text}" - names = defined_by_path.setdefault(path, _defined_test_names(path)) - assert leaf_name in names, f"Missing regression test: {nodeid}" - - for gap in ledger["known_gaps"]: - path_text, separator, test_name = gap["bounded_by"].partition("::") - assert separator and test_name.startswith("test_") - path = _REPOSITORY_ROOT / path_text - assert path.is_file() - assert test_name in _defined_test_names(path) + General feature contracts require exact parameterized nodes at execution. + This inventory check is not a substitute for their executing gate. + """ + ledger = _load_ledger() + references = {nodeid for row in ledger["bugs"] for nodeid in row["regression_tests"]} | { + gap["bounded_by"] for gap in ledger["known_gaps"] + } + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "-p", + "no:cacheprovider", + "--collect-only", + "-q", + "-o", + "addopts=", + *sorted(references), + ], + cwd=_REPOSITORY_ROOT, + env={**os.environ, "PYTEST_ADDOPTS": ""}, + text=True, + capture_output=True, + timeout=120, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + collected = {line.strip() for line in result.stdout.splitlines() if line.startswith("tests/")} + for nodeid in references: + assert any( + candidate == nodeid or candidate.startswith(f"{nodeid}[") for candidate in collected + ), f"Uncollected regression reference: {nodeid}" diff --git a/tests/unit/scripts/test_lifecycle_evidence.py b/tests/unit/scripts/test_lifecycle_evidence.py new file mode 100644 index 0000000000..3646e1169b --- /dev/null +++ b/tests/unit/scripts/test_lifecycle_evidence.py @@ -0,0 +1,479 @@ +"""Fail-closed regressions for authored plans and fresh pytest observations.""" + +from __future__ import annotations + +import argparse +import copy +import json +import subprocess +import sys +from pathlib import Path +from typing import Any + +import click +import pytest + +from scripts.check_lifecycle_evidence import candidate, execute, main, source_profile +from scripts.lifecycle_contracts import ( + EvidenceError, + command_inventory, + command_path, + runtime_candidate, + select_contracts, + validate_contracts, + validate_execution, +) +from tests.utils.lifecycle_evidence import LifecycleEvidencePlugin + +pytestmark = pytest.mark.component +pytest_plugins = ["pytester"] +RATCHET_TEST_SCOPE = "fixture" + + +def _ledger() -> dict[str, Any]: + witnesses = [ + { + "id": kind, + "nodeid": f"tests/test_sample.py::test_{kind}[global]", + "kind": kind, + "dimensions": {"variant": "global"}, + "transitions": [ + {"command": "install", "argv_contains": [], "returncode": 0, "state": state} + for state in ("changed", "unchanged") + ], + } + for kind in ("deterministic", "generated") + ] + return { + "schema_version": 1, + "property_catalog": [{"id": "ownership.preserve_unowned"}], + "lifecycle_contracts": [ + { + "id": "feature", + "assessment": "Shared global ownership affects install and repeated install.", + "paths": ["src/new.py"], + "properties": ["ownership.preserve_unowned"], + "dimensions": {"variant": ["global"]}, + "commands": { + "install": { + "disposition": "applicable", + "reason": "Installs this feature.", + "deterministic": ["deterministic"], + "generated": ["generated"], + } + }, + "witnesses": witnesses, + } + ], + } + + +def _inventory() -> dict[str, click.Command]: + return {"install": click.Command("install")} + + +def _evidence() -> dict[str, Any]: + return { + "collected": True, + "dimensions": {"variant": "global"}, + "phases": {"setup": "passed", "call": "passed", "teardown": "passed"}, + "models": 1, + "events": [ + { + "command": "install", + "args": [], + "returncode": 0, + "cwd": "/fixture", + "model": 1, + "before": "a", + "after": after, + } + for after in ("b", "a") + ], + } + + +@pytest.mark.parametrize( + "path", + [ + "src/new.py", + "src/renamed.py", + "unknown/feature.md", + "packages/skill/SKILL.md", + "install.sh", + "uv.lock", + "pyproject.toml", + "templates/new.yml", + "scripts/new.py", + ], +) +def test_unknown_and_payload_paths_conservatively_require_assessment(path: str) -> None: + assert runtime_candidate(path) + empty = {**_ledger(), "lifecycle_contracts": []} + with pytest.raises(EvidenceError, match="Missing fresh lifecycle"): + select_contracts(empty, empty, {path}, _inventory()) + + +def test_fresh_exact_assessment_and_renames() -> None: + head = _ledger() + empty = {**head, "lifecycle_contracts": []} + assert select_contracts(empty, head, {"src/new.py"}, _inventory()) + with pytest.raises(EvidenceError, match="Missing fresh lifecycle"): + select_contracts(head, head, {"src/new.py"}, _inventory()) + with pytest.raises(EvidenceError, match=r"src/old.py"): + select_contracts(empty, head, {"src/new.py", "src/old.py"}, _inventory()) + + +def test_non_runtime_exception_is_exact_reasoned_and_not_reusable() -> None: + base = {**_ledger(), "lifecycle_contracts": []} + head = {**base, "non_runtime_changes": [{"paths": ["scripts/tool.py"], "reason": "CI tooling"}]} + assert select_contracts(base, head, {"scripts/tool.py"}, _inventory()) == [] + with pytest.raises(EvidenceError, match="Missing fresh"): + select_contracts(head, head, {"scripts/tool.py"}, _inventory()) + head["non_runtime_changes"][0]["paths"] = ["scripts/*"] + with pytest.raises(EvidenceError, match="exact repository"): + select_contracts(base, head, {"scripts/tool.py"}, _inventory()) + + +@pytest.mark.parametrize("mutation", ["contract", "witness", "dimension", "command", "path"]) +def test_coordinated_obligation_removal_fails(mutation: str) -> None: + base = _ledger() + head = copy.deepcopy(base) + row = head["lifecycle_contracts"][0] + if mutation == "contract": + head["lifecycle_contracts"] = [] + elif mutation == "witness": + row["witnesses"][0]["nodeid"] = "tests/test_other.py::test_other" + elif mutation == "dimension": + row["dimensions"] = {} + elif mutation == "command": + row["commands"]["install"] = {"disposition": "semantic_na", "reason": "Removed"} + else: + row["paths"] = ["src/replacement.py"] + with pytest.raises(EvidenceError): + select_contracts(base, head, set(), _inventory()) + + +@pytest.mark.parametrize("mutation", ["inventory", "kind", "dimension", "transition", "property"]) +def test_incomplete_contract_fails(mutation: str) -> None: + ledger = _ledger() + row = ledger["lifecycle_contracts"][0] + if mutation == "inventory": + row["commands"] = {} + elif mutation == "kind": + row["witnesses"][1]["kind"] = "deterministic" + elif mutation == "dimension": + row["dimensions"]["variant"].append("aliased") + elif mutation == "transition": + row["witnesses"][0]["transitions"] = [] + else: + row["properties"] = ["invented.property"] + with pytest.raises(EvidenceError): + validate_contracts(ledger, _inventory()) + + +@pytest.mark.parametrize( + "mutation", + [ + "uncollected", + "skip", + "xfail", + "setup", + "call", + "teardown", + "dimension", + "model", + "missing-transition", + "status", + "workspace", + "state", + "json-only", + "outside-model", + ], +) +def test_execution_obligations_fail_closed(mutation: str) -> None: + witness = _ledger()["lifecycle_contracts"][0]["witnesses"][1] + evidence = _evidence() + validate_execution(witness, evidence) + if mutation == "uncollected": + evidence["collected"] = False + elif mutation in {"setup", "call", "teardown"}: + evidence["phases"][mutation] = "failed" + elif mutation in {"skip", "xfail"}: + evidence["phases"]["call"] = mutation + elif mutation == "dimension": + evidence["dimensions"] = {} + elif mutation == "model": + evidence["models"] = 0 + elif mutation == "missing-transition": + evidence["events"].pop() + elif mutation == "status": + evidence["events"][1]["returncode"] = 1 + elif mutation == "workspace": + evidence["events"][1]["cwd"] = "/other" + elif mutation == "state": + evidence["events"][1]["after"] = "different" + elif mutation == "outside-model": + evidence["events"][1]["model"] = None + else: + evidence = {"status": "passed"} + with pytest.raises(EvidenceError): + validate_execution(witness, evidence) + + +def test_inventory_uses_recursive_registrations_and_aliases( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from apm_cli.cli import cli + + monkeypatch.setitem(cli.commands, "fixture-alias", cli.commands["install"]) + inventory = command_inventory() + assert inventory["fixture-alias"] is inventory["install"] + assert "marketplace package add" in inventory + assert "" in inventory and "lock" in inventory and "lock export" in inventory + assert command_path(["--verbose", "cache", "prune", "--help"], inventory) == "cache prune" + assert command_path(["lock"], inventory) == "lock" + assert command_path(["--help"], inventory) == "" + + +def test_candidate_rejects_stale_head_and_dirty_tree(tmp_path: Path) -> None: + def run(*args: str) -> str: + return subprocess.check_output(["git", *args], cwd=tmp_path, text=True).strip() + + run("init", "-q") + run("config", "user.email", "fixture@example.invalid") + run("config", "user.name", "Fixture") + run("commit", "--allow-empty", "-qm", "base") + base = run("rev-parse", "HEAD") + run("commit", "--allow-empty", "-qm", "head") + head = run("rev-parse", "HEAD") + assert candidate(tmp_path, base, head)["tested_tree"] == run("rev-parse", "HEAD^{tree}") + with pytest.raises(EvidenceError, match="checked-out"): + candidate(tmp_path, base, base) + (tmp_path / "unknown").write_text("dirty") + with pytest.raises(EvidenceError, match="clean"): + candidate(tmp_path, base, head) + + +def test_plugin_observes_actual_execution_and_rejects_deselection( + pytester: pytest.Pytester, +) -> None: + module = pytester.makepyfile(""" + import os + import sys + import pytest + from tests.utils.apm_lifecycle_runner import ApmLifecycleRunner + from hypothesis import settings + from hypothesis.stateful import RuleBasedStateMachine, rule, run_state_machine_as_test + @pytest.mark.parametrize("variant", ["global"]) + def test_real(tmp_path, variant): + env = {**os.environ, "HOME": str(tmp_path / "home")} + runner = ApmLifecycleRunner((sys.executable, "-m", "apm_cli.cli")) + for _ in range(2): + result = runner.run(["--version"], cwd=tmp_path, env=env) + assert result.returncode == 0 + def test_skip(): + pytest.skip("contract skip") + @pytest.mark.xfail(reason="contract xfail") + def test_xfail(): + assert False + @pytest.mark.parametrize("variant", ["global"]) + def test_generated(tmp_path, variant): + class Model(RuleBasedStateMachine): + @rule() + def version(self): + env = {**os.environ, "HOME": str(tmp_path / "home")} + runner = ApmLifecycleRunner((sys.executable, "-m", "apm_cli.cli")) + for _ in range(2): + assert runner.run(["--version"], cwd=tmp_path, env=env).returncode == 0 + run_state_machine_as_test( + Model, settings=settings( + max_examples=1, stateful_step_count=1, deadline=None, database=None + ) + ) + """) + path = module.name + real = f"{path}::test_real[global]" + skipped, xfailed = f"{path}::test_skip", f"{path}::test_xfail" + generated = f"{path}::test_generated[global]" + executable = Path(sys.executable).parent / "apm" + plugin = LifecycleEvidencePlugin([real, skipped, xfailed, generated], executable) + result = pytester.runpytest_inprocess("-q", "-o", "addopts=", plugins=[plugin]) + result.assert_outcomes(passed=2, skipped=1, xfailed=1) + record = plugin.records[real] + assert record["dimensions"] == {"variant": "global"} + assert len(record["events"]) == 2 + assert record["phases"] == {"setup": "passed", "call": "passed", "teardown": "passed"} + assert plugin.records[skipped]["phases"]["call"] == "skipped" + assert plugin.records[xfailed]["phases"]["call"] == "xfail" + assert plugin.records[generated]["models"] == 1 + assert len(plugin.records[generated]["events"]) >= 2 + assert all(event["model"] == 1 for event in plugin.records[generated]["events"]) + deselected = LifecycleEvidencePlugin([real], executable) + result = pytester.runpytest_inprocess("-q", "-k", "skip", plugins=[deselected]) + assert real not in deselected.records + assert result.ret == 0 + assert json.dumps(record) + + +@pytest.mark.parametrize("phase", ["setup", "teardown"]) +def test_plugin_records_fixture_failures(pytester: pytest.Pytester, phase: str) -> None: + module = pytester.makepyfile(f""" + import pytest + @pytest.fixture + def broken(): + if {phase!r} == "setup": + raise RuntimeError("fixture failed") + yield + raise RuntimeError("fixture failed") + def test_fixture(broken): + assert True + """) + nodeid = f"{module.name}::test_fixture" + plugin = LifecycleEvidencePlugin([nodeid], Path(sys.executable).parent / "apm") + result = pytester.runpytest_inprocess("-q", plugins=[plugin]) + assert result.ret != 0 + assert plugin.records[nodeid]["phases"][phase] == "failed" + + +@pytest.mark.parametrize("selector", ["::WrongClass::test_exists", "::test_exists[missing]"]) +def test_plugin_rejects_wrong_class_or_parameter(pytester: pytest.Pytester, selector: str) -> None: + module = pytester.makepyfile(""" + import pytest + @pytest.mark.parametrize("variant", ["actual"]) + def test_exists(variant): + assert variant + """) + nodeid = f"{module.name}{selector}" + plugin = LifecycleEvidencePlugin([nodeid], Path(sys.executable).parent / "apm") + result = pytester.runpytest_inprocess("-q", nodeid, plugins=[plugin]) + assert result.ret != 0 + assert not plugin.records + + +def test_source_profile_rejects_foreign_source_and_stale_launcher( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import apm_cli + + monkeypatch.setattr(apm_cli, "__file__", str(tmp_path / "src/apm_cli/__init__.py")) + python = tmp_path / "python" + python.write_text("fixture interpreter", encoding="ascii") + monkeypatch.setattr(sys, "executable", str(python)) + with pytest.raises(EvidenceError, match="Unix installed"): + source_profile(tmp_path) + launcher = tmp_path / "apm" + launcher.write_text("#!/foreign/python\nfrom apm_cli.cli import cli\n", encoding="ascii") + with pytest.raises(EvidenceError, match="this Python environment"): + source_profile(tmp_path) + launcher.write_text(f"#!{python}\nfrom apm_cli.cli import cli\n", encoding="ascii") + _, initial = source_profile(tmp_path) + launcher.write_text(f"#!{python}\nfrom apm_cli.cli import cli\n# changed\n", encoding="ascii") + assert source_profile(tmp_path)[1] != initial + with pytest.raises(EvidenceError, match="candidate checkout"): + source_profile(tmp_path / "different") + + +def test_cli_rejects_in_tree_receipts(monkeypatch: pytest.MonkeyPatch) -> None: + from scripts.check_lifecycle_evidence import ROOT + + monkeypatch.setattr( + sys, + "argv", + [ + "check_lifecycle_evidence.py", + "--base", + "a", + "--head", + "b", + "--lane", + "full", + "--report", + str(ROOT / "receipt.json"), + ], + ) + with pytest.raises(SystemExit) as exc: + main() + assert exc.value.code == 2 + + +@pytest.mark.parametrize("lane,status", [("pr", "pending"), ("full", "passed")]) +def test_execute_produces_candidate_bound_native_receipt( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, lane: str, status: str +) -> None: + """Exercise the gate, git comparison, real collection, subprocesses and model together.""" + from scripts import check_lifecycle_evidence as gate + + profile = source_profile(gate.ROOT) + monkeypatch.setattr(gate, "ROOT", tmp_path) + monkeypatch.setattr(gate, "source_profile", lambda _root: profile) + monkeypatch.chdir(tmp_path) + tests = tmp_path / "tests" + tests.mkdir() + filename = f"test_native_probe_{lane}.py" + (tests / filename).write_text( + """ +import os +import sys +import pytest +from hypothesis import settings +from hypothesis.stateful import RuleBasedStateMachine, rule, run_state_machine_as_test +from tests.utils.apm_lifecycle_runner import ApmLifecycleRunner +@pytest.mark.parametrize("kind", ["deterministic", "generated"]) +def test_probe(tmp_path, kind): + def commands(): + env = {"PATH": os.environ["PATH"], "HOME": str(tmp_path / "home")} + runner = ApmLifecycleRunner((sys.executable, "-m", "apm_cli.cli")) + for _ in range(2): + assert runner.run(["--version"], cwd=tmp_path, env=env).returncode == 0 + if kind == "deterministic": + commands() + else: + class Model(RuleBasedStateMachine): + @rule() + def version(self): + commands() + run_state_machine_as_test(Model, settings=settings( + max_examples=1, stateful_step_count=1, database=None, deadline=None)) +""", + encoding="ascii", + ) + ledger = _ledger() + path = tests / "fixtures/lifecycle_bug_ledger.json" + path.parent.mkdir() + path.write_text(json.dumps({**ledger, "lifecycle_contracts": []}), encoding="ascii") + + def commit(message: str) -> str: + subprocess.run(["git", "add", "."], check=True) + subprocess.run(["git", "commit", "-qm", message], check=True) + return subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() + + subprocess.run(["git", "init", "-q"], check=True) + subprocess.run(["git", "config", "user.email", "fixture@example.invalid"], check=True) + subprocess.run(["git", "config", "user.name", "Fixture"], check=True) + (tmp_path / ".gitignore").write_text("__pycache__/\n.hypothesis/\n", encoding="ascii") + base = commit("base") + contract = ledger["lifecycle_contracts"][0] + contract["dimensions"] = {} + contract["commands"] = { + name: {"disposition": "semantic_na", "reason": "Fixture only tests version reporting."} + for name in command_inventory() + } + contract["commands"][""] = { + "disposition": "applicable", + "reason": "Version reporting fixture.", + "deterministic": ["deterministic"], + "generated": ["generated"], + } + for witness in contract["witnesses"]: + witness["nodeid"] = f"tests/{filename}::test_probe[{witness['kind']}]" + witness["dimensions"] = {"kind": witness["kind"]} + for transition in witness["transitions"]: + transition.update(command="", argv_contains=["--version"], state="observed") + path.write_text(json.dumps(ledger), encoding="ascii") + head = commit("contract") + report = execute(argparse.Namespace(base=base, head=head, lane=lane)) + assert report["status"] == status, report + assert report["base"] == base and report["head"] == head + assert len(report["witnesses"]) == (2 if lane == "full" else 1) + assert all(value["events"] for value in report["witnesses"].values()) diff --git a/tests/utils/isolated_apm_environment.py b/tests/utils/isolated_apm_environment.py index 58c28b9f3f..c954cadabd 100644 --- a/tests/utils/isolated_apm_environment.py +++ b/tests/utils/isolated_apm_environment.py @@ -262,7 +262,7 @@ def _guarded_import_module(name, package=None): | _APM_REMOTE_CONTROL_ENV_NAMES ) _STRIPPED_ENV_PREFIXES = _APM_AUTH_ENV_PREFIXES + _GIT_CONFIG_INJECTION_PREFIXES -_PINNED_ENVIRONMENT_NAMES = ( +DURABLE_ENVIRONMENT_ROOTS = ( "HOME", "USERPROFILE", "XDG_CONFIG_HOME", @@ -271,6 +271,9 @@ def _guarded_import_module(name, package=None): "LOCALAPPDATA", "APM_HOME", "APM_CACHE_DIR", +) +_PINNED_ENVIRONMENT_NAMES = ( + *DURABLE_ENVIRONMENT_ROOTS, "APM_TEMP_DIR", "GH_CONFIG_DIR", "AZURE_CONFIG_DIR", diff --git a/tests/utils/lifecycle_evidence.py b/tests/utils/lifecycle_evidence.py new file mode 100644 index 0000000000..2e2ff43985 --- /dev/null +++ b/tests/utils/lifecycle_evidence.py @@ -0,0 +1,145 @@ +"""Scoped pytest observer for freshly executed lifecycle contract witnesses.""" + +from __future__ import annotations + +import hashlib +import json +import sys +from dataclasses import asdict +from pathlib import Path +from typing import Any + +import pytest + +from scripts.lifecycle_contracts import EvidenceError, command_inventory, command_path +from tests.utils.apm_lifecycle_runner import ApmLifecycleRunner +from tests.utils.artifact_snapshot import ArtifactSnapshotSet +from tests.utils.isolated_apm_environment import DURABLE_ENVIRONMENT_ROOTS + + +def fingerprint(path: Path) -> str: + """Hash an executable or source artifact without trusting its filename.""" + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def snapshot(cwd: Path, env: dict[str, str]) -> str: + """Use the existing open-world oracle for project and isolated user roots.""" + candidates = [cwd, *(Path(env[key]) for key in DURABLE_ENVIRONMENT_ROOTS if env.get(key))] + roots: list[Path] = [] + for candidate in sorted({path.resolve() for path in candidates}, key=lambda p: len(p.parts)): + if not any(candidate.is_relative_to(root) for root in roots): + roots.append(candidate) + captured = ArtifactSnapshotSet.capture({str(path): path for path in roots}) + payload = [ + (name, value.root_existed, [asdict(entry) for entry in value.entries]) + for name, value in captured.snapshots + ] + return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest() + + +class LifecycleEvidencePlugin: + """Observe exact nodes and runner invocations, never authored outcome strings.""" + + def __init__(self, nodeids: list[str], executable: Path) -> None: + self.nodeids = set(nodeids) + self.executable = executable.resolve() + self.executable_hash = fingerprint(self.executable) + self.python_hash = fingerprint(Path(sys.executable).resolve()) + self.inventory = command_inventory() + self.records: dict[str, dict[str, Any]] = {} + self.active: str | None = None + self.phase: str | None = None + self.model_run: int | None = None + self.patch = pytest.MonkeyPatch() + + def pytest_sessionstart(self, session: pytest.Session) -> None: + """Install observers before test-module imports bind runner/model functions.""" + from hypothesis import stateful + + original = ApmLifecycleRunner._run_with_timeout + model = stateful.run_state_machine_as_test + + def observe(runner: ApmLifecycleRunner, args: Any, **kwargs: Any) -> Any: + if self.active not in self.nodeids or self.phase != "call": + return original(runner, args, **kwargs) + command = runner._command + source_command = (sys.executable, "-m", "apm_cli.cli") + if command != source_command and command != (str(self.executable),): + raise EvidenceError(f"Unverified source executable: {command}") + if ( + fingerprint(self.executable) != self.executable_hash + or fingerprint(Path(sys.executable).resolve()) != self.python_hash + ): + raise EvidenceError("Executable changed during evidence execution") + cwd, env = kwargs["cwd"], kwargs["env"] + before = snapshot(cwd, env) + result = original(runner, args, **kwargs) + after = snapshot(cwd, env) + self.records[self.active]["events"].append( + { + "command": command_path(list(args), self.inventory), + "args": list(args), + "returncode": result.returncode, + "cwd": str(cwd.resolve()), + "before": before, + "after": after, + "scenario_id": kwargs["scenario_id"], + "entrypoint": list(command), + "model": self.model_run, + } + ) + return result + + def observe_model(*args: Any, **kwargs: Any) -> Any: + if self.active not in self.nodeids or self.phase != "call": + return model(*args, **kwargs) + previous = self.model_run + self.model_run = self.records[self.active]["models"] + 1 + try: + result = model(*args, **kwargs) + self.records[self.active]["models"] += 1 + return result + finally: + self.model_run = previous + + self.patch.setattr(ApmLifecycleRunner, "_run_with_timeout", observe) + self.patch.setattr(stateful, "run_state_machine_as_test", observe_model) + + def pytest_collection_finish(self, session: pytest.Session) -> None: + """Final selection, not an AST function lookup or pre-deselection inventory.""" + for item in session.items: + if item.nodeid not in self.nodeids: + continue + callspec = getattr(item, "callspec", None) + self.records[item.nodeid] = { + "collected": True, + "dimensions": dict(callspec.params) if callspec else {}, + "phases": {}, + "events": [], + "models": 0, + } + + @pytest.hookimpl(hookwrapper=True) + def pytest_runtest_protocol(self, item: pytest.Item, nextitem: pytest.Item | None) -> Any: + """Keep fixture execution associated with its exact collected node.""" + self.active = item.nodeid + yield + self.active, self.phase = None, None + + def pytest_runtest_setup(self, item: pytest.Item) -> None: + self.phase = "setup" + + def pytest_runtest_call(self, item: pytest.Item) -> None: + self.phase = "call" + + def pytest_runtest_teardown(self, item: pytest.Item, nextitem: pytest.Item | None) -> None: + self.phase = "teardown" + + def pytest_runtest_logreport(self, report: pytest.TestReport) -> None: + if report.nodeid in self.records: + outcome = "xfail" if hasattr(report, "wasxfail") else report.outcome + self.records[report.nodeid]["phases"][report.when] = outcome + + def pytest_sessionfinish(self, session: pytest.Session, exitstatus: int) -> None: + """Restore instrumentation even when collection or test setup fails.""" + self.patch.undo() From 7635cef31ccefc489735e38823a727a14fbbaa6d Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Mon, 7 Sep 2026 17:19:27 +0200 Subject: [PATCH 2/8] fix(ci): bind lifecycle trajectories to continuous isolated state Reject mixed durable roots and generated model calls; require reviewed preparation reasons for fixture-side state changes. Support canonical source-module evidence without a Unix launcher and register the now-tracked gate regression suite in the existing ratchet job. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci.yml | 1 + scripts/check_lifecycle_evidence.py | 23 +++++----- scripts/lifecycle_contracts.py | 41 ++++++++++++++---- tests/unit/scripts/test_lifecycle_evidence.py | 43 +++++++++++++++---- tests/utils/lifecycle_evidence.py | 23 +++++++--- 5 files changed, 97 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 48c58c3806..3988b70fe0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -195,6 +195,7 @@ jobs: uv run --frozen --extra dev pytest -p no:cacheprovider -q \ tests/unit/scripts/test_check_test_assertions.py \ tests/unit/scripts/test_check_exact_test_duplicates.py \ + tests/unit/scripts/test_lifecycle_evidence.py \ tests/quality/test_test_taxonomy.py \ tests/quality/test_quality_baselines.py \ tests/quality/test_lifecycle_bug_ledger.py \ diff --git a/scripts/check_lifecycle_evidence.py b/scripts/check_lifecycle_evidence.py index 784317468e..017d3c7b8d 100644 --- a/scripts/check_lifecycle_evidence.py +++ b/scripts/check_lifecycle_evidence.py @@ -33,23 +33,23 @@ ] -def source_profile(root: Path) -> tuple[Path, dict[str, str]]: +def source_profile(root: Path) -> tuple[Path | None, dict[str, Any]]: """Identify the installed editable source and its exact interpreter/launcher.""" import apm_cli from tests.utils.lifecycle_evidence import fingerprint if Path(apm_cli.__file__).resolve().parent != root / "src/apm_cli": raise EvidenceError("Installed source does not resolve to the candidate checkout") - executable = Path(sys.executable).parent / ("apm.exe" if os.name == "nt" else "apm") - if os.name == "nt" or not executable.is_file(): - raise EvidenceError("source-python profile requires the Unix installed apm script") - script = executable.read_text(encoding="utf-8") - if "apm_cli.cli" not in script or not script.startswith(f"#!{sys.executable}\n"): - raise EvidenceError("Installed apm entrypoint is not this Python environment") + launcher = Path(sys.executable).parent / "apm" + executable = launcher if os.name != "nt" and launcher.is_file() else None + if executable: + script = executable.read_text(encoding="utf-8") + if "apm_cli.cli" not in script or not script.startswith(f"#!{sys.executable}\n"): + raise EvidenceError("Installed apm entrypoint is not this Python environment") return executable, { "kind": "source-python", - "executable": str(executable), - "executable_sha256": fingerprint(executable), + "executable": str(executable) if executable else None, + "executable_sha256": fingerprint(executable) if executable else None, "python": str(Path(sys.executable).resolve()), "python_sha256": fingerprint(Path(sys.executable).resolve()), } @@ -127,7 +127,10 @@ def execute(args: argparse.Namespace) -> dict[str, Any]: executable, report["profile"] = source_profile(ROOT) nodeids = sorted({witness["nodeid"] for witness in witnesses}) plugin = LifecycleEvidencePlugin(nodeids, executable) - os.environ["APM_BINARY_PATH"] = str(executable) + if executable: + os.environ["APM_BINARY_PATH"] = str(executable) + else: + os.environ.pop("APM_BINARY_PATH", None) os.environ["APM_E2E_TESTS"] = "1" # No user-provided pytest options, deselection filters or receipt inputs. os.environ.pop("PYTEST_ADDOPTS", None) diff --git a/scripts/lifecycle_contracts.py b/scripts/lifecycle_contracts.py index c95d75bc76..bc84f6378d 100644 --- a/scripts/lifecycle_contracts.py +++ b/scripts/lifecycle_contracts.py @@ -145,6 +145,12 @@ def validate_contracts( and all(isinstance(arg, str) for arg in transition["argv_contains"]), f"{nodeid}: invalid argv constraints", ) + if "preparation" in transition: + _require( + isinstance(transition["preparation"], str) + and transition["preparation"].strip(), + f"{nodeid}: preparation needs a reviewed mutation rationale", + ) for param, values in row["dimensions"].items(): _require(values and isinstance(values, list), f"{identifier}: empty dimension") for kind in ("deterministic", "generated"): @@ -293,21 +299,38 @@ def validate_execution(witness: dict[str, Any], evidence: dict[str, Any]) -> Non for event in events if event.get("model") is not None and 0 < event["model"] <= evidence["models"] ] + # One trajectory cannot be assembled out of independent temporary projects. - workspaces = {event["cwd"] for event in events} + def trajectory(event: dict[str, Any]) -> tuple[str, str, int | None]: + _require(event.get("roots"), f"{nodeid}: missing durable-root identity") + return event["cwd"], json.dumps(event["roots"], sort_keys=True), event.get("model") + + def matches(event: dict[str, Any], expected: dict[str, Any]) -> bool: + return ( + event["command"] == expected["command"] + and event["returncode"] == expected["returncode"] + and all(arg in event["args"] for arg in expected["argv_contains"]) + and ( + expected["state"] == "observed" + or (event["before"] == event["after"]) == (expected["state"] == "unchanged") + ) + ) + + workspaces = {trajectory(event) for event in events} for workspace in workspaces: cursor = 0 - for event in (event for event in events if event["cwd"] == workspace): + previous_after = None + for event in (event for event in events if trajectory(event) == workspace): expected = witness["transitions"][cursor] if ( - event["command"] == expected["command"] - and event["returncode"] == expected["returncode"] - and all(arg in event["args"] for arg in expected["argv_contains"]) - and ( - expected["state"] == "observed" - or (event["before"] == event["after"]) == (expected["state"] == "unchanged") - ) + previous_after is not None + and previous_after != event["before"] + and not (expected.get("preparation") and matches(event, expected)) ): + cursor = 0 + expected = witness["transitions"][cursor] + previous_after = event["after"] + if matches(event, expected): cursor += 1 if cursor == len(witness["transitions"]): return diff --git a/tests/unit/scripts/test_lifecycle_evidence.py b/tests/unit/scripts/test_lifecycle_evidence.py index 3646e1169b..80d7c53fe0 100644 --- a/tests/unit/scripts/test_lifecycle_evidence.py +++ b/tests/unit/scripts/test_lifecycle_evidence.py @@ -5,6 +5,7 @@ import argparse import copy import json +import os import subprocess import sys from pathlib import Path @@ -72,6 +73,11 @@ def _inventory() -> dict[str, click.Command]: return {"install": click.Command("install")} +def _launcher() -> Path | None: + path = Path(sys.executable).parent / "apm" + return path if path.is_file() else None + + def _evidence() -> dict[str, Any]: return { "collected": True, @@ -84,11 +90,12 @@ def _evidence() -> dict[str, Any]: "args": [], "returncode": 0, "cwd": "/fixture", + "roots": {"HOME": "/fixture/home", "workspace": "/fixture"}, "model": 1, - "before": "a", + "before": before, "after": after, } - for after in ("b", "a") + for before, after in (("a", "b"), ("b", "b")) ], } @@ -189,6 +196,9 @@ def test_incomplete_contract_fails(mutation: str) -> None: "state", "json-only", "outside-model", + "roots", + "continuity", + "model-splice", ], ) def test_execution_obligations_fail_closed(mutation: str) -> None: @@ -215,12 +225,29 @@ def test_execution_obligations_fail_closed(mutation: str) -> None: evidence["events"][1]["after"] = "different" elif mutation == "outside-model": evidence["events"][1]["model"] = None + elif mutation == "roots": + evidence["events"][1]["roots"]["HOME"] = "/other/home" + elif mutation == "continuity": + evidence["events"][1].update(before="different", after="different") + elif mutation == "model-splice": + evidence["models"] = 2 + evidence["events"][1]["model"] = 2 else: evidence = {"status": "passed"} with pytest.raises(EvidenceError): validate_execution(witness, evidence) +def test_intentional_fixture_mutation_requires_reviewed_preparation() -> None: + witness = _ledger()["lifecycle_contracts"][0]["witnesses"][1] + evidence = _evidence() + evidence["events"][1].update(before="tampered", after="tampered") + with pytest.raises(EvidenceError, match="trajectory"): + validate_execution(witness, evidence) + witness["transitions"][1]["preparation"] = "User tampers with deployed bytes before audit." + validate_execution(witness, evidence) + + def test_inventory_uses_recursive_registrations_and_aliases( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -296,7 +323,7 @@ def version(self): real = f"{path}::test_real[global]" skipped, xfailed = f"{path}::test_skip", f"{path}::test_xfail" generated = f"{path}::test_generated[global]" - executable = Path(sys.executable).parent / "apm" + executable = _launcher() plugin = LifecycleEvidencePlugin([real, skipped, xfailed, generated], executable) result = pytester.runpytest_inprocess("-q", "-o", "addopts=", plugins=[plugin]) result.assert_outcomes(passed=2, skipped=1, xfailed=1) @@ -330,7 +357,7 @@ def test_fixture(broken): assert True """) nodeid = f"{module.name}::test_fixture" - plugin = LifecycleEvidencePlugin([nodeid], Path(sys.executable).parent / "apm") + plugin = LifecycleEvidencePlugin([nodeid], _launcher()) result = pytester.runpytest_inprocess("-q", plugins=[plugin]) assert result.ret != 0 assert plugin.records[nodeid]["phases"][phase] == "failed" @@ -345,12 +372,13 @@ def test_exists(variant): assert variant """) nodeid = f"{module.name}{selector}" - plugin = LifecycleEvidencePlugin([nodeid], Path(sys.executable).parent / "apm") + plugin = LifecycleEvidencePlugin([nodeid], _launcher()) result = pytester.runpytest_inprocess("-q", nodeid, plugins=[plugin]) assert result.ret != 0 assert not plugin.records +@pytest.mark.skipif(os.name == "nt", reason="Unix script launcher; Windows uses the source module") def test_source_profile_rejects_foreign_source_and_stale_launcher( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -360,8 +388,7 @@ def test_source_profile_rejects_foreign_source_and_stale_launcher( python = tmp_path / "python" python.write_text("fixture interpreter", encoding="ascii") monkeypatch.setattr(sys, "executable", str(python)) - with pytest.raises(EvidenceError, match="Unix installed"): - source_profile(tmp_path) + assert source_profile(tmp_path)[0] is None launcher = tmp_path / "apm" launcher.write_text("#!/foreign/python\nfrom apm_cli.cli import cli\n", encoding="ascii") with pytest.raises(EvidenceError, match="this Python environment"): @@ -422,7 +449,7 @@ def test_execute_produces_candidate_bound_native_receipt( @pytest.mark.parametrize("kind", ["deterministic", "generated"]) def test_probe(tmp_path, kind): def commands(): - env = {"PATH": os.environ["PATH"], "HOME": str(tmp_path / "home")} + env = {**os.environ, "HOME": str(tmp_path / "home"), "USERPROFILE": str(tmp_path / "home")} runner = ApmLifecycleRunner((sys.executable, "-m", "apm_cli.cli")) for _ in range(2): assert runner.run(["--version"], cwd=tmp_path, env=env).returncode == 0 diff --git a/tests/utils/lifecycle_evidence.py b/tests/utils/lifecycle_evidence.py index 2e2ff43985..d3b7c64f55 100644 --- a/tests/utils/lifecycle_evidence.py +++ b/tests/utils/lifecycle_evidence.py @@ -40,10 +40,10 @@ def snapshot(cwd: Path, env: dict[str, str]) -> str: class LifecycleEvidencePlugin: """Observe exact nodes and runner invocations, never authored outcome strings.""" - def __init__(self, nodeids: list[str], executable: Path) -> None: + def __init__(self, nodeids: list[str], executable: Path | None) -> None: self.nodeids = set(nodeids) - self.executable = executable.resolve() - self.executable_hash = fingerprint(self.executable) + self.executable = executable.resolve() if executable else None + self.executable_hash = fingerprint(self.executable) if self.executable else None self.python_hash = fingerprint(Path(sys.executable).resolve()) self.inventory = command_inventory() self.records: dict[str, dict[str, Any]] = {} @@ -64,12 +64,13 @@ def observe(runner: ApmLifecycleRunner, args: Any, **kwargs: Any) -> Any: return original(runner, args, **kwargs) command = runner._command source_command = (sys.executable, "-m", "apm_cli.cli") - if command != source_command and command != (str(self.executable),): + if command != source_command and not ( + self.executable and command == (str(self.executable),) + ): raise EvidenceError(f"Unverified source executable: {command}") if ( - fingerprint(self.executable) != self.executable_hash - or fingerprint(Path(sys.executable).resolve()) != self.python_hash - ): + self.executable and fingerprint(self.executable) != self.executable_hash + ) or fingerprint(Path(sys.executable).resolve()) != self.python_hash: raise EvidenceError("Executable changed during evidence execution") cwd, env = kwargs["cwd"], kwargs["env"] before = snapshot(cwd, env) @@ -81,6 +82,14 @@ def observe(runner: ApmLifecycleRunner, args: Any, **kwargs: Any) -> Any: "args": list(args), "returncode": result.returncode, "cwd": str(cwd.resolve()), + "roots": { + "workspace": str(cwd.absolute()), + **{ + key: str(Path(env[key]).absolute()) + for key in DURABLE_ENVIRONMENT_ROOTS + if env.get(key) + }, + }, "before": before, "after": after, "scenario_id": kwargs["scenario_id"], From cbbad528cf15b88833e9588d4d7bb3893b14490c Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Mon, 7 Sep 2026 18:49:20 +0200 Subject: [PATCH 3/8] feat(governance): make lifecycle completeness a shipping invariant Bind P8 to canonical agent instructions, exact lifecycle report summaries, independent terminal verification and contributor guidance. Preserve deterministic and generated-model obligations across rebases. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .agents/skills/apm-issue-autopilot/SKILL.md | 9 +- .../assets/acceptance-observer.md | 7 + .../assets/ideate-prompt.md | 6 + .../assets/solution-pipeline-prompt.md | 5 + .../assets/task-implement-prompt.md | 7 + .agents/skills/apm-review-panel/SKILL.md | 152 +++--------------- .../assets/panelist-return-schema.json | 23 ++- .../references/conditional-panelists.md | 97 +++++++++++ .agents/skills/batch-bug-shepherd/SKILL.md | 8 +- .../batch-bug-shepherd/assets/fix-prompt.md | 8 + .../assets/scenario-evidence-rubric.md | 11 ++ .agents/skills/shepherd-driver/SKILL.md | 6 + .../assets/completion-schema.json | 28 +++- .../assets/conflict-resolution-prompt.md | 9 +- .../assets/shepherd-driver-prompt.md | 46 +++++- .../references/mergeability-gate.md | 9 +- .apm/agents/apm-ceo.agent.md | 10 +- .apm/agents/test-coverage-expert.agent.md | 46 +++--- .apm/instructions/lifecycle.instructions.md | 118 ++++++++++++++ .github/agents/apm-ceo.agent.md | 10 +- .github/agents/test-coverage-expert.agent.md | 46 +++--- .../instructions/lifecycle.instructions.md | 118 ++++++++++++++ MANIFESTO.md | 5 + PRINCIPLES.md | 43 +++++ .../docs/contributing/integration-testing.md | 126 +++++++++++---- .../.apm/skills/apm-issue-autopilot/SKILL.md | 9 +- .../assets/acceptance-observer.md | 7 + .../assets/ideate-prompt.md | 6 + .../assets/solution-pipeline-prompt.md | 5 + .../assets/task-implement-prompt.md | 7 + packages/apm-review-panel/SKILL.md | 152 +++--------------- .../assets/panelist-return-schema.json | 23 ++- .../references/conditional-panelists.md | 97 +++++++++++ .../.apm/skills/batch-bug-shepherd/SKILL.md | 8 +- .../batch-bug-shepherd/assets/fix-prompt.md | 8 + .../assets/scenario-evidence-rubric.md | 11 ++ packages/shepherd-driver/SKILL.md | 6 + .../assets/completion-schema.json | 28 +++- .../assets/conflict-resolution-prompt.md | 9 +- .../assets/shepherd-driver-prompt.md | 46 +++++- .../references/mergeability-gate.md | 9 +- tests/fixtures/lifecycle_bug_ledger.json | 43 +++++ tests/unit/test_lifecycle_agent_contract.py | 101 ++++++++++++ .../test_shepherd_driver_completion_schema.py | 54 +++++++ 44 files changed, 1227 insertions(+), 355 deletions(-) create mode 100644 .agents/skills/apm-review-panel/references/conditional-panelists.md create mode 100644 .apm/instructions/lifecycle.instructions.md create mode 100644 .github/instructions/lifecycle.instructions.md create mode 100644 packages/apm-review-panel/references/conditional-panelists.md create mode 100644 tests/unit/test_lifecycle_agent_contract.py diff --git a/.agents/skills/apm-issue-autopilot/SKILL.md b/.agents/skills/apm-issue-autopilot/SKILL.md index 59508714dd..e75023393d 100644 --- a/.agents/skills/apm-issue-autopilot/SKILL.md +++ b/.agents/skills/apm-issue-autopilot/SKILL.md @@ -290,12 +290,17 @@ uv run python /.agents/skills/shepherd-driver/scripts/owner_touch_ --head --completion ``` -Do not write terminal state until both gates pass. A schema or semantic +Load `/.apm/instructions/lifecycle.instructions.md` and +independently execute its repository provider with `--lane full`, +`--base $BASE_SHA --head `, +`--completion `, and a new parent report path. +Do not write terminal state until schema, owner and lifecycle gates pass. +A schema or semantic failure gets one re-spawn; a second failure marks the row `blocked` with the verifier diagnostic. This prevents stale evidence or child self-classification from bypassing canonical owner detection. -After both gates pass, write `head_sha` and the +After all gates pass, write `head_sha` and the `mergeable/merge_state_status/ci_status` projection into the row's `head_sha` and `merge_state` columns (the crash-survivable A11 stop evidence), and remove ONLY the `status/shepherding` labels listed in diff --git a/.agents/skills/apm-issue-autopilot/assets/acceptance-observer.md b/.agents/skills/apm-issue-autopilot/assets/acceptance-observer.md index 853a938d80..cdf7e3a335 100644 --- a/.agents/skills/apm-issue-autopilot/assets/acceptance-observer.md +++ b/.agents/skills/apm-issue-autopilot/assets/acceptance-observer.md @@ -7,6 +7,13 @@ only place a PR is opened in Phase 4 (task children never open PRs). ## Procedure (pipeline child, in the issue worktree) +Load `$REPO_ROOT/.apm/instructions/lifecycle.instructions.md` before +acceptance. P8 applies even if the original `acceptance_shape` omitted it. +Execute its full lifecycle gate on the integrated candidate; preliminary +PR-lane, missing, skipped, failed, or stale evidence is not completion. +Re-plan missing lifecycle work under the same failure cap below; do not +open a ready PR or defer that work as out of scope. + 1. Reload plan.json (B8 ATTENTION ANCHOR). Read `acceptance_shape`. 2. Run the full lint contract one last time (must be silent) and the full test suite (`uv run --extra dev pytest -q`, must be green). diff --git a/.agents/skills/apm-issue-autopilot/assets/ideate-prompt.md b/.agents/skills/apm-issue-autopilot/assets/ideate-prompt.md index ba4cef5834..eb189c9578 100644 --- a/.agents/skills/apm-issue-autopilot/assets/ideate-prompt.md +++ b/.agents/skills/apm-issue-autopilot/assets/ideate-prompt.md @@ -25,6 +25,12 @@ OBSERVER), so make it concrete and testable, not aspirational. ## What to produce +First load `$REPO_ROOT/.apm/instructions/lifecycle.instructions.md`. +Include its P8 command map, deterministic trajectory and generated-model +obligations in `acceptance_shape` for every applicable change, regardless +of issue type. A brief that excludes required lifecycle work must be +re-planned or escalated, not accepted as install-only scope. + 1. Survey the touched surface read-only (`grep`, `view`, `gh issue view`). Understand the user-facing shape and where it lives. 2. Frame the **design brief** through the DevX lens: who the change diff --git a/.agents/skills/apm-issue-autopilot/assets/solution-pipeline-prompt.md b/.agents/skills/apm-issue-autopilot/assets/solution-pipeline-prompt.md index d9c6c43ee1..a415fa7ddf 100644 --- a/.agents/skills/apm-issue-autopilot/assets/solution-pipeline-prompt.md +++ b/.agents/skills/apm-issue-autopilot/assets/solution-pipeline-prompt.md @@ -26,6 +26,11 @@ parallelizing the work within an issue. Reload plan.md and (once it exists) plan.json before EACH stage and before EACH spawn (B8 ATTENTION ANCHOR). Never drive from recall. +Load `$REPO_ROOT/.apm/instructions/lifecycle.instructions.md` before +Ideate. Carry P8 into `acceptance_shape` and wave `integration_check`; +assign shared lifecycle/model files to one wave owner. Task-local success +does not certify the integrated candidate's lifecycle. + ## Model routing (B12) Every child below is spawned with an EXPLICIT model, resolved from diff --git a/.agents/skills/apm-issue-autopilot/assets/task-implement-prompt.md b/.agents/skills/apm-issue-autopilot/assets/task-implement-prompt.md index e0704628de..8be04477eb 100644 --- a/.agents/skills/apm-issue-autopilot/assets/task-implement-prompt.md +++ b/.agents/skills/apm-issue-autopilot/assets/task-implement-prompt.md @@ -37,6 +37,13 @@ Read TASK.type and load the matching lens; do not load the others: The lens defines the coverage discipline -- follow it exactly. +All lenses also inherit P8 from +`$WORKTREE/.apm/instructions/lifecycle.instructions.md`; load it before +editing. Keep the assigned lifecycle obligations executable across task +boundaries. If they are absent from the plan, escalate for re-planning +rather than silently narrowing to one command. Acceptance close verifies +the integrated candidate, not independent per-task success claims. + ## Discipline 1. Work ONLY inside WORKTREE on TASK_BRANCH. Never touch another diff --git a/.agents/skills/apm-review-panel/SKILL.md b/.agents/skills/apm-review-panel/SKILL.md index 403845415f..a38bb5e41f 100644 --- a/.agents/skills/apm-review-panel/SKILL.md +++ b/.agents/skills/apm-review-panel/SKILL.md @@ -4,7 +4,7 @@ description: >- Use this skill to run a multi-persona expert advisory review on a labelled pull request in microsoft/apm. The panel fans out to five mandatory specialists plus a test-coverage specialist (active on every PR that - touches src/) plus three conditional specialists (auth, doc-writer, + touches runtime behavior) plus three conditional specialists (auth, doc-writer, performance-expert), all running in their own agent threads, and a CEO synthesizer. The orchestrator is the sole writer to the PR: ONE recommendation comment, no verdict labels, no merge gating. The panel @@ -132,126 +132,27 @@ surfaces findings; the maintainer and the PR author decide ship. ## Conditional panelists -Three personas are conditional (auth, doc-writer, performance-expert). A -fourth (test-coverage) is mandatory on every PR that touches `src/` and -only skipped on documentation-only PRs -- see its section below for why. -The orchestrator ALWAYS spawns ALL four tasks to keep the schema -return shape uniform; the prompt instructs the subagent to set -`active: false` with an `inactive_reason` if the condition does not -hold. - -### Auth Expert - -Activate when the PR changes any of: -- `src/apm_cli/core/auth.py` -- `src/apm_cli/core/token_manager.py` -- `src/apm_cli/core/azure_cli.py` -- `src/apm_cli/deps/github_downloader.py` -- `src/apm_cli/marketplace/client.py` -- `src/apm_cli/utils/github_host.py` -- `src/apm_cli/install/validation.py` -- `src/apm_cli/install/pipeline.py` -- `src/apm_cli/deps/registry_proxy.py` - -Fallback self-check (when no fast-path file matched): "Does this PR -change authentication behavior, token management, credential resolution, -host classification used by `AuthResolver`, git or HTTP authorization -headers, or remote-host fallback semantics? If unsure, answer YES." - -### Doc Writer - -Activate when the PR changes any of: -- `README.md` -- `CHANGELOG.md` -- `MANIFESTO.md` -- `docs/src/content/docs/**` -- `.apm/skills/**/*.md` -- `.apm/agents/**/*.md` -- `.github/skills/**/*.md` -- `.github/agents/**/*.md` -- `.github/instructions/**/*.md` -- `.github/workflows/*.md` (gh-aw natural-language workflows) -- `packages/apm-guide/**` - -Fallback self-check (when no fast-path file matched): "Does this PR -change user-facing documentation, agent or skill prose, instruction -files, CHANGELOG entries, README claims, or any natural-language -artifact a reader will rely on? If unsure, answer YES." - -When the doc-writer is active and the PR includes documentation changes, -the persona reviews them for: (a) consistency with the existing voice -and structure, (b) accuracy against the code being changed, (c) -completeness for the typical reader (no orphan claims, no missing -prerequisites), (d) discoverability (cross-links, sidebar order if -Starlight content). When the doc-writer is active because of code -changes that SHOULD have updated docs but did not, the persona surfaces -that gap as a finding. - -### Performance Expert - -Activate when the PR changes any of: -- `src/apm_cli/cache/**` -- `src/apm_cli/deps/**` -- `src/apm_cli/install/phases/**` -- `src/apm_cli/install/pipeline.py` -- `src/apm_cli/install/resolve.py` -- `src/apm_cli/utils/**` -- `src/apm_cli/marketplace/**` -- `src/apm_cli/compilation/**` -- `scripts/perf/**` -- `src/apm_cli/core/command_logger.py` (when the diff adds perf-instrumentation logs) - -Also activate when: -- The PR description claims a performance win (speedup ratio, latency - reduction, bytes-on-disk reduction, throughput improvement) or - attaches a perf-harness measurement table. -- The diff introduces loops over collections (`for x in collection`) - where the collection may grow with dependency count or file count. -- The diff adds `os.scandir`, `os.walk`, `os.listdir`, or - `subprocess.run` calls on a path that executes per-package or - per-dependency. -- The diff adds `x in list_variable` inside a loop body. - -Fallback self-check (when no fast-path file matched): "Does this PR -change the hot path for dependency download, materialization, cache -layout, transport (git protocol, partial clone, sparse checkout), -parallelism, or any user-visible install/update wall-time? Does it -introduce an algorithmic complexity regression (O(n^2) loops, repeated -I/O, missing indexes, unconditional full scans, blocking synchronous -calls, heavy top-level imports)? If unsure, answer YES." - -When active, the performance-expert reviews against BOTH: -1. The package-manager performance playbook: transport minimization - (depth, filter, sparse scope), cache layering and dedup keys, - parallelism and lock contention, working-tree materialization cost, - perf-harness methodology (cache wipe, warm/cold separation, - statistical noise), and pervasive application of the chosen - technique across install / update / run surfaces. -2. The algorithmic performance lens (Big O analysis): complexity class - of every loop/lookup in the diff, index vs linear scan patterns, - unconditional expensive operations, import startup costs, redundant - computation, and parallelism opportunities. See the agent's - `references/algorithmic-patterns.md` for the full pattern catalogue. +Before selecting activation, load +[references/conditional-panelists.md](references/conditional-panelists.md) +for auth, doc-writer and performance triggers. Test coverage follows the +P8 rule below. ALWAYS spawn all four tasks to keep the return shape +uniform; an inactive persona returns `active: false` and a specific +`inactive_reason`. ### Test Coverage Expert -**Active by default on every PR that touches `src/**/*.py`.** The only -condition that flips this persona to `active: false` is a -documentation-only PR -- the diff contains zero `src/**/*.py` files. -In that case set `inactive_reason: "documentation-only PR -- no -runtime code paths to defend"`. - -The activation rule is intentionally narrow: under the advisory regime, -test outcomes are LOAD-BEARING for CEO arbitration (passed / failed / -missing test evidence outranks opinion-only findings -- see -`apm-ceo.agent.md` and `panelist-return-schema.json` evidence block). -A persona whose findings carry that weight cannot be silently skipped -on a heuristic. Better to spawn it on a pure refactor and have it -return a single `nit`-severity "no behavior surface touched -- no -coverage finding" line than to skip it and leave the CEO without -evidence to weigh. (Earlier revisions of this skill paired test-coverage -with auth and doc-writer as conditional for symmetry; that symmetry -broke when test evidence became load-bearing.) +**Active on every PR touching `src/**/*.py` or a P8 lifecycle surface.** +Before classifying, load the consumer repository's +`.apm/instructions/lifecycle.instructions.md`. Absence of Python changes +does not exempt installed payloads, manifests, shared state or tooling +that alters lifecycle behavior. Only a demonstrated unrelated change +may set `active: false`, with a specific `inactive_reason`. + +Pass that rule and the candidate-bound lifecycle evidence to the +coverage reviewer and CEO. Missing required deterministic or generated +coverage is a blocking P8 gap, not a deferrable recommendation. +The panel remains advisory; the executing repository gate, not a +`ship_now` opinion, discharges the shipping precondition. The test-coverage-expert is paired with the devx-ux-expert lens and defends the user-promise contracts the DevX persona enumerates (CLI @@ -259,7 +160,7 @@ surface, error wording, install idempotency, lockfile determinism, auth resolution). It MUST verify "no test exists" claims with `view`/`grep` on the test tree before emitting a finding -- false-positive coverage findings destroy trust in the field. It does NOT compute coverage -percentages, does NOT flag tests for pure refactors, and does NOT +percentages, does NOT demand artificial new tests for refactors, and does NOT duplicate python-architect on test-code design. ## Routing matrix (CEO synthesis emphasis only) @@ -482,12 +383,8 @@ no comment can be rendered, an explicit `noop` (step 9) -- are emitted. a diff changes how a remote host, org, token source, or fallback path is selected and you are not certain it is auth-neutral, activate auth-expert as `active: true`. -- **Test-coverage probe is mandatory.** The test-coverage-expert MUST - verify "no test exists for X" via `view`/`grep` on the `tests/` tree - before emitting a finding. A false-positive coverage finding (test - exists but persona claimed it does not) destroys maintainer trust in - the field. The persona scope file enforces this; the orchestrator - passes the diff and trusts the persona to probe. +- **Test-coverage probe is mandatory.** Confirm missing-test claims + against `tests/`, as required by the coverage persona. - **Subagent write enforcement is contract-based, not sandbox-based.** Tool permissions are workflow-scoped, not subagent-scoped, so every spawned task technically inherits the same `gh` toolset. The @@ -505,7 +402,4 @@ no comment can be rendered, an explicit `noop` (step 9) -- are emitted. the turn MUST end with a safe output -- the comment, or an explicit `noop`. See the "Synchronous fan-out" and "Non-empty turn exit" architecture invariants and step 9. -- **No verdict-label reset workflow.** The previous regime had a - companion workflow `pr-panel-label-reset.yml` that stripped verdict - labels on every push. The advisory regime has no verdict labels to - strip; that workflow is removed. +- **No verdict-label reset.** Advisory panels emit no verdict labels. diff --git a/.agents/skills/apm-review-panel/assets/panelist-return-schema.json b/.agents/skills/apm-review-panel/assets/panelist-return-schema.json index 0bf67df88c..bf2babd927 100644 --- a/.agents/skills/apm-review-panel/assets/panelist-return-schema.json +++ b/.agents/skills/apm-review-panel/assets/panelist-return-schema.json @@ -115,7 +115,7 @@ }, "tier": { "type": "string", - "enum": ["unit", "integration-with-fixtures", "e2e", "manual-only", "static"], + "enum": ["unit", "integration-with-fixtures", "e2e", "lifecycle-state-machine", "manual-only", "static"], "description": "Tier of evidence. unit: function-level test with mocks at the boundary; cheap, fast, narrow. integration-with-fixtures: real I/O against real fixtures (real files, real subprocess, real network when tagged), no mocked surface for the asserted contract. e2e: full CLI invocation end-to-end, real artifacts, real exit codes. manual-only: only a manual procedure; no automated guardrail. static: lint / type-check / schema validation only. The CEO weights tier against the SURFACE FLOOR named in the test-coverage-expert tier-floor matrix: a `unit` passed evidence on a critical-promise surface (CLI / install / lockfile / auth / hooks / marketplace / cross-module) does NOT silence an opinion-finding from another panelist asking for `integration-with-fixtures` coverage. Required on every evidence block." }, "run_evidence": { @@ -146,7 +146,26 @@ ] } } - } + }, + "allOf": [ + { + "if": { + "properties": { + "outcome": { "const": "passed" }, + "tier": { "const": "lifecycle-state-machine" } + }, + "required": ["outcome", "tier"] + }, + "then": { + "required": ["test_file", "run_evidence", "assertion_excerpt"], + "properties": { + "test_file": { "minLength": 1 }, + "run_evidence": { "minLength": 1 }, + "assertion_excerpt": { "minLength": 1 } + } + } + } + ] } } } diff --git a/.agents/skills/apm-review-panel/references/conditional-panelists.md b/.agents/skills/apm-review-panel/references/conditional-panelists.md new file mode 100644 index 0000000000..309cc19368 --- /dev/null +++ b/.agents/skills/apm-review-panel/references/conditional-panelists.md @@ -0,0 +1,97 @@ +# Conditional specialist activation + +Load this reference before the review panel selects persona activation. +Test-coverage activation is owned by the P8 section in `../SKILL.md`. + +## Auth Expert + +Activate when the PR changes any of: +- `src/apm_cli/core/auth.py` +- `src/apm_cli/core/token_manager.py` +- `src/apm_cli/core/azure_cli.py` +- `src/apm_cli/deps/github_downloader.py` +- `src/apm_cli/marketplace/client.py` +- `src/apm_cli/utils/github_host.py` +- `src/apm_cli/install/validation.py` +- `src/apm_cli/install/pipeline.py` +- `src/apm_cli/deps/registry_proxy.py` + +Fallback self-check (when no fast-path file matched): "Does this PR +change authentication behavior, token management, credential resolution, +host classification used by `AuthResolver`, git or HTTP authorization +headers, or remote-host fallback semantics? If unsure, answer YES." + +## Doc Writer + +Activate when the PR changes any of: +- `README.md` +- `CHANGELOG.md` +- `MANIFESTO.md` +- `docs/src/content/docs/**` +- `.apm/skills/**/*.md` +- `.apm/agents/**/*.md` +- `.github/skills/**/*.md` +- `.github/agents/**/*.md` +- `.github/instructions/**/*.md` +- `.github/workflows/*.md` (gh-aw natural-language workflows) +- `packages/apm-guide/**` + +Fallback self-check (when no fast-path file matched): "Does this PR +change user-facing documentation, agent or skill prose, instruction +files, CHANGELOG entries, README claims, or any natural-language +artifact a reader will rely on? If unsure, answer YES." + +When the doc-writer is active and the PR includes documentation changes, +the persona reviews them for: (a) consistency with the existing voice +and structure, (b) accuracy against the code being changed, (c) +completeness for the typical reader (no orphan claims, no missing +prerequisites), (d) discoverability (cross-links, sidebar order if +Starlight content). When the doc-writer is active because of code +changes that SHOULD have updated docs but did not, the persona surfaces +that gap as a finding. + +## Performance Expert + +Activate when the PR changes any of: +- `src/apm_cli/cache/**` +- `src/apm_cli/deps/**` +- `src/apm_cli/install/phases/**` +- `src/apm_cli/install/pipeline.py` +- `src/apm_cli/install/resolve.py` +- `src/apm_cli/utils/**` +- `src/apm_cli/marketplace/**` +- `src/apm_cli/compilation/**` +- `scripts/perf/**` +- `src/apm_cli/core/command_logger.py` (when the diff adds perf-instrumentation logs) + +Also activate when: +- The PR description claims a performance win (speedup ratio, latency + reduction, bytes-on-disk reduction, throughput improvement) or + attaches a perf-harness measurement table. +- The diff introduces loops over collections (`for x in collection`) + where the collection may grow with dependency count or file count. +- The diff adds `os.scandir`, `os.walk`, `os.listdir`, or + `subprocess.run` calls on a path that executes per-package or + per-dependency. +- The diff adds `x in list_variable` inside a loop body. + +Fallback self-check (when no fast-path file matched): "Does this PR +change the hot path for dependency download, materialization, cache +layout, transport (git protocol, partial clone, sparse checkout), +parallelism, or any user-visible install/update wall-time? Does it +introduce an algorithmic complexity regression (O(n^2) loops, repeated +I/O, missing indexes, unconditional full scans, blocking synchronous +calls, heavy top-level imports)? If unsure, answer YES." + +When active, the performance-expert reviews against BOTH: +1. The package-manager performance playbook: transport minimization + (depth, filter, sparse scope), cache layering and dedup keys, + parallelism and lock contention, working-tree materialization cost, + perf-harness methodology (cache wipe, warm/cold separation, + statistical noise), and pervasive application of the chosen + technique across install / update / run surfaces. +2. The algorithmic performance lens (Big O analysis): complexity class + of every loop/lookup in the diff, index vs linear scan patterns, + unconditional expensive operations, import startup costs, redundant + computation, and parallelism opportunities. See the agent's + `references/algorithmic-patterns.md` for the full pattern catalogue. diff --git a/.agents/skills/batch-bug-shepherd/SKILL.md b/.agents/skills/batch-bug-shepherd/SKILL.md index 525b0b288c..026b8605cb 100644 --- a/.agents/skills/batch-bug-shepherd/SKILL.md +++ b/.agents/skills/batch-bug-shepherd/SKILL.md @@ -325,14 +325,18 @@ uv run python /.agents/skills/shepherd-driver/scripts/owner_touch_ --head --completion ``` -Do not update the table or labels until schema AND semantic +Load `/.apm/instructions/lifecycle.instructions.md` and +independently execute its repository provider with `--lane full`, +`--base $BASE_SHA --head `, +`--completion `, and a new parent report path. +Do not update the table or labels until schema, owner AND lifecycle verification pass. A non-zero verifier result gets the same retry-once treatment as malformed schema; on a second failure mark the row `blocked` with the diagnostic. This parent re-probe prevents a child from bypassing deterministic owner detection or presenting stale functional evidence. -After both gates pass, write `head_sha`, `mergeable`, +After all gates pass, write `head_sha`, `mergeable`, `merge_state_status`, and `ci_status` from the return into the table, and remove the `status/shepherding` label from the driven issue (assignment stays). The orchestrator owns only validation, table diff --git a/.agents/skills/batch-bug-shepherd/assets/fix-prompt.md b/.agents/skills/batch-bug-shepherd/assets/fix-prompt.md index f09b4b1006..2035368ee5 100644 --- a/.agents/skills/batch-bug-shepherd/assets/fix-prompt.md +++ b/.agents/skills/batch-bug-shepherd/assets/fix-prompt.md @@ -14,6 +14,14 @@ under microsoft/apm. ## Procedure +Before planning, load `$REPO_ROOT/.apm/instructions/lifecycle.instructions.md`. +P8 acceptance covers the entire applicable command state machine, with +separate deterministic and generated-model evidence. Run its full +executing gate at the final integrated head before opening a ready PR. +Missing infrastructure or evidence blocks shipping; install-only +success cannot discharge the issue. Do not defer lifecycle gaps as +outside the proposed patch's scope. + 1. Re-read the issue and the repro steps. Confirm the bug still reproduces on HEAD. 2. Design the minimum fix. Identify the canonical sibling code paths diff --git a/.agents/skills/pr-description-skill/assets/scenario-evidence-rubric.md b/.agents/skills/pr-description-skill/assets/scenario-evidence-rubric.md index 3a3388aec3..e9ad87844e 100644 --- a/.agents/skills/pr-description-skill/assets/scenario-evidence-rubric.md +++ b/.agents/skills/pr-description-skill/assets/scenario-evidence-rubric.md @@ -27,6 +27,13 @@ SCENARIO coverage: the user-facing flows that, if they silently break, erode trust in APM as a tool. Pure coverage metrics are not the gate; scenario-test mapping is. +Before synthesis, load `.apm/instructions/lifecycle.instructions.md` +from the consumer repository. For a P8-applicable change, present the +complete command applicability, connected deterministic trajectory, +generated-model dimensions and actual candidate-bound gate result. +Missing required evidence is a shipping blocker, not a trade-off row. +This table describes evidence; it does not replace executing the gate. + ## APM principle taxonomy Every scenario in a PR maps to AT LEAST ONE of these principles @@ -159,3 +166,7 @@ following is true (state which one in trade-offs): evidence; cite the suite that ran green). Any other PR that omits the table is incomplete. + +These are presentation exemptions only. A lifecycle-affecting refactor +still needs current P8 execution; a lockfile or installed-markdown change +can alter runtime behavior and is not automatically exempt. diff --git a/.agents/skills/shepherd-driver/SKILL.md b/.agents/skills/shepherd-driver/SKILL.md index fe2da09cdd..8d3efa42e6 100644 --- a/.agents/skills/shepherd-driver/SKILL.md +++ b/.agents/skills/shepherd-driver/SKILL.md @@ -52,6 +52,12 @@ ISSUE_NUMBER, AUTHOR, HEAD_REPO, HEAD_BRANCH, MAINTAINER_CAN_MODIFY, REPO_ROOT, ORIGIN, optional PANEL_PRIOR). The subagent owns the whole convergence loop end-to-end and returns a `completion_return`. +The consumer repository supplies `.apm/instructions/lifecycle.instructions.md` +and `scripts/check_lifecycle_evidence.py`; probe both before execution. +The driver and parent independently run the full P8 gate; the parent uses +`--completion` before accepting either shipping-eligible status. Missing +assets or evidence block completion; do not bundle another lifecycle owner. + After every shepherded PR returns `ready-to-merge`, the orchestrator runs the conflict-resolution phase: probe mergeability and, on DIRTY / BEHIND / CONFLICTING, spawn one conflict-resolution subagent diff --git a/.agents/skills/shepherd-driver/assets/completion-schema.json b/.agents/skills/shepherd-driver/assets/completion-schema.json index 17894d1b0f..c2f93abad1 100644 --- a/.agents/skills/shepherd-driver/assets/completion-schema.json +++ b/.agents/skills/shepherd-driver/assets/completion-schema.json @@ -8,6 +8,22 @@ { "$ref": "#/definitions/conflict_resolution_return" } ], "definitions": { + "lifecycle_evidence": { + "type": "object", + "additionalProperties": false, + "description": "Summary of a native full-lane lifecycle run. The executing repository provider verifies it with --completion and independently reruns the candidate; schema validity alone is not proof.", + "required": ["version", "base_sha", "head_sha", "tested_tree", "lane", "status", "report_path", "report_sha256"], + "properties": { + "version": { "const": 1 }, + "base_sha": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, + "head_sha": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, + "tested_tree": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, + "lane": { "const": "full" }, + "status": { "enum": ["passed", "not_applicable"] }, + "report_path": { "type": "string", "minLength": 1 }, + "report_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" } + } + }, "owner_touch_report": { "type": "object", "additionalProperties": false, @@ -335,16 +351,19 @@ }, "architecture_evidence": { "$ref": "#/definitions/architecture_evidence_v2" + }, + "lifecycle_evidence": { + "$ref": "#/definitions/lifecycle_evidence" } }, "allOf": [ { "if": { "properties": { "status": { "const": "ready-to-merge" } } }, - "then": { "required": ["ci_evidence", "lint_evidence", "head_sha", "mergeable", "merge_state_status", "ci_status", "architecture_evidence"] } + "then": { "required": ["ci_evidence", "lint_evidence", "head_sha", "mergeable", "merge_state_status", "ci_status", "architecture_evidence", "lifecycle_evidence"] } }, { "if": { "properties": { "status": { "const": "advisory-with-deferred" } } }, - "then": { "required": ["ci_evidence", "deferred_items", "head_sha", "mergeable", "merge_state_status", "ci_status", "architecture_evidence"] } + "then": { "required": ["ci_evidence", "deferred_items", "head_sha", "mergeable", "merge_state_status", "ci_status", "architecture_evidence", "lifecycle_evidence"] } }, { "if": { "properties": { "status": { "const": "blocked" } } }, @@ -384,6 +403,9 @@ "type": "string", "description": "Brief human-readable narrative of the rebase: base SHA, head SHA pre-rebase, head SHA post-rebase, conflicts resolved, fallback used (merge-of-main) if any. Required when status=resolved." }, + "lifecycle_evidence": { + "$ref": "#/definitions/lifecycle_evidence" + }, "push_command": { "type": "string", "pattern": "^(?!.*--force(?!-with-lease)).*$", @@ -442,7 +464,7 @@ { "if": { "properties": { "status": { "const": "resolved" } } }, "then": { - "required": ["rebase_evidence", "push_command", "mergeStateStatus_post", "lint_evidence", "comment_url"], + "required": ["rebase_evidence", "push_command", "mergeStateStatus_post", "lint_evidence", "comment_url", "lifecycle_evidence"], "properties": { "mergeStateStatus_post": { "enum": ["CLEAN", "UNSTABLE", "HAS_HOOKS"] } } diff --git a/.agents/skills/shepherd-driver/assets/conflict-resolution-prompt.md b/.agents/skills/shepherd-driver/assets/conflict-resolution-prompt.md index fd464602fb..f9d11c55a2 100644 --- a/.agents/skills/shepherd-driver/assets/conflict-resolution-prompt.md +++ b/.agents/skills/shepherd-driver/assets/conflict-resolution-prompt.md @@ -106,6 +106,13 @@ regression-trap test, `--force-with-lease` only (never bare resolution-failed`, `blocker: "lint noisy post-rebase: "`. Do not push. + Also load `$REPO_ROOT/.apm/instructions/lifecycle.instructions.md`. + Rebase invalidates the previous lifecycle result. Execute its full + gate against the new base/head before claiming resolution; include + the native report summary as `lifecycle_evidence` in the return. + Missing, skipped, failed or stale P8 evidence means + `resolution-failed`, not `resolved`. + 7. PUSH WITH `--force-with-lease`. Choose the remote based on HEAD_REPO: - Head on a fork: `git push --force-with-lease=: HEAD:`. @@ -151,7 +158,7 @@ regression-trap test, `--force-with-lease` only (never bare return shapes on this const). Required fields by status: - resolved: pr, status, mergeStateStatus_pre, mergeStateStatus_post, rebase_evidence, push_command, - lint_evidence, comment_url. Plus mutation_break_evidence + lint_evidence, comment_url, lifecycle_evidence. Plus mutation_break_evidence and rebase_touched_regression_test=true if step 5 ran. - requires-author-action: pr, status, mergeStateStatus_pre, fork_url, head_branch, maintainer_can_modify=false, diff --git a/.agents/skills/shepherd-driver/assets/shepherd-driver-prompt.md b/.agents/skills/shepherd-driver/assets/shepherd-driver-prompt.md index 8e51d8ae88..5bf4336ad8 100644 --- a/.agents/skills/shepherd-driver/assets/shepherd-driver-prompt.md +++ b/.agents/skills/shepherd-driver/assets/shepherd-driver-prompt.md @@ -45,6 +45,8 @@ are part of your contract: table parsed by the deterministic gate (Step X.2.5) - `owner_touch_gate.py` -- exact-revision owner detection and terminal evidence verification (Step X.2.5) +- `$REPO_ROOT/.apm/instructions/lifecycle.instructions.md` -- P8 shipping + precondition and repository executing provider (Step X.2.6) - `../apm-review-panel/SKILL.md` -- panel composition contract - `../pr-description-skill/SKILL.md` -- superseding-PR body author (Path B) @@ -57,6 +59,7 @@ X.0 fetch + classify Copilot X.1 invoke apm-review-panel skill X.2 merge follow-ups, apply fold-vs-defer rubric X.2.5 canonical-owner gate (classify + evidence, FAIL CLOSED) +X.2.6 lifecycle gate (execute full candidate evidence, FAIL CLOSED) X.3 edit code, fold foldable items X.4 lint contract (silent) X.5 push (author fork or superseding PR) @@ -351,6 +354,31 @@ classification may interpret its result but may not override it. If evidence cannot be produced within the loop cap, return `blocked` with the missing owner/test named. Never defer the gate. +### Step X.2.6 -- lifecycle shipping gate (FAIL CLOSED) + +Load `$REPO_ROOT/.apm/instructions/lifecycle.instructions.md` and probe +`$REPO_ROOT/scripts/check_lifecycle_evidence.py`. Either missing means +`blocked`; do not recreate the provider inside this package. At loop +entry, plan any missing P8 obligations for Step X.3. Once the integrated +candidate is committed and clean, execute the rule's full-lane gate +before certifying completion. Re-run after any later edit or rebase. + +Record `lifecycle_evidence` from the native report: version 1, +`base_sha` from `base`, `head_sha` from `head`, `tested_tree`, lane +`full`, status `passed` or `not_applicable`, the report's absolute +`report_path`, and its tool-computed byte `report_sha256`. Never author +a passing report or relabel an old head. The preliminary `pr` lane +cannot supply terminal evidence. + +Schema-validate the completion referencing the report just produced; do +not repeat an identical native run just to serialize its summary. The +parent independently executes the same full gate with +`--completion ` and a new report path, checking the reference +and freshly running the candidate rather than importing a receipt. +Missing lifecycle work is in scope even if the original brief omitted it. +Repair it within the loop or return `blocked`; neither a panel stance +nor `advisory-with-deferred` can waive P8. + ### Step X.3 -- edit code, fold foldable items For each FOLD item: @@ -482,6 +510,8 @@ On cap hit: `status: blocked` with failing job + log excerpt in - CEO stance is `ship_now`, OR `ship_with_followups` where all remaining followups are tagged DEFER with valid scope-boundary notes. +- The full executing lifecycle gate (Step X.2.6) passed on this exact + candidate; no required deterministic or generated-model gap remains. In this case: re-run the apm-review-panel ONE LAST TIME so the visible comment reflects the converged state. That final run posts its @@ -491,7 +521,9 @@ the panel always posts its result via `gh`). Move to "Finalize" below. **Terminal `status: advisory-with-deferred`** when: - Iteration cap (4) is hit, AND -- Foldable items remain unresolved. +- Foldable items remain unresolved, AND +- P8 full-lane lifecycle evidence is complete. Any missing required + lifecycle work instead returns `blocked`, regardless of the cap. In this case: re-run the apm-review-panel one last time so its final recommendation comment reflects the converged (capped) state, carrying @@ -603,6 +635,16 @@ most one short clause (e.g. `pending required review`, "ci_evidence": "string (required for ready-to-merge or advisory-with-deferred)", "lint_evidence": "string (required when status=ready-to-merge)", "mutation_break_evidence": [...], + "lifecycle_evidence": { + "version": 1, + "base_sha": "40-char comparison base", + "head_sha": "40-char final candidate", + "tested_tree": "40-char tested tree", + "lane": "full", + "status": "passed|not_applicable", + "report_path": "absolute native report path outside checkout", + "report_sha256": "64-char byte digest" + }, "architecture_evidence": { "version": "2", "classification": "ordinary-fix|owner-extension|new-owner|split-authority-repair|not-applicable", @@ -672,6 +714,8 @@ Use `status` (NOT `terminal_state`) and `pr` (NOT `pr_number`). migration: version 1 self-classified `decisions[]` returns now fail and force one re-spawn under v2. Blocked and superseded returns remain compatible because they do not require architecture evidence. +Both terminal success statuses also require `lifecycle_evidence`; +older returns without it must re-run P8, not invent the new fields. ## Hard rules diff --git a/.agents/skills/shepherd-driver/references/mergeability-gate.md b/.agents/skills/shepherd-driver/references/mergeability-gate.md index 02367dfa9a..1d9110fb40 100644 --- a/.agents/skills/shepherd-driver/references/mergeability-gate.md +++ b/.agents/skills/shepherd-driver/references/mergeability-gate.md @@ -114,6 +114,13 @@ table + progress diagram (P5c done, P6 active). ## Trust-but-verify discipline +Every `resolved` return also requires post-rebase `lifecycle_evidence`. +Load the row worktree's `.apm/instructions/lifecycle.instructions.md`; +derive the new comparison base and independently execute the full +provider with `--completion ` and a fresh parent +report path. A failed P8 gate changes the result to `resolution-failed`, +even if GitHub reports CLEAN. Do not reuse pre-rebase lifecycle proof. + The orchestrator's 5c re-probe is mandatory even when the subagent returned `status: resolved` with `mergeStateStatus_post: CLEAN`. Two reasons: @@ -135,7 +142,7 @@ re-probe is the gate. belong to the maintainer's policy workflow, not to the bbs. - Does NOT re-run the apm-review-panel. The shepherd-driver convergence loop already owned review, fold/defer, implementation, - push, and CI; Phase 5 owns ONLY rebase and re-probe. Conflict + push, and CI; Phase 5 owns rebase, fresh P8 evidence and re-probe. Conflict resolution is a mechanical merge, not a fresh judgment pass. - Does NOT open new PRs. Supersession is a Phase 4 affordance; if a fork-with-flag-false PR shows up here, it routes to diff --git a/.apm/agents/apm-ceo.agent.md b/.apm/agents/apm-ceo.agent.md index 88a45c3129..d8db4ab05d 100644 --- a/.apm/agents/apm-ceo.agent.md +++ b/.apm/agents/apm-ceo.agent.md @@ -43,6 +43,12 @@ If a release or strategic call would invalidate something in these files, the fi 4. **Ground every claim in evidence.** Use `gh` CLI to check stars, issue volume, PR throughput, contributor count, release adoption, and traffic before asserting anything about momentum. +5. **Lifecycle completeness is a shipping precondition.** Load + `PRINCIPLES.md` P8 and `.apm/instructions/lifecycle.instructions.md` + for every lifecycle-affecting change. Missing required command, + deterministic-trajectory or generated-model evidence cannot be + waived by a trade-off, a green unrelated suite, or an advisory + `ship_now`. Return the gap to implementation; do not defer it. ## Tools you use @@ -153,7 +159,9 @@ commit. Apply this weighting in `arbitration`: Manual verification does not survive the next refactor. - `outcome: unknown` carries NO weight. If a panelist returned `unknown` without explaining why, note the gap in `dissent_notes` - and weight that finding as opinion only. + and weight that finding as opinion only. Exception: unknown required + P8 lifecycle evidence leaves the shipping precondition unsatisfied; + it cannot be downweighted into permission to ship. Two-tier guidance for the `recommended_followups[:5]`: 1. Failed-test evidence rows ALWAYS rank above opinion-only rows of diff --git a/.apm/agents/test-coverage-expert.agent.md b/.apm/agents/test-coverage-expert.agent.md index af76364d38..18f89ad381 100644 --- a/.apm/agents/test-coverage-expert.agent.md +++ b/.apm/agents/test-coverage-expert.agent.md @@ -9,8 +9,8 @@ description: >- scenario coverage on changed behavior, regression-trap tests on bug fixes, integration coverage on cross-module flows, and silent-drift risk where code paths exist but no assertion would notice if they broke. Boundary: - never demands 100% line coverage, never flags tests for pure refactors - that preserve behavior, never duplicates code-style review. + never demands 100% line coverage or artificial new tests for refactors; + requires current lifecycle evidence and never duplicates code-style review. model: claude-opus-4.6 --- @@ -31,6 +31,18 @@ If yes -- no finding. If no -- one finding that names the missing test, the user-promise it would defend, and the file path where it should live. +## Mandatory lifecycle shipping rule + +Before reviewing, load `.apm/instructions/lifecycle.instructions.md` +from the consumer repository root. P8 governs applicability, complete +command mapping, deterministic trajectories, generated-model dimensions, +and candidate-bound execution. Audit both layers separately; an install, +audit and uninstall scenario does not also prove update or reinstall. +Missing, unknown, skipped, stale or unexecuted required lifecycle evidence +is a `blocking` finding, not an optional follow-up. A refactor may reuse +existing coverage but cannot omit current execution. Do not duplicate +the rule's command inventory or accept a panel opinion in its place. + ## North star (inherited from DevX UX) A new user types `apm init`, `apm install`, then `apm run` and ships @@ -112,18 +124,20 @@ Three new disciplines follow from this matrix: exists, return TWO evidence rows: one `outcome: passed, tier: unit` for the unit coverage you found, and one `outcome: missing, tier: integration-with-fixtures` for the floor gap. Severity on the - missing row is `recommended` by default; promote to `blocking` only + missing row is `blocking` for a P8 lifecycle obligation; otherwise + it is `recommended` by default. Promote other gaps to `blocking` only when the surface change is a security/auth/install promise AND there is no reasonable fixture path. Do NOT silence the unit row -- the unit test still defends the function in isolation; you are saying the user-promise is not yet certified end-to-end. 2. **S7 PROBE RULE on integration evidence.** When you return - `outcome: passed` at `tier: integration-with-fixtures` or `e2e` on + `outcome: passed` at `tier: integration-with-fixtures`, `e2e`, or + `lifecycle-state-machine` on a critical-promise surface, you MUST have RUN the test (not just read it) within this review. Capture the pytest invocation + the pass/fail line + duration in `evidence.run_evidence` (verbatim, under 240 chars). Reading test code is LLM assertion; running it - against real fixtures is irrefutable proof. Skip-condition: if the + against real fixtures supplies bounded execution evidence. If the test requires a credential you don't have (e.g. `GITHUB_APM_PAT`), note the skip in `evidence.run_evidence` and downgrade `outcome` to `unknown` for that row -- do NOT certify on a read. @@ -164,16 +178,10 @@ exactly the class of promise that unit and integration tests cannot certify. In full agentic coding, where a change lands without a human walking the CLI by hand, this harness IS the human walkthrough. -**Trigger list -- signal REQUIRED when the diff touches any of:** - -- install / uninstall / update / compile / audit command paths -- the deployment ledger, lockfile write path, or `deployed_files` -- target resolution, target widening or narrowing, primitive-to-target - layout mapping -- anything writing into a target root (`.github/`, `.claude/`, - `.cursor/`, `.agents/`, ...) or into user scope (`~/.apm/`) -- `--dry-run` behavior on any command -- cleanup, pruning, reconciliation, drift, or orphan handling +**Applicability authority:** use the complete assessment required by +`.apm/instructions/lifecycle.instructions.md`, including indirect shared +state and observational changes. Do not restrict it to the entry +command or to a hardcoded list of runtime paths. **What you must return on a trigger.** One of exactly these, and never silence: @@ -184,9 +192,8 @@ silence: - `outcome: missing`, `tier: lifecycle-state-machine` -- no lifecycle coverage for this transition. Give the exact test file, the test name you would use, the command SEQUENCE it should drive, and the - assertion pseudocode. Severity `recommended` by default; `blocking` - when the change touches an ownership boundary, a destructive path - (uninstall / cleanup / prune), or a dry-run no-write promise. + assertion pseudocode. Severity `blocking` for any required P8 + lifecycle gap, including a missing generated-model dimension. - `outcome: passed`, `tier: ` PLUS a second `outcome: missing`, `tier: lifecycle-state-machine` row -- the tier-floor compliance shape, when lower-tier tests exist but the floor is unmet. @@ -239,7 +246,8 @@ it via tool calls before emitting it as a finding. The procedure: row's test. If a file is touched but no scenario row exercises a path through it, that is a coverage gap. Refactors that produce identical user-visible behavior are exempt -- but the author - should have stated this in trade-offs. + should have stated this in trade-offs. That presentation exemption + does not exempt lifecycle-affecting refactors from P8 execution. 4. **Audit deterministic owner evidence when supplied.** For every `touched_owners[].decision`, require at least one executed test row whose `owner_decisions` includes that exact decision, whose diff --git a/.apm/instructions/lifecycle.instructions.md b/.apm/instructions/lifecycle.instructions.md new file mode 100644 index 0000000000..86957d8a2b --- /dev/null +++ b/.apm/instructions/lifecycle.instructions.md @@ -0,0 +1,118 @@ +--- +applyTo: "**" +description: "P8 shipping requirement: prove features across the applicable command state machine, not just installation." +--- + +# Lifecycle completeness + +[PRINCIPLES.md P8](../../PRINCIPLES.md#p8----lifecycle-completeness) +is mandatory for developers, coding agents, reviewers, and completion +drivers. Read this rule before planning, implementing, reviewing, or +certifying a change. It is the single operational lifecycle rule; +specialist prompts reference it rather than keeping narrower checklists. + +## Establish the whole contract before editing + +For every feature, fix, or lifecycle-affecting refactor: + +1. Derive the command/subcommand inventory from the executable CLI at + the candidate revision. Do not use a remembered list or documentation + headings as the authority. +2. Map the changed feature and its dimensions to every command. Include + observers, no-ops, safe refusal, and failure/recovery, not only commands + that write files. Explain genuinely unrelated commands specifically. + "Install-only change", "read-only", "no fixture", "not modeled", or an + unsupported flag is not an applicability exemption. +3. Plan a connected real-CLI trajectory through all applicable transitions + in one isolated workspace. Reuse the existing APMLifecycle runner, + local repositories, state snapshots, and physical artifact snapshots. + Preserve state between commands; do not reset the project to make each + command pass independently. +4. Integrate relevant dimensions and changed interactions into the + existing generated state-machine model. Require deterministic replay + of mandatory transitions as well as bounded generated exploration. + A parameter or rule that never executes is not coverage. + +Feature dimensions include scope, path spelling, source/ref type, target, +primitive shape, ownership, policy and authentication where relevant. +Installed markdown payloads, lockfile-only changes, and shared helpers +can affect lifecycle behavior. Do not classify them by filename alone. +Purely unrelated contributor prose can be not applicable; uncertainty +must be resolved, not silently classified away. + +## Prove transitions, not success messages + +The trajectory must exercise relevant install, reinstall/idempotency, +version publication and update, lock/frozen replay, compile, audit, +outdated, removal, and reinstall-after-removal behavior. This is a +planning spine, not a substitute for the complete command map or a +license to invent unsupported command flags. + +At command boundaries, assert the contract on real artifacts: deployed +bytes and layout, lock and ownership consistency, permitted write sets, +and survival of unrelated user files. For observers and previews, +compare before/after durable state. Use independent physical-root +snapshots as well as ownership-derived snapshots; a missing ownership +record must not hide collateral writes. + +For failures, inject a deterministic fault, assert the documented +allowed/forbidden effects, restore the precondition, then retry in the +same workspace. Do not assume every operation is atomic. Input aliases +remain aliases while observations use the physical roots; never relax +containment checks to make a scenario pass. + +Use hermetic fixtures without personal credentials or live services. +Distinguish installed Python CLI evidence from packaged-binary evidence. +Run the required platform/capability lane when a feature depends on it; +a skipped required case is not a pass. + +For a bug fix, remove the relevant production correction and show the +lifecycle regression fails for the intended reason. Restore it and +rerun the clean candidate. Unit or component tests remain useful, but +their totals cannot substitute for lifecycle assertions. + +## Execute the shipping gate + +The deterministic entrypoint is +`scripts/check_lifecycle_evidence.py`, backed by the existing lifecycle +ledger and test machinery. Read its `--help` and the contributor testing +guide before first use. A missing entrypoint or rule is a blocker, not +permission to invent an evidence document. + +At the final integrated revision, execute: + +```sh +uv run --extra dev python scripts/check_lifecycle_evidence.py \ + --base --head \ + --lane full --report +``` + +Run from a clean committed candidate checkout; write reports outside it. +The gate executes verification; do not substitute imported, hand-written +or earlier-head reports. Use its computed applicability, collection, +execution and candidate identity. The bounded `pr` lane is preliminary: +pending generated-model evidence cannot certify shipping. Required +merge-candidate execution is distinct from local PR-head evidence. + +Missing cases, deselection, skips, xfail, failures, stale evidence, and +unexercised required variants/transitions block completion. A lifecycle +gap remains in the feature's scope even if the original brief omitted +it. Re-plan or return blocked; do not defer it to a follow-up issue. +Existing adequate tests may satisfy a refactor without being rewritten. + +## Review and completion + +The coverage reviewer audits the command map, both lifecycle layers, +feature activation, state assertions, and actual execution identity. +Reviewers must not infer a pass from filenames, source inspection, +exit-code-only assertions, or the author's narrative. + +Neither `ready-to-merge` nor `advisory-with-deferred` is valid while +required lifecycle evidence is missing. An advisory panel cannot waive +P8. The driver executes the gate at its final head; its parent +independently executes it before accepting terminal completion. +The parent uses `--completion ` with a fresh report path to check the +driver's summary and digest while independently re-executing the full gate. +Integrating main, rebasing, or changing code/tests invalidates older +evidence. Keep applicability and fixture limitations explicit; bounded +execution does not prove every possible permutation. diff --git a/.github/agents/apm-ceo.agent.md b/.github/agents/apm-ceo.agent.md index 88a45c3129..d8db4ab05d 100644 --- a/.github/agents/apm-ceo.agent.md +++ b/.github/agents/apm-ceo.agent.md @@ -43,6 +43,12 @@ If a release or strategic call would invalidate something in these files, the fi 4. **Ground every claim in evidence.** Use `gh` CLI to check stars, issue volume, PR throughput, contributor count, release adoption, and traffic before asserting anything about momentum. +5. **Lifecycle completeness is a shipping precondition.** Load + `PRINCIPLES.md` P8 and `.apm/instructions/lifecycle.instructions.md` + for every lifecycle-affecting change. Missing required command, + deterministic-trajectory or generated-model evidence cannot be + waived by a trade-off, a green unrelated suite, or an advisory + `ship_now`. Return the gap to implementation; do not defer it. ## Tools you use @@ -153,7 +159,9 @@ commit. Apply this weighting in `arbitration`: Manual verification does not survive the next refactor. - `outcome: unknown` carries NO weight. If a panelist returned `unknown` without explaining why, note the gap in `dissent_notes` - and weight that finding as opinion only. + and weight that finding as opinion only. Exception: unknown required + P8 lifecycle evidence leaves the shipping precondition unsatisfied; + it cannot be downweighted into permission to ship. Two-tier guidance for the `recommended_followups[:5]`: 1. Failed-test evidence rows ALWAYS rank above opinion-only rows of diff --git a/.github/agents/test-coverage-expert.agent.md b/.github/agents/test-coverage-expert.agent.md index b2dca69c4e..ec421483e2 100644 --- a/.github/agents/test-coverage-expert.agent.md +++ b/.github/agents/test-coverage-expert.agent.md @@ -9,8 +9,8 @@ description: >- scenario coverage on changed behavior, regression-trap tests on bug fixes, integration coverage on cross-module flows, and silent-drift risk where code paths exist but no assertion would notice if they broke. Boundary: - never demands 100% line coverage, never flags tests for pure refactors - that preserve behavior, never duplicates code-style review. + never demands 100% line coverage or artificial new tests for refactors; + requires current lifecycle evidence and never duplicates code-style review. model: claude-opus-4.6 --- @@ -31,6 +31,18 @@ If yes -- no finding. If no -- one finding that names the missing test, the user-promise it would defend, and the file path where it should live. +## Mandatory lifecycle shipping rule + +Before reviewing, load `.apm/instructions/lifecycle.instructions.md` +from the consumer repository root. P8 governs applicability, complete +command mapping, deterministic trajectories, generated-model dimensions, +and candidate-bound execution. Audit both layers separately; an install, +audit and uninstall scenario does not also prove update or reinstall. +Missing, unknown, skipped, stale or unexecuted required lifecycle evidence +is a `blocking` finding, not an optional follow-up. A refactor may reuse +existing coverage but cannot omit current execution. Do not duplicate +the rule's command inventory or accept a panel opinion in its place. + ## North star (inherited from DevX UX) A new user types `apm init`, `apm install`, then `apm run` and ships @@ -112,18 +124,20 @@ Three new disciplines follow from this matrix: exists, return TWO evidence rows: one `outcome: passed, tier: unit` for the unit coverage you found, and one `outcome: missing, tier: integration-with-fixtures` for the floor gap. Severity on the - missing row is `recommended` by default; promote to `blocking` only + missing row is `blocking` for a P8 lifecycle obligation; otherwise + it is `recommended` by default. Promote other gaps to `blocking` only when the surface change is a security/auth/install promise AND there is no reasonable fixture path. Do NOT silence the unit row -- the unit test still defends the function in isolation; you are saying the user-promise is not yet certified end-to-end. 2. **S7 PROBE RULE on integration evidence.** When you return - `outcome: passed` at `tier: integration-with-fixtures` or `e2e` on + `outcome: passed` at `tier: integration-with-fixtures`, `e2e`, or + `lifecycle-state-machine` on a critical-promise surface, you MUST have RUN the test (not just read it) within this review. Capture the pytest invocation + the pass/fail line + duration in `evidence.run_evidence` (verbatim, under 240 chars). Reading test code is LLM assertion; running it - against real fixtures is irrefutable proof. Skip-condition: if the + against real fixtures supplies bounded execution evidence. If the test requires a credential you don't have (e.g. `GITHUB_APM_PAT`), note the skip in `evidence.run_evidence` and downgrade `outcome` to `unknown` for that row -- do NOT certify on a read. @@ -164,16 +178,10 @@ exactly the class of promise that unit and integration tests cannot certify. In full agentic coding, where a change lands without a human walking the CLI by hand, this harness IS the human walkthrough. -**Trigger list -- signal REQUIRED when the diff touches any of:** - -- install / uninstall / update / compile / audit command paths -- the deployment ledger, lockfile write path, or `deployed_files` -- target resolution, target widening or narrowing, primitive-to-target - layout mapping -- anything writing into a target root (`.github/`, `.claude/`, - `.cursor/`, `.agents/`, ...) or into user scope (`~/.apm/`) -- `--dry-run` behavior on any command -- cleanup, pruning, reconciliation, drift, or orphan handling +**Applicability authority:** use the complete assessment required by +`.apm/instructions/lifecycle.instructions.md`, including indirect shared +state and observational changes. Do not restrict it to the entry +command or to a hardcoded list of runtime paths. **What you must return on a trigger.** One of exactly these, and never silence: @@ -184,9 +192,8 @@ silence: - `outcome: missing`, `tier: lifecycle-state-machine` -- no lifecycle coverage for this transition. Give the exact test file, the test name you would use, the command SEQUENCE it should drive, and the - assertion pseudocode. Severity `recommended` by default; `blocking` - when the change touches an ownership boundary, a destructive path - (uninstall / cleanup / prune), or a dry-run no-write promise. + assertion pseudocode. Severity `blocking` for any required P8 + lifecycle gap, including a missing generated-model dimension. - `outcome: passed`, `tier: ` PLUS a second `outcome: missing`, `tier: lifecycle-state-machine` row -- the tier-floor compliance shape, when lower-tier tests exist but the floor is unmet. @@ -239,7 +246,8 @@ it via tool calls before emitting it as a finding. The procedure: row's test. If a file is touched but no scenario row exercises a path through it, that is a coverage gap. Refactors that produce identical user-visible behavior are exempt -- but the author - should have stated this in trade-offs. + should have stated this in trade-offs. That presentation exemption + does not exempt lifecycle-affecting refactors from P8 execution. 4. **Audit deterministic owner evidence when supplied.** For every `touched_owners[].decision`, require at least one executed test row whose `owner_decisions` includes that exact decision, whose diff --git a/.github/instructions/lifecycle.instructions.md b/.github/instructions/lifecycle.instructions.md new file mode 100644 index 0000000000..86957d8a2b --- /dev/null +++ b/.github/instructions/lifecycle.instructions.md @@ -0,0 +1,118 @@ +--- +applyTo: "**" +description: "P8 shipping requirement: prove features across the applicable command state machine, not just installation." +--- + +# Lifecycle completeness + +[PRINCIPLES.md P8](../../PRINCIPLES.md#p8----lifecycle-completeness) +is mandatory for developers, coding agents, reviewers, and completion +drivers. Read this rule before planning, implementing, reviewing, or +certifying a change. It is the single operational lifecycle rule; +specialist prompts reference it rather than keeping narrower checklists. + +## Establish the whole contract before editing + +For every feature, fix, or lifecycle-affecting refactor: + +1. Derive the command/subcommand inventory from the executable CLI at + the candidate revision. Do not use a remembered list or documentation + headings as the authority. +2. Map the changed feature and its dimensions to every command. Include + observers, no-ops, safe refusal, and failure/recovery, not only commands + that write files. Explain genuinely unrelated commands specifically. + "Install-only change", "read-only", "no fixture", "not modeled", or an + unsupported flag is not an applicability exemption. +3. Plan a connected real-CLI trajectory through all applicable transitions + in one isolated workspace. Reuse the existing APMLifecycle runner, + local repositories, state snapshots, and physical artifact snapshots. + Preserve state between commands; do not reset the project to make each + command pass independently. +4. Integrate relevant dimensions and changed interactions into the + existing generated state-machine model. Require deterministic replay + of mandatory transitions as well as bounded generated exploration. + A parameter or rule that never executes is not coverage. + +Feature dimensions include scope, path spelling, source/ref type, target, +primitive shape, ownership, policy and authentication where relevant. +Installed markdown payloads, lockfile-only changes, and shared helpers +can affect lifecycle behavior. Do not classify them by filename alone. +Purely unrelated contributor prose can be not applicable; uncertainty +must be resolved, not silently classified away. + +## Prove transitions, not success messages + +The trajectory must exercise relevant install, reinstall/idempotency, +version publication and update, lock/frozen replay, compile, audit, +outdated, removal, and reinstall-after-removal behavior. This is a +planning spine, not a substitute for the complete command map or a +license to invent unsupported command flags. + +At command boundaries, assert the contract on real artifacts: deployed +bytes and layout, lock and ownership consistency, permitted write sets, +and survival of unrelated user files. For observers and previews, +compare before/after durable state. Use independent physical-root +snapshots as well as ownership-derived snapshots; a missing ownership +record must not hide collateral writes. + +For failures, inject a deterministic fault, assert the documented +allowed/forbidden effects, restore the precondition, then retry in the +same workspace. Do not assume every operation is atomic. Input aliases +remain aliases while observations use the physical roots; never relax +containment checks to make a scenario pass. + +Use hermetic fixtures without personal credentials or live services. +Distinguish installed Python CLI evidence from packaged-binary evidence. +Run the required platform/capability lane when a feature depends on it; +a skipped required case is not a pass. + +For a bug fix, remove the relevant production correction and show the +lifecycle regression fails for the intended reason. Restore it and +rerun the clean candidate. Unit or component tests remain useful, but +their totals cannot substitute for lifecycle assertions. + +## Execute the shipping gate + +The deterministic entrypoint is +`scripts/check_lifecycle_evidence.py`, backed by the existing lifecycle +ledger and test machinery. Read its `--help` and the contributor testing +guide before first use. A missing entrypoint or rule is a blocker, not +permission to invent an evidence document. + +At the final integrated revision, execute: + +```sh +uv run --extra dev python scripts/check_lifecycle_evidence.py \ + --base --head \ + --lane full --report +``` + +Run from a clean committed candidate checkout; write reports outside it. +The gate executes verification; do not substitute imported, hand-written +or earlier-head reports. Use its computed applicability, collection, +execution and candidate identity. The bounded `pr` lane is preliminary: +pending generated-model evidence cannot certify shipping. Required +merge-candidate execution is distinct from local PR-head evidence. + +Missing cases, deselection, skips, xfail, failures, stale evidence, and +unexercised required variants/transitions block completion. A lifecycle +gap remains in the feature's scope even if the original brief omitted +it. Re-plan or return blocked; do not defer it to a follow-up issue. +Existing adequate tests may satisfy a refactor without being rewritten. + +## Review and completion + +The coverage reviewer audits the command map, both lifecycle layers, +feature activation, state assertions, and actual execution identity. +Reviewers must not infer a pass from filenames, source inspection, +exit-code-only assertions, or the author's narrative. + +Neither `ready-to-merge` nor `advisory-with-deferred` is valid while +required lifecycle evidence is missing. An advisory panel cannot waive +P8. The driver executes the gate at its final head; its parent +independently executes it before accepting terminal completion. +The parent uses `--completion ` with a fresh report path to check the +driver's summary and digest while independently re-executing the full gate. +Integrating main, rebasing, or changing code/tests invalidates older +evidence. Keep applicability and fixture limitations explicit; bounded +execution does not prove every possible permutation. diff --git a/MANIFESTO.md b/MANIFESTO.md index df0b46f4cd..ec76c01ca0 100644 --- a/MANIFESTO.md +++ b/MANIFESTO.md @@ -14,6 +14,11 @@ Shareable workflow packages with dependency management. Community-driven knowled ### **Reliability over Magic** Predictable, step-by-step execution over black-box AI Agent behavior. Transparent processes that teams can audit, understand, and trust. No surprises, just results. +[P8: Lifecycle completeness](PRINCIPLES.md#p8----lifecycle-completeness) +makes this a shipping requirement: a feature must remain correct across +its applicable command state machine, not merely succeed at installation. +Developers and agents supply executable lifecycle evidence before shipping. + ### **Developer Experience over AI Sophistication** Simple CLI tooling over complex agent frameworks. Fast iteration and local execution over cloud-dependent workflows. diff --git a/PRINCIPLES.md b/PRINCIPLES.md index 7f8a644654..9c7831311b 100644 --- a/PRINCIPLES.md +++ b/PRINCIPLES.md @@ -11,6 +11,10 @@ principle here conflicts with a feature request, the principle wins by default and shifts only via an explicit, written trade-off in the same PR. +P8 is a shipping precondition, not a trade-off: a paragraph in a PR +cannot waive required lifecycle evidence. Weakening it requires an +explicit strategic policy change. + ## P1 -- No invented primitive frontmatter APM emits to canonical schemas defined by upstream ecosystems: @@ -88,6 +92,45 @@ External-contributor PRs and issues triage before internal nice-to-haves. A contributor lost is worse than a feature delayed. Surface every external interaction at the top of the queue. +## P8 -- Lifecycle completeness + +A feature is not complete when its entry command succeeds. Every +feature, fix, and lifecycle-affecting refactor must work across the +entire applicable command state machine before it ships. This binds +human developers and coding agents equally, including changes to +shared state that are not described as lifecycle work. + +Account for every command and subcommand from the executable command +inventory: state transitions, observations, no-ops, refusals, and +failure/recovery. A command is not applicable only when it has no +relationship to the changed behavior or state, with a specific, +reviewable rationale. Missing test infrastructure is work to complete, +not an exemption. + +Require both connected, deterministic real-CLI trajectories over +persistent isolated state and relevant dimensions exercised in the +generated state-machine model. Assert deployed content, ownership, +preservation of user data, and supported failure/recovery behavior. +Neither isolated command tests nor one of these two lifecycle layers +substitutes for the other. Refactors may reuse adequate existing +coverage; they need not manufacture new tests. + +Missing, uncollected, skipped, xfailed, failed, stale, or merely asserted +required evidence blocks shipping. Evidence must apply to the actual +candidate. Unit-test totals, an unrelated green suite, and an advisory +ship recommendation cannot discharge this requirement. A missing +relevant model dimension cannot be deferred outside a feature's scope. + +The operational authority is +[the lifecycle shipping rule](.apm/instructions/lifecycle.instructions.md). +Required CI enforces executable proof obligations; maintainers review +their applicability and assertion semantics. This is bounded evidence, +not a claim of exhaustive permutations or mathematical correctness. + +Rejection example: "Global install works with this path format; update +and uninstall can be covered later." NO. Complete the lifecycle before +shipping the path format. + ## How this file is used - `apm-ceo` cites by number in arbitration prose. diff --git a/docs/src/content/docs/contributing/integration-testing.md b/docs/src/content/docs/contributing/integration-testing.md index 64bbdaf30c..80d399e30c 100644 --- a/docs/src/content/docs/contributing/integration-testing.md +++ b/docs/src/content/docs/contributing/integration-testing.md @@ -34,24 +34,91 @@ APM uses a tiered approach to integration testing: - **Trigger**: merge queue integration workflow, plus tag, schedule, and manual promotion runs ### 3. **Lifecycle Smoke** (PR-time required check) -- **Location**: selected declaratively via `lifecycle_smoke and not lifecycle_merge_group`. Tests marked `lifecycle_merge_group` remain outside the bounded required set. -- **Purpose**: Promote a stable, hermetic slice of Consume/Produce/Govern lifecycle contracts onto the PR-time critical path, so regressions in install, lock, deployment ownership, compile, pack, prune, uninstall, audit, and repair fail the PR. -- **Scope**: the family contains a static authority guard plus content-hash, policy, hook, virtual-package, audit, auth, and installed-console rows. Real subprocess cases use the uv-installed `apm` command and local Git. This is not frozen PyInstaller coverage. -- **Prerequisites**: the pytest step sets `APM_E2E_TESTS=1` so subprocess rows execute. `APM_RUN_INTEGRATION_TESTS` remains unset, the socket guard denies network sockets, and the job binds no credentials. -- **Duration**: the required expression must remain inside its hard 6-minute job timeout; hosted duration is authoritative. -- **Trigger**: every pull request and merge queue run (`ci.yml`'s `lifecycle-smoke` job, required via `merge-gate.yml`) -- **Selection mechanism**: `pytest --strict-markers -m 'lifecycle_smoke and not lifecycle_merge_group' tests/integration` -- declarative, not a file/node-id list. No central count or membership list is maintained. -- **Full-coverage path**: merge-group workflow `ci-integration.yml`, job `integration-tests-shard`, step `Run integration tests (sharded + parallelized)`, calls `uv run ./scripts/test-integration.sh`; that script runs unfiltered `pytest tests/integration/`, so the complete lifecycle family remains exercised. -- **Drift guard**: `tests/quality/test_ci_topology.py` independently collects the full, merge-group-only, and required selections; verifies their set partition; and preserves the required expression, full-integration execution path, step-level `APM_E2E_TESTS: "1"` binding, network/credential prohibitions, and required-check membership. -- **Fixture controls**: lifecycle helpers set `APM_TEST_LOOPBACK_PORTS` for a port-scoped local registry and `APM_TEST_FAIL_LOCK_REPLACE=1` for atomic-write fault injection. These are internal test controls, not user-facing APM settings. -- **Learning ledger**: `tests/fixtures/lifecycle_bug_ledger.json` maps representative escaped defects to generalized laws, oracle tiers, phases, and executable regression node IDs, including coverage of already-correct behavior. Use same-workspace transitions to prove survivor ownership and scoped cleanup. It is not a bug-count census; `tests/quality/test_lifecycle_bug_ledger.py` validates its taxonomy and links. -- **Generated lifecycle model**: `test_generated_lifecycle_state_machine.py` uses Hypothesis to generate guarded install, dry-run, audit, tamper, repair, declaration, and prune sequences against the real CLI. The model tracks declaration, materialization, integrity, and lock state independently of the product lockfile. Every transition captures complete project and user roots, and mutating commands must stay inside reviewed write sets. It stays in the merge-group family until hosted runtime supports promotion to the bounded PR-time smoke set. -- **Known gap**: a late lockfile replacement failure can leave target files on the newly declared target while retaining the prior lockfile. The required lifecycle suite bounds that blast radius and proves the next install converges; expanding the install transaction is a separate design decision recorded in the ledger. -- **Run it locally** (the exact command CI runs): - ```bash - APM_E2E_TESTS=1 uv run --extra dev pytest -p no:cacheprovider -q --strict-markers \ - -m 'lifecycle_smoke and not lifecycle_merge_group' tests/integration - ``` + +Every human or agent change requires lifecycle applicability assessment under +[P8](https://github.com/microsoft/apm/blob/main/PRINCIPLES.md#p8----lifecycle-completeness) +and the [lifecycle shipping rule](https://github.com/microsoft/apm/blob/main/.apm/instructions/lifecycle.instructions.md). +Applicable changes need both deterministic real-CLI trajectories and generated +state-machine evidence; install-only success is insufficient. + +#### Author the contract + +Add or extend a `lifecycle_contracts` row in +`tests/fixtures/lifecycle_bug_ledger.json`; `scripts/lifecycle_contracts.py` +validates it. Preserve existing obligations. + +1. Give the row an `id`, fresh reviewed `assessment`, exact changed `paths`, + and `properties` from the ledger's `property_catalog`. +2. Populate `commands` from `command_inventory()` in + `scripts/lifecycle_contracts.py`. It recursively walks actual Click + registrations, including hidden commands, groups, aliases, and the root + (`""`). Every entry needs a `reason` and a `disposition`: `applicable` or + `semantic_na`. Reserve `semantic_na` for commands unrelated to the changed + behavior or state. Observers, no-ops, refusals, and failure/recovery count; + missing fixtures or model support do not justify N/A. +3. Reference both `deterministic` and `generated` witnesses for each applicable + command. Each witness needs an `id`, `kind`, exact collected pytest `nodeid` + (including parameter IDs), `dimensions` matching actual pytest parameters, + and ordered `transitions`. Cover every required dimension value in both + layers and review changed interactions—not a blanket Cartesian product. +4. Each transition records `command`, `argv_contains`, `returncode`, and + `state` (`observed`, `unchanged`, or `changed`). Use at least two connected + transitions in one persistent isolated workspace. Record intentional + between-command fixture mutations with a reviewed `preparation` rationale. + +Use the [hermetic lifecycle fixtures](#hermetic-lifecycle-fixtures) and +`ApmLifecycleRunner`. Assert property semantics, deployed bytes, lock/ownership +consistency, permitted writes, and survival of unowned sentinel files. Observe +physical project/user roots independently of ownership records. Inject relevant +faults, assert allowed effects, restore the precondition, and retry without +resetting state; do not assume universal atomicity. + +Extend `test_generated_lifecycle_state_machine.py` for relevant feature +dimensions. A generated witness must execute a genuine Hypothesis state +machine and its required CLI trajectory inside that model; a generated label +or an unexecuted parameter is not evidence. + +#### Execute and interpret evidence + +Replace the placeholders below. The checkout must be clean and committed, +including no untracked files; `--head` must resolve to its exact current +commit, and `--base` must be a distinct ancestor. Keep reports outside it. + +```bash +uv run --extra dev python scripts/check_lifecycle_evidence.py \ + --base --head \ + --lane full --report +``` + +The gate executes tests, not imported receipts. `full` runs both witness kinds +and rejects missing, uncollected, deselected, skipped, xfailed, failed, stale, +or unexercised required evidence. Re-run after changing the candidate. +`pr` executes deterministic witnesses only: applicable changes return +`pending`, **never shipping proof**, even with exit code zero. +`not_applicable` means an assessed repository-only change, not proof of runtime +behavior. + +The profile is source-Python, with the verified installed Unix `apm` +entrypoint where available—not frozen-binary certification. Semantic test +relevance and N/A rationales still require human review. Bounded execution +provides neither exhaustive coverage nor a mathematical guarantee. +[Architecture lint and ownership evidence](../development-guide/#architecture-guardrails) +remain separate obligations. + +#### CI lanes + +The required `ci.yml` `lifecycle-smoke` job retains its six-minute limit, +hermetic marker selection, and deterministic `pr` evidence lane. +`ci-integration.yml` runs the complete integration family and the `full` +evidence lane on merge-group candidates. `tests/quality/test_ci_topology.py` +guards this wiring; do not change jobs or required-check policy to waive P8. + +For focused smoke feedback only: + +```bash +APM_E2E_TESTS=1 uv run --extra dev pytest -p no:cacheprovider -q --strict-markers \ + -m 'lifecycle_smoke and not lifecycle_merge_group' tests/integration +``` ### 4. **Live Guardrailing Hero** (scheduled/manual) - **Location**: `tests/integration/test_guardrailing_hero_e2e.py` @@ -466,13 +533,8 @@ Promotion integration tests run on: - [+] Release pipeline integrity (GitHub Release -> PyPI) ### Lifecycle Smoke Verifies: -- Install content-hash roundtrip (Consume contract) -- Virtual-skill lock convergence (Produce contract, adjacent to the #2226 ADO lock-coordinate fix) -- Policy pinned-constraint enforcement (Govern contract) -- The virtual/manifestless lifecycle matrix: install, lock, frozen-install, update, and audit stay consistent (the direct #2240 regression) -- The ADO lock-coordinate single-owner guard (the direct #2226 regression) -- Prune's merged-hook and ownership-sidecar reconciliation for the `claude` target (the direct #2249 regression -- an orphaned package's merged hook entries and sidecar markers must be cleaned up, not left pointing at deleted scripts) -- No network, no credentials, no built binary required for any of the above +See [Tier 3](#3-lifecycle-smoke-pr-time-required-check) for the lifecycle +contract and evidence lanes; the ledger owns regression-to-property mappings. ## Benefits @@ -513,7 +575,8 @@ Promotion integration tests run on: ### Lifecycle Smoke Failures - These tests are hermetic -- no credentials, no built binary, no network (a real socket attempt raises `OSError`, it does not hang or retry). A failure is a genuine regression, not an environment issue. -- Run the exact CI command from the "Run it locally" block under Tier 3 above to reproduce. +- Use [Tier 3's focused smoke command](#ci-lanes) to reproduce smoke failures; + run the full evidence lane before lifecycle completion. - If the failure is about the CI job's shape (marker not registered, wrong `-m`/`--strict-markers` invocation, unbounded root, timeout, empty marker family, or required-check wiring) rather than test logic, check `tests/quality/test_ci_topology.py` -- that guard pins the job's contract and its own failure message will point at what drifted. - For hanging issues: Check command transformation in script runner (codex expects prompt content, not file paths) @@ -531,10 +594,7 @@ Promotion integration tests run on: support. ### For New Features: -1. Add a smoke test for compilation/validation. -2. Add an E2E test if the feature requires API calls -- pick the - smallest set of markers that captures its real preconditions - (`requires_github_token`, `requires_network_integration`, etc.) - so contributors without those credentials still get a clean - `SKIPPED` rather than a hard failure. -3. Keep tests focused and fast. +Follow [Tier 3's contract workflow](#author-the-contract), including both +lifecycle layers. Declare real preconditions through the +[marker registry](#the-marker-registry); a skipped required witness blocks +lifecycle completion. diff --git a/packages/apm-issue-autopilot/.apm/skills/apm-issue-autopilot/SKILL.md b/packages/apm-issue-autopilot/.apm/skills/apm-issue-autopilot/SKILL.md index 59508714dd..e75023393d 100644 --- a/packages/apm-issue-autopilot/.apm/skills/apm-issue-autopilot/SKILL.md +++ b/packages/apm-issue-autopilot/.apm/skills/apm-issue-autopilot/SKILL.md @@ -290,12 +290,17 @@ uv run python /.agents/skills/shepherd-driver/scripts/owner_touch_ --head --completion ``` -Do not write terminal state until both gates pass. A schema or semantic +Load `/.apm/instructions/lifecycle.instructions.md` and +independently execute its repository provider with `--lane full`, +`--base $BASE_SHA --head `, +`--completion `, and a new parent report path. +Do not write terminal state until schema, owner and lifecycle gates pass. +A schema or semantic failure gets one re-spawn; a second failure marks the row `blocked` with the verifier diagnostic. This prevents stale evidence or child self-classification from bypassing canonical owner detection. -After both gates pass, write `head_sha` and the +After all gates pass, write `head_sha` and the `mergeable/merge_state_status/ci_status` projection into the row's `head_sha` and `merge_state` columns (the crash-survivable A11 stop evidence), and remove ONLY the `status/shepherding` labels listed in diff --git a/packages/apm-issue-autopilot/.apm/skills/apm-issue-autopilot/assets/acceptance-observer.md b/packages/apm-issue-autopilot/.apm/skills/apm-issue-autopilot/assets/acceptance-observer.md index 853a938d80..cdf7e3a335 100644 --- a/packages/apm-issue-autopilot/.apm/skills/apm-issue-autopilot/assets/acceptance-observer.md +++ b/packages/apm-issue-autopilot/.apm/skills/apm-issue-autopilot/assets/acceptance-observer.md @@ -7,6 +7,13 @@ only place a PR is opened in Phase 4 (task children never open PRs). ## Procedure (pipeline child, in the issue worktree) +Load `$REPO_ROOT/.apm/instructions/lifecycle.instructions.md` before +acceptance. P8 applies even if the original `acceptance_shape` omitted it. +Execute its full lifecycle gate on the integrated candidate; preliminary +PR-lane, missing, skipped, failed, or stale evidence is not completion. +Re-plan missing lifecycle work under the same failure cap below; do not +open a ready PR or defer that work as out of scope. + 1. Reload plan.json (B8 ATTENTION ANCHOR). Read `acceptance_shape`. 2. Run the full lint contract one last time (must be silent) and the full test suite (`uv run --extra dev pytest -q`, must be green). diff --git a/packages/apm-issue-autopilot/.apm/skills/apm-issue-autopilot/assets/ideate-prompt.md b/packages/apm-issue-autopilot/.apm/skills/apm-issue-autopilot/assets/ideate-prompt.md index ba4cef5834..eb189c9578 100644 --- a/packages/apm-issue-autopilot/.apm/skills/apm-issue-autopilot/assets/ideate-prompt.md +++ b/packages/apm-issue-autopilot/.apm/skills/apm-issue-autopilot/assets/ideate-prompt.md @@ -25,6 +25,12 @@ OBSERVER), so make it concrete and testable, not aspirational. ## What to produce +First load `$REPO_ROOT/.apm/instructions/lifecycle.instructions.md`. +Include its P8 command map, deterministic trajectory and generated-model +obligations in `acceptance_shape` for every applicable change, regardless +of issue type. A brief that excludes required lifecycle work must be +re-planned or escalated, not accepted as install-only scope. + 1. Survey the touched surface read-only (`grep`, `view`, `gh issue view`). Understand the user-facing shape and where it lives. 2. Frame the **design brief** through the DevX lens: who the change diff --git a/packages/apm-issue-autopilot/.apm/skills/apm-issue-autopilot/assets/solution-pipeline-prompt.md b/packages/apm-issue-autopilot/.apm/skills/apm-issue-autopilot/assets/solution-pipeline-prompt.md index d9c6c43ee1..a415fa7ddf 100644 --- a/packages/apm-issue-autopilot/.apm/skills/apm-issue-autopilot/assets/solution-pipeline-prompt.md +++ b/packages/apm-issue-autopilot/.apm/skills/apm-issue-autopilot/assets/solution-pipeline-prompt.md @@ -26,6 +26,11 @@ parallelizing the work within an issue. Reload plan.md and (once it exists) plan.json before EACH stage and before EACH spawn (B8 ATTENTION ANCHOR). Never drive from recall. +Load `$REPO_ROOT/.apm/instructions/lifecycle.instructions.md` before +Ideate. Carry P8 into `acceptance_shape` and wave `integration_check`; +assign shared lifecycle/model files to one wave owner. Task-local success +does not certify the integrated candidate's lifecycle. + ## Model routing (B12) Every child below is spawned with an EXPLICIT model, resolved from diff --git a/packages/apm-issue-autopilot/.apm/skills/apm-issue-autopilot/assets/task-implement-prompt.md b/packages/apm-issue-autopilot/.apm/skills/apm-issue-autopilot/assets/task-implement-prompt.md index e0704628de..8be04477eb 100644 --- a/packages/apm-issue-autopilot/.apm/skills/apm-issue-autopilot/assets/task-implement-prompt.md +++ b/packages/apm-issue-autopilot/.apm/skills/apm-issue-autopilot/assets/task-implement-prompt.md @@ -37,6 +37,13 @@ Read TASK.type and load the matching lens; do not load the others: The lens defines the coverage discipline -- follow it exactly. +All lenses also inherit P8 from +`$WORKTREE/.apm/instructions/lifecycle.instructions.md`; load it before +editing. Keep the assigned lifecycle obligations executable across task +boundaries. If they are absent from the plan, escalate for re-planning +rather than silently narrowing to one command. Acceptance close verifies +the integrated candidate, not independent per-task success claims. + ## Discipline 1. Work ONLY inside WORKTREE on TASK_BRANCH. Never touch another diff --git a/packages/apm-review-panel/SKILL.md b/packages/apm-review-panel/SKILL.md index 403845415f..a38bb5e41f 100644 --- a/packages/apm-review-panel/SKILL.md +++ b/packages/apm-review-panel/SKILL.md @@ -4,7 +4,7 @@ description: >- Use this skill to run a multi-persona expert advisory review on a labelled pull request in microsoft/apm. The panel fans out to five mandatory specialists plus a test-coverage specialist (active on every PR that - touches src/) plus three conditional specialists (auth, doc-writer, + touches runtime behavior) plus three conditional specialists (auth, doc-writer, performance-expert), all running in their own agent threads, and a CEO synthesizer. The orchestrator is the sole writer to the PR: ONE recommendation comment, no verdict labels, no merge gating. The panel @@ -132,126 +132,27 @@ surfaces findings; the maintainer and the PR author decide ship. ## Conditional panelists -Three personas are conditional (auth, doc-writer, performance-expert). A -fourth (test-coverage) is mandatory on every PR that touches `src/` and -only skipped on documentation-only PRs -- see its section below for why. -The orchestrator ALWAYS spawns ALL four tasks to keep the schema -return shape uniform; the prompt instructs the subagent to set -`active: false` with an `inactive_reason` if the condition does not -hold. - -### Auth Expert - -Activate when the PR changes any of: -- `src/apm_cli/core/auth.py` -- `src/apm_cli/core/token_manager.py` -- `src/apm_cli/core/azure_cli.py` -- `src/apm_cli/deps/github_downloader.py` -- `src/apm_cli/marketplace/client.py` -- `src/apm_cli/utils/github_host.py` -- `src/apm_cli/install/validation.py` -- `src/apm_cli/install/pipeline.py` -- `src/apm_cli/deps/registry_proxy.py` - -Fallback self-check (when no fast-path file matched): "Does this PR -change authentication behavior, token management, credential resolution, -host classification used by `AuthResolver`, git or HTTP authorization -headers, or remote-host fallback semantics? If unsure, answer YES." - -### Doc Writer - -Activate when the PR changes any of: -- `README.md` -- `CHANGELOG.md` -- `MANIFESTO.md` -- `docs/src/content/docs/**` -- `.apm/skills/**/*.md` -- `.apm/agents/**/*.md` -- `.github/skills/**/*.md` -- `.github/agents/**/*.md` -- `.github/instructions/**/*.md` -- `.github/workflows/*.md` (gh-aw natural-language workflows) -- `packages/apm-guide/**` - -Fallback self-check (when no fast-path file matched): "Does this PR -change user-facing documentation, agent or skill prose, instruction -files, CHANGELOG entries, README claims, or any natural-language -artifact a reader will rely on? If unsure, answer YES." - -When the doc-writer is active and the PR includes documentation changes, -the persona reviews them for: (a) consistency with the existing voice -and structure, (b) accuracy against the code being changed, (c) -completeness for the typical reader (no orphan claims, no missing -prerequisites), (d) discoverability (cross-links, sidebar order if -Starlight content). When the doc-writer is active because of code -changes that SHOULD have updated docs but did not, the persona surfaces -that gap as a finding. - -### Performance Expert - -Activate when the PR changes any of: -- `src/apm_cli/cache/**` -- `src/apm_cli/deps/**` -- `src/apm_cli/install/phases/**` -- `src/apm_cli/install/pipeline.py` -- `src/apm_cli/install/resolve.py` -- `src/apm_cli/utils/**` -- `src/apm_cli/marketplace/**` -- `src/apm_cli/compilation/**` -- `scripts/perf/**` -- `src/apm_cli/core/command_logger.py` (when the diff adds perf-instrumentation logs) - -Also activate when: -- The PR description claims a performance win (speedup ratio, latency - reduction, bytes-on-disk reduction, throughput improvement) or - attaches a perf-harness measurement table. -- The diff introduces loops over collections (`for x in collection`) - where the collection may grow with dependency count or file count. -- The diff adds `os.scandir`, `os.walk`, `os.listdir`, or - `subprocess.run` calls on a path that executes per-package or - per-dependency. -- The diff adds `x in list_variable` inside a loop body. - -Fallback self-check (when no fast-path file matched): "Does this PR -change the hot path for dependency download, materialization, cache -layout, transport (git protocol, partial clone, sparse checkout), -parallelism, or any user-visible install/update wall-time? Does it -introduce an algorithmic complexity regression (O(n^2) loops, repeated -I/O, missing indexes, unconditional full scans, blocking synchronous -calls, heavy top-level imports)? If unsure, answer YES." - -When active, the performance-expert reviews against BOTH: -1. The package-manager performance playbook: transport minimization - (depth, filter, sparse scope), cache layering and dedup keys, - parallelism and lock contention, working-tree materialization cost, - perf-harness methodology (cache wipe, warm/cold separation, - statistical noise), and pervasive application of the chosen - technique across install / update / run surfaces. -2. The algorithmic performance lens (Big O analysis): complexity class - of every loop/lookup in the diff, index vs linear scan patterns, - unconditional expensive operations, import startup costs, redundant - computation, and parallelism opportunities. See the agent's - `references/algorithmic-patterns.md` for the full pattern catalogue. +Before selecting activation, load +[references/conditional-panelists.md](references/conditional-panelists.md) +for auth, doc-writer and performance triggers. Test coverage follows the +P8 rule below. ALWAYS spawn all four tasks to keep the return shape +uniform; an inactive persona returns `active: false` and a specific +`inactive_reason`. ### Test Coverage Expert -**Active by default on every PR that touches `src/**/*.py`.** The only -condition that flips this persona to `active: false` is a -documentation-only PR -- the diff contains zero `src/**/*.py` files. -In that case set `inactive_reason: "documentation-only PR -- no -runtime code paths to defend"`. - -The activation rule is intentionally narrow: under the advisory regime, -test outcomes are LOAD-BEARING for CEO arbitration (passed / failed / -missing test evidence outranks opinion-only findings -- see -`apm-ceo.agent.md` and `panelist-return-schema.json` evidence block). -A persona whose findings carry that weight cannot be silently skipped -on a heuristic. Better to spawn it on a pure refactor and have it -return a single `nit`-severity "no behavior surface touched -- no -coverage finding" line than to skip it and leave the CEO without -evidence to weigh. (Earlier revisions of this skill paired test-coverage -with auth and doc-writer as conditional for symmetry; that symmetry -broke when test evidence became load-bearing.) +**Active on every PR touching `src/**/*.py` or a P8 lifecycle surface.** +Before classifying, load the consumer repository's +`.apm/instructions/lifecycle.instructions.md`. Absence of Python changes +does not exempt installed payloads, manifests, shared state or tooling +that alters lifecycle behavior. Only a demonstrated unrelated change +may set `active: false`, with a specific `inactive_reason`. + +Pass that rule and the candidate-bound lifecycle evidence to the +coverage reviewer and CEO. Missing required deterministic or generated +coverage is a blocking P8 gap, not a deferrable recommendation. +The panel remains advisory; the executing repository gate, not a +`ship_now` opinion, discharges the shipping precondition. The test-coverage-expert is paired with the devx-ux-expert lens and defends the user-promise contracts the DevX persona enumerates (CLI @@ -259,7 +160,7 @@ surface, error wording, install idempotency, lockfile determinism, auth resolution). It MUST verify "no test exists" claims with `view`/`grep` on the test tree before emitting a finding -- false-positive coverage findings destroy trust in the field. It does NOT compute coverage -percentages, does NOT flag tests for pure refactors, and does NOT +percentages, does NOT demand artificial new tests for refactors, and does NOT duplicate python-architect on test-code design. ## Routing matrix (CEO synthesis emphasis only) @@ -482,12 +383,8 @@ no comment can be rendered, an explicit `noop` (step 9) -- are emitted. a diff changes how a remote host, org, token source, or fallback path is selected and you are not certain it is auth-neutral, activate auth-expert as `active: true`. -- **Test-coverage probe is mandatory.** The test-coverage-expert MUST - verify "no test exists for X" via `view`/`grep` on the `tests/` tree - before emitting a finding. A false-positive coverage finding (test - exists but persona claimed it does not) destroys maintainer trust in - the field. The persona scope file enforces this; the orchestrator - passes the diff and trusts the persona to probe. +- **Test-coverage probe is mandatory.** Confirm missing-test claims + against `tests/`, as required by the coverage persona. - **Subagent write enforcement is contract-based, not sandbox-based.** Tool permissions are workflow-scoped, not subagent-scoped, so every spawned task technically inherits the same `gh` toolset. The @@ -505,7 +402,4 @@ no comment can be rendered, an explicit `noop` (step 9) -- are emitted. the turn MUST end with a safe output -- the comment, or an explicit `noop`. See the "Synchronous fan-out" and "Non-empty turn exit" architecture invariants and step 9. -- **No verdict-label reset workflow.** The previous regime had a - companion workflow `pr-panel-label-reset.yml` that stripped verdict - labels on every push. The advisory regime has no verdict labels to - strip; that workflow is removed. +- **No verdict-label reset.** Advisory panels emit no verdict labels. diff --git a/packages/apm-review-panel/assets/panelist-return-schema.json b/packages/apm-review-panel/assets/panelist-return-schema.json index 0bf67df88c..bf2babd927 100644 --- a/packages/apm-review-panel/assets/panelist-return-schema.json +++ b/packages/apm-review-panel/assets/panelist-return-schema.json @@ -115,7 +115,7 @@ }, "tier": { "type": "string", - "enum": ["unit", "integration-with-fixtures", "e2e", "manual-only", "static"], + "enum": ["unit", "integration-with-fixtures", "e2e", "lifecycle-state-machine", "manual-only", "static"], "description": "Tier of evidence. unit: function-level test with mocks at the boundary; cheap, fast, narrow. integration-with-fixtures: real I/O against real fixtures (real files, real subprocess, real network when tagged), no mocked surface for the asserted contract. e2e: full CLI invocation end-to-end, real artifacts, real exit codes. manual-only: only a manual procedure; no automated guardrail. static: lint / type-check / schema validation only. The CEO weights tier against the SURFACE FLOOR named in the test-coverage-expert tier-floor matrix: a `unit` passed evidence on a critical-promise surface (CLI / install / lockfile / auth / hooks / marketplace / cross-module) does NOT silence an opinion-finding from another panelist asking for `integration-with-fixtures` coverage. Required on every evidence block." }, "run_evidence": { @@ -146,7 +146,26 @@ ] } } - } + }, + "allOf": [ + { + "if": { + "properties": { + "outcome": { "const": "passed" }, + "tier": { "const": "lifecycle-state-machine" } + }, + "required": ["outcome", "tier"] + }, + "then": { + "required": ["test_file", "run_evidence", "assertion_excerpt"], + "properties": { + "test_file": { "minLength": 1 }, + "run_evidence": { "minLength": 1 }, + "assertion_excerpt": { "minLength": 1 } + } + } + } + ] } } } diff --git a/packages/apm-review-panel/references/conditional-panelists.md b/packages/apm-review-panel/references/conditional-panelists.md new file mode 100644 index 0000000000..309cc19368 --- /dev/null +++ b/packages/apm-review-panel/references/conditional-panelists.md @@ -0,0 +1,97 @@ +# Conditional specialist activation + +Load this reference before the review panel selects persona activation. +Test-coverage activation is owned by the P8 section in `../SKILL.md`. + +## Auth Expert + +Activate when the PR changes any of: +- `src/apm_cli/core/auth.py` +- `src/apm_cli/core/token_manager.py` +- `src/apm_cli/core/azure_cli.py` +- `src/apm_cli/deps/github_downloader.py` +- `src/apm_cli/marketplace/client.py` +- `src/apm_cli/utils/github_host.py` +- `src/apm_cli/install/validation.py` +- `src/apm_cli/install/pipeline.py` +- `src/apm_cli/deps/registry_proxy.py` + +Fallback self-check (when no fast-path file matched): "Does this PR +change authentication behavior, token management, credential resolution, +host classification used by `AuthResolver`, git or HTTP authorization +headers, or remote-host fallback semantics? If unsure, answer YES." + +## Doc Writer + +Activate when the PR changes any of: +- `README.md` +- `CHANGELOG.md` +- `MANIFESTO.md` +- `docs/src/content/docs/**` +- `.apm/skills/**/*.md` +- `.apm/agents/**/*.md` +- `.github/skills/**/*.md` +- `.github/agents/**/*.md` +- `.github/instructions/**/*.md` +- `.github/workflows/*.md` (gh-aw natural-language workflows) +- `packages/apm-guide/**` + +Fallback self-check (when no fast-path file matched): "Does this PR +change user-facing documentation, agent or skill prose, instruction +files, CHANGELOG entries, README claims, or any natural-language +artifact a reader will rely on? If unsure, answer YES." + +When the doc-writer is active and the PR includes documentation changes, +the persona reviews them for: (a) consistency with the existing voice +and structure, (b) accuracy against the code being changed, (c) +completeness for the typical reader (no orphan claims, no missing +prerequisites), (d) discoverability (cross-links, sidebar order if +Starlight content). When the doc-writer is active because of code +changes that SHOULD have updated docs but did not, the persona surfaces +that gap as a finding. + +## Performance Expert + +Activate when the PR changes any of: +- `src/apm_cli/cache/**` +- `src/apm_cli/deps/**` +- `src/apm_cli/install/phases/**` +- `src/apm_cli/install/pipeline.py` +- `src/apm_cli/install/resolve.py` +- `src/apm_cli/utils/**` +- `src/apm_cli/marketplace/**` +- `src/apm_cli/compilation/**` +- `scripts/perf/**` +- `src/apm_cli/core/command_logger.py` (when the diff adds perf-instrumentation logs) + +Also activate when: +- The PR description claims a performance win (speedup ratio, latency + reduction, bytes-on-disk reduction, throughput improvement) or + attaches a perf-harness measurement table. +- The diff introduces loops over collections (`for x in collection`) + where the collection may grow with dependency count or file count. +- The diff adds `os.scandir`, `os.walk`, `os.listdir`, or + `subprocess.run` calls on a path that executes per-package or + per-dependency. +- The diff adds `x in list_variable` inside a loop body. + +Fallback self-check (when no fast-path file matched): "Does this PR +change the hot path for dependency download, materialization, cache +layout, transport (git protocol, partial clone, sparse checkout), +parallelism, or any user-visible install/update wall-time? Does it +introduce an algorithmic complexity regression (O(n^2) loops, repeated +I/O, missing indexes, unconditional full scans, blocking synchronous +calls, heavy top-level imports)? If unsure, answer YES." + +When active, the performance-expert reviews against BOTH: +1. The package-manager performance playbook: transport minimization + (depth, filter, sparse scope), cache layering and dedup keys, + parallelism and lock contention, working-tree materialization cost, + perf-harness methodology (cache wipe, warm/cold separation, + statistical noise), and pervasive application of the chosen + technique across install / update / run surfaces. +2. The algorithmic performance lens (Big O analysis): complexity class + of every loop/lookup in the diff, index vs linear scan patterns, + unconditional expensive operations, import startup costs, redundant + computation, and parallelism opportunities. See the agent's + `references/algorithmic-patterns.md` for the full pattern catalogue. diff --git a/packages/batch-bug-shepherd/.apm/skills/batch-bug-shepherd/SKILL.md b/packages/batch-bug-shepherd/.apm/skills/batch-bug-shepherd/SKILL.md index 525b0b288c..026b8605cb 100644 --- a/packages/batch-bug-shepherd/.apm/skills/batch-bug-shepherd/SKILL.md +++ b/packages/batch-bug-shepherd/.apm/skills/batch-bug-shepherd/SKILL.md @@ -325,14 +325,18 @@ uv run python /.agents/skills/shepherd-driver/scripts/owner_touch_ --head --completion ``` -Do not update the table or labels until schema AND semantic +Load `/.apm/instructions/lifecycle.instructions.md` and +independently execute its repository provider with `--lane full`, +`--base $BASE_SHA --head `, +`--completion `, and a new parent report path. +Do not update the table or labels until schema, owner AND lifecycle verification pass. A non-zero verifier result gets the same retry-once treatment as malformed schema; on a second failure mark the row `blocked` with the diagnostic. This parent re-probe prevents a child from bypassing deterministic owner detection or presenting stale functional evidence. -After both gates pass, write `head_sha`, `mergeable`, +After all gates pass, write `head_sha`, `mergeable`, `merge_state_status`, and `ci_status` from the return into the table, and remove the `status/shepherding` label from the driven issue (assignment stays). The orchestrator owns only validation, table diff --git a/packages/batch-bug-shepherd/.apm/skills/batch-bug-shepherd/assets/fix-prompt.md b/packages/batch-bug-shepherd/.apm/skills/batch-bug-shepherd/assets/fix-prompt.md index f09b4b1006..2035368ee5 100644 --- a/packages/batch-bug-shepherd/.apm/skills/batch-bug-shepherd/assets/fix-prompt.md +++ b/packages/batch-bug-shepherd/.apm/skills/batch-bug-shepherd/assets/fix-prompt.md @@ -14,6 +14,14 @@ under microsoft/apm. ## Procedure +Before planning, load `$REPO_ROOT/.apm/instructions/lifecycle.instructions.md`. +P8 acceptance covers the entire applicable command state machine, with +separate deterministic and generated-model evidence. Run its full +executing gate at the final integrated head before opening a ready PR. +Missing infrastructure or evidence blocks shipping; install-only +success cannot discharge the issue. Do not defer lifecycle gaps as +outside the proposed patch's scope. + 1. Re-read the issue and the repro steps. Confirm the bug still reproduces on HEAD. 2. Design the minimum fix. Identify the canonical sibling code paths diff --git a/packages/pr-description-skill/assets/scenario-evidence-rubric.md b/packages/pr-description-skill/assets/scenario-evidence-rubric.md index 3a3388aec3..e9ad87844e 100644 --- a/packages/pr-description-skill/assets/scenario-evidence-rubric.md +++ b/packages/pr-description-skill/assets/scenario-evidence-rubric.md @@ -27,6 +27,13 @@ SCENARIO coverage: the user-facing flows that, if they silently break, erode trust in APM as a tool. Pure coverage metrics are not the gate; scenario-test mapping is. +Before synthesis, load `.apm/instructions/lifecycle.instructions.md` +from the consumer repository. For a P8-applicable change, present the +complete command applicability, connected deterministic trajectory, +generated-model dimensions and actual candidate-bound gate result. +Missing required evidence is a shipping blocker, not a trade-off row. +This table describes evidence; it does not replace executing the gate. + ## APM principle taxonomy Every scenario in a PR maps to AT LEAST ONE of these principles @@ -159,3 +166,7 @@ following is true (state which one in trade-offs): evidence; cite the suite that ran green). Any other PR that omits the table is incomplete. + +These are presentation exemptions only. A lifecycle-affecting refactor +still needs current P8 execution; a lockfile or installed-markdown change +can alter runtime behavior and is not automatically exempt. diff --git a/packages/shepherd-driver/SKILL.md b/packages/shepherd-driver/SKILL.md index fe2da09cdd..8d3efa42e6 100644 --- a/packages/shepherd-driver/SKILL.md +++ b/packages/shepherd-driver/SKILL.md @@ -52,6 +52,12 @@ ISSUE_NUMBER, AUTHOR, HEAD_REPO, HEAD_BRANCH, MAINTAINER_CAN_MODIFY, REPO_ROOT, ORIGIN, optional PANEL_PRIOR). The subagent owns the whole convergence loop end-to-end and returns a `completion_return`. +The consumer repository supplies `.apm/instructions/lifecycle.instructions.md` +and `scripts/check_lifecycle_evidence.py`; probe both before execution. +The driver and parent independently run the full P8 gate; the parent uses +`--completion` before accepting either shipping-eligible status. Missing +assets or evidence block completion; do not bundle another lifecycle owner. + After every shepherded PR returns `ready-to-merge`, the orchestrator runs the conflict-resolution phase: probe mergeability and, on DIRTY / BEHIND / CONFLICTING, spawn one conflict-resolution subagent diff --git a/packages/shepherd-driver/assets/completion-schema.json b/packages/shepherd-driver/assets/completion-schema.json index 17894d1b0f..c2f93abad1 100644 --- a/packages/shepherd-driver/assets/completion-schema.json +++ b/packages/shepherd-driver/assets/completion-schema.json @@ -8,6 +8,22 @@ { "$ref": "#/definitions/conflict_resolution_return" } ], "definitions": { + "lifecycle_evidence": { + "type": "object", + "additionalProperties": false, + "description": "Summary of a native full-lane lifecycle run. The executing repository provider verifies it with --completion and independently reruns the candidate; schema validity alone is not proof.", + "required": ["version", "base_sha", "head_sha", "tested_tree", "lane", "status", "report_path", "report_sha256"], + "properties": { + "version": { "const": 1 }, + "base_sha": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, + "head_sha": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, + "tested_tree": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, + "lane": { "const": "full" }, + "status": { "enum": ["passed", "not_applicable"] }, + "report_path": { "type": "string", "minLength": 1 }, + "report_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" } + } + }, "owner_touch_report": { "type": "object", "additionalProperties": false, @@ -335,16 +351,19 @@ }, "architecture_evidence": { "$ref": "#/definitions/architecture_evidence_v2" + }, + "lifecycle_evidence": { + "$ref": "#/definitions/lifecycle_evidence" } }, "allOf": [ { "if": { "properties": { "status": { "const": "ready-to-merge" } } }, - "then": { "required": ["ci_evidence", "lint_evidence", "head_sha", "mergeable", "merge_state_status", "ci_status", "architecture_evidence"] } + "then": { "required": ["ci_evidence", "lint_evidence", "head_sha", "mergeable", "merge_state_status", "ci_status", "architecture_evidence", "lifecycle_evidence"] } }, { "if": { "properties": { "status": { "const": "advisory-with-deferred" } } }, - "then": { "required": ["ci_evidence", "deferred_items", "head_sha", "mergeable", "merge_state_status", "ci_status", "architecture_evidence"] } + "then": { "required": ["ci_evidence", "deferred_items", "head_sha", "mergeable", "merge_state_status", "ci_status", "architecture_evidence", "lifecycle_evidence"] } }, { "if": { "properties": { "status": { "const": "blocked" } } }, @@ -384,6 +403,9 @@ "type": "string", "description": "Brief human-readable narrative of the rebase: base SHA, head SHA pre-rebase, head SHA post-rebase, conflicts resolved, fallback used (merge-of-main) if any. Required when status=resolved." }, + "lifecycle_evidence": { + "$ref": "#/definitions/lifecycle_evidence" + }, "push_command": { "type": "string", "pattern": "^(?!.*--force(?!-with-lease)).*$", @@ -442,7 +464,7 @@ { "if": { "properties": { "status": { "const": "resolved" } } }, "then": { - "required": ["rebase_evidence", "push_command", "mergeStateStatus_post", "lint_evidence", "comment_url"], + "required": ["rebase_evidence", "push_command", "mergeStateStatus_post", "lint_evidence", "comment_url", "lifecycle_evidence"], "properties": { "mergeStateStatus_post": { "enum": ["CLEAN", "UNSTABLE", "HAS_HOOKS"] } } diff --git a/packages/shepherd-driver/assets/conflict-resolution-prompt.md b/packages/shepherd-driver/assets/conflict-resolution-prompt.md index fd464602fb..f9d11c55a2 100644 --- a/packages/shepherd-driver/assets/conflict-resolution-prompt.md +++ b/packages/shepherd-driver/assets/conflict-resolution-prompt.md @@ -106,6 +106,13 @@ regression-trap test, `--force-with-lease` only (never bare resolution-failed`, `blocker: "lint noisy post-rebase: "`. Do not push. + Also load `$REPO_ROOT/.apm/instructions/lifecycle.instructions.md`. + Rebase invalidates the previous lifecycle result. Execute its full + gate against the new base/head before claiming resolution; include + the native report summary as `lifecycle_evidence` in the return. + Missing, skipped, failed or stale P8 evidence means + `resolution-failed`, not `resolved`. + 7. PUSH WITH `--force-with-lease`. Choose the remote based on HEAD_REPO: - Head on a fork: `git push --force-with-lease=: HEAD:`. @@ -151,7 +158,7 @@ regression-trap test, `--force-with-lease` only (never bare return shapes on this const). Required fields by status: - resolved: pr, status, mergeStateStatus_pre, mergeStateStatus_post, rebase_evidence, push_command, - lint_evidence, comment_url. Plus mutation_break_evidence + lint_evidence, comment_url, lifecycle_evidence. Plus mutation_break_evidence and rebase_touched_regression_test=true if step 5 ran. - requires-author-action: pr, status, mergeStateStatus_pre, fork_url, head_branch, maintainer_can_modify=false, diff --git a/packages/shepherd-driver/assets/shepherd-driver-prompt.md b/packages/shepherd-driver/assets/shepherd-driver-prompt.md index 8e51d8ae88..5bf4336ad8 100644 --- a/packages/shepherd-driver/assets/shepherd-driver-prompt.md +++ b/packages/shepherd-driver/assets/shepherd-driver-prompt.md @@ -45,6 +45,8 @@ are part of your contract: table parsed by the deterministic gate (Step X.2.5) - `owner_touch_gate.py` -- exact-revision owner detection and terminal evidence verification (Step X.2.5) +- `$REPO_ROOT/.apm/instructions/lifecycle.instructions.md` -- P8 shipping + precondition and repository executing provider (Step X.2.6) - `../apm-review-panel/SKILL.md` -- panel composition contract - `../pr-description-skill/SKILL.md` -- superseding-PR body author (Path B) @@ -57,6 +59,7 @@ X.0 fetch + classify Copilot X.1 invoke apm-review-panel skill X.2 merge follow-ups, apply fold-vs-defer rubric X.2.5 canonical-owner gate (classify + evidence, FAIL CLOSED) +X.2.6 lifecycle gate (execute full candidate evidence, FAIL CLOSED) X.3 edit code, fold foldable items X.4 lint contract (silent) X.5 push (author fork or superseding PR) @@ -351,6 +354,31 @@ classification may interpret its result but may not override it. If evidence cannot be produced within the loop cap, return `blocked` with the missing owner/test named. Never defer the gate. +### Step X.2.6 -- lifecycle shipping gate (FAIL CLOSED) + +Load `$REPO_ROOT/.apm/instructions/lifecycle.instructions.md` and probe +`$REPO_ROOT/scripts/check_lifecycle_evidence.py`. Either missing means +`blocked`; do not recreate the provider inside this package. At loop +entry, plan any missing P8 obligations for Step X.3. Once the integrated +candidate is committed and clean, execute the rule's full-lane gate +before certifying completion. Re-run after any later edit or rebase. + +Record `lifecycle_evidence` from the native report: version 1, +`base_sha` from `base`, `head_sha` from `head`, `tested_tree`, lane +`full`, status `passed` or `not_applicable`, the report's absolute +`report_path`, and its tool-computed byte `report_sha256`. Never author +a passing report or relabel an old head. The preliminary `pr` lane +cannot supply terminal evidence. + +Schema-validate the completion referencing the report just produced; do +not repeat an identical native run just to serialize its summary. The +parent independently executes the same full gate with +`--completion ` and a new report path, checking the reference +and freshly running the candidate rather than importing a receipt. +Missing lifecycle work is in scope even if the original brief omitted it. +Repair it within the loop or return `blocked`; neither a panel stance +nor `advisory-with-deferred` can waive P8. + ### Step X.3 -- edit code, fold foldable items For each FOLD item: @@ -482,6 +510,8 @@ On cap hit: `status: blocked` with failing job + log excerpt in - CEO stance is `ship_now`, OR `ship_with_followups` where all remaining followups are tagged DEFER with valid scope-boundary notes. +- The full executing lifecycle gate (Step X.2.6) passed on this exact + candidate; no required deterministic or generated-model gap remains. In this case: re-run the apm-review-panel ONE LAST TIME so the visible comment reflects the converged state. That final run posts its @@ -491,7 +521,9 @@ the panel always posts its result via `gh`). Move to "Finalize" below. **Terminal `status: advisory-with-deferred`** when: - Iteration cap (4) is hit, AND -- Foldable items remain unresolved. +- Foldable items remain unresolved, AND +- P8 full-lane lifecycle evidence is complete. Any missing required + lifecycle work instead returns `blocked`, regardless of the cap. In this case: re-run the apm-review-panel one last time so its final recommendation comment reflects the converged (capped) state, carrying @@ -603,6 +635,16 @@ most one short clause (e.g. `pending required review`, "ci_evidence": "string (required for ready-to-merge or advisory-with-deferred)", "lint_evidence": "string (required when status=ready-to-merge)", "mutation_break_evidence": [...], + "lifecycle_evidence": { + "version": 1, + "base_sha": "40-char comparison base", + "head_sha": "40-char final candidate", + "tested_tree": "40-char tested tree", + "lane": "full", + "status": "passed|not_applicable", + "report_path": "absolute native report path outside checkout", + "report_sha256": "64-char byte digest" + }, "architecture_evidence": { "version": "2", "classification": "ordinary-fix|owner-extension|new-owner|split-authority-repair|not-applicable", @@ -672,6 +714,8 @@ Use `status` (NOT `terminal_state`) and `pr` (NOT `pr_number`). migration: version 1 self-classified `decisions[]` returns now fail and force one re-spawn under v2. Blocked and superseded returns remain compatible because they do not require architecture evidence. +Both terminal success statuses also require `lifecycle_evidence`; +older returns without it must re-run P8, not invent the new fields. ## Hard rules diff --git a/packages/shepherd-driver/references/mergeability-gate.md b/packages/shepherd-driver/references/mergeability-gate.md index 02367dfa9a..1d9110fb40 100644 --- a/packages/shepherd-driver/references/mergeability-gate.md +++ b/packages/shepherd-driver/references/mergeability-gate.md @@ -114,6 +114,13 @@ table + progress diagram (P5c done, P6 active). ## Trust-but-verify discipline +Every `resolved` return also requires post-rebase `lifecycle_evidence`. +Load the row worktree's `.apm/instructions/lifecycle.instructions.md`; +derive the new comparison base and independently execute the full +provider with `--completion ` and a fresh parent +report path. A failed P8 gate changes the result to `resolution-failed`, +even if GitHub reports CLEAN. Do not reuse pre-rebase lifecycle proof. + The orchestrator's 5c re-probe is mandatory even when the subagent returned `status: resolved` with `mergeStateStatus_post: CLEAN`. Two reasons: @@ -135,7 +142,7 @@ re-probe is the gate. belong to the maintainer's policy workflow, not to the bbs. - Does NOT re-run the apm-review-panel. The shepherd-driver convergence loop already owned review, fold/defer, implementation, - push, and CI; Phase 5 owns ONLY rebase and re-probe. Conflict + push, and CI; Phase 5 owns rebase, fresh P8 evidence and re-probe. Conflict resolution is a mechanical merge, not a fresh judgment pass. - Does NOT open new PRs. Supersession is a Phase 4 affordance; if a fork-with-flag-false PR shows up here, it routes to diff --git a/tests/fixtures/lifecycle_bug_ledger.json b/tests/fixtures/lifecycle_bug_ledger.json index 2b64f79740..b3a7094839 100644 --- a/tests/fixtures/lifecycle_bug_ledger.json +++ b/tests/fixtures/lifecycle_bug_ledger.json @@ -6,6 +6,49 @@ { "paths": ["scripts/check_lifecycle_evidence.py", "scripts/lifecycle_contracts.py"], "reason": "Repository-only lifecycle evidence provider and authored-contract validator; neither module is shipped in apm_cli or invoked by an APM command. Exercised by tests/unit/scripts/test_lifecycle_evidence.py and the existing quality/CI topology gates." + }, + { + "paths": [ + ".apm/instructions/lifecycle.instructions.md", + ".github/instructions/lifecycle.instructions.md", + ".apm/agents/apm-ceo.agent.md", + ".github/agents/apm-ceo.agent.md", + ".apm/agents/test-coverage-expert.agent.md", + ".github/agents/test-coverage-expert.agent.md", + "packages/apm-issue-autopilot/.apm/skills/apm-issue-autopilot/SKILL.md", + ".agents/skills/apm-issue-autopilot/SKILL.md", + "packages/apm-issue-autopilot/.apm/skills/apm-issue-autopilot/assets/acceptance-observer.md", + ".agents/skills/apm-issue-autopilot/assets/acceptance-observer.md", + "packages/apm-issue-autopilot/.apm/skills/apm-issue-autopilot/assets/ideate-prompt.md", + ".agents/skills/apm-issue-autopilot/assets/ideate-prompt.md", + "packages/apm-issue-autopilot/.apm/skills/apm-issue-autopilot/assets/solution-pipeline-prompt.md", + ".agents/skills/apm-issue-autopilot/assets/solution-pipeline-prompt.md", + "packages/apm-issue-autopilot/.apm/skills/apm-issue-autopilot/assets/task-implement-prompt.md", + ".agents/skills/apm-issue-autopilot/assets/task-implement-prompt.md", + "packages/batch-bug-shepherd/.apm/skills/batch-bug-shepherd/SKILL.md", + ".agents/skills/batch-bug-shepherd/SKILL.md", + "packages/batch-bug-shepherd/.apm/skills/batch-bug-shepherd/assets/fix-prompt.md", + ".agents/skills/batch-bug-shepherd/assets/fix-prompt.md", + "packages/apm-review-panel/SKILL.md", + ".agents/skills/apm-review-panel/SKILL.md", + "packages/apm-review-panel/references/conditional-panelists.md", + ".agents/skills/apm-review-panel/references/conditional-panelists.md", + "packages/apm-review-panel/assets/panelist-return-schema.json", + ".agents/skills/apm-review-panel/assets/panelist-return-schema.json", + "packages/pr-description-skill/assets/scenario-evidence-rubric.md", + ".agents/skills/pr-description-skill/assets/scenario-evidence-rubric.md", + "packages/shepherd-driver/SKILL.md", + ".agents/skills/shepherd-driver/SKILL.md", + "packages/shepherd-driver/assets/completion-schema.json", + ".agents/skills/shepherd-driver/assets/completion-schema.json", + "packages/shepherd-driver/assets/conflict-resolution-prompt.md", + ".agents/skills/shepherd-driver/assets/conflict-resolution-prompt.md", + "packages/shepherd-driver/assets/shepherd-driver-prompt.md", + ".agents/skills/shepherd-driver/assets/shepherd-driver-prompt.md", + "packages/shepherd-driver/references/mergeability-gate.md", + ".agents/skills/shepherd-driver/references/mergeability-gate.md" + ], + "reason": "This change binds maintainer and coding-agent planning, review and completion to P8 and tightens their evidence JSON contracts. These exact source files and regenerated mirrors are contributor/shipping workflow content; no APM command implementation, dependency pin, deployed primitive format, target layout or package ownership semantics changes here. Schema and consumer-link regressions exercise this tooling change. This assessment does not exempt future payload, format or runtime changes to these paths." } ], "property_catalog": [ diff --git a/tests/unit/test_lifecycle_agent_contract.py b/tests/unit/test_lifecycle_agent_contract.py new file mode 100644 index 0000000000..418ca06574 --- /dev/null +++ b/tests/unit/test_lifecycle_agent_contract.py @@ -0,0 +1,101 @@ +"""Keep P8's operational owner connected to its agent consumers.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest +import yaml +from jsonschema import Draft7Validator + +pytestmark = pytest.mark.component +ROOT = Path(__file__).parents[2] +RULE = ".apm/instructions/lifecycle.instructions.md" +CONSUMERS = [ + ".apm/agents/apm-ceo.agent.md", + ".apm/agents/test-coverage-expert.agent.md", + "packages/shepherd-driver/SKILL.md", + "packages/shepherd-driver/assets/shepherd-driver-prompt.md", + "packages/shepherd-driver/assets/conflict-resolution-prompt.md", + "packages/shepherd-driver/references/mergeability-gate.md", + "packages/apm-review-panel/SKILL.md", + "packages/pr-description-skill/assets/scenario-evidence-rubric.md", + "packages/batch-bug-shepherd/.apm/skills/batch-bug-shepherd/SKILL.md", + "packages/batch-bug-shepherd/.apm/skills/batch-bug-shepherd/assets/fix-prompt.md", + "packages/apm-issue-autopilot/.apm/skills/apm-issue-autopilot/SKILL.md", + *[ + f"packages/apm-issue-autopilot/.apm/skills/apm-issue-autopilot/assets/{name}.md" + for name in ( + "ideate-prompt", + "task-implement-prompt", + "solution-pipeline-prompt", + "acceptance-observer", + ) + ], +] + + +def test_lifecycle_rule_is_universally_scoped_and_deployed() -> None: + """Payloads and indirect changes cannot escape a source-only instruction glob.""" + content = (ROOT / RULE).read_text(encoding="ascii") + frontmatter = yaml.safe_load(content.split("---", 2)[1]) + assert frontmatter["applyTo"] == "**" + assert (ROOT / ".github/instructions/lifecycle.instructions.md").read_text( + encoding="ascii" + ) == content + + +@pytest.mark.parametrize("path", CONSUMERS) +def test_lifecycle_consumers_load_one_operational_rule(path: str) -> None: + """Implementation, review, completion and rebase must retain the owner link.""" + assert "lifecycle.instructions.md" in (ROOT / path).read_text(encoding="ascii") + + +def test_panel_retains_lazy_specialist_reference_in_deployed_bundle() -> None: + """Progressive disclosure must not remove conditional specialists' inputs.""" + reference = "references/conditional-panelists.md" + source = ROOT / "packages/apm-review-panel" + assert reference in (source / "SKILL.md").read_text(encoding="ascii") + assert (source / reference).read_bytes() == ( + ROOT / ".agents/skills/apm-review-panel" / reference + ).read_bytes() + + +def _panel_evidence() -> dict[str, Any]: + """Build the existing coverage persona's lifecycle evidence return shape.""" + return { + "persona": "test-coverage-expert", + "active": True, + "summary": "Lifecycle evidence is present.", + "findings": [ + { + "severity": "nit", + "summary": "Required trajectory executed.", + "rationale": "P8 requires actual lifecycle assertions.", + "evidence": { + "tier": "lifecycle-state-machine", + "outcome": "passed", + "test_file": "tests/integration/test_generated_lifecycle_state_machine.py", + "run_evidence": "Native full report for the candidate.", + "assertion_excerpt": "assert state.deployed_bytes == expected", + }, + } + ], + } + + +@pytest.mark.parametrize("missing", [None, "test_file", "run_evidence", "assertion_excerpt"]) +def test_panel_can_carry_only_complete_passing_lifecycle_evidence( + missing: str | None, +) -> None: + """The declared persona tier must serialize without accepting empty proof.""" + schema_path = ROOT / "packages/apm-review-panel/assets/panelist-return-schema.json" + schema = json.loads(schema_path.read_text(encoding="ascii")) + Draft7Validator.check_schema(schema) + document = _panel_evidence() + if missing: + del document["findings"][0]["evidence"][missing] + errors = list(Draft7Validator(schema).iter_errors(document)) + assert bool(errors) is (missing is not None) diff --git a/tests/unit/test_shepherd_driver_completion_schema.py b/tests/unit/test_shepherd_driver_completion_schema.py index 5960e23b37..ad48b96e8a 100644 --- a/tests/unit/test_shepherd_driver_completion_schema.py +++ b/tests/unit/test_shepherd_driver_completion_schema.py @@ -9,6 +9,7 @@ import pytest from jsonschema import Draft7Validator +pytestmark = pytest.mark.component ROOT = Path(__file__).parents[2] CANONICAL_SCHEMA = ROOT / "packages/shepherd-driver/assets/completion-schema.json" MIRROR_SCHEMA = ROOT / ".agents/skills/shepherd-driver/assets/completion-schema.json" @@ -78,6 +79,16 @@ def _ready_completion( "mergeable": "MERGEABLE", "merge_state_status": "CLEAN", "ci_status": "green", + "lifecycle_evidence": { + "version": 1, + "base_sha": BASE_SHA, + "head_sha": HEAD_SHA, + "tested_tree": "d" * 40, + "lane": "full", + "status": "not_applicable", + "report_path": "/session/lifecycle-evidence.json", + "report_sha256": "e" * 64, + }, "architecture_evidence": { "version": "2", "classification": classification, @@ -135,6 +146,49 @@ def test_owner_touch_requires_functional_evidence() -> None: assert list(_validator().iter_errors(document)) +@pytest.mark.parametrize("status", ["ready-to-merge", "advisory-with-deferred"]) +def test_terminal_completion_cannot_omit_lifecycle_evidence(status: str) -> None: + """A panel recommendation or deferral cannot waive the P8 precondition.""" + document = _ready_completion("ordinary-fix") + document["status"] = status + document["deferred_items"] = [] + del document["lifecycle_evidence"] + + assert list(_validator().iter_errors(document)) + + +@pytest.mark.parametrize( + ("field", "value"), + [("status", "pending"), ("status", "blocked"), ("lane", "pr"), ("report_sha256", "")], +) +def test_terminal_lifecycle_summary_rejects_nonshipping_evidence(field: str, value: str) -> None: + """Only a full-lane report reference can be handed to native verification.""" + document = _ready_completion("ordinary-fix") + document["lifecycle_evidence"][field] = value + + assert list(_validator().iter_errors(document)) + + +@pytest.mark.parametrize("include_evidence", [False, True]) +def test_resolved_rebase_requires_fresh_lifecycle_summary(include_evidence: bool) -> None: + """Resolving conflicts cannot carry forward the old candidate's eligibility.""" + document = { + "kind": "conflict-resolution", + "pr": 1, + "status": "resolved", + "mergeStateStatus_pre": "DIRTY", + "mergeStateStatus_post": "CLEAN", + "rebase_evidence": "Resolved against the new base.", + "push_command": "git push --force-with-lease", + "lint_evidence": "Exit 0.", + "comment_url": "https://example.com/rebase", + } + if include_evidence: + document["lifecycle_evidence"] = _ready_completion("ordinary-fix")["lifecycle_evidence"] + + assert bool(list(_validator().iter_errors(document))) is not include_evidence + + @pytest.mark.parametrize("classification", ["ordinary-fix", "not-applicable"]) def test_owner_touch_rejects_false_self_classification(classification: str) -> None: """LLM classification cannot self-exempt a deterministic owner touch.""" From efdafc3f6680e3a5547eac9a058b65751f8f756d Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Mon, 7 Sep 2026 18:59:55 +0200 Subject: [PATCH 4/8] fix(governance): preserve honest lifecycle review outcomes Allow advisory panels to report unmet P8 requirements, resolve native evidence without redundant reviewer reruns, and use the installed scenario-rubric path. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .agents/skills/apm-review-panel/SKILL.md | 6 +- .../assets/panelist-return-schema.json | 6 +- .../assets/recommendation-template.md | 5 +- .apm/agents/test-coverage-expert.agent.md | 55 ++++++++++--------- .github/agents/test-coverage-expert.agent.md | 55 ++++++++++--------- packages/apm-review-panel/SKILL.md | 6 +- .../assets/panelist-return-schema.json | 6 +- .../assets/recommendation-template.md | 5 +- tests/fixtures/lifecycle_bug_ledger.json | 2 + tests/unit/test_lifecycle_agent_contract.py | 11 +++- 10 files changed, 88 insertions(+), 69 deletions(-) diff --git a/.agents/skills/apm-review-panel/SKILL.md b/.agents/skills/apm-review-panel/SKILL.md index a38bb5e41f..70a8b481ee 100644 --- a/.agents/skills/apm-review-panel/SKILL.md +++ b/.agents/skills/apm-review-panel/SKILL.md @@ -88,7 +88,7 @@ surfaces findings; the maintainer and the PR author decide ship. | [OSS Growth Hacker](../../agents/oss-growth-hacker.agent.md) | Adoption Strategist | Yes | | [Auth Expert](../../agents/auth-expert.agent.md) | Auth / Token Reviewer | Conditional (see below) | | [Doc Writer](../../agents/doc-writer.agent.md) | Documentation Reviewer | Conditional (see below) | -| [Test Coverage Expert](../../agents/test-coverage-expert.agent.md) | Test-Presence Reviewer (paired with DevX UX) | Yes (skipped only on docs-only PRs -- see below) | +| [Test Coverage Expert](../../agents/test-coverage-expert.agent.md) | Test-Presence Reviewer (paired with DevX UX) | Yes for src or P8 lifecycle surfaces (see below) | | [Performance Expert](../../agents/performance-expert.agent.md) | Package-Manager Performance Reviewer | Conditional (see below) | | [APM CEO](../../agents/apm-ceo.agent.md) | Strategic Arbiter / Synthesizer | Yes | @@ -311,8 +311,8 @@ no comment can be rendered, an explicit `noop` (step 9) -- are emitted. invent diagrams). - The recommended follow-ups list renders the CEO's curated subset, not every finding. Full per-persona findings collapse at the bottom. - - NEVER render the words "Verdict", "APPROVE", "REJECT", "blocked", - "merge gate", or any equivalent. The panel is advisory. + - Follow the template's advisory-versus-P8 distinction. Reporting an + unmet repository shipping precondition is not a panel verdict. 8. **Sweep labels** via `safe-outputs.remove-labels`. The list MUST be `[panel-review, panel-approved, panel-rejected]` -- always all three, diff --git a/.agents/skills/apm-review-panel/assets/panelist-return-schema.json b/.agents/skills/apm-review-panel/assets/panelist-return-schema.json index bf2babd927..dfb0a179e9 100644 --- a/.agents/skills/apm-review-panel/assets/panelist-return-schema.json +++ b/.agents/skills/apm-review-panel/assets/panelist-return-schema.json @@ -96,7 +96,7 @@ }, "evidence": { "type": "object", - "description": "Optional irrefutable evidence backing this finding -- a real test that exists (passed/failed) or a real test that should exist but does not (missing). When present, the apm-ceo synthesizer treats this as LOAD-BEARING: a passed test proves the asserted user promise holds AT THE STATED TIER; a failed test proves it does not; a missing test on a critical-promise surface proves the regression-trap gap. Tier matters: a `unit` passed does NOT certify the promise at integration-with-fixtures or e2e tier, and a critical user-promise surface (CLI command, install pipeline, lockfile, auth, hooks, marketplace, cross-module integration -- see test-coverage-expert tier-floor matrix) requires `integration-with-fixtures` or `e2e` to fully certify. The CEO does not arbitrate against a passed/failed outcome except by naming a specific reason (test was wrong, tier was below the surface floor, environment issue, flakiness with run-count). REQUIRED on every test-coverage-expert finding (the persona's contract); STRONGLY ENCOURAGED on any finding from any other persona that points at a test (e.g. supply-chain-security citing a test proving an exploit, devx-ux citing a test proving an error wording).", + "description": "Candidate-bound evidence at the stated tier, not a claim of exhaustive correctness. REQUIRED on every test-coverage-expert finding; encouraged on other test-backed findings. Preserve actual passed, failed, missing or unknown states. Unit evidence cannot substitute for the surface's integration or lifecycle floor. The CEO may question assertion semantics, tier, environment or demonstrated flakiness, but cannot waive missing required P8 evidence.", "required": ["outcome", "tier"], "additionalProperties": false, "properties": { @@ -120,7 +120,7 @@ }, "run_evidence": { "type": "string", - "description": "Optional. For `outcome=passed` at `tier=integration-with-fixtures` or `e2e` on a critical-promise surface, the persona SHOULD have actually run the test (not just read it) and recorded the pytest invocation + pass/fail line + duration here. Verbatim, under 240 chars. Reading code is not running code (S7 DETERMINISTIC TOOL BRIDGE: facts-that-must-be-true do not survive as LLM assertions)." + "description": "Native candidate-bound invocation, outcome and duration, or a tool-verified native report reference. Under 240 chars. Required for passed lifecycle evidence. Reviewers may resolve another thread's native run through tools; reading test source alone does not establish execution." }, "assertion_excerpt": { "type": "string", @@ -128,7 +128,7 @@ }, "proves": { "type": "string", - "description": "One sentence in USER words naming the promise the test proves (or fails to prove if missing). MUST mirror a Scenario Evidence row from the PR body when one exists -- the rubric (.github/skills/pr-description-skill/assets/scenario-evidence-rubric.md) is the shared vocabulary." + "description": "One sentence in USER words naming the observed promise or missing proof. MUST mirror a Scenario Evidence row when present. Resolve and probe .agents/skills/pr-description-skill/assets/scenario-evidence-rubric.md from the consumer repository root for the shared vocabulary." }, "principles": { "type": "array", diff --git a/.agents/skills/apm-review-panel/assets/recommendation-template.md b/.agents/skills/apm-review-panel/assets/recommendation-template.md index d28efe2bcb..b786ebcb83 100644 --- a/.agents/skills/apm-review-panel/assets/recommendation-template.md +++ b/.agents/skills/apm-review-panel/assets/recommendation-template.md @@ -11,8 +11,9 @@ Everything else is collapsed or omitted unless it adds signal. RENDERING RULES (the orchestrator follows these literally): - ASCII only. No emojis, no Unicode dashes, no box-drawing characters. -- The panel is ADVISORY. NEVER render the words "Verdict", "APPROVE", - "REJECT", "blocked", "merge gate", or any equivalent. +- The panel emits no verdict labels or merge checks. Report unmet P8 + evidence explicitly: "P8 shipping precondition unsatisfied; return to + implementation." This reports repository policy, not a panel-imposed gate. - Sections are SKIPPED (not rendered as empty placeholders) when their source field is empty or missing. A section that adds no signal is worse than no section. diff --git a/.apm/agents/test-coverage-expert.agent.md b/.apm/agents/test-coverage-expert.agent.md index 18f89ad381..fb153c16c6 100644 --- a/.apm/agents/test-coverage-expert.agent.md +++ b/.apm/agents/test-coverage-expert.agent.md @@ -132,27 +132,23 @@ Three new disciplines follow from this matrix: the user-promise is not yet certified end-to-end. 2. **S7 PROBE RULE on integration evidence.** When you return `outcome: passed` at `tier: integration-with-fixtures`, `e2e`, or - `lifecycle-state-machine` on - a critical-promise surface, you MUST have RUN the test (not just - read it) within this review. Capture the pytest invocation + the - pass/fail line + duration in `evidence.run_evidence` (verbatim, - under 240 chars). Reading test code is LLM assertion; running it - against real fixtures supplies bounded execution evidence. If the - test requires a credential you don't have (e.g. `GITHUB_APM_PAT`), - note the skip in `evidence.run_evidence` and downgrade `outcome` - to `unknown` for that row -- do NOT certify on a read. + `lifecycle-state-machine` on a critical-promise surface, resolve + candidate-bound native execution through tools. Record its invocation, + outcome and duration, or a verified native report reference, in + `evidence.run_evidence`. Source inspection alone never establishes a + pass. Rerun targeted diagnostics when needed, not an identical suite + solely because another thread ran it. Unavailable, skipped or stale + execution is `unknown`, with the reason recorded; do not certify it. 3. **ApmLifecycle contract-test signal (LOAD-BEARING).** When the diff touches durable state, you MUST return an explicit signal about whether an ApmLifecycle contract test is added, adjusted, or missing. Silence is not an acceptable answer on these surfaces. See the next section for the engine and the trigger list. -## The ApmLifecycle engine (your highest-proof instrument) +## The ApmLifecycle engine -APM ships a real-CLI state-machine harness. It is the most powerful -verification tool in this repo and the one most often skipped, because -unit tests are cheaper to write and LOOK like proof. Know it and -demand it. +APM ships a real-CLI state-machine harness for cross-command contracts. +Use it alongside lower-tier tests, not as a claim of exhaustive proof. **What it is.** `tests/integration/test_required_lifecycle_state_machine.py` is the canonical module. Its machinery: @@ -169,26 +165,29 @@ is the canonical module. Its machinery: - markers: `integration`, `e2e`, `lifecycle_smoke`, `requires_apm_binary`, `requires_e2e_mode` -**Why it is different in kind.** It drives real STATE TRANSITIONS +**Why it matters.** It drives real STATE TRANSITIONS (install -> compile -> audit -> uninstall, target widen-then-narrow, -lock-then-prune) and snapshots durable state on both sides. It is the -only tier that can prove a NEGATIVE -- "this file was not written", -"this file was not deleted", "this user content survived" -- which is -exactly the class of promise that unit and integration tests cannot -certify. In full agentic coding, where a change lands without a human -walking the CLI by hand, this harness IS the human walkthrough. +lock-then-prune) and snapshots durable state on both sides. This exposes +cross-command corruption that isolated command tests do not exercise. +Independent snapshots establish observed no-write and user-content +preservation properties at tested boundaries, not correctness in every +possible state. **Applicability authority:** use the complete assessment required by `.apm/instructions/lifecycle.instructions.md`, including indirect shared state and observational changes. Do not restrict it to the entry command or to a hardcoded list of runtime paths. -**What you must return on a trigger.** One of exactly these, and never -silence: +**What you must return on a trigger.** Report the actual evidence state, +never silence: - `outcome: passed`, `tier: lifecycle-state-machine` -- an ApmLifecycle test covers this transition. `run_evidence` REQUIRED - (S7 PROBE RULE: you RAN it). Name the test and quote the assertion. + from tool-verified native execution. Name the test and quote the assertion. +- `outcome: failed`, `tier: lifecycle-state-machine` -- execution failed. + Record the failure; do not describe an existing test as absent. +- `outcome: unknown`, `tier: lifecycle-state-machine` -- execution is + unavailable, skipped or stale. Record why it cannot certify this head. - `outcome: missing`, `tier: lifecycle-state-machine` -- no lifecycle coverage for this transition. Give the exact test file, the test name you would use, the command SEQUENCE it should drive, and the @@ -198,6 +197,9 @@ silence: `tier: lifecycle-state-machine` row -- the tier-floor compliance shape, when lower-tier tests exist but the floor is unmet. +Every nonpassing required P8 row is blocking. Reviewer report consumption +does not replace the driver's full run or its parent's independent run. + **Adjusting beats adding.** Prefer extending an EXISTING lifecycle test (a new assertion, a new parametrize case) over authoring a new module. These tests are expensive in CI shard time; a sibling PR in @@ -227,8 +229,9 @@ You are the panelist who makes claims about TEST PRESENCE. Every claim of "no test exists for X" is a fact-that-must-be-true. You MUST verify it via tool calls before emitting it as a finding. The procedure: -1. **Read the PR body's Scenario Evidence table FIRST** (governed by - `.github/skills/pr-description-skill/assets/scenario-evidence-rubric.md`). +1. **Read the PR body's Scenario Evidence table FIRST**. Resolve and + probe `.agents/skills/pr-description-skill/assets/scenario-evidence-rubric.md` + from the consumer repository root before using its vocabulary. It is the author's stated proof that the change works for each user-promise scenario, mapped to the APM principle the scenario serves (Portability / Secure by default / Governed by policy / diff --git a/.github/agents/test-coverage-expert.agent.md b/.github/agents/test-coverage-expert.agent.md index ec421483e2..323f17c951 100644 --- a/.github/agents/test-coverage-expert.agent.md +++ b/.github/agents/test-coverage-expert.agent.md @@ -132,27 +132,23 @@ Three new disciplines follow from this matrix: the user-promise is not yet certified end-to-end. 2. **S7 PROBE RULE on integration evidence.** When you return `outcome: passed` at `tier: integration-with-fixtures`, `e2e`, or - `lifecycle-state-machine` on - a critical-promise surface, you MUST have RUN the test (not just - read it) within this review. Capture the pytest invocation + the - pass/fail line + duration in `evidence.run_evidence` (verbatim, - under 240 chars). Reading test code is LLM assertion; running it - against real fixtures supplies bounded execution evidence. If the - test requires a credential you don't have (e.g. `GITHUB_APM_PAT`), - note the skip in `evidence.run_evidence` and downgrade `outcome` - to `unknown` for that row -- do NOT certify on a read. + `lifecycle-state-machine` on a critical-promise surface, resolve + candidate-bound native execution through tools. Record its invocation, + outcome and duration, or a verified native report reference, in + `evidence.run_evidence`. Source inspection alone never establishes a + pass. Rerun targeted diagnostics when needed, not an identical suite + solely because another thread ran it. Unavailable, skipped or stale + execution is `unknown`, with the reason recorded; do not certify it. 3. **ApmLifecycle contract-test signal (LOAD-BEARING).** When the diff touches durable state, you MUST return an explicit signal about whether an ApmLifecycle contract test is added, adjusted, or missing. Silence is not an acceptable answer on these surfaces. See the next section for the engine and the trigger list. -## The ApmLifecycle engine (your highest-proof instrument) +## The ApmLifecycle engine -APM ships a real-CLI state-machine harness. It is the most powerful -verification tool in this repo and the one most often skipped, because -unit tests are cheaper to write and LOOK like proof. Know it and -demand it. +APM ships a real-CLI state-machine harness for cross-command contracts. +Use it alongside lower-tier tests, not as a claim of exhaustive proof. **What it is.** `tests/integration/test_required_lifecycle_state_machine.py` is the canonical module. Its machinery: @@ -169,26 +165,29 @@ is the canonical module. Its machinery: - markers: `integration`, `e2e`, `lifecycle_smoke`, `requires_apm_binary`, `requires_e2e_mode` -**Why it is different in kind.** It drives real STATE TRANSITIONS +**Why it matters.** It drives real STATE TRANSITIONS (install -> compile -> audit -> uninstall, target widen-then-narrow, -lock-then-prune) and snapshots durable state on both sides. It is the -only tier that can prove a NEGATIVE -- "this file was not written", -"this file was not deleted", "this user content survived" -- which is -exactly the class of promise that unit and integration tests cannot -certify. In full agentic coding, where a change lands without a human -walking the CLI by hand, this harness IS the human walkthrough. +lock-then-prune) and snapshots durable state on both sides. This exposes +cross-command corruption that isolated command tests do not exercise. +Independent snapshots establish observed no-write and user-content +preservation properties at tested boundaries, not correctness in every +possible state. **Applicability authority:** use the complete assessment required by `.apm/instructions/lifecycle.instructions.md`, including indirect shared state and observational changes. Do not restrict it to the entry command or to a hardcoded list of runtime paths. -**What you must return on a trigger.** One of exactly these, and never -silence: +**What you must return on a trigger.** Report the actual evidence state, +never silence: - `outcome: passed`, `tier: lifecycle-state-machine` -- an ApmLifecycle test covers this transition. `run_evidence` REQUIRED - (S7 PROBE RULE: you RAN it). Name the test and quote the assertion. + from tool-verified native execution. Name the test and quote the assertion. +- `outcome: failed`, `tier: lifecycle-state-machine` -- execution failed. + Record the failure; do not describe an existing test as absent. +- `outcome: unknown`, `tier: lifecycle-state-machine` -- execution is + unavailable, skipped or stale. Record why it cannot certify this head. - `outcome: missing`, `tier: lifecycle-state-machine` -- no lifecycle coverage for this transition. Give the exact test file, the test name you would use, the command SEQUENCE it should drive, and the @@ -198,6 +197,9 @@ silence: `tier: lifecycle-state-machine` row -- the tier-floor compliance shape, when lower-tier tests exist but the floor is unmet. +Every nonpassing required P8 row is blocking. Reviewer report consumption +does not replace the driver's full run or its parent's independent run. + **Adjusting beats adding.** Prefer extending an EXISTING lifecycle test (a new assertion, a new parametrize case) over authoring a new module. These tests are expensive in CI shard time; a sibling PR in @@ -227,8 +229,9 @@ You are the panelist who makes claims about TEST PRESENCE. Every claim of "no test exists for X" is a fact-that-must-be-true. You MUST verify it via tool calls before emitting it as a finding. The procedure: -1. **Read the PR body's Scenario Evidence table FIRST** (governed by - `.github/skills/pr-description-skill/assets/scenario-evidence-rubric.md`). +1. **Read the PR body's Scenario Evidence table FIRST**. Resolve and + probe `.agents/skills/pr-description-skill/assets/scenario-evidence-rubric.md` + from the consumer repository root before using its vocabulary. It is the author's stated proof that the change works for each user-promise scenario, mapped to the APM principle the scenario serves (Portability / Secure by default / Governed by policy / diff --git a/packages/apm-review-panel/SKILL.md b/packages/apm-review-panel/SKILL.md index a38bb5e41f..70a8b481ee 100644 --- a/packages/apm-review-panel/SKILL.md +++ b/packages/apm-review-panel/SKILL.md @@ -88,7 +88,7 @@ surfaces findings; the maintainer and the PR author decide ship. | [OSS Growth Hacker](../../agents/oss-growth-hacker.agent.md) | Adoption Strategist | Yes | | [Auth Expert](../../agents/auth-expert.agent.md) | Auth / Token Reviewer | Conditional (see below) | | [Doc Writer](../../agents/doc-writer.agent.md) | Documentation Reviewer | Conditional (see below) | -| [Test Coverage Expert](../../agents/test-coverage-expert.agent.md) | Test-Presence Reviewer (paired with DevX UX) | Yes (skipped only on docs-only PRs -- see below) | +| [Test Coverage Expert](../../agents/test-coverage-expert.agent.md) | Test-Presence Reviewer (paired with DevX UX) | Yes for src or P8 lifecycle surfaces (see below) | | [Performance Expert](../../agents/performance-expert.agent.md) | Package-Manager Performance Reviewer | Conditional (see below) | | [APM CEO](../../agents/apm-ceo.agent.md) | Strategic Arbiter / Synthesizer | Yes | @@ -311,8 +311,8 @@ no comment can be rendered, an explicit `noop` (step 9) -- are emitted. invent diagrams). - The recommended follow-ups list renders the CEO's curated subset, not every finding. Full per-persona findings collapse at the bottom. - - NEVER render the words "Verdict", "APPROVE", "REJECT", "blocked", - "merge gate", or any equivalent. The panel is advisory. + - Follow the template's advisory-versus-P8 distinction. Reporting an + unmet repository shipping precondition is not a panel verdict. 8. **Sweep labels** via `safe-outputs.remove-labels`. The list MUST be `[panel-review, panel-approved, panel-rejected]` -- always all three, diff --git a/packages/apm-review-panel/assets/panelist-return-schema.json b/packages/apm-review-panel/assets/panelist-return-schema.json index bf2babd927..dfb0a179e9 100644 --- a/packages/apm-review-panel/assets/panelist-return-schema.json +++ b/packages/apm-review-panel/assets/panelist-return-schema.json @@ -96,7 +96,7 @@ }, "evidence": { "type": "object", - "description": "Optional irrefutable evidence backing this finding -- a real test that exists (passed/failed) or a real test that should exist but does not (missing). When present, the apm-ceo synthesizer treats this as LOAD-BEARING: a passed test proves the asserted user promise holds AT THE STATED TIER; a failed test proves it does not; a missing test on a critical-promise surface proves the regression-trap gap. Tier matters: a `unit` passed does NOT certify the promise at integration-with-fixtures or e2e tier, and a critical user-promise surface (CLI command, install pipeline, lockfile, auth, hooks, marketplace, cross-module integration -- see test-coverage-expert tier-floor matrix) requires `integration-with-fixtures` or `e2e` to fully certify. The CEO does not arbitrate against a passed/failed outcome except by naming a specific reason (test was wrong, tier was below the surface floor, environment issue, flakiness with run-count). REQUIRED on every test-coverage-expert finding (the persona's contract); STRONGLY ENCOURAGED on any finding from any other persona that points at a test (e.g. supply-chain-security citing a test proving an exploit, devx-ux citing a test proving an error wording).", + "description": "Candidate-bound evidence at the stated tier, not a claim of exhaustive correctness. REQUIRED on every test-coverage-expert finding; encouraged on other test-backed findings. Preserve actual passed, failed, missing or unknown states. Unit evidence cannot substitute for the surface's integration or lifecycle floor. The CEO may question assertion semantics, tier, environment or demonstrated flakiness, but cannot waive missing required P8 evidence.", "required": ["outcome", "tier"], "additionalProperties": false, "properties": { @@ -120,7 +120,7 @@ }, "run_evidence": { "type": "string", - "description": "Optional. For `outcome=passed` at `tier=integration-with-fixtures` or `e2e` on a critical-promise surface, the persona SHOULD have actually run the test (not just read it) and recorded the pytest invocation + pass/fail line + duration here. Verbatim, under 240 chars. Reading code is not running code (S7 DETERMINISTIC TOOL BRIDGE: facts-that-must-be-true do not survive as LLM assertions)." + "description": "Native candidate-bound invocation, outcome and duration, or a tool-verified native report reference. Under 240 chars. Required for passed lifecycle evidence. Reviewers may resolve another thread's native run through tools; reading test source alone does not establish execution." }, "assertion_excerpt": { "type": "string", @@ -128,7 +128,7 @@ }, "proves": { "type": "string", - "description": "One sentence in USER words naming the promise the test proves (or fails to prove if missing). MUST mirror a Scenario Evidence row from the PR body when one exists -- the rubric (.github/skills/pr-description-skill/assets/scenario-evidence-rubric.md) is the shared vocabulary." + "description": "One sentence in USER words naming the observed promise or missing proof. MUST mirror a Scenario Evidence row when present. Resolve and probe .agents/skills/pr-description-skill/assets/scenario-evidence-rubric.md from the consumer repository root for the shared vocabulary." }, "principles": { "type": "array", diff --git a/packages/apm-review-panel/assets/recommendation-template.md b/packages/apm-review-panel/assets/recommendation-template.md index d28efe2bcb..b786ebcb83 100644 --- a/packages/apm-review-panel/assets/recommendation-template.md +++ b/packages/apm-review-panel/assets/recommendation-template.md @@ -11,8 +11,9 @@ Everything else is collapsed or omitted unless it adds signal. RENDERING RULES (the orchestrator follows these literally): - ASCII only. No emojis, no Unicode dashes, no box-drawing characters. -- The panel is ADVISORY. NEVER render the words "Verdict", "APPROVE", - "REJECT", "blocked", "merge gate", or any equivalent. +- The panel emits no verdict labels or merge checks. Report unmet P8 + evidence explicitly: "P8 shipping precondition unsatisfied; return to + implementation." This reports repository policy, not a panel-imposed gate. - Sections are SKIPPED (not rendered as empty placeholders) when their source field is empty or missing. A section that adds no signal is worse than no section. diff --git a/tests/fixtures/lifecycle_bug_ledger.json b/tests/fixtures/lifecycle_bug_ledger.json index b3a7094839..6a264b65a5 100644 --- a/tests/fixtures/lifecycle_bug_ledger.json +++ b/tests/fixtures/lifecycle_bug_ledger.json @@ -35,6 +35,8 @@ ".agents/skills/apm-review-panel/references/conditional-panelists.md", "packages/apm-review-panel/assets/panelist-return-schema.json", ".agents/skills/apm-review-panel/assets/panelist-return-schema.json", + "packages/apm-review-panel/assets/recommendation-template.md", + ".agents/skills/apm-review-panel/assets/recommendation-template.md", "packages/pr-description-skill/assets/scenario-evidence-rubric.md", ".agents/skills/pr-description-skill/assets/scenario-evidence-rubric.md", "packages/shepherd-driver/SKILL.md", diff --git a/tests/unit/test_lifecycle_agent_contract.py b/tests/unit/test_lifecycle_agent_contract.py index 418ca06574..3a57fc9f40 100644 --- a/tests/unit/test_lifecycle_agent_contract.py +++ b/tests/unit/test_lifecycle_agent_contract.py @@ -63,6 +63,12 @@ def test_panel_retains_lazy_specialist_reference_in_deployed_bundle() -> None: ).read_bytes() +def test_panel_template_can_report_unmet_repository_precondition() -> None: + """Advisory rendering must not conceal a required shipping failure.""" + template = ROOT / "packages/apm-review-panel/assets/recommendation-template.md" + assert "P8 shipping precondition unsatisfied" in template.read_text(encoding="ascii") + + def _panel_evidence() -> dict[str, Any]: """Build the existing coverage persona's lifecycle evidence return shape.""" return { @@ -86,16 +92,19 @@ def _panel_evidence() -> dict[str, Any]: } +@pytest.mark.parametrize("outcome", ["passed", "failed", "unknown", "missing"]) @pytest.mark.parametrize("missing", [None, "test_file", "run_evidence", "assertion_excerpt"]) def test_panel_can_carry_only_complete_passing_lifecycle_evidence( missing: str | None, + outcome: str, ) -> None: """The declared persona tier must serialize without accepting empty proof.""" schema_path = ROOT / "packages/apm-review-panel/assets/panelist-return-schema.json" schema = json.loads(schema_path.read_text(encoding="ascii")) Draft7Validator.check_schema(schema) document = _panel_evidence() + document["findings"][0]["evidence"]["outcome"] = outcome if missing: del document["findings"][0]["evidence"][missing] errors = list(Draft7Validator(schema).iter_errors(document)) - assert bool(errors) is (missing is not None) + assert bool(errors) is (outcome == "passed" and missing is not None) From ba13ce588bbab14c66601632b87c17b6a1ccc3ca Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Mon, 7 Sep 2026 19:12:56 +0200 Subject: [PATCH 5/8] fix(ci): verify lifecycle contexts and completion against native execution Keep physical snapshot domains stable across reviewed APM_HOME cwd transitions, verify actual child source imports, compare terminal completion to fresh full execution, and defer duplicate deterministic smoke witnesses to the required native provider. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci-integration.yml | 6 +- .github/workflows/ci.yml | 11 +- scripts/check_lifecycle_evidence.py | 108 ++++- scripts/lifecycle_contracts.py | 43 +- tests/quality/test_ci_topology.py | 42 +- tests/unit/scripts/test_lifecycle_evidence.py | 407 +++++++++++++++++- tests/utils/isolated_apm_environment.py | 9 +- tests/utils/lifecycle_evidence.py | 162 ++++++- 8 files changed, 739 insertions(+), 49 deletions(-) diff --git a/.github/workflows/ci-integration.yml b/.github/workflows/ci-integration.yml index 40b99e0989..eb2ea12c65 100644 --- a/.github/workflows/ci-integration.yml +++ b/.github/workflows/ci-integration.yml @@ -242,7 +242,7 @@ jobs: timeout-minutes: 20 - name: Execute full lifecycle contracts - if: matrix.shard == 1 + if: github.event_name == 'merge_group' && matrix.shard == 1 env: LIFECYCLE_BASE: ${{ github.event.merge_group.base_sha }} LIFECYCLE_HEAD: ${{ github.event.merge_group.head_sha }} @@ -251,10 +251,10 @@ jobs: uv run --extra dev python scripts/check_lifecycle_evidence.py \ --base "$LIFECYCLE_BASE" --head "$LIFECYCLE_HEAD" --lane full \ --report "$RUNNER_TEMP/lifecycle-evidence-full.json" - timeout-minutes: 6 + timeout-minutes: 20 - name: Upload full lifecycle evidence - if: always() && matrix.shard == 1 + if: always() && github.event_name == 'merge_group' && matrix.shard == 1 uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: lifecycle-evidence-full diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3988b70fe0..82e6c8ef9d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -515,7 +515,9 @@ jobs: lifecycle-smoke: name: Lifecycle Smoke (Linux) runs-on: ubuntu-24.04 - timeout-minutes: 6 + # Five deterministic witnesses alone measured 201s; leave bounded room for + # the deferred-smoke remainder and dependency/runtime setup. + timeout-minutes: 12 permissions: contents: read @@ -550,10 +552,13 @@ jobs: - name: Run required lifecycle smoke subset env: APM_E2E_TESTS: "1" + LIFECYCLE_BASE: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} + LIFECYCLE_HEAD: ${{ github.sha }} run: | - uv run --extra dev pytest -p no:cacheprovider -q --strict-markers \ + uv run --extra dev python -m pytest -p no:cacheprovider -p tests.utils.lifecycle_evidence -q --strict-markers \ -n 2 --dist loadgroup \ - -m 'lifecycle_smoke and not lifecycle_merge_group' tests/integration + -m 'lifecycle_smoke and not lifecycle_merge_group' tests/integration \ + --lifecycle-defer-base "$LIFECYCLE_BASE" --lifecycle-defer-head "$LIFECYCLE_HEAD" - name: Execute changed lifecycle contracts env: diff --git a/scripts/check_lifecycle_evidence.py b/scripts/check_lifecycle_evidence.py index 017d3c7b8d..342f9b5653 100644 --- a/scripts/check_lifecycle_evidence.py +++ b/scripts/check_lifecycle_evidence.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import hashlib import json import os import subprocess @@ -16,12 +17,10 @@ sys.path.insert(0, str(ROOT)) from scripts.lifecycle_contracts import ( # noqa: E402 - LEDGER, EvidenceError, + candidate_contracts, command_inventory, git, - load_ledger, - select_contracts, validate_execution, ) @@ -33,6 +32,16 @@ ] +class CompletionOutputConflict(EvidenceError): + """The proposed independent report would overwrite the driver's inputs.""" + + +def _same_file(left: Path, right: Path) -> bool: + return left.resolve() == right.resolve() or ( + left.exists() and right.exists() and left.samefile(right) + ) + + def source_profile(root: Path) -> tuple[Path | None, dict[str, Any]]: """Identify the installed editable source and its exact interpreter/launcher.""" import apm_cli @@ -48,6 +57,7 @@ def source_profile(root: Path) -> tuple[Path | None, dict[str, Any]]: raise EvidenceError("Installed apm entrypoint is not this Python environment") return executable, { "kind": "source-python", + "source_root": str(root / "src/apm_cli"), "executable": str(executable) if executable else None, "executable_sha256": fingerprint(executable) if executable else None, "python": str(Path(sys.executable).resolve()), @@ -73,6 +83,53 @@ def candidate(root: Path, base: str, head: str) -> dict[str, str]: } +def validate_completion(path: Path, native: dict[str, Any], output: Path) -> None: + """Compare a driver's summary to independent execution, never use it as proof.""" + if _same_file(output, path): + raise CompletionOutputConflict("Independent report must not overwrite completion") + completion = json.loads(path.read_text(encoding="utf-8")) + summary = completion["lifecycle_evidence"] + if not isinstance(summary, dict): + raise EvidenceError("Malformed lifecycle completion summary") + report_path = summary.get("report_path") + if not isinstance(report_path, str) or not report_path.strip(): + raise EvidenceError("Completion requires its driver's report_path") + driver_path = Path(report_path) + if not driver_path.is_absolute(): + driver_path = path.parent / driver_path + if _same_file(output, driver_path): + raise CompletionOutputConflict("Independent report must not overwrite driver evidence") + if type(summary.get("version")) is not int: + raise EvidenceError("Malformed lifecycle completion version") + if summary["version"] != 1 or summary.get("lane") != "full": + raise EvidenceError("Completion requires version 1 full-lane evidence") + if summary.get("status") not in {"passed", "not_applicable"}: + raise EvidenceError("Completion requires passed or not_applicable evidence") + fields = { + "version": "version", + "base_sha": "base", + "head_sha": "head", + "tested_tree": "tested_tree", + "lane": "lane", + "status": "status", + } + for claim, field in fields.items(): + if summary.get(claim) != native.get(field): + raise EvidenceError(f"Completion disagrees with fresh execution: {claim}") + if "head_sha" in completion and completion["head_sha"] != native["head"]: + raise EvidenceError("Completion top-level head_sha is stale") + raw = driver_path.read_bytes() + if summary.get("report_sha256") != hashlib.sha256(raw).hexdigest(): + raise EvidenceError("Completion driver report digest mismatch") + driver = json.loads(raw) + if ( + not isinstance(driver, dict) + or type(driver.get("version")) is not int + or any(driver.get(field) != native.get(field) for field in fields.values()) + ): + raise EvidenceError("Completion driver report header disagrees with fresh execution") + + def execute(args: argparse.Namespace) -> dict[str, Any]: """Generate native evidence from this process's own fresh pytest execution.""" identity = candidate(ROOT, args.base, args.head) @@ -88,21 +145,9 @@ def execute(args: argparse.Namespace) -> dict[str, Any]: } try: inventory = command_inventory() - base_text = git(ROOT, "show", f"{identity['base']}:{LEDGER}") - base = load_ledger(base_text) - head = load_ledger((ROOT / LEDGER).read_text(encoding="utf-8")) - changed = set( - git( - ROOT, - "diff", - "--name-only", - "--no-renames", - "-z", - identity["base"], - identity["head"], - ).split("\0") - ) - {""} - contracts = select_contracts(base, head, changed, inventory) + changed, contracts = candidate_contracts( + ROOT, identity["base"], identity["head"], inventory + ) report["contracts"] = [contract["id"] for contract in contracts] report["command_inventory"] = sorted(inventory) report["changed_paths"] = sorted(changed) @@ -126,7 +171,18 @@ def execute(args: argparse.Namespace) -> dict[str, Any]: executable, report["profile"] = source_profile(ROOT) nodeids = sorted({witness["nodeid"] for witness in witnesses}) - plugin = LifecycleEvidencePlugin(nodeids, executable) + contexts = { + nodeid: { + transition.get("context", "initial") + for witness in witnesses + if witness["nodeid"] == nodeid + for transition in witness["transitions"] + } + for nodeid in nodeids + } + plugin = LifecycleEvidencePlugin( + nodeids, executable, contexts, Path(report["profile"]["source_root"]) + ) if executable: os.environ["APM_BINARY_PATH"] = str(executable) else: @@ -160,11 +216,25 @@ def main() -> int: parser.add_argument("--head", required=True) parser.add_argument("--lane", choices=("pr", "full"), required=True) parser.add_argument("--report", type=Path, required=True) + parser.add_argument("--completion", type=Path) args = parser.parse_args() if args.report.resolve().is_relative_to(ROOT): parser.error("--report must be outside the candidate checkout") + if args.completion and args.lane != "full": + parser.error("--completion requires --lane full") try: report = execute(args) + if args.completion: + try: + validate_completion(args.completion, report, args.report) + except CompletionOutputConflict as exc: + print(f"Completion verification failed: {exc}") + return 1 + except (OSError, KeyError, TypeError, ValueError) as exc: + report["status"] = "blocked" + report["error"] = ( + f"{report.get('error', '')} Completion verification failed: {exc}" + ).strip() except (EvidenceError, subprocess.CalledProcessError) as exc: report = { "version": 1, diff --git a/scripts/lifecycle_contracts.py b/scripts/lifecycle_contracts.py index bc84f6378d..1b7996a64f 100644 --- a/scripts/lifecycle_contracts.py +++ b/scripts/lifecycle_contracts.py @@ -134,6 +134,10 @@ def validate_contracts( _paths([nodeid.split("::")[0]]) _require(len(witness["transitions"]) >= 2, f"{nodeid}: missing trajectory") for transition in witness["transitions"]: + _require( + transition.get("context", "initial") in {"initial", "APM_HOME"}, + f"{nodeid}: unsupported command context", + ) _require(transition["command"] in inventory, f"{nodeid}: unknown command") _require(type(transition["returncode"]) is int, f"{nodeid}: invalid exit status") _require( @@ -277,6 +281,18 @@ def select_contracts( return [current[key] for key in sorted(selected)] +def candidate_contracts( + root: Path, base: str, head: str, inventory: dict[str, click.Command] +) -> tuple[set[str], list[dict[str, Any]]]: + """One diff-to-obligation authority for smoke deferral and native execution.""" + previous = load_ledger(git(root, "show", f"{base}:{LEDGER}")) + current = load_ledger((root / LEDGER).read_text(encoding="utf-8")) + changed = set( + git(root, "diff", "--name-only", "--no-renames", "-z", base, head).split("\0") + ) - {""} + return changed, select_contracts(previous, current, changed, inventory) + + def validate_execution(witness: dict[str, Any], evidence: dict[str, Any]) -> None: """Check actual collection, phases, model execution and persistent trajectory.""" nodeid = witness["nodeid"] @@ -301,13 +317,34 @@ def validate_execution(witness: dict[str, Any], evidence: dict[str, Any]) -> Non ] # One trajectory cannot be assembled out of independent temporary projects. - def trajectory(event: dict[str, Any]) -> tuple[str, str, int | None]: + def trajectory(event: dict[str, Any]) -> tuple[str, int | None]: _require(event.get("roots"), f"{nodeid}: missing durable-root identity") - return event["cwd"], json.dumps(event["roots"], sort_keys=True), event.get("model") + environment = event.get("environment") + _require( + isinstance(environment, dict) + and set(environment) == set(event["roots"]) - {"workspace"} + and all(isinstance(value, str) and value for value in environment.values()), + f"{nodeid}: missing or inconsistent environment identity", + ) + context = event.get("context", "initial") + _require(context in {"initial", "APM_HOME"}, f"{nodeid}: unknown observed context") + root = "workspace" if context == "initial" else context + _require( + event["cwd"] == event["roots"].get(root), + f"{nodeid}: cwd is outside its observed context", + ) + return ( + json.dumps([event["roots"], environment], sort_keys=True), + event.get("model"), + ) def matches(event: dict[str, Any], expected: dict[str, Any]) -> bool: return ( - event["command"] == expected["command"] + event["cwd"] + == event["roots"].get( + "workspace" if expected.get("context", "initial") == "initial" else "APM_HOME" + ) + and event["command"] == expected["command"] and event["returncode"] == expected["returncode"] and all(arg in event["args"] for arg in expected["argv_contains"]) and ( diff --git a/tests/quality/test_ci_topology.py b/tests/quality/test_ci_topology.py index 54add7b5af..2db06b2001 100644 --- a/tests/quality/test_ci_topology.py +++ b/tests/quality/test_ci_topology.py @@ -63,7 +63,7 @@ LIFECYCLE_SMOKE_JOB = "lifecycle-smoke" LIFECYCLE_SMOKE_CHECK = "Lifecycle Smoke (Linux)" LIFECYCLE_SMOKE_RUN_STEP = "Run required lifecycle smoke subset" -LIFECYCLE_SMOKE_MAX_TIMEOUT_MINUTES = 6 +LIFECYCLE_SMOKE_MAX_TIMEOUT_MINUTES = 12 LIFECYCLE_SMOKE_MARKER = "lifecycle_smoke" LIFECYCLE_SMOKE_ROOT = "tests/integration" LIFECYCLE_SMOKE_E2E_ENV = "APM_E2E_TESTS" @@ -85,9 +85,26 @@ FORBIDDEN_CREDENTIAL_EXPRESSIONS = (*FORBIDDEN_CREDENTIAL_ENV, "github.token") +def _assert_lifecycle_provider_pair(job: WorkflowNode) -> None: + smoke = workflow_step(job, LIFECYCLE_SMOKE_RUN_STEP) + native = workflow_step(job, "Execute changed lifecycle contracts") + assert job["steps"].index(native) == job["steps"].index(smoke) + 1 + assert "if" not in native and "continue-on-error" not in native + assert "scripts/check_lifecycle_evidence.py" in native["run"] + assert '--base "$LIFECYCLE_BASE" --head "$LIFECYCLE_HEAD" --lane pr' in native["run"] + assert "-p tests.utils.lifecycle_evidence" in smoke["run"] + assert "python -m pytest" in smoke["run"] + assert '--lifecycle-defer-base "$LIFECYCLE_BASE"' in smoke["run"] + assert '--lifecycle-defer-head "$LIFECYCLE_HEAD"' in smoke["run"] + assert all( + smoke["env"][key] == native["env"][key] for key in ("LIFECYCLE_BASE", "LIFECYCLE_HEAD") + ) + + def test_lifecycle_evidence_is_executed_in_required_candidate_lanes() -> None: """A saved receipt cannot replace either required executing provider lane.""" smoke = workflow_job(load_workflow(REPO_ROOT / ".github/workflows/ci.yml"), LIFECYCLE_SMOKE_JOB) + _assert_lifecycle_provider_pair(smoke) step = workflow_step(smoke, "Execute changed lifecycle contracts") assert "scripts/check_lifecycle_evidence.py" in step["run"] assert '--base "$LIFECYCLE_BASE" --head "$LIFECYCLE_HEAD" --lane pr' in step["run"] @@ -100,7 +117,8 @@ def test_lifecycle_evidence_is_executed_in_required_candidate_lanes() -> None: load_workflow(MERGE_GROUP_INTEGRATION_WORKFLOW), MERGE_GROUP_INTEGRATION_JOB ) full = workflow_step(integration, "Execute full lifecycle contracts") - assert full["if"] == "matrix.shard == 1" + assert full["if"] == "github.event_name == 'merge_group' && matrix.shard == 1" + assert full["timeout-minutes"] == 20 assert "scripts/check_lifecycle_evidence.py" in full["run"] assert '--base "$LIFECYCLE_BASE" --head "$LIFECYCLE_HEAD" --lane full' in full["run"] assert full["env"]["LIFECYCLE_BASE"] == "${{ github.event.merge_group.base_sha }}" @@ -109,6 +127,22 @@ def test_lifecycle_evidence_is_executed_in_required_candidate_lanes() -> None: assert "continue-on-error" not in step and "continue-on-error" not in full +@pytest.mark.parametrize("mutation", ["missing", "conditional", "base", "head"]) +def test_smoke_deferral_cannot_outlive_its_mandatory_provider(mutation: str) -> None: + job = deepcopy( + workflow_job(load_workflow(REPO_ROOT / ".github/workflows/ci.yml"), LIFECYCLE_SMOKE_JOB) + ) + native = workflow_step(job, "Execute changed lifecycle contracts") + if mutation == "missing": + job["steps"].remove(native) + elif mutation == "conditional": + native["if"] = "false" + else: + native["env"][f"LIFECYCLE_{mutation.upper()}"] = "different-candidate" + with pytest.raises((AssertionError, KeyError)): + _assert_lifecycle_provider_pair(job) + + def _pytest_ini_markers() -> list[str]: """The registered marker declarations from [tool.pytest.ini_options].markers in pyproject.toml -- the single @@ -528,7 +562,7 @@ def _assert_lifecycle_smoke_command(job: WorkflowNode) -> None: "lifecycle-smoke must use loadgroup to preserve xdist_group serialization" ) assert "-m" in tokens - marker_index = tokens.index("-m") + marker_index = tokens.index("-m", tokens.index("pytest") + 1) assert tokens[marker_index + 1] == LIFECYCLE_SMOKE_REQUIRED_EXPRESSION targets = [token for token in tokens if token.startswith("tests/")] assert targets == [LIFECYCLE_SMOKE_ROOT], ( @@ -580,6 +614,8 @@ def _assert_lifecycle_smoke_e2e_mode(job: WorkflowNode) -> None: run_step = workflow_step(job, LIFECYCLE_SMOKE_RUN_STEP) assert run_step.get("env") == { LIFECYCLE_SMOKE_E2E_ENV: LIFECYCLE_SMOKE_E2E_VALUE, + "LIFECYCLE_BASE": "${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}", + "LIFECYCLE_HEAD": "${{ github.sha }}", }, ( f"{LIFECYCLE_SMOKE_RUN_STEP!r} must bind exactly " f"{LIFECYCLE_SMOKE_E2E_ENV}={LIFECYCLE_SMOKE_E2E_VALUE!r}" diff --git a/tests/unit/scripts/test_lifecycle_evidence.py b/tests/unit/scripts/test_lifecycle_evidence.py index 80d7c53fe0..0a58c01453 100644 --- a/tests/unit/scripts/test_lifecycle_evidence.py +++ b/tests/unit/scripts/test_lifecycle_evidence.py @@ -4,17 +4,25 @@ import argparse import copy +import hashlib import json import os import subprocess import sys from pathlib import Path +from types import SimpleNamespace from typing import Any import click import pytest -from scripts.check_lifecycle_evidence import candidate, execute, main, source_profile +from scripts.check_lifecycle_evidence import ( + candidate, + execute, + main, + source_profile, + validate_completion, +) from scripts.lifecycle_contracts import ( EvidenceError, command_inventory, @@ -24,7 +32,7 @@ validate_contracts, validate_execution, ) -from tests.utils.lifecycle_evidence import LifecycleEvidencePlugin +from tests.utils.lifecycle_evidence import LifecycleEvidencePlugin, snapshot pytestmark = pytest.mark.component pytest_plugins = ["pytester"] @@ -91,6 +99,7 @@ def _evidence() -> dict[str, Any]: "returncode": 0, "cwd": "/fixture", "roots": {"HOME": "/fixture/home", "workspace": "/fixture"}, + "environment": {"HOME": "/fixture/home"}, "model": 1, "before": before, "after": after, @@ -199,6 +208,7 @@ def test_incomplete_contract_fails(mutation: str) -> None: "roots", "continuity", "model-splice", + "environment", ], ) def test_execution_obligations_fail_closed(mutation: str) -> None: @@ -232,6 +242,8 @@ def test_execution_obligations_fail_closed(mutation: str) -> None: elif mutation == "model-splice": evidence["models"] = 2 evidence["events"][1]["model"] = 2 + elif mutation == "environment": + del evidence["events"][1]["environment"] else: evidence = {"status": "passed"} with pytest.raises(EvidenceError): @@ -248,6 +260,213 @@ def test_intentional_fixture_mutation_requires_reviewed_preparation() -> None: validate_execution(witness, evidence) +def test_reviewed_apm_home_context_retains_one_trajectory() -> None: + witness = _ledger()["lifecycle_contracts"][0]["witnesses"][1] + evidence = _evidence() + for event in evidence["events"]: + event["roots"]["APM_HOME"] = "/fixture/home/.apm" + event["environment"]["APM_HOME"] = "/fixture/home/.apm" + evidence["events"][1].update(cwd="/fixture/home/.apm", context="APM_HOME") + with pytest.raises(EvidenceError, match="trajectory"): + validate_execution(witness, evidence) + witness["transitions"][1]["context"] = "APM_HOME" + validate_execution(witness, evidence) + evidence["events"][1]["cwd"] = "/fixture/unrelated" + with pytest.raises(EvidenceError, match="outside its observed context"): + validate_execution(witness, evidence) + + +def test_domain_keeps_snapshot_roots_stable_across_physical_apm_home( + tmp_path: Path, +) -> None: + caller = tmp_path / "caller" + home = tmp_path / "home" + caller.mkdir() + (home / ".apm").mkdir(parents=True) + env = { + "HOME": str(home), + "APM_HOME": str(home / ".apm"), + "CLAUDE_CONFIG_DIR": str(tmp_path / "claude"), + "HERMES_HOME": str(tmp_path / "hermes"), + } + plugin = LifecycleEvidencePlugin(["test"], None, {"test": {"APM_HOME"}}) + plugin.active = "test" + initial, _, first_context = plugin._domain(caller, env) + before = snapshot(initial) + global_roots, _, context = plugin._domain(home / ".apm", env) + assert first_context == "initial" and context == "APM_HOME" + assert global_roots == initial + assert snapshot(global_roots) == before + for key in ("CLAUDE_CONFIG_DIR", "HERMES_HOME"): + root = Path(env[key]) + root.mkdir() + (root / "sentinel").write_text("external target bytes", encoding="ascii") + assert snapshot(global_roots) != before + before = snapshot(global_roots) + with pytest.raises(EvidenceError, match="Unreviewed"): + plugin._domain(tmp_path / "unrelated", env) + for key in ("HOME", "APM_HOME"): + with pytest.raises(EvidenceError, match="durable roots changed"): + plugin._domain(caller, {**env, key: str(tmp_path / "other")}) + undeclared = LifecycleEvidencePlugin(["test"], None) + undeclared.active = "test" + undeclared._domain(caller, env) + with pytest.raises(EvidenceError, match="Undeclared"): + undeclared._domain(home / ".apm", env) + + +def test_domain_does_not_splice_distinct_hypothesis_instances(tmp_path: Path) -> None: + plugin = LifecycleEvidencePlugin(["test"], None) + plugin.active, plugin.model_run = "test", 1 + first, _, _ = plugin._domain(tmp_path / "first/caller", {"HOME": str(tmp_path / "first/home")}) + second, _, _ = plugin._domain( + tmp_path / "second/caller", {"HOME": str(tmp_path / "second/home")} + ) + assert first != second + witness = _ledger()["lifecycle_contracts"][0]["witnesses"][1] + evidence = _evidence() + for event, roots in zip(evidence["events"], (first, second), strict=True): + event.update(roots=roots, cwd=roots["workspace"]) + with pytest.raises(EvidenceError, match="trajectory"): + validate_execution(witness, evidence) + + +def test_child_source_probe_rejects_old_checkout_even_when_cli_would_pass( + tmp_path: Path, +) -> None: + old = tmp_path / "old" + package = old / "apm_cli" + package.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="ascii") + (package / "cli.py").write_text( + "if __name__ == '__main__':\n print('expected-success')\n", encoding="ascii" + ) + env = {**os.environ, "PYTHONPATH": str(old), "HOME": str(tmp_path / "home")} + result = subprocess.run( + [sys.executable, "-m", "apm_cli.cli", "--version"], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0 and result.stdout.strip() == "expected-success" + plugin = LifecycleEvidencePlugin(["test"], None) + assert plugin.candidate_source != package + with pytest.raises(EvidenceError, match="Child source identity"): + plugin._source_identity(tmp_path, env, 10) + good = {**env, "PYTHONPATH": str(plugin.candidate_source.parent)} + assert plugin._source_identity(tmp_path, good, 10)["package"] == str(plugin.candidate_source) + with pytest.raises(EvidenceError, match="Child source identity"): + plugin._source_identity(tmp_path, env, 10) + + +@pytest.mark.parametrize("has_contract", [False, True]) +def test_smoke_deferral_leaves_proof_to_fresh_native_execution( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, has_contract: bool +) -> None: + from scripts import check_lifecycle_evidence as gate + from tests.utils import lifecycle_evidence as observer + + contract = _ledger()["lifecycle_contracts"][0] + deferred = contract["witnesses"][0] + removed = [] + config = SimpleNamespace( + rootpath=tmp_path, + getoption=lambda name: "base" if name.endswith("base") else "head", + hook=SimpleNamespace(pytest_deselected=lambda items: removed.extend(items)), + ) + monkeypatch.setattr(gate, "candidate", lambda *args: {"base": "base", "head": "head"}) + monkeypatch.setattr( + observer, "candidate_contracts", lambda *args: (set(), [contract] if has_contract else []) + ) + items = [SimpleNamespace(nodeid=w["nodeid"]) for w in contract["witnesses"]] + items.append(SimpleNamespace(nodeid="tests/test_unrelated.py::test_other")) + observer.pytest_collection_modifyitems(config, items) + assert len(removed) == int(has_contract) + assert any("unrelated" in item.nodeid for item in items) + assert any("test_generated" in item.nodeid for item in items) + if has_contract: + assert removed[0].nodeid == deferred["nodeid"] + with pytest.raises(EvidenceError, match="not collected"): + validate_execution(deferred, {}) + failed = _evidence() + failed["phases"]["call"] = "failed" + with pytest.raises(EvidenceError, match="must pass"): + validate_execution(deferred, failed) + + +def test_smoke_deferral_rejects_stale_comparison( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from scripts import check_lifecycle_evidence as gate + from tests.utils import lifecycle_evidence as observer + + config = SimpleNamespace(rootpath=tmp_path, getoption=lambda _name: "stale") + + def reject(*args: object) -> None: + raise EvidenceError("Requested head is not the checked-out candidate") + + monkeypatch.setattr(gate, "candidate", reject) + with pytest.raises(pytest.UsageError, match="checked-out candidate"): + observer.pytest_collection_modifyitems(config, []) + + +def test_smoke_plugin_loads_before_collection_from_repository() -> None: + from scripts.check_lifecycle_evidence import ROOT + + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "-p", + "tests.utils.lifecycle_evidence", + "--help", + ], + cwd=ROOT, + env={key: value for key, value in os.environ.items() if key != "PYTHONPATH"}, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + assert result.returncode == 0, result.stderr + assert "--lifecycle-defer-base" in result.stdout + assert "--lifecycle-defer-head" in result.stdout + + +@pytest.mark.skipif(os.name == "nt", reason="Installed Unix launcher import-path control") +def test_source_probe_distinguishes_launcher_and_module_search_paths(tmp_path: Path) -> None: + package = tmp_path / "apm_cli" + package.mkdir() + (package / "__init__.py").write_text("", encoding="ascii") + (package / "cli.py").write_text( + "if __name__ == '__main__':\n print('shadow-cli')\n", encoding="ascii" + ) + env = { + key: value + for key, value in os.environ.items() + if key not in {"PYTHONPATH", "PYTHONSAFEPATH"} + } + plugin = LifecycleEvidencePlugin(["test"], _launcher()) + assert plugin.executable is not None + launched = subprocess.run( + [str(plugin.executable), "--version"], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + check=False, + ) + assert launched.returncode == 0 and "shadow-cli" not in launched.stdout + assert plugin._source_identity(tmp_path, env, 10, (str(plugin.executable),))["package"] == str( + plugin.candidate_source + ) + with pytest.raises(EvidenceError, match="Child source identity"): + plugin._source_identity(tmp_path, env, 10, (sys.executable, "-m", "apm_cli.cli")) + + def test_inventory_uses_recursive_registrations_and_aliases( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -424,6 +643,190 @@ def test_cli_rejects_in_tree_receipts(monkeypatch: pytest.MonkeyPatch) -> None: assert exc.value.code == 2 +def _completion(tmp_path: Path) -> tuple[Path, dict[str, Any], dict[str, Any]]: + native = { + "version": 1, + "base": "a" * 40, + "head": "b" * 40, + "tested_tree": "c" * 40, + "lane": "full", + "status": "passed", + } + driver = tmp_path / "driver.json" + driver.write_text(json.dumps(native), encoding="ascii") + data = { + "head_sha": native["head"], + "lifecycle_evidence": { + "version": 1, + "base_sha": native["base"], + "head_sha": native["head"], + "tested_tree": native["tested_tree"], + "lane": "full", + "status": "passed", + "report_path": str(driver), + "report_sha256": hashlib.sha256(driver.read_bytes()).hexdigest(), + }, + } + completion = tmp_path / "completion.json" + completion.write_text(json.dumps(data), encoding="ascii") + return completion, data, native + + +@pytest.mark.parametrize( + "mutation", + [ + "base_sha", + "head_sha", + "tested_tree", + "lane", + "status", + "version", + "report_sha256", + "report_path", + "top_head", + "header", + "malformed", + "missing", + ], +) +def test_completion_rejects_stale_or_malformed_claims(tmp_path: Path, mutation: str) -> None: + path, data, native = _completion(tmp_path) + output = tmp_path / "independent.json" + validate_completion(path, native, output) + summary = data["lifecycle_evidence"] + if mutation in {"base_sha", "head_sha", "tested_tree", "report_sha256"}: + summary[mutation] = "d" * len(summary[mutation]) + elif mutation == "lane": + summary["lane"] = "pr" + elif mutation == "status": + summary["status"] = "pending" + elif mutation == "version": + summary["version"] = True + elif mutation == "report_path": + summary["report_path"] = "" + elif mutation == "top_head": + data["head_sha"] = "d" * 40 + elif mutation == "header": + driver = Path(summary["report_path"]) + driver.write_text(json.dumps({**native, "tested_tree": "d" * 40}), encoding="ascii") + summary["report_sha256"] = hashlib.sha256(driver.read_bytes()).hexdigest() + elif mutation == "malformed": + data["lifecycle_evidence"] = [] + else: + del data["lifecycle_evidence"] + path.write_text(json.dumps(data), encoding="ascii") + with pytest.raises((EvidenceError, KeyError, TypeError)): + validate_completion(path, native, output) + + +def test_completion_accepts_resolved_summary_and_protects_driver_bytes(tmp_path: Path) -> None: + path, data, native = _completion(tmp_path) + del data["head_sha"] + data["lifecycle_evidence"]["report_path"] = "driver.json" + path.write_text(json.dumps(data), encoding="ascii") + validate_completion(path, native, tmp_path / "independent.json") + for output in (path, tmp_path / "driver.json"): + before = output.read_bytes() + with pytest.raises(EvidenceError, match="overwrite"): + validate_completion(path, native, output) + assert output.read_bytes() == before + hardlink = tmp_path / "driver-hardlink.json" + hardlink.hardlink_to(tmp_path / "driver.json") + with pytest.raises(EvidenceError, match="overwrite"): + validate_completion(path, native, hardlink) + + +def test_completion_always_executes_native_gate_first( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from scripts import check_lifecycle_evidence as gate + + completion, _data, native = _completion(tmp_path) + calls = [] + + def fresh(args: argparse.Namespace) -> dict[str, Any]: + calls.append(args.lane) + return dict(native) + + monkeypatch.setattr(gate, "execute", fresh) + output = tmp_path / "independent.json" + monkeypatch.setattr( + sys, + "argv", + [ + "gate", + "--base", + native["base"], + "--head", + native["head"], + "--lane", + "full", + "--report", + str(output), + "--completion", + str(completion), + ], + ) + assert main() == 0 + assert calls == ["full"] + completion.write_text("not json", encoding="ascii") + assert main() == 1 + assert calls == ["full", "full"] + assert json.loads(output.read_text())["status"] == "blocked" + monkeypatch.setattr( + sys, + "argv", + [ + "gate", + "--base", + native["base"], + "--head", + native["head"], + "--lane", + "pr", + "--report", + str(output), + "--completion", + str(completion), + ], + ) + with pytest.raises(SystemExit): + main() + assert calls == ["full", "full"] + + +def test_completion_preserves_driver_even_when_claims_are_stale( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from scripts import check_lifecycle_evidence as gate + + completion, data, native = _completion(tmp_path) + driver = Path(data["lifecycle_evidence"]["report_path"]) + data["lifecycle_evidence"]["status"] = "pending" + completion.write_text(json.dumps(data), encoding="ascii") + original = driver.read_bytes() + monkeypatch.setattr(gate, "execute", lambda _args: dict(native)) + monkeypatch.setattr( + sys, + "argv", + [ + "gate", + "--base", + native["base"], + "--head", + native["head"], + "--lane", + "full", + "--report", + str(driver), + "--completion", + str(completion), + ], + ) + assert main() == 1 + assert driver.read_bytes() == original + + @pytest.mark.parametrize("lane,status", [("pr", "pending"), ("full", "passed")]) def test_execute_produces_candidate_bound_native_receipt( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, lane: str, status: str diff --git a/tests/utils/isolated_apm_environment.py b/tests/utils/isolated_apm_environment.py index c954cadabd..8d43a100e5 100644 --- a/tests/utils/isolated_apm_environment.py +++ b/tests/utils/isolated_apm_environment.py @@ -262,7 +262,7 @@ def _guarded_import_module(name, package=None): | _APM_REMOTE_CONTROL_ENV_NAMES ) _STRIPPED_ENV_PREFIXES = _APM_AUTH_ENV_PREFIXES + _GIT_CONFIG_INJECTION_PREFIXES -DURABLE_ENVIRONMENT_ROOTS = ( +_PINNED_DURABLE_ROOTS = ( "HOME", "USERPROFILE", "XDG_CONFIG_HOME", @@ -272,8 +272,13 @@ def _guarded_import_module(name, package=None): "APM_HOME", "APM_CACHE_DIR", ) +DURABLE_ENVIRONMENT_ROOTS = ( + *_PINNED_DURABLE_ROOTS, + "CLAUDE_CONFIG_DIR", + "HERMES_HOME", +) _PINNED_ENVIRONMENT_NAMES = ( - *DURABLE_ENVIRONMENT_ROOTS, + *_PINNED_DURABLE_ROOTS, "APM_TEMP_DIR", "GH_CONFIG_DIR", "AZURE_CONFIG_DIR", diff --git a/tests/utils/lifecycle_evidence.py b/tests/utils/lifecycle_evidence.py index d3b7c64f55..f924105e1e 100644 --- a/tests/utils/lifecycle_evidence.py +++ b/tests/utils/lifecycle_evidence.py @@ -4,6 +4,7 @@ import hashlib import json +import subprocess import sys from dataclasses import asdict from pathlib import Path @@ -11,20 +12,61 @@ import pytest -from scripts.lifecycle_contracts import EvidenceError, command_inventory, command_path +from scripts.lifecycle_contracts import ( + EvidenceError, + candidate_contracts, + command_inventory, + command_path, +) from tests.utils.apm_lifecycle_runner import ApmLifecycleRunner from tests.utils.artifact_snapshot import ArtifactSnapshotSet from tests.utils.isolated_apm_environment import DURABLE_ENVIRONMENT_ROOTS +def pytest_addoption(parser: pytest.Parser) -> None: + """Explicit candidate arguments enable smoke-only deferral, never proof reuse.""" + parser.addoption("--lifecycle-defer-base") + parser.addoption("--lifecycle-defer-head") + + +@pytest.hookimpl(trylast=True) +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + """Defer exact deterministic nodes to the mandatory subsequent native gate.""" + base = config.getoption("lifecycle_defer_base") + head = config.getoption("lifecycle_defer_head") + if base is None and head is None: + return + if not base or not head: + raise pytest.UsageError("Lifecycle smoke deferral requires both base and head") + from scripts.check_lifecycle_evidence import candidate + + try: + root = Path(config.rootpath) + identity = candidate(root, base, head) + _, contracts = candidate_contracts( + root, identity["base"], identity["head"], command_inventory() + ) + except (EvidenceError, KeyError, TypeError, ValueError, subprocess.CalledProcessError) as exc: + raise pytest.UsageError(f"Invalid lifecycle smoke deferral: {exc}") from exc + deferred = { + witness["nodeid"] + for contract in contracts + for witness in contract["witnesses"] + if witness["kind"] == "deterministic" + } + removed = [item for item in items if item.nodeid in deferred] + items[:] = [item for item in items if item.nodeid not in deferred] + config.hook.pytest_deselected(items=removed) + + def fingerprint(path: Path) -> str: """Hash an executable or source artifact without trusting its filename.""" return hashlib.sha256(path.read_bytes()).hexdigest() -def snapshot(cwd: Path, env: dict[str, str]) -> str: +def snapshot(domain: dict[str, str]) -> str: """Use the existing open-world oracle for project and isolated user roots.""" - candidates = [cwd, *(Path(env[key]) for key in DURABLE_ENVIRONMENT_ROOTS if env.get(key))] + candidates = [Path(value) for value in domain.values()] roots: list[Path] = [] for candidate in sorted({path.resolve() for path in candidates}, key=lambda p: len(p.parts)): if not any(candidate.is_relative_to(root) for root in roots): @@ -40,8 +82,19 @@ def snapshot(cwd: Path, env: dict[str, str]) -> str: class LifecycleEvidencePlugin: """Observe exact nodes and runner invocations, never authored outcome strings.""" - def __init__(self, nodeids: list[str], executable: Path | None) -> None: + def __init__( + self, + nodeids: list[str], + executable: Path | None, + contexts: dict[str, set[str]] | None = None, + candidate_source: Path | None = None, + ) -> None: + import apm_cli + self.nodeids = set(nodeids) + self.contexts = contexts or {} + self.candidate_source = (candidate_source or Path(apm_cli.__file__).parent).resolve() + self.source_hash = fingerprint(self.candidate_source / "cli.py") self.executable = executable.resolve() if executable else None self.executable_hash = fingerprint(self.executable) if self.executable else None self.python_hash = fingerprint(Path(sys.executable).resolve()) @@ -50,8 +103,87 @@ def __init__(self, nodeids: list[str], executable: Path | None) -> None: self.active: str | None = None self.phase: str | None = None self.model_run: int | None = None + self.domains: dict[tuple[str, int | None, str], dict[str, str]] = {} self.patch = pytest.MonkeyPatch() + def _source_identity( + self, + cwd: Path, + env: dict[str, str], + timeout: float, + entrypoint: tuple[str, ...] | None = None, + ) -> dict[str, str]: + """Probe actual child import resolution; isolated fixtures replace PYTHONPATH.""" + path_head = ( + self.executable.parent + if self.executable and entrypoint == (str(self.executable),) + else cwd.resolve() + ) + probe = subprocess.run( + [ + sys.executable, + "-c", + "import sys; " + "sys.path[:] = sys.path if getattr(sys.flags, 'safe_path', False) " + "else [sys.argv[1], *sys.path[1:]]; " + "import hashlib, importlib, importlib.util, json; from pathlib import Path; " + "package = importlib.import_module('apm_cli'); " + "spec = importlib.util.find_spec('apm_cli.cli'); " + "path = Path(spec.origin).resolve(); " + "print(json.dumps({'package': str(Path(package.__file__).resolve().parent), " + "'cli': str(path), 'sha256': hashlib.sha256(path.read_bytes()).hexdigest()}))", + str(path_head), + ], + cwd=cwd, + env=dict(env), + capture_output=True, + text=True, + timeout=min(timeout, 30), + check=False, + ) + if probe.returncode != 0: + raise EvidenceError(f"Child source identity probe failed: exit {probe.returncode}") + try: + source = json.loads(probe.stdout) + except json.JSONDecodeError as exc: + raise EvidenceError("Child source identity probe produced invalid output") from exc + expected = { + "package": str(self.candidate_source), + "cli": str(self.candidate_source / "cli.py"), + "sha256": self.source_hash, + } + if source != expected: + raise EvidenceError("Child source identity does not match the candidate checkout") + return source + + def _domain(self, cwd: Path, env: dict[str, str]) -> tuple[dict[str, str], dict[str, str], str]: + """Pin an isolated domain while allowing only reviewed command contexts.""" + if self.active is None: + raise EvidenceError("Lifecycle context requires an active test node") + environment = { + key: str(Path(env[key]).absolute()) for key in DURABLE_ENVIRONMENT_ROOTS if env.get(key) + } + physical = {key: str(Path(value).resolve()) for key, value in environment.items()} + identity = json.dumps([environment, physical], sort_keys=True) + key = (self.active, self.model_run, identity) + location = str(cwd.resolve()) + if key not in self.domains: + if any( + node == self.active and model == self.model_run and roots["workspace"] == location + for (node, model, _identity), roots in self.domains.items() + ): + raise EvidenceError("Lifecycle durable roots changed for the same caller workspace") + self.domains[key] = {"workspace": location, **physical} + roots = self.domains[key] + context = "initial" + if location != roots["workspace"]: + if location != roots.get("APM_HOME"): + raise EvidenceError(f"Unreviewed lifecycle command cwd: {cwd}") + context = "APM_HOME" + if context not in self.contexts.get(self.active, set()) | {"initial"}: + raise EvidenceError(f"Undeclared lifecycle command context: {context}") + return roots, environment, context + def pytest_sessionstart(self, session: pytest.Session) -> None: """Install observers before test-module imports bind runner/model functions.""" from hypothesis import stateful @@ -73,23 +205,23 @@ def observe(runner: ApmLifecycleRunner, args: Any, **kwargs: Any) -> Any: ) or fingerprint(Path(sys.executable).resolve()) != self.python_hash: raise EvidenceError("Executable changed during evidence execution") cwd, env = kwargs["cwd"], kwargs["env"] - before = snapshot(cwd, env) + roots, environment, context = self._domain(cwd, env) + source = self._source_identity(cwd, env, kwargs["timeout_seconds"], command) + before = snapshot(roots) result = original(runner, args, **kwargs) - after = snapshot(cwd, env) + if any(str(Path(value).resolve()) != roots[key] for key, value in environment.items()): + raise EvidenceError("Lifecycle durable root identity changed during command") + after = snapshot(roots) self.records[self.active]["events"].append( { "command": command_path(list(args), self.inventory), "args": list(args), "returncode": result.returncode, "cwd": str(cwd.resolve()), - "roots": { - "workspace": str(cwd.absolute()), - **{ - key: str(Path(env[key]).absolute()) - for key in DURABLE_ENVIRONMENT_ROOTS - if env.get(key) - }, - }, + "roots": roots, + "environment": environment, + "context": context, + "source": source, "before": before, "after": after, "scenario_id": kwargs["scenario_id"], @@ -124,6 +256,7 @@ def pytest_collection_finish(self, session: pytest.Session) -> None: "collected": True, "dimensions": dict(callspec.params) if callspec else {}, "phases": {}, + "durations": {}, "events": [], "models": 0, } @@ -148,6 +281,7 @@ def pytest_runtest_logreport(self, report: pytest.TestReport) -> None: if report.nodeid in self.records: outcome = "xfail" if hasattr(report, "wasxfail") else report.outcome self.records[report.nodeid]["phases"][report.when] = outcome + self.records[report.nodeid]["durations"][report.when] = report.duration def pytest_sessionfinish(self, session: pytest.Session, exitstatus: int) -> None: """Restore instrumentation even when collection or test setup fails.""" From 5de467e53fbd51e655a20b7dc6bff9b331fc6960 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Mon, 7 Sep 2026 20:31:23 +0200 Subject: [PATCH 6/8] fix(ci): run PR checks across stacked base branches Retain existing path filters and merge-queue trust boundaries. Align the contributor guide with the native evidence timeouts and witness deferral. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- .github/workflows/codeql.yml | 1 - .github/workflows/crlf-invariance.yml | 1 - .../workflows/devcontainer-feature-test.yml | 1 - .github/workflows/merge-gate.yml | 2 -- .github/workflows/notice-drift.yml | 1 - .github/workflows/spec-conformance.yml | 1 - .../docs/contributing/integration-testing.md | 15 ++++++---- .../unit/test_security_workflow_contracts.py | 30 +++++++++++++++++-- 9 files changed, 39 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 82e6c8ef9d..56e1d629fd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,8 +4,8 @@ env: PYTHON_VERSION: '3.12' on: + # Upper stacked PRs need their own head checks before their base reaches main. pull_request: - branches: [ main ] # Tier 1 also runs in merge queue context so the same unit + build checks # execute against the tentative merge commit that the queue creates. See # microsoft/apm#770 for the design. diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index a9dace6af3..1927778d14 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -4,7 +4,6 @@ on: push: branches: [ main ] pull_request: - branches: [ main ] merge_group: branches: [ main ] types: [ checks_requested ] diff --git a/.github/workflows/crlf-invariance.yml b/.github/workflows/crlf-invariance.yml index 6c1e332e5f..5cff59e88a 100644 --- a/.github/workflows/crlf-invariance.yml +++ b/.github/workflows/crlf-invariance.yml @@ -14,7 +14,6 @@ name: CRLF Invariance (apm#1952) on: pull_request: - branches: [ main ] paths: - 'src/apm_cli/utils/content_hash.py' - 'src/apm_cli/utils/atomic_io.py' diff --git a/.github/workflows/devcontainer-feature-test.yml b/.github/workflows/devcontainer-feature-test.yml index 25bfc9db5b..019dac390f 100644 --- a/.github/workflows/devcontainer-feature-test.yml +++ b/.github/workflows/devcontainer-feature-test.yml @@ -14,7 +14,6 @@ name: Devcontainer Feature - Test on: pull_request: - branches: [ main ] paths: - 'devcontainer/**' - '.github/workflows/devcontainer-feature-test.yml' diff --git a/.github/workflows/merge-gate.yml b/.github/workflows/merge-gate.yml index 914454d6a7..342fc9e0f9 100644 --- a/.github/workflows/merge-gate.yml +++ b/.github/workflows/merge-gate.yml @@ -41,7 +41,6 @@ name: Merge Gate on: pull_request: - branches: [ main ] merge_group: branches: [ main ] types: [ checks_requested ] @@ -147,4 +146,3 @@ jobs: chmod +x .github/scripts/ci/merge_gate_wait.sh .github/scripts/ci/merge_gate_wait.sh - diff --git a/.github/workflows/notice-drift.yml b/.github/workflows/notice-drift.yml index 305602e1a1..87c31b346c 100644 --- a/.github/workflows/notice-drift.yml +++ b/.github/workflows/notice-drift.yml @@ -53,7 +53,6 @@ name: NOTICE Drift Check on: pull_request: - branches: [ main ] merge_group: branches: [ main ] types: [ checks_requested ] diff --git a/.github/workflows/spec-conformance.yml b/.github/workflows/spec-conformance.yml index 0daa414319..1d0f227cd3 100644 --- a/.github/workflows/spec-conformance.yml +++ b/.github/workflows/spec-conformance.yml @@ -8,7 +8,6 @@ name: Spec conformance on: pull_request: - branches: [ main ] paths: - 'docs/src/content/docs/specs/**' - 'tests/fixtures/spec-conformance/**' diff --git a/docs/src/content/docs/contributing/integration-testing.md b/docs/src/content/docs/contributing/integration-testing.md index 80d399e30c..e6a5a57024 100644 --- a/docs/src/content/docs/contributing/integration-testing.md +++ b/docs/src/content/docs/contributing/integration-testing.md @@ -107,11 +107,16 @@ remain separate obligations. #### CI lanes -The required `ci.yml` `lifecycle-smoke` job retains its six-minute limit, -hermetic marker selection, and deterministic `pr` evidence lane. -`ci-integration.yml` runs the complete integration family and the `full` -evidence lane on merge-group candidates. `tests/quality/test_ci_topology.py` -guards this wiring; do not change jobs or required-check policy to waive P8. +The required `ci.yml` `lifecycle-smoke` job has a 12-minute whole-job timeout. +Hermetic smoke defers exact contracted deterministic nodes to the immediately +following native `pr` gate, avoiding duplicate execution without importing +receipts. `ci-integration.yml` runs the complete merge-group integration family; +its native `full` evidence step has a 20-minute timeout. +Standard PR checks cover all base branches, including stacked PRs, subject to +existing path filters; secret-bearing integration remains merge-group-only. +`tests/quality/test_ci_topology.py` and +`tests/unit/test_security_workflow_contracts.py` guard this wiring; do not change +jobs or required-check policy to waive P8. For focused smoke feedback only: diff --git a/tests/unit/test_security_workflow_contracts.py b/tests/unit/test_security_workflow_contracts.py index 587f0379ad..4934988050 100644 --- a/tests/unit/test_security_workflow_contracts.py +++ b/tests/unit/test_security_workflow_contracts.py @@ -2,6 +2,8 @@ from pathlib import Path +import pytest + from tests.workflow_contracts import ( assert_unconditional, load_workflow, @@ -12,6 +14,31 @@ ROOT = Path(__file__).resolve().parents[2] +@pytest.mark.parametrize( + "filename", + [ + "ci.yml", + "merge-gate.yml", + "notice-drift.yml", + "codeql.yml", + "spec-conformance.yml", + "devcontainer-feature-test.yml", + "crlf-invariance.yml", + "docs.yml", + ], +) +def test_pr_checks_cover_stacked_bases(filename: str) -> None: + """An upper PR needs checks on its own head before its base reaches main.""" + workflow = load_workflow(ROOT / ".github" / "workflows" / filename) + events = workflow["on"] + assert "pull_request" in events + assert "pull_request_target" not in events + filters = events["pull_request"] or {} + assert not {"branches", "branches-ignore"}.intersection(filters) + if filename in {"ci.yml", "merge-gate.yml", "notice-drift.yml"}: + assert not {"paths", "paths-ignore"}.intersection(filters) + + def test_codeql_covers_merge_queue_with_existing_analysis_configurations() -> None: """Required scans must run on the queue SHA, not just the earlier PR SHA.""" workflow = load_workflow(ROOT / ".github" / "workflows" / "codeql.yml") @@ -19,8 +46,7 @@ def test_codeql_covers_merge_queue_with_existing_analysis_configurations() -> No "branches": ["main"], "types": ["checks_requested"], } - for event in ("pull_request", "push"): - assert workflow["on"][event] == {"branches": ["main"]} + assert workflow["on"]["push"] == {"branches": ["main"]} analyze = workflow_job(workflow, "analyze") assert analyze["strategy"]["matrix"]["language"] == ["python", "actions"] From 3e7b3deca88dd94da524ad8eb712c58c6dbf3709 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Mon, 7 Sep 2026 20:39:16 +0200 Subject: [PATCH 7/8] docs(ci): clarify preliminary lifecycle evidence in the merge gate Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/merge-gate.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/merge-gate.yml b/.github/workflows/merge-gate.yml index 342fc9e0f9..0cc5788a76 100644 --- a/.github/workflows/merge-gate.yml +++ b/.github/workflows/merge-gate.yml @@ -117,14 +117,16 @@ jobs: # allocation latency on the critical path. # merge_group context: Coverage Combine IS required, so the # global 80% floor blocks the actual merge. - # Lifecycle Smoke (Linux) is required in BOTH contexts: it's a - # hermetic, ~65-70s ci.yml job (runs on pull_request AND + # Lifecycle Smoke (Linux) is required in BOTH contexts: its + # bounded ci.yml job runs hermetic smoke and fresh deterministic + # lifecycle evidence (runs on pull_request AND # merge_group, same as APM Self-Check / NOTICE Drift Check), # promoting the smallest stable subset of the real # Consume/Produce/Govern lifecycle contracts -- selected # declaratively via the `lifecycle_smoke` pytest marker, # including the direct #2226/#2240 regression tests -- onto - # the PR-time critical path. + # the PR-time critical path. Preliminary evidence is not full + # shipping proof; ci-integration.yml runs the full queue lane. # "Windows Compatibility Gate" is also required at BOTH # PR-time and merge-queue-time: it is the only Windows signal # that runs before merge (build-release.yml's full Windows @@ -145,4 +147,3 @@ jobs: run: | chmod +x .github/scripts/ci/merge_gate_wait.sh .github/scripts/ci/merge_gate_wait.sh - From 103029508a8fbf863c55fdd224be63461e115a2c Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Mon, 7 Sep 2026 20:38:14 +0200 Subject: [PATCH 8/8] fix(ci): correct lifecycle root identity and shard execution Fetch complete integration history, resolve durable roots against the child context, accept interpreter aliases only within one Python environment, and defer canonical witness IDs before xdist loadgroup decoration. Cover each review defect with executable regression tests. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci-integration.yml | 2 + scripts/check_lifecycle_evidence.py | 10 +- tests/quality/test_ci_topology.py | 2 + tests/unit/scripts/test_lifecycle_evidence.py | 181 +++++++++++++++++- tests/utils/lifecycle_evidence.py | 61 +++++- 5 files changed, 247 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci-integration.yml b/.github/workflows/ci-integration.yml index eb2ea12c65..a0b3398005 100644 --- a/.github/workflows/ci-integration.yml +++ b/.github/workflows/ci-integration.yml @@ -169,6 +169,8 @@ jobs: shard: [1, 2, 3, 4] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 # Artifact root is ./dist/ (least common ancestor of the uploaded paths in # the build job), so download to ./dist/ to preserve dist/apm-linux-x86_64/apm. diff --git a/scripts/check_lifecycle_evidence.py b/scripts/check_lifecycle_evidence.py index 342f9b5653..0ad917a8ea 100644 --- a/scripts/check_lifecycle_evidence.py +++ b/scripts/check_lifecycle_evidence.py @@ -53,7 +53,15 @@ def source_profile(root: Path) -> tuple[Path | None, dict[str, Any]]: executable = launcher if os.name != "nt" and launcher.is_file() else None if executable: script = executable.read_text(encoding="utf-8") - if "apm_cli.cli" not in script or not script.startswith(f"#!{sys.executable}\n"): + interpreter = Path(script.splitlines()[0][2:]) if script.startswith("#!") else None + if ( + "apm_cli.cli" not in script + or interpreter is None + or not interpreter.is_absolute() + or not interpreter.is_file() + or not interpreter.samefile(sys.executable) + or interpreter.parent.resolve() != Path(sys.executable).parent.resolve() + ): raise EvidenceError("Installed apm entrypoint is not this Python environment") return executable, { "kind": "source-python", diff --git a/tests/quality/test_ci_topology.py b/tests/quality/test_ci_topology.py index 2db06b2001..0fdbe7d75f 100644 --- a/tests/quality/test_ci_topology.py +++ b/tests/quality/test_ci_topology.py @@ -116,6 +116,8 @@ def test_lifecycle_evidence_is_executed_in_required_candidate_lanes() -> None: integration = workflow_job( load_workflow(MERGE_GROUP_INTEGRATION_WORKFLOW), MERGE_GROUP_INTEGRATION_JOB ) + checkout = next(s for s in integration["steps"] if "actions/checkout@" in s.get("uses", "")) + assert checkout["with"]["fetch-depth"] == 0 full = workflow_step(integration, "Execute full lifecycle contracts") assert full["if"] == "github.event_name == 'merge_group' && matrix.shard == 1" assert full["timeout-minutes"] == 20 diff --git a/tests/unit/scripts/test_lifecycle_evidence.py b/tests/unit/scripts/test_lifecycle_evidence.py index 0a58c01453..ef3ad068a2 100644 --- a/tests/unit/scripts/test_lifecycle_evidence.py +++ b/tests/unit/scripts/test_lifecycle_evidence.py @@ -331,6 +331,87 @@ def test_domain_does_not_splice_distinct_hypothesis_instances(tmp_path: Path) -> validate_execution(witness, evidence) +@pytest.mark.parametrize("override", ["../outside-hermes", " ~/external-hermes "]) +@pytest.mark.parametrize( + ("key", "resolver"), + [ + ("HERMES_HOME", "resolve_hermes_root()"), + ("CLAUDE_CONFIG_DIR", "Path.home() / KNOWN_TARGETS['claude'].for_scope(True).root_dir"), + ("APM_CACHE_DIR", "get_cache_root()"), + ("XDG_CACHE_HOME", "get_cache_root().parent"), + ], +) +def test_domain_observes_actual_child_resolved_root( + tmp_path: Path, override: str, key: str, resolver: str +) -> None: + from tests.utils.isolated_apm_environment import DURABLE_ENVIRONMENT_ROOTS + + caller = tmp_path / "caller" + caller.mkdir() + env = { + **{ + name: value + for name, value in os.environ.items() + if name not in DURABLE_ENVIRONMENT_ROOTS + }, + "HOME": str(tmp_path / "home"), + "USERPROFILE": str(tmp_path / "home"), + key: override, + } + plugin = LifecycleEvidencePlugin(["test"], None) + plugin.active = "test" + roots, environment, _ = plugin._domain(caller, env) + before = snapshot(roots) + result = subprocess.run( + [ + sys.executable, + "-c", + "from pathlib import Path; " + "from apm_cli.integration.targets import KNOWN_TARGETS, resolve_hermes_root; " + "from apm_cli.cache.paths import get_cache_root; " + f"root = ({resolver}).resolve(); root.mkdir(parents=True, exist_ok=True); " + "(root / 'sentinel').write_text('child wrote here'); print(root)", + ], + cwd=caller, + env=env, + capture_output=True, + text=True, + check=True, + ) + assert environment[key] == override + assert roots[key] == result.stdout.strip() + assert snapshot(roots) != before + assert plugin._domain(caller, env)[0] == roots + + +def test_relative_roots_cannot_move_with_reviewed_cwd(tmp_path: Path) -> None: + caller = tmp_path / "caller" + apm_home = tmp_path / "home/apm" + caller.mkdir() + apm_home.mkdir(parents=True) + env = {"APM_HOME": str(apm_home), "HERMES_HOME": "../hermes"} + plugin = LifecycleEvidencePlugin(["test"], None, {"test": {"APM_HOME"}}) + plugin.active = "test" + plugin._domain(caller, env) + with pytest.raises(EvidenceError, match="durable roots changed"): + plugin._domain(apm_home, env) + + +@pytest.mark.parametrize("output", ["not json", "[]", "{}", '{"HERMES_HOME": null}']) +def test_root_expansion_rejects_malformed_probe( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, output: str +) -> None: + from tests.utils import lifecycle_evidence as observer + + monkeypatch.setattr( + observer.subprocess, + "run", + lambda *args, **kwargs: SimpleNamespace(returncode=0, stdout=output), + ) + with pytest.raises(EvidenceError, match="Child durable root expansion"): + observer._physical_roots(tmp_path, {"HERMES_HOME": "~/hermes"}) + + def test_child_source_probe_rejects_old_checkout_even_when_cli_would_pass( tmp_path: Path, ) -> None: @@ -436,6 +517,52 @@ def test_smoke_plugin_loads_before_collection_from_repository() -> None: assert "--lifecycle-defer-head" in result.stdout +def test_smoke_defers_exact_node_before_real_loadgroup_decoration( + pytester: pytest.Pytester, +) -> None: + from scripts.check_lifecycle_evidence import ROOT + + pytester.makeconftest(f""" + import sys + sys.path.insert(0, {str(ROOT)!r}) + pytest_plugins = ["tests.utils.lifecycle_evidence"] + def pytest_configure(config): + from scripts import check_lifecycle_evidence as gate + from tests.utils import lifecycle_evidence as observer + gate.candidate = lambda *args: {{"base": "base", "head": "head"}} + observer.candidate_contracts = lambda *args: (set(), [{{ + "witnesses": [{{ + "nodeid": "test_grouped.py::test_deferred[global]", + "kind": "deterministic", + }}], + }}]) + """) + pytester.makepyfile( + test_grouped=""" + import pytest + pytestmark = pytest.mark.xdist_group("home_env") + @pytest.mark.parametrize("variant", ["global"]) + def test_deferred(variant): + pytest.fail("must execute only in the subsequent native gate") + def test_retained(request): + assert request.node.nodeid.endswith("@home_env") + """ + ) + result = pytester.runpytest_subprocess( + "-q", + "-n", + "2", + "--dist", + "loadgroup", + "--lifecycle-defer-base", + "base", + "--lifecycle-defer-head", + "head", + ) + result.assert_outcomes(passed=1) + assert result.ret == 0 + + @pytest.mark.skipif(os.name == "nt", reason="Installed Unix launcher import-path control") def test_source_probe_distinguishes_launcher_and_module_search_paths(tmp_path: Path) -> None: package = tmp_path / "apm_cli" @@ -501,6 +628,28 @@ def run(*args: str) -> str: candidate(tmp_path, base, head) +def test_candidate_requires_history_beyond_fetched_shallow_endpoints(tmp_path: Path) -> None: + origin, checkout = tmp_path / "origin", tmp_path / "checkout" + origin.mkdir() + + def run(root: Path, *args: str) -> str: + return subprocess.check_output(["git", *args], cwd=root, text=True).strip() + + run(origin, "init", "-q") + run(origin, "config", "user.email", "fixture@example.invalid") + run(origin, "config", "user.name", "Fixture") + run(origin, "commit", "--allow-empty", "-qm", "base") + base = run(origin, "rev-parse", "HEAD") + run(origin, "commit", "--allow-empty", "-qm", "head") + head = run(origin, "rev-parse", "HEAD") + run(tmp_path, "clone", "-q", "--depth=1", origin.as_uri(), str(checkout)) + run(checkout, "fetch", "-q", "origin", base, head) + with pytest.raises(subprocess.CalledProcessError): + candidate(checkout, base, head) + run(checkout, "fetch", "-q", "--unshallow", "origin") + assert candidate(checkout, base, head)["head"] == head + + def test_plugin_observes_actual_execution_and_rejects_deselection( pytester: pytest.Pytester, ) -> None: @@ -513,7 +662,10 @@ def test_plugin_observes_actual_execution_and_rejects_deselection( from hypothesis.stateful import RuleBasedStateMachine, rule, run_state_machine_as_test @pytest.mark.parametrize("variant", ["global"]) def test_real(tmp_path, variant): - env = {**os.environ, "HOME": str(tmp_path / "home")} + env = { + **os.environ, "HOME": str(tmp_path / "home"), + "HERMES_HOME": "../external-hermes", + } runner = ApmLifecycleRunner((sys.executable, "-m", "apm_cli.cli")) for _ in range(2): result = runner.run(["--version"], cwd=tmp_path, env=env) @@ -549,6 +701,11 @@ def version(self): record = plugin.records[real] assert record["dimensions"] == {"variant": "global"} assert len(record["events"]) == 2 + for event in record["events"]: + assert event["environment"]["HERMES_HOME"] == "../external-hermes" + assert event["roots"]["HERMES_HOME"] == str( + (Path(event["cwd"]).parent / "external-hermes").resolve() + ) assert record["phases"] == {"setup": "passed", "call": "passed", "teardown": "passed"} assert plugin.records[skipped]["phases"]["call"] == "skipped" assert plugin.records[xfailed]["phases"]["call"] == "xfail" @@ -620,6 +777,28 @@ def test_source_profile_rejects_foreign_source_and_stale_launcher( source_profile(tmp_path / "different") +@pytest.mark.skipif(os.name == "nt", reason="Unix launcher interpreter aliases") +def test_source_profile_accepts_only_same_environment_interpreter_aliases( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import venv + + from scripts.check_lifecycle_evidence import ROOT + + for name in ("current", "foreign"): + venv.EnvBuilder(with_pip=False, symlinks=True).create(tmp_path / name) + current = tmp_path / "current/bin" + foreign = tmp_path / "foreign/bin" + monkeypatch.setattr(sys, "executable", str(current / "python3")) + launcher = current / "apm" + launcher.write_text(f"#!{current / 'python'}\nfrom apm_cli.cli import cli\n", encoding="ascii") + assert source_profile(ROOT)[0] == launcher + assert (current / "python").samefile(foreign / "python") + launcher.write_text(f"#!{foreign / 'python'}\nfrom apm_cli.cli import cli\n", encoding="ascii") + with pytest.raises(EvidenceError, match="this Python environment"): + source_profile(ROOT) + + def test_cli_rejects_in_tree_receipts(monkeypatch: pytest.MonkeyPatch) -> None: from scripts.check_lifecycle_evidence import ROOT diff --git a/tests/utils/lifecycle_evidence.py b/tests/utils/lifecycle_evidence.py index f924105e1e..9dbe40835f 100644 --- a/tests/utils/lifecycle_evidence.py +++ b/tests/utils/lifecycle_evidence.py @@ -29,7 +29,7 @@ def pytest_addoption(parser: pytest.Parser) -> None: parser.addoption("--lifecycle-defer-head") -@pytest.hookimpl(trylast=True) +@pytest.hookimpl(tryfirst=True) def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: """Defer exact deterministic nodes to the mandatory subsequent native gate.""" base = config.getoption("lifecycle_defer_base") @@ -79,6 +79,53 @@ def snapshot(domain: dict[str, str]) -> str: return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest() +def _physical_roots(cwd: Path, env: dict[str, str]) -> dict[str, str]: + """Resolve overrides like the child, without mutating the observer's environment.""" + expanded_keys = {"CLAUDE_CONFIG_DIR", "HERMES_HOME", "APM_CACHE_DIR", "XDG_CACHE_HOME"} + values = { + key: env[key].strip() if key in expanded_keys else env[key] + for key in DURABLE_ENVIRONMENT_ROOTS + if env.get(key) + } + tilde = { + key: value + for key, value in values.items() + if key in expanded_keys and value.startswith("~") + } + if tilde: + # Only tilde roots need child HOME/USERPROFILE and platform account lookup. + probe = subprocess.run( + [ + sys.executable, + "-c", + "import json, sys; from pathlib import Path; " + "print(json.dumps({k: str(Path(v).expanduser()) " + "for k, v in json.load(sys.stdin).items()}))", + ], + input=json.dumps(tilde), + cwd=cwd, + env=dict(env), + capture_output=True, + text=True, + timeout=30, + check=False, + ) + if probe.returncode != 0: + raise EvidenceError("Child durable root expansion failed") + try: + expanded = json.loads(probe.stdout) + except json.JSONDecodeError as exc: + raise EvidenceError("Child durable root expansion produced invalid output") from exc + if ( + not isinstance(expanded, dict) + or expanded.keys() != tilde.keys() + or any(not isinstance(value, str) or not value for value in expanded.values()) + ): + raise EvidenceError("Child durable root expansion produced invalid paths") + values.update(expanded) + return {key: str((cwd / value).resolve()) for key, value in values.items()} + + class LifecycleEvidencePlugin: """Observe exact nodes and runner invocations, never authored outcome strings.""" @@ -160,16 +207,16 @@ def _domain(self, cwd: Path, env: dict[str, str]) -> tuple[dict[str, str], dict[ """Pin an isolated domain while allowing only reviewed command contexts.""" if self.active is None: raise EvidenceError("Lifecycle context requires an active test node") - environment = { - key: str(Path(env[key]).absolute()) for key in DURABLE_ENVIRONMENT_ROOTS if env.get(key) - } - physical = {key: str(Path(value).resolve()) for key, value in environment.items()} + environment = {key: env[key] for key in DURABLE_ENVIRONMENT_ROOTS if env.get(key)} + physical = _physical_roots(cwd, env) identity = json.dumps([environment, physical], sort_keys=True) key = (self.active, self.model_run, identity) location = str(cwd.resolve()) if key not in self.domains: if any( - node == self.active and model == self.model_run and roots["workspace"] == location + node == self.active + and model == self.model_run + and location in {roots["workspace"], roots.get("APM_HOME")} for (node, model, _identity), roots in self.domains.items() ): raise EvidenceError("Lifecycle durable roots changed for the same caller workspace") @@ -209,7 +256,7 @@ def observe(runner: ApmLifecycleRunner, args: Any, **kwargs: Any) -> Any: source = self._source_identity(cwd, env, kwargs["timeout_seconds"], command) before = snapshot(roots) result = original(runner, args, **kwargs) - if any(str(Path(value).resolve()) != roots[key] for key, value in environment.items()): + if _physical_roots(cwd, env) != {key: roots[key] for key in environment}: raise EvidenceError("Lifecycle durable root identity changed during command") after = snapshot(roots) self.records[self.active]["events"].append(