From 81ef05ecd75c4c2af9f3e37d9566d2514c9ae052 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Fri, 2 Oct 2026 15:06:05 +0200 Subject: [PATCH 01/11] fix(audit): validate skill subsets against prepared lock-pinned replay Use the shared CI replay dependency tree when available without weakening subset, integrity or deployed drift checks. Cover warm and cold checkouts, stale modules, invalid selections and manifest/lock mismatches, with a static replay-root regression guard. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../content/docs/enterprise/enforce-in-ci.md | 2 +- docs/src/content/docs/integrations/ci-cd.md | 6 +- .../content/docs/reference/baseline-checks.md | 6 +- docs/src/content/docs/reference/cli/audit.md | 2 +- .../.apm/skills/apm-usage/commands.md | 2 + .../checks/install_frozen_and_audit.py | 4 + src/apm_cli/install/audit_replay.py | 5 +- src/apm_cli/policy/ci_checks.py | 25 ++- ...architecture_install_compound_mutations.py | 9 + .../test_audit_skill_subset_replay.py | 138 +++++++++++++++ .../policy/test_ci_skill_subset_replay.py | 159 ++++++++++++++++++ 11 files changed, 344 insertions(+), 14 deletions(-) create mode 100644 tests/integration/test_audit_skill_subset_replay.py create mode 100644 tests/unit/policy/test_ci_skill_subset_replay.py diff --git a/docs/src/content/docs/enterprise/enforce-in-ci.md b/docs/src/content/docs/enterprise/enforce-in-ci.md index bb0e7613ff..02424195a5 100644 --- a/docs/src/content/docs/enterprise/enforce-in-ci.md +++ b/docs/src/content/docs/enterprise/enforce-in-ci.md @@ -120,7 +120,7 @@ jobs: `setup-only: true` leaves every deployed file exactly as checked out. `apm audit --ci` now self-hydrates its scratch replay from `apm.lock.yaml`, -so drift and `config-consistency` still run even when the checkout has no +so drift, `config-consistency`, and `skill-subset-consistency` still run even when the checkout has no live `apm_modules/` tree. If the scratch replay itself cannot be materialized, the audit fails closed instead of reporting a green skip. The `content-integrity` check still verifies that every deployed file's SHA-256 diff --git a/docs/src/content/docs/integrations/ci-cd.md b/docs/src/content/docs/integrations/ci-cd.md index 730aed13ee..3029995902 100644 --- a/docs/src/content/docs/integrations/ci-cd.md +++ b/docs/src/content/docs/integrations/ci-cd.md @@ -92,7 +92,11 @@ provides the CLI, then run the full CI gate: In setup-only CI, `apm audit --ci` now self-hydrates a lock-pinned scratch install when `apm_modules/` is absent, so drift and `config-consistency` -still run without mutating the checkout. Repos that gitignore deployed +still run without mutating the checkout. `skill-subset-consistency` also +checks selected skills against this lock-pinned tree, not the absent checkout +dependencies. Invalid selections and manifest/lock mismatches still fail; +deployed-file integrity and drift checks still inspect the checkout. +Repos that gitignore deployed outputs can still use the audit-only pattern: `deployed-files-present` skips gitignored paths automatically, so a fresh checkout of a repo that gitignores a deploy directory (e.g. `.agents/`) passes the check without diff --git a/docs/src/content/docs/reference/baseline-checks.md b/docs/src/content/docs/reference/baseline-checks.md index ab6898df14..49c6a9d9be 100644 --- a/docs/src/content/docs/reference/baseline-checks.md +++ b/docs/src/content/docs/reference/baseline-checks.md @@ -118,8 +118,8 @@ the [policy schema](../policy-schema/). ### `skill-subset-consistency` -- **What it verifies.** That each `skills:` selection in `apm.yml` matches the `skill_subset` recorded in the lockfile, and that every recorded skill path exists in the resolved package tree. -- **Fails when.** The sorted manifest skill list differs from the sorted lockfile `skill_subset`, or a recorded subset path no longer maps to a deployable skill in the installed package. +- **What it verifies.** That each `skills:` selection in `apm.yml` matches the `skill_subset` recorded in the lockfile, and that every recorded skill path exists in the resolved package tree. When CI audit has prepared a lock-pinned scratch replay, this check uses its dependency tree instead of checkout-local `apm_modules/`; no checkout install is required. +- **Fails when.** The sorted manifest skill list differs from the sorted lockfile `skill_subset`, or a recorded subset path does not map to a deployable skill in the dependency tree being checked. - **Remediation.** Run `apm install` to regenerate the lockfile against the current selection. ### `config-consistency` @@ -153,7 +153,7 @@ the [policy schema](../policy-schema/). ## Run order and fail-fast -The aggregate runner in `run_baseline_checks` evaluates checks in this order: `manifest-parse` (only when `apm.yml` is unparseable), `lockfile-exists`, `ref-consistency`, `deployment-ledger-owners`, `deployed-files-present`, `no-orphaned-packages`, `skill-subset-consistency`, `config-consistency`, `content-integrity`, `includes-consent`. Drift is invoked separately by the audit command after the baseline batch, but in `--ci` mode it shares the same cold-cache scratch materialization with `config-consistency`. +The aggregate runner in `run_baseline_checks` evaluates checks in this order: `manifest-parse` (only when `apm.yml` is unparseable), `lockfile-exists`, `ref-consistency`, `deployment-ledger-owners`, `deployed-files-present`, `no-orphaned-packages`, `skill-subset-consistency`, `config-consistency`, `content-integrity`, `includes-consent`. Drift is invoked separately by the audit command after the baseline batch, but in `--ci` mode it shares the same cold-cache scratch materialization with `skill-subset-consistency` and `config-consistency`. With fail-fast on (the default), the runner stops at the first failing check. `apm audit --ci --no-fail-fast` evaluates every check so the report lists every problem at once. diff --git a/docs/src/content/docs/reference/cli/audit.md b/docs/src/content/docs/reference/cli/audit.md index b688ac3aa5..bd194f9239 100644 --- a/docs/src/content/docs/reference/cli/audit.md +++ b/docs/src/content/docs/reference/cli/audit.md @@ -16,7 +16,7 @@ apm audit [PACKAGE] [OPTIONS] `apm audit` is the explicit security and integrity tool. It runs in two modes: - **Content scan mode** (default). Discovers recognized deployed primitives and checks applicable prompt content for hidden Unicode, including untracked primitives and recorded files outside currently selected target directories. It replays the install pipeline into a scratch tree to detect drift (hand-edits to deployed files, missing integrations, orphaned files vs the lockfile). Can also remediate regular prompt documents with `--strip` or scan an arbitrary file with `--file`. -- **CI gate mode** (`--ci`). Runs lockfile consistency checks plus drift in machine-readable form (text, JSON, or SARIF) suitable for branch-protection gates. When `apm_modules/` is absent but `apm.lock.yaml` is present, CI mode self-hydrates a lock-pinned scratch install for `config-consistency` and drift without mutating the checkout. Auto-discovers org policy from your project's git remote unless `--no-policy` is set. +- **CI gate mode** (`--ci`). Runs lockfile consistency checks plus drift in machine-readable form (text, JSON, or SARIF) suitable for branch-protection gates. When `apm_modules/` is absent but `apm.lock.yaml` is present, CI mode self-hydrates a lock-pinned scratch install for `skill-subset-consistency`, `config-consistency`, and drift without mutating the checkout. Auto-discovers org policy from your project's git remote unless `--no-policy` is set. Global audit also checks resolved external deployment roots such as `HERMES_HOME` and `CLAUDE_CONFIG_DIR`. Default audit compares tracked files in diff --git a/packages/apm-guide/.apm/skills/apm-usage/commands.md b/packages/apm-guide/.apm/skills/apm-usage/commands.md index f8055348c3..f84b4271ce 100644 --- a/packages/apm-guide/.apm/skills/apm-usage/commands.md +++ b/packages/apm-guide/.apm/skills/apm-usage/commands.md @@ -309,6 +309,8 @@ rewriting files. Explicit `--file` remains user-directed. |---------|---------|-----------| | `apm audit [PKG]` | Scan installed primitives for hidden Unicode, drift, and lockfile/policy violations | `--file PATH`, `--strip`, `--dry-run`, `-v`, `-f [text\|json\|sarif\|md]`, `-o PATH`, `--ci`, `--policy SOURCE`, `--no-cache`, `--no-fail-fast`, `--no-drift`, `--external NAME` (experimental; ingest a third-party SARIF scanner, e.g. `skillspector`), `--external-sarif PATH`, `--external-llm/--no-external-llm`, `--external-args TEXT` | +For `apm audit --ci` in a checkout without `apm_modules/`, skill-subset validation uses the prepared lock-pinned scratch dependency tree. No checkout install is required. Invalid selections, manifest/lock mismatches, deployed-file integrity failures, and drift still fail the audit. + `apm audit` runs **drift detection by default** (issue #1071). It replays `apm install` into a temporary scratch tree and diffs the result against your working tree. Catches three failure modes: (1) `.apm/` source added without re-running `apm install`, (2) hand-edits to deployed files that diverge from canonical source, (3) orphan files left after their source was removed. The scan is read-only -- never writes to your project, lockfile, or live `apm_modules/`. Build IDs, CRLF line endings, and BOMs are normalized away so they cannot trigger false positives. Bare `apm audit` still uses the warmed local cache and skips with an informational message when the cache is absent. `apm audit --ci` is stricter: when `apm_modules/` is missing but `apm.lock.yaml` is present, it self-hydrates a lock-pinned scratch install for `config-consistency` and drift without touching the checkout. That closes the setup-only CI gap for repos that commit deployed files. Repos that gitignore deployed outputs still need those files present on disk for `deployed-files-present`, so keep the full-install CI pattern there. Use `--no-drift` to opt out (e.g. fast inner loops); the flag is mutually exclusive with `--strip`/`--file`. Ordinary drift remains advisory in bare audit and fails only in `--ci` mode or when policy promotes it. A stale canonical deployment owner is different: `deployment-ledger-owners` is a hard integrity failure in both modes, exits 1, names the owner and path in text/JSON/SARIF, and blocks `--strip`. Remediate it with `apm prune`, then rerun `apm audit`. Drift output is integrated into JSON (top-level `drift` key) and SARIF (rule IDs `apm/drift/` where kind is `modified`/`unintegrated`/`orphaned`). `apm audit` runs **drift detection by default** (issue #1071). It replays `apm install` cache-only into a temporary scratch tree and diffs the result against your working tree. It catches four failure modes: (1) `.apm/` source added without re-running `apm install`, (2) hand-edits to deployed files that diverge from canonical source, (3) orphan files left after their source was removed, and (4) `unrecorded` files that install deploys but no lockfile entry claims. The scan is read-only -- never writes to your project, lockfile, or `apm_modules/`. Build IDs, CRLF line endings, and BOMs are normalized away so they cannot trigger false positives. If the install cache has not been warmed (e.g. a fresh checkout before the first `apm install`), the drift check is skipped with an informational message and can still exit 0; run `apm install` before relying on drift until cold-cache replay lands. Use `--no-drift` to opt out with reduced coverage; the flag is mutually exclusive with `--strip`/`--file`. Ordinary drift remains advisory in bare audit and fails in `--ci` mode. Remediate `unrecorded` with `apm install`, then commit the regenerated `apm.lock.yaml`. A stale canonical deployment owner is different: `deployment-ledger-owners` is a hard integrity failure in both modes, exits 1, names the owner and path in text/JSON/SARIF, and blocks `--strip`. Remediate it with `apm prune`, then rerun `apm audit`. Drift output is integrated into JSON (top-level `drift` key) and SARIF (rule IDs `apm/drift/` where kind is `modified`/`unintegrated`/`orphaned`/`unrecorded`). `apm audit` runs **drift detection by default** (issue #1071). It replays `apm install` cache-only into a temporary scratch tree and diffs the result against your working tree. It catches four failure modes: (1) `.apm/` source added without re-running `apm install`, (2) hand-edits to deployed files that diverge from canonical source, (3) orphan files left after their source was removed, and (4) `unrecorded` files that install deploys but no lockfile entry claims. The scan is read-only -- never writes to your project, lockfile, or `apm_modules/`. Build IDs, CRLF line endings, and BOMs are normalized away so they cannot trigger false positives. If the install cache has not been warmed (e.g. a fresh checkout before the first `apm install`), the drift check is skipped with an informational message and can still exit 0; run `apm install` before relying on drift until cold-cache replay lands. Use `--no-drift` to opt out with reduced coverage; the flag is mutually exclusive with `--strip`/`--file`. Drift is advisory in bare audit by default unless policy enables `security.audit.fail_on_drift`; `--ci` always gates on drift. Remediate `unrecorded` with `apm install`, then commit the regenerated `apm.lock.yaml`. A stale canonical deployment owner is different: `deployment-ledger-owners` is a hard integrity failure in both modes, exits 1, names the owner and path in text/JSON/SARIF, and blocks `--strip`. Remediate it with `apm prune`, then rerun `apm audit`. Drift output is integrated into JSON (top-level `drift` key) and SARIF (rule IDs `apm/drift/` where kind is `modified`/`unintegrated`/`orphaned`/`unrecorded`). diff --git a/scripts/architecture_linter/checks/install_frozen_and_audit.py b/scripts/architecture_linter/checks/install_frozen_and_audit.py index 67bcfc36fa..14ee777fea 100644 --- a/scripts/architecture_linter/checks/install_frozen_and_audit.py +++ b/scripts/architecture_linter/checks/install_frozen_and_audit.py @@ -283,11 +283,15 @@ def check_audit_replay(provider: FactsProvider) -> tuple[Violation, ...]: config_body = _awk_body( ci_checks, re.compile(r"^def _check_config_consistency\("), re.compile(r"^def ") ) + subset_body = _awk_body( + ci_checks, re.compile(r"^def _check_skill_subset_consistency\("), re.compile(r"^def ") + ) if ( not _present_re(owner, re.compile(r"^def prepare_ci_audit_replay\(")) or "prepare_ci_audit_replay" not in audit_gate_calls or "run_replay" in audit_gate_calls or not _body_has(config_body, "prepared_replay.modules_root") + or not _body_has(subset_body, "prepared_replay.modules_root") ): findings.append( _summary( diff --git a/src/apm_cli/install/audit_replay.py b/src/apm_cli/install/audit_replay.py index d5dc5df647..d00d625e0f 100644 --- a/src/apm_cli/install/audit_replay.py +++ b/src/apm_cli/install/audit_replay.py @@ -2,8 +2,9 @@ This module owns the one-shot orchestration that turns a checkout with a lockfile but no live ``apm_modules/`` tree into a prepared, lock-pinned scratch -replay. ``commands/audit.py`` creates the replay once, then both -``config-consistency`` and ``drift`` consume the same materialized state. +replay. ``commands/audit.py`` creates the replay once, then +``skill-subset-consistency``, ``config-consistency`` and ``drift`` consume +the same materialized state. """ from __future__ import annotations diff --git a/src/apm_cli/policy/ci_checks.py b/src/apm_cli/policy/ci_checks.py index 26b62f3168..4d3a1c6ccc 100644 --- a/src/apm_cli/policy/ci_checks.py +++ b/src/apm_cli/policy/ci_checks.py @@ -27,6 +27,7 @@ from ..install.drift import DriftFinding from ..integration.targets import TargetProfile from ..models.apm_package import APMPackage + from ..models.dependency.reference import DependencyReference _logger = logging.getLogger(__name__) @@ -342,8 +343,17 @@ def _check_skill_subset_consistency( manifest: APMPackage, lock: LockFile, project_root: Path, + *, + prepared_replay: PreparedCiAuditReplay | None = None, ) -> CheckResult: """Verify skill subsets match the lockfile and real package tree.""" + from ..constants import APM_MODULES_DIR + + modules_root = ( + prepared_replay.modules_root + if prepared_replay is not None + else project_root / APM_MODULES_DIR + ) mismatches: list[str] = [] for dep_ref in manifest.get_all_apm_dependencies(): key = dep_ref.get_unique_key() @@ -360,7 +370,7 @@ def _check_skill_subset_consistency( ) continue missing = _missing_recorded_skill_subset_paths( - project_root, + modules_root, dep_ref, locked_dep.package_type, lock_subset, @@ -388,8 +398,8 @@ def _check_skill_subset_consistency( def _missing_recorded_skill_subset_paths( - project_root: Path, - dep_ref, + modules_root: Path, + dep_ref: DependencyReference, package_type: str | None, subset: list[str], ) -> tuple[str, ...]: @@ -399,7 +409,6 @@ def _missing_recorded_skill_subset_paths( from types import SimpleNamespace - from ..constants import APM_MODULES_DIR from ..install.outcome import missing_requested_components from ..integration.skill_integrator import SkillIntegrator from ..models.validation import PackageType @@ -407,7 +416,7 @@ def _missing_recorded_skill_subset_paths( try: resolved_package_type = PackageType(package_type) if package_type else None package_info = SimpleNamespace( - install_path=dep_ref.get_install_path(project_root / APM_MODULES_DIR), + install_path=dep_ref.get_install_path(modules_root), package_type=resolved_package_type, ) available = SkillIntegrator.available_skill_names(package_info) @@ -1005,7 +1014,11 @@ def _run(check: CheckResult) -> bool: return result # Check 6: Skill subset consistency (manifest vs lockfile) - if _run(_check_skill_subset_consistency(manifest, lock, project_root)): + if _run( + _check_skill_subset_consistency( + manifest, lock, project_root, prepared_replay=prepared_replay + ) + ): return result # Check 7: Config consistency (MCP) diff --git a/tests/integration/test_architecture_install_compound_mutations.py b/tests/integration/test_architecture_install_compound_mutations.py index cf7048ade6..4e1f4c26ec 100644 --- a/tests/integration/test_architecture_install_compound_mutations.py +++ b/tests/integration/test_architecture_install_compound_mutations.py @@ -482,6 +482,15 @@ def _replace(old: str, new: str) -> tuple[tuple[str, str], ...]: " prepared_replay = prepare_ci_audit_replay(", ), ), + CompoundMutation( + "audit-replay-subset-checkout-root", + AUDIT_RULE, + "src/apm_cli/policy/ci_checks.py", + _replace( + " modules_root = (\n prepared_replay.modules_root\n", + " modules_root = (\n project_root / APM_MODULES_DIR\n", + ), + ), CompoundMutation( "audit-replay-config-root", AUDIT_RULE, diff --git a/tests/integration/test_audit_skill_subset_replay.py b/tests/integration/test_audit_skill_subset_replay.py new file mode 100644 index 0000000000..9c8904dbf2 --- /dev/null +++ b/tests/integration/test_audit_skill_subset_replay.py @@ -0,0 +1,138 @@ +"""Hermetic source-CLI proof of subset audit in a fresh committed checkout.""" + +import json +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +from apm_cli.utils.yaml_io import dump_yaml, load_yaml +from tests.utils.apm_lifecycle_runner import ApmLifecycleRunner +from tests.utils.isolated_apm_environment import IsolatedApmEnvironment +from tests.utils.local_git_repository import LocalGitRepositoryFactory +from tests.utils.local_package import LocalPackageFactory + +pytestmark = [pytest.mark.integration, pytest.mark.e2e] + +_AUDIT = ("audit", "--ci", "--no-policy", "--no-fail-fast", "--format", "json") + + +def _checkout_bytes(project: Path) -> dict[str, bytes]: + """Snapshot every non-git file, including any accidentally created modules.""" + return { + path.relative_to(project).as_posix(): path.read_bytes() + for path in project.rglob("*") + if path.is_file() and ".git" not in path.relative_to(project).parts + } + + +@pytest.mark.parametrize( + ("scenario", "failed_checks"), + [ + ("clean", set()), + ("tampered-deployment", {"content-integrity", "drift"}), + ("subset-mismatch", {"skill-subset-consistency"}), + ("invalid-selection", {"skill-subset-consistency", "drift"}), + ( + "ref-mismatch", + {"ref-consistency", "skill-subset-consistency", "config-consistency", "drift"}, + ), + ], +) +def test_fresh_subset_audit_uses_locked_commit_without_checkout_writes( + tmp_path: Path, + apm_engine_command: tuple[str, ...], + scenario: str, + failed_checks: set[str], +) -> None: + """A real install/commit/clone audit stays pinned after the remote advances.""" + isolated = IsolatedApmEnvironment.create(tmp_path / "isolated", base_env=dict(os.environ)) + repositories = LocalGitRepositoryFactory( + isolated.repository_root, env=isolated.subprocess_env() + ) + packages = LocalPackageFactory(isolated.package_root) + dependency = packages.create("subset-tools") + packages.add_skill( + dependency, "alpha", "---\nname: alpha\ndescription: Selected skill\n---\n# Alpha\n" + ) + packages.add_skill( + dependency, "beta", "---\nname: beta\ndescription: Unselected skill\n---\n# Beta\n" + ) + repository = repositories.create("subset-tools", source_tree=dependency.root) + commit = repositories.commit(repository, message="seed selected skill") + remote = "https://github.com/test/subset-tools" + environment = repositories.url_rewrite_subprocess_env(repository, remote) + consumer = packages.create( + "consumer", + dependencies=({"git": remote, "ref": commit.sha, "skills": ["alpha"]},), + targets=("copilot",), + ) + (consumer.root / ".gitignore").write_text("apm_modules/\n", encoding="utf-8") + runner = ApmLifecycleRunner(apm_engine_command, timeout_seconds=120) + runner.run_sequence( + (("install", "--no-policy"), _AUDIT), + expected_returncodes=(0, 0), + scenario_id="subset-warm", + cwd=consumer.root, + env=environment, + ) + consumer_repo = repositories.create("consumer", source_tree=consumer.root) + repositories.commit(consumer_repo, message="commit generated outputs") + fresh = isolated.work_root / "fresh" + subprocess.run( + ("git", "clone", "--no-local", consumer_repo.file_url, str(fresh)), + env=environment, + check=True, + capture_output=True, + timeout=30, + ) + shutil.rmtree(repository.worktree / "skills" / "alpha") + repositories.commit(repository, message="remove selected skill on latest main") + shutil.rmtree(isolated.cache_root) + isolated.cache_root.mkdir() + assert not (fresh / "apm_modules").exists() + if scenario == "tampered-deployment": + deployed = list(fresh.rglob("SKILL.md")) + assert len(deployed) == 1 + deployed[0].write_text("# Tampered selected skill\n", encoding="utf-8") + elif scenario in {"subset-mismatch", "invalid-selection", "ref-mismatch"}: + manifest_path = fresh / "apm.yml" + manifest = load_yaml(manifest_path) + declaration = manifest["dependencies"]["apm"][0] + if scenario == "ref-mismatch": + declaration["ref"] = "b" * 40 + else: + declaration["skills"] = ["beta" if scenario == "subset-mismatch" else "nonexistent"] + dump_yaml(manifest, manifest_path) + if scenario == "invalid-selection": + lock_path = fresh / "apm.lock.yaml" + lock = load_yaml(lock_path) + lock["dependencies"][0]["skill_subset"] = ["nonexistent"] + dump_yaml(lock, lock_path) + before = _checkout_bytes(fresh) + + result = runner.run(_AUDIT, scenario_id="subset-cold", cwd=fresh, env=environment) + + payload = json.loads(result.stdout) + assert result.returncode == int(bool(failed_checks)), (payload, result.stderr) + assert payload["passed"] is (not failed_checks) + assert {check["name"] for check in payload["checks"] if not check["passed"]} == failed_checks + subset = next( + check for check in payload["checks"] if check["name"] == "skill-subset-consistency" + ) + assert subset["passed"] is ("skill-subset-consistency" not in failed_checks) + if scenario == "clean": + assert payload["drift"]["drift"] == [] + elif scenario == "tampered-deployment": + assert {finding["kind"] for finding in payload["drift"]["drift"]} == {"modified"} + elif scenario == "subset-mismatch": + assert "manifest skills ['beta'] != lockfile skill_subset ['alpha']" in subset["details"][0] + elif scenario == "invalid-selection": + assert ( + "recorded skill subset path(s) not found in package tree: nonexistent" + in subset["details"][0] + ) + assert not (fresh / "apm_modules").exists() + assert _checkout_bytes(fresh) == before diff --git a/tests/unit/policy/test_ci_skill_subset_replay.py b/tests/unit/policy/test_ci_skill_subset_replay.py new file mode 100644 index 0000000000..8b1c4f0d0a --- /dev/null +++ b/tests/unit/policy/test_ci_skill_subset_replay.py @@ -0,0 +1,159 @@ +"""Subset audit must inspect prepared dependencies, not incidental checkout state.""" + +from pathlib import Path + +import pytest + +from apm_cli.install.audit_replay import PreparedCiAuditReplay +from apm_cli.models.dependency.reference import DependencyReference +from apm_cli.policy.ci_checks import run_baseline_checks +from apm_cli.utils.yaml_io import dump_yaml, load_yaml + +pytestmark = pytest.mark.component + + +@pytest.fixture(params=["dependencies", "devDependencies"]) +def subset_project(tmp_path: Path, request: pytest.FixtureRequest) -> Path: + """Write matching manifest and lock selections for a repository subdirectory.""" + project = tmp_path / "checkout" + project.mkdir() + dump_yaml( + { + "name": "subset-audit", + "version": "1.0.0", + "targets": ["copilot"], + request.param: { + "apm": [ + { + "git": "owner/repo", + "path": "plugins/tools", + "ref": "v1", + "skills": ["alpha"], + } + ] + }, + }, + project / "apm.yml", + ) + dump_yaml( + { + "lockfile_version": "1", + "dependencies": [ + { + "repo_url": "owner/repo", + "virtual_path": "plugins/tools", + "is_virtual": True, + "resolved_ref": "v1", + "package_type": "skill_bundle", + "skill_subset": ["alpha"], + } + ], + }, + project / "apm.lock.yaml", + ) + return project + + +def _write_skills(modules_root: Path, names: tuple[str, ...]) -> None: + """Populate a real dependency install path without mocking path resolution.""" + ref = DependencyReference.parse_from_dict({"git": "owner/repo", "path": "plugins/tools"}) + package = ref.get_install_path(modules_root) + package.mkdir(parents=True) + for name in names: + skill = package / "skills" / name / "SKILL.md" + skill.parent.mkdir(parents=True) + skill.write_text(f"---\nname: {name}\ndescription: Test skill\n---\n", encoding="utf-8") + + +@pytest.mark.parametrize( + ("checkout_skills", "replay_skills", "passed"), + [ + (None, ("alpha",), True), + (("alpha",), ("alpha",), True), + (("stale",), ("alpha",), True), + (("alpha",), ("beta",), False), + (None, (), False), + ], + ids=["absent", "present", "stale", "invalid-despite-checkout", "invalid-absent"], +) +def test_baseline_subset_uses_prepared_tree( + subset_project: Path, + tmp_path: Path, + checkout_skills: tuple[str, ...] | None, + replay_skills: tuple[str, ...], + passed: bool, +) -> None: + """Prepared package contents take precedence in both pass and failure cases.""" + if checkout_skills is not None: + _write_skills(subset_project / "apm_modules", checkout_skills) + scratch = tmp_path / "scratch" + modules = scratch / "apm_modules" + _write_skills(modules, replay_skills) + prepared = PreparedCiAuditReplay( + scratch_root=scratch, + modules_root=modules, + lockfile_path=subset_project / "apm.lock.yaml", + tracked_files=None, + targets=(), + ) + before = {path: path.read_bytes() for path in subset_project.rglob("*") if path.is_file()} + + result = run_baseline_checks(subset_project, fail_fast=False, prepared_replay=prepared) + + check = next(check for check in result.checks if check.name == "skill-subset-consistency") + assert next(check for check in result.checks if check.name == "ref-consistency").passed + assert check.passed is passed, check.details + assert check.details == ( + [] + if passed + else [ + "owner/repo/plugins/tools: recorded skill subset path(s) " + "not found in package tree: alpha" + ] + ) + assert { + path: path.read_bytes() for path in subset_project.rglob("*") if path.is_file() + } == before + assert (subset_project / "apm_modules").exists() is (checkout_skills is not None) + + +@pytest.mark.parametrize("checkout_skills", [None, ("alpha",), ("stale",)]) +def test_subset_without_prepared_replay_checks_checkout( + subset_project: Path, checkout_skills: tuple[str, ...] | None +) -> None: + """Callers without a prepared replay retain real local subset validation.""" + if checkout_skills is not None: + _write_skills(subset_project / "apm_modules", checkout_skills) + + result = run_baseline_checks(subset_project, fail_fast=False) + + check = next(check for check in result.checks if check.name == "skill-subset-consistency") + assert check.passed is (checkout_skills == ("alpha",)) + + +def test_prepared_tree_does_not_override_manifest_lock_subset_mismatch( + subset_project: Path, tmp_path: Path +) -> None: + """Even a tree containing both selections cannot hide a manifest/lock mismatch.""" + lock_path = subset_project / "apm.lock.yaml" + lock = load_yaml(lock_path) + lock["dependencies"][0]["skill_subset"] = ["beta"] + dump_yaml(lock, lock_path) + scratch = tmp_path / "scratch" + modules = scratch / "apm_modules" + _write_skills(modules, ("alpha", "beta")) + prepared = PreparedCiAuditReplay( + scratch_root=scratch, + modules_root=modules, + lockfile_path=lock_path, + tracked_files=None, + targets=(), + ) + + result = run_baseline_checks(subset_project, fail_fast=False, prepared_replay=prepared) + + check = next(check for check in result.checks if check.name == "skill-subset-consistency") + assert check.passed is False + assert check.details == [ + "owner/repo/plugins/tools: manifest skills ['alpha'] != lockfile skill_subset ['beta']" + ] From 9169fb70e025de2f696f95124a1c8b1148c8b877 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Fri, 2 Oct 2026 15:52:11 +0200 Subject: [PATCH 02/11] fix(audit): fail closed on prepared_replay_error in skill-subset-consistency The Check 6 skill-subset-consistency gate ignored prepared_replay_error from the scratch-install replay, unlike the sibling config-consistency check. A prepared-replay failure (missing module, integrity mismatch, drift) silently fell through to re-derive from the checkout instead of failing closed, masking the exact fault the replay surfaced. - ci_checks.py: thread prepared_replay_error through _check_skill_subset_consistency with the same fail-closed early return used by _check_config_consistency. - New regression test covering both checkout-skills parametrizations. - Extend the install-deployment-audit-replay static architecture guard to require the fail-closed branch's behavioral marker (not just the parameter name, which already existed in the signature), plus a matching CompoundMutation case; mutation-break proven for both the new test and the new guard clause. - Reconcile two stale doc summaries (commands.md, enforce-in-ci.md) that omitted skill-subset-consistency from the cold-cache/self- hydration description, contradicting the correct list elsewhere on the same page. Fold items surfaced by a full advisory panel review (python-architect, test-coverage-expert, doc-writer, supply-chain-security-expert) that independently converged on the same root cause. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../content/docs/enterprise/enforce-in-ci.md | 3 ++- .../.apm/skills/apm-usage/commands.md | 2 +- .../checks/install_frozen_and_audit.py | 1 + src/apm_cli/policy/ci_checks.py | 14 +++++++++++++- ...architecture_install_compound_mutations.py | 17 +++++++++++++++++ .../policy/test_ci_skill_subset_replay.py | 19 +++++++++++++++++++ 6 files changed, 53 insertions(+), 3 deletions(-) diff --git a/docs/src/content/docs/enterprise/enforce-in-ci.md b/docs/src/content/docs/enterprise/enforce-in-ci.md index 02424195a5..345c61a047 100644 --- a/docs/src/content/docs/enterprise/enforce-in-ci.md +++ b/docs/src/content/docs/enterprise/enforce-in-ci.md @@ -94,7 +94,8 @@ check then compares the freshly restored file against a hash that matches, and the tampering goes undetected. For repos that **commit** their deployed files, the CI gate can now run in -setup-only mode and still execute drift plus `config-consistency` from a cold +setup-only mode and still execute drift plus `config-consistency` and +`skill-subset-consistency` from a cold cache. `apm audit --ci` self-hydrates a lock-pinned scratch install, compares the tracked checkout against that replay, and never rewrites the working tree or live `apm_modules/`. diff --git a/packages/apm-guide/.apm/skills/apm-usage/commands.md b/packages/apm-guide/.apm/skills/apm-usage/commands.md index f84b4271ce..e80ef5903a 100644 --- a/packages/apm-guide/.apm/skills/apm-usage/commands.md +++ b/packages/apm-guide/.apm/skills/apm-usage/commands.md @@ -311,7 +311,7 @@ rewriting files. Explicit `--file` remains user-directed. For `apm audit --ci` in a checkout without `apm_modules/`, skill-subset validation uses the prepared lock-pinned scratch dependency tree. No checkout install is required. Invalid selections, manifest/lock mismatches, deployed-file integrity failures, and drift still fail the audit. -`apm audit` runs **drift detection by default** (issue #1071). It replays `apm install` into a temporary scratch tree and diffs the result against your working tree. Catches three failure modes: (1) `.apm/` source added without re-running `apm install`, (2) hand-edits to deployed files that diverge from canonical source, (3) orphan files left after their source was removed. The scan is read-only -- never writes to your project, lockfile, or live `apm_modules/`. Build IDs, CRLF line endings, and BOMs are normalized away so they cannot trigger false positives. Bare `apm audit` still uses the warmed local cache and skips with an informational message when the cache is absent. `apm audit --ci` is stricter: when `apm_modules/` is missing but `apm.lock.yaml` is present, it self-hydrates a lock-pinned scratch install for `config-consistency` and drift without touching the checkout. That closes the setup-only CI gap for repos that commit deployed files. Repos that gitignore deployed outputs still need those files present on disk for `deployed-files-present`, so keep the full-install CI pattern there. Use `--no-drift` to opt out (e.g. fast inner loops); the flag is mutually exclusive with `--strip`/`--file`. Ordinary drift remains advisory in bare audit and fails only in `--ci` mode or when policy promotes it. A stale canonical deployment owner is different: `deployment-ledger-owners` is a hard integrity failure in both modes, exits 1, names the owner and path in text/JSON/SARIF, and blocks `--strip`. Remediate it with `apm prune`, then rerun `apm audit`. Drift output is integrated into JSON (top-level `drift` key) and SARIF (rule IDs `apm/drift/` where kind is `modified`/`unintegrated`/`orphaned`). +`apm audit` runs **drift detection by default** (issue #1071). It replays `apm install` into a temporary scratch tree and diffs the result against your working tree. Catches three failure modes: (1) `.apm/` source added without re-running `apm install`, (2) hand-edits to deployed files that diverge from canonical source, (3) orphan files left after their source was removed. The scan is read-only -- never writes to your project, lockfile, or live `apm_modules/`. Build IDs, CRLF line endings, and BOMs are normalized away so they cannot trigger false positives. Bare `apm audit` still uses the warmed local cache and skips with an informational message when the cache is absent. `apm audit --ci` is stricter: when `apm_modules/` is missing but `apm.lock.yaml` is present, it self-hydrates a lock-pinned scratch install for `skill-subset-consistency`, `config-consistency`, and drift without touching the checkout. That closes the setup-only CI gap for repos that commit deployed files. Repos that gitignore deployed outputs still need those files present on disk for `deployed-files-present`, so keep the full-install CI pattern there. Use `--no-drift` to opt out (e.g. fast inner loops); the flag is mutually exclusive with `--strip`/`--file`. Ordinary drift remains advisory in bare audit and fails only in `--ci` mode or when policy promotes it. A stale canonical deployment owner is different: `deployment-ledger-owners` is a hard integrity failure in both modes, exits 1, names the owner and path in text/JSON/SARIF, and blocks `--strip`. Remediate it with `apm prune`, then rerun `apm audit`. Drift output is integrated into JSON (top-level `drift` key) and SARIF (rule IDs `apm/drift/` where kind is `modified`/`unintegrated`/`orphaned`). `apm audit` runs **drift detection by default** (issue #1071). It replays `apm install` cache-only into a temporary scratch tree and diffs the result against your working tree. It catches four failure modes: (1) `.apm/` source added without re-running `apm install`, (2) hand-edits to deployed files that diverge from canonical source, (3) orphan files left after their source was removed, and (4) `unrecorded` files that install deploys but no lockfile entry claims. The scan is read-only -- never writes to your project, lockfile, or `apm_modules/`. Build IDs, CRLF line endings, and BOMs are normalized away so they cannot trigger false positives. If the install cache has not been warmed (e.g. a fresh checkout before the first `apm install`), the drift check is skipped with an informational message and can still exit 0; run `apm install` before relying on drift until cold-cache replay lands. Use `--no-drift` to opt out with reduced coverage; the flag is mutually exclusive with `--strip`/`--file`. Ordinary drift remains advisory in bare audit and fails in `--ci` mode. Remediate `unrecorded` with `apm install`, then commit the regenerated `apm.lock.yaml`. A stale canonical deployment owner is different: `deployment-ledger-owners` is a hard integrity failure in both modes, exits 1, names the owner and path in text/JSON/SARIF, and blocks `--strip`. Remediate it with `apm prune`, then rerun `apm audit`. Drift output is integrated into JSON (top-level `drift` key) and SARIF (rule IDs `apm/drift/` where kind is `modified`/`unintegrated`/`orphaned`/`unrecorded`). `apm audit` runs **drift detection by default** (issue #1071). It replays `apm install` cache-only into a temporary scratch tree and diffs the result against your working tree. It catches four failure modes: (1) `.apm/` source added without re-running `apm install`, (2) hand-edits to deployed files that diverge from canonical source, (3) orphan files left after their source was removed, and (4) `unrecorded` files that install deploys but no lockfile entry claims. The scan is read-only -- never writes to your project, lockfile, or `apm_modules/`. Build IDs, CRLF line endings, and BOMs are normalized away so they cannot trigger false positives. If the install cache has not been warmed (e.g. a fresh checkout before the first `apm install`), the drift check is skipped with an informational message and can still exit 0; run `apm install` before relying on drift until cold-cache replay lands. Use `--no-drift` to opt out with reduced coverage; the flag is mutually exclusive with `--strip`/`--file`. Drift is advisory in bare audit by default unless policy enables `security.audit.fail_on_drift`; `--ci` always gates on drift. Remediate `unrecorded` with `apm install`, then commit the regenerated `apm.lock.yaml`. A stale canonical deployment owner is different: `deployment-ledger-owners` is a hard integrity failure in both modes, exits 1, names the owner and path in text/JSON/SARIF, and blocks `--strip`. Remediate it with `apm prune`, then rerun `apm audit`. Drift output is integrated into JSON (top-level `drift` key) and SARIF (rule IDs `apm/drift/` where kind is `modified`/`unintegrated`/`orphaned`/`unrecorded`). `apm audit` runs **drift detection by default** (issue #1071). It replays `apm install` cache-only into a temporary scratch tree and diffs the result against your working tree. It catches four failure modes: (1) `.apm/` source added without re-running `apm install`, (2) hand-edits to deployed files that diverge from canonical source, (3) orphan files left after their source was removed, and (4) `unrecorded` files that install deploys but no lockfile entry claims. The scan is read-only -- never writes to your project, lockfile, or `apm_modules/`. Build IDs, CRLF line endings, and BOMs are normalized away so they cannot trigger false positives. diff --git a/scripts/architecture_linter/checks/install_frozen_and_audit.py b/scripts/architecture_linter/checks/install_frozen_and_audit.py index 14ee777fea..d5af5b8773 100644 --- a/scripts/architecture_linter/checks/install_frozen_and_audit.py +++ b/scripts/architecture_linter/checks/install_frozen_and_audit.py @@ -292,6 +292,7 @@ def check_audit_replay(provider: FactsProvider) -> tuple[Violation, ...]: or "run_replay" in audit_gate_calls or not _body_has(config_body, "prepared_replay.modules_root") or not _body_has(subset_body, "prepared_replay.modules_root") + or not _body_has(subset_body, "prepared_replay_error is not None") ): findings.append( _summary( diff --git a/src/apm_cli/policy/ci_checks.py b/src/apm_cli/policy/ci_checks.py index 4d3a1c6ccc..3b9a164b6b 100644 --- a/src/apm_cli/policy/ci_checks.py +++ b/src/apm_cli/policy/ci_checks.py @@ -345,10 +345,18 @@ def _check_skill_subset_consistency( project_root: Path, *, prepared_replay: PreparedCiAuditReplay | None = None, + prepared_replay_error: str | None = None, ) -> CheckResult: """Verify skill subsets match the lockfile and real package tree.""" from ..constants import APM_MODULES_DIR + if prepared_replay_error is not None: + return CheckResult( + name="skill-subset-consistency", + passed=False, + message=f"skill-subset-consistency replay failed: {prepared_replay_error}", + details=[prepared_replay_error], + ) modules_root = ( prepared_replay.modules_root if prepared_replay is not None @@ -1016,7 +1024,11 @@ def _run(check: CheckResult) -> bool: # Check 6: Skill subset consistency (manifest vs lockfile) if _run( _check_skill_subset_consistency( - manifest, lock, project_root, prepared_replay=prepared_replay + manifest, + lock, + project_root, + prepared_replay=prepared_replay, + prepared_replay_error=prepared_replay_error, ) ): return result diff --git a/tests/integration/test_architecture_install_compound_mutations.py b/tests/integration/test_architecture_install_compound_mutations.py index 4e1f4c26ec..f65449ad1f 100644 --- a/tests/integration/test_architecture_install_compound_mutations.py +++ b/tests/integration/test_architecture_install_compound_mutations.py @@ -497,6 +497,23 @@ def _replace(old: str, new: str) -> tuple[tuple[str, str], ...]: "src/apm_cli/policy/ci_checks.py", _replace("prepared_replay.modules_root", "prepared_replay.project_root"), ), + CompoundMutation( + "audit-replay-subset-error-fail-closed", + AUDIT_RULE, + "src/apm_cli/policy/ci_checks.py", + _replace( + " if prepared_replay_error is not None:\n" + " return CheckResult(\n" + ' name="skill-subset-consistency",\n' + " passed=False,\n" + ' message=f"skill-subset-consistency replay failed: ' + '{prepared_replay_error}",\n' + " details=[prepared_replay_error],\n" + " )\n" + " modules_root = (\n", + " modules_root = (\n", + ), + ), CompoundMutation( "uninstall-select-owner", UNINSTALL_RULE, diff --git a/tests/unit/policy/test_ci_skill_subset_replay.py b/tests/unit/policy/test_ci_skill_subset_replay.py index 8b1c4f0d0a..59cd9517ff 100644 --- a/tests/unit/policy/test_ci_skill_subset_replay.py +++ b/tests/unit/policy/test_ci_skill_subset_replay.py @@ -131,6 +131,25 @@ def test_subset_without_prepared_replay_checks_checkout( assert check.passed is (checkout_skills == ("alpha",)) +@pytest.mark.parametrize("checkout_skills", [None, ("alpha",)]) +def test_subset_fails_closed_on_prepared_replay_error( + subset_project: Path, checkout_skills: tuple[str, ...] | None +) -> None: + """A failed replay preparation must fail the check, not fall back to checkout state.""" + if checkout_skills is not None: + _write_skills(subset_project / "apm_modules", checkout_skills) + + result = run_baseline_checks( + subset_project, + fail_fast=False, + prepared_replay_error="scratch materialization failed: disk quota exceeded", + ) + + check = next(check for check in result.checks if check.name == "skill-subset-consistency") + assert check.passed is False + assert check.details == ["scratch materialization failed: disk quota exceeded"] + + def test_prepared_tree_does_not_override_manifest_lock_subset_mismatch( subset_project: Path, tmp_path: Path ) -> None: From 2b4a08db68a4f4352984b010d4d31a7d312c82f8 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Fri, 2 Oct 2026 16:24:45 +0200 Subject: [PATCH 03/11] test(lifecycle): update global-audit expectations for fail-closed subset check The previous fold made skill-subset-consistency fail closed on prepared_replay_error, matching the existing config-consistency and drift behaviour. This e2e lifecycle-smoke test (gated behind APM_E2E_TESTS + a packaged binary, so not exercised by the targeted unit/integration selection run earlier in this recovery) still asserted the pre-fold two-check failure set for the "package materialization missing" scenarios. Update both affected assertions to include skill-subset-consistency, consistent with the fail-closed design. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../integration/test_required_lifecycle_state_machine.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_required_lifecycle_state_machine.py b/tests/integration/test_required_lifecycle_state_machine.py index bb18ff1db2..5f385e0c08 100644 --- a/tests/integration/test_required_lifecycle_state_machine.py +++ b/tests/integration/test_required_lifecycle_state_machine.py @@ -3162,7 +3162,7 @@ def assert_clean(scenario_id: str) -> dict[str, object]: ) package_removed_audit = audit_row( "global-audit-package-dir-removed", - failed={"config-consistency", "drift"}, + failed={"config-consistency", "drift", "skill-subset-consistency"}, ) package_removed_after_audit = LifecycleStateSnapshot.capture( cwd, @@ -3171,7 +3171,7 @@ def assert_clean(scenario_id: str) -> dict[str, object]: assert _check(package_removed_audit, "no-orphaned-packages")["passed"] is True assert _check(package_removed_audit, "deployed-files-present")["passed"] is True assert _check(package_removed_audit, "content-integrity")["passed"] is True - for check_name in ("config-consistency", "drift"): + for check_name in ("config-consistency", "drift", "skill-subset-consistency"): message = str(_check(package_removed_audit, check_name)["message"]) assert "installed package materialization is missing" in message assert "apm install --global" in message @@ -3254,7 +3254,10 @@ def assert_clean(scenario_id: str) -> dict[str, object]: run(("deps", "clean", "--yes"), "global-deps-clean", command_cwd=physical_apm_home) assert not modules_dir.exists() assert capture().deployment_records == before_clean.deployment_records - audit_row("global-audit-after-deps-clean", failed={"config-consistency", "drift"}) + audit_row( + "global-audit-after-deps-clean", + failed={"config-consistency", "drift", "skill-subset-consistency"}, + ) run(install_args, "global-rehydrate-after-deps-clean") assert_revision(commit_b, "b") assert_clean("global-audit-after-rehydrate") From 6514451b18c0038dbb2676ac204a85d7f1c5a7b7 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Fri, 2 Oct 2026 16:51:07 +0200 Subject: [PATCH 04/11] test(audit): assert replay-failure message text; docs: clarify gitignored audit-only coverage Fold two in-scope delta-panel findings (test-coverage-expert, supply-chain-security-expert): - test_subset_fails_closed_on_prepared_replay_error now asserts check.message contains both the fail-closed prefix and the concrete replay error text, not just check.details. Mutation-break verified: dropping the error text from the message makes this test fail. - ci-cd.md's audit-only-for-gitignored-deploy-roots guidance now states explicitly that content-integrity and drift have no committed bytes to compare in that case, so coverage there is limited to lockfile/subset consistency. Both touch only files already modified by this PR; no new scope. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/src/content/docs/integrations/ci-cd.md | 5 ++++- tests/unit/policy/test_ci_skill_subset_replay.py | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/src/content/docs/integrations/ci-cd.md b/docs/src/content/docs/integrations/ci-cd.md index 3029995902..6294641ecf 100644 --- a/docs/src/content/docs/integrations/ci-cd.md +++ b/docs/src/content/docs/integrations/ci-cd.md @@ -100,7 +100,10 @@ Repos that gitignore deployed outputs can still use the audit-only pattern: `deployed-files-present` skips gitignored paths automatically, so a fresh checkout of a repo that gitignores a deploy directory (e.g. `.agents/`) passes the check without -an `apm install` step. See +an `apm install` step. Note that `content-integrity` and drift have no +committed deployed bytes to compare in that case, so coverage is limited +to lockfile/subset consistency for gitignored deploy roots -- commit +deployed outputs if you need full integrity and drift coverage. See [Audit-only CI pattern](../../enterprise/enforce-in-ci/#audit-only-ci-pattern) for the full recipe and when to use each approach. diff --git a/tests/unit/policy/test_ci_skill_subset_replay.py b/tests/unit/policy/test_ci_skill_subset_replay.py index 59cd9517ff..1c19670146 100644 --- a/tests/unit/policy/test_ci_skill_subset_replay.py +++ b/tests/unit/policy/test_ci_skill_subset_replay.py @@ -148,6 +148,8 @@ def test_subset_fails_closed_on_prepared_replay_error( check = next(check for check in result.checks if check.name == "skill-subset-consistency") assert check.passed is False assert check.details == ["scratch materialization failed: disk quota exceeded"] + assert "skill-subset-consistency replay failed" in check.message + assert "scratch materialization failed: disk quota exceeded" in check.message def test_prepared_tree_does_not_override_manifest_lock_subset_mismatch( From 55616b44428807fd5980def06f42a94dd59d10d1 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Fri, 2 Oct 2026 17:26:02 +0200 Subject: [PATCH 05/11] docs(apm-usage): reconcile --ci skill-subset no-checkout-install claim with drift cold-cache caveat Copilot review flagged the new skill-subset-consistency paragraph as contradicting the pre-existing drift-detection paragraphs below it: the new line said 'no checkout install is required' for --ci, while the adjacent paragraphs say drift is skipped until cold-cache replay lands on a fresh checkout. The two are not actually in conflict -- the --ci scratch-install path (already documented a few lines down) covers skill-subset-consistency, config-consistency, AND drift without a checkout install -- but the juxtaposition read as mutually exclusive CI guidance. Scope this sentence to --ci explicitly so it is clear the bare (non-CI) cold-cache caveat below is unaffected. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- packages/apm-guide/.apm/skills/apm-usage/commands.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apm-guide/.apm/skills/apm-usage/commands.md b/packages/apm-guide/.apm/skills/apm-usage/commands.md index e80ef5903a..3513396de5 100644 --- a/packages/apm-guide/.apm/skills/apm-usage/commands.md +++ b/packages/apm-guide/.apm/skills/apm-usage/commands.md @@ -309,7 +309,7 @@ rewriting files. Explicit `--file` remains user-directed. |---------|---------|-----------| | `apm audit [PKG]` | Scan installed primitives for hidden Unicode, drift, and lockfile/policy violations | `--file PATH`, `--strip`, `--dry-run`, `-v`, `-f [text\|json\|sarif\|md]`, `-o PATH`, `--ci`, `--policy SOURCE`, `--no-cache`, `--no-fail-fast`, `--no-drift`, `--external NAME` (experimental; ingest a third-party SARIF scanner, e.g. `skillspector`), `--external-sarif PATH`, `--external-llm/--no-external-llm`, `--external-args TEXT` | -For `apm audit --ci` in a checkout without `apm_modules/`, skill-subset validation uses the prepared lock-pinned scratch dependency tree. No checkout install is required. Invalid selections, manifest/lock mismatches, deployed-file integrity failures, and drift still fail the audit. +For `apm audit --ci` in a checkout without `apm_modules/`, skill-subset validation uses the prepared lock-pinned scratch dependency tree. No checkout install is required for `--ci` specifically: the cold-cache scratch replay described below already covers `--ci` drift too, so this is not the "until cold-cache replay lands" caveat that still applies to bare (non-CI) `apm audit` drift checks. Invalid selections, manifest/lock mismatches, deployed-file integrity failures, and drift still fail the audit. `apm audit` runs **drift detection by default** (issue #1071). It replays `apm install` into a temporary scratch tree and diffs the result against your working tree. Catches three failure modes: (1) `.apm/` source added without re-running `apm install`, (2) hand-edits to deployed files that diverge from canonical source, (3) orphan files left after their source was removed. The scan is read-only -- never writes to your project, lockfile, or live `apm_modules/`. Build IDs, CRLF line endings, and BOMs are normalized away so they cannot trigger false positives. Bare `apm audit` still uses the warmed local cache and skips with an informational message when the cache is absent. `apm audit --ci` is stricter: when `apm_modules/` is missing but `apm.lock.yaml` is present, it self-hydrates a lock-pinned scratch install for `skill-subset-consistency`, `config-consistency`, and drift without touching the checkout. That closes the setup-only CI gap for repos that commit deployed files. Repos that gitignore deployed outputs still need those files present on disk for `deployed-files-present`, so keep the full-install CI pattern there. Use `--no-drift` to opt out (e.g. fast inner loops); the flag is mutually exclusive with `--strip`/`--file`. Ordinary drift remains advisory in bare audit and fails only in `--ci` mode or when policy promotes it. A stale canonical deployment owner is different: `deployment-ledger-owners` is a hard integrity failure in both modes, exits 1, names the owner and path in text/JSON/SARIF, and blocks `--strip`. Remediate it with `apm prune`, then rerun `apm audit`. Drift output is integrated into JSON (top-level `drift` key) and SARIF (rule IDs `apm/drift/` where kind is `modified`/`unintegrated`/`orphaned`). `apm audit` runs **drift detection by default** (issue #1071). It replays `apm install` cache-only into a temporary scratch tree and diffs the result against your working tree. It catches four failure modes: (1) `.apm/` source added without re-running `apm install`, (2) hand-edits to deployed files that diverge from canonical source, (3) orphan files left after their source was removed, and (4) `unrecorded` files that install deploys but no lockfile entry claims. The scan is read-only -- never writes to your project, lockfile, or `apm_modules/`. Build IDs, CRLF line endings, and BOMs are normalized away so they cannot trigger false positives. If the install cache has not been warmed (e.g. a fresh checkout before the first `apm install`), the drift check is skipped with an informational message and can still exit 0; run `apm install` before relying on drift until cold-cache replay lands. Use `--no-drift` to opt out with reduced coverage; the flag is mutually exclusive with `--strip`/`--file`. Ordinary drift remains advisory in bare audit and fails in `--ci` mode. Remediate `unrecorded` with `apm install`, then commit the regenerated `apm.lock.yaml`. A stale canonical deployment owner is different: `deployment-ledger-owners` is a hard integrity failure in both modes, exits 1, names the owner and path in text/JSON/SARIF, and blocks `--strip`. Remediate it with `apm prune`, then rerun `apm audit`. Drift output is integrated into JSON (top-level `drift` key) and SARIF (rule IDs `apm/drift/` where kind is `modified`/`unintegrated`/`orphaned`/`unrecorded`). From 4963591e2065841b1f6b38fb48fd7c35f5d38e89 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Mon, 5 Oct 2026 14:58:58 +0200 Subject: [PATCH 06/11] docs: fold panel review doc-writer findings for skill-subset-consistency replay - Reorder/merge the new disambiguation paragraph in commands.md to follow the pre-existing self-hydration paragraph it depends on, removing the forward-reference and duplicate explanation. - Qualify the drift-checks-inspect-the-checkout claim in ci-cd.md with 'when outputs are committed' for accuracy. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/src/content/docs/integrations/ci-cd.md | 3 ++- packages/apm-guide/.apm/skills/apm-usage/commands.md | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/src/content/docs/integrations/ci-cd.md b/docs/src/content/docs/integrations/ci-cd.md index 6294641ecf..1d2f78cb81 100644 --- a/docs/src/content/docs/integrations/ci-cd.md +++ b/docs/src/content/docs/integrations/ci-cd.md @@ -95,7 +95,8 @@ install when `apm_modules/` is absent, so drift and `config-consistency` still run without mutating the checkout. `skill-subset-consistency` also checks selected skills against this lock-pinned tree, not the absent checkout dependencies. Invalid selections and manifest/lock mismatches still fail; -deployed-file integrity and drift checks still inspect the checkout. +deployed-file integrity and drift checks still inspect the checkout when +outputs are committed. Repos that gitignore deployed outputs can still use the audit-only pattern: `deployed-files-present` skips gitignored paths automatically, so a fresh checkout of a repo that diff --git a/packages/apm-guide/.apm/skills/apm-usage/commands.md b/packages/apm-guide/.apm/skills/apm-usage/commands.md index 3513396de5..0ac43d52fc 100644 --- a/packages/apm-guide/.apm/skills/apm-usage/commands.md +++ b/packages/apm-guide/.apm/skills/apm-usage/commands.md @@ -309,9 +309,9 @@ rewriting files. Explicit `--file` remains user-directed. |---------|---------|-----------| | `apm audit [PKG]` | Scan installed primitives for hidden Unicode, drift, and lockfile/policy violations | `--file PATH`, `--strip`, `--dry-run`, `-v`, `-f [text\|json\|sarif\|md]`, `-o PATH`, `--ci`, `--policy SOURCE`, `--no-cache`, `--no-fail-fast`, `--no-drift`, `--external NAME` (experimental; ingest a third-party SARIF scanner, e.g. `skillspector`), `--external-sarif PATH`, `--external-llm/--no-external-llm`, `--external-args TEXT` | -For `apm audit --ci` in a checkout without `apm_modules/`, skill-subset validation uses the prepared lock-pinned scratch dependency tree. No checkout install is required for `--ci` specifically: the cold-cache scratch replay described below already covers `--ci` drift too, so this is not the "until cold-cache replay lands" caveat that still applies to bare (non-CI) `apm audit` drift checks. Invalid selections, manifest/lock mismatches, deployed-file integrity failures, and drift still fail the audit. - `apm audit` runs **drift detection by default** (issue #1071). It replays `apm install` into a temporary scratch tree and diffs the result against your working tree. Catches three failure modes: (1) `.apm/` source added without re-running `apm install`, (2) hand-edits to deployed files that diverge from canonical source, (3) orphan files left after their source was removed. The scan is read-only -- never writes to your project, lockfile, or live `apm_modules/`. Build IDs, CRLF line endings, and BOMs are normalized away so they cannot trigger false positives. Bare `apm audit` still uses the warmed local cache and skips with an informational message when the cache is absent. `apm audit --ci` is stricter: when `apm_modules/` is missing but `apm.lock.yaml` is present, it self-hydrates a lock-pinned scratch install for `skill-subset-consistency`, `config-consistency`, and drift without touching the checkout. That closes the setup-only CI gap for repos that commit deployed files. Repos that gitignore deployed outputs still need those files present on disk for `deployed-files-present`, so keep the full-install CI pattern there. Use `--no-drift` to opt out (e.g. fast inner loops); the flag is mutually exclusive with `--strip`/`--file`. Ordinary drift remains advisory in bare audit and fails only in `--ci` mode or when policy promotes it. A stale canonical deployment owner is different: `deployment-ledger-owners` is a hard integrity failure in both modes, exits 1, names the owner and path in text/JSON/SARIF, and blocks `--strip`. Remediate it with `apm prune`, then rerun `apm audit`. Drift output is integrated into JSON (top-level `drift` key) and SARIF (rule IDs `apm/drift/` where kind is `modified`/`unintegrated`/`orphaned`). + +Because of that self-hydration, no `apm install` is required before `apm audit --ci` even in a fresh checkout -- this is not the "run `apm install` until cold-cache replay lands" caveat below, which is specific to bare (non-CI) `apm audit` drift. Invalid selections, manifest/lock mismatches, deployed-file integrity failures, and drift still fail the audit. `apm audit` runs **drift detection by default** (issue #1071). It replays `apm install` cache-only into a temporary scratch tree and diffs the result against your working tree. It catches four failure modes: (1) `.apm/` source added without re-running `apm install`, (2) hand-edits to deployed files that diverge from canonical source, (3) orphan files left after their source was removed, and (4) `unrecorded` files that install deploys but no lockfile entry claims. The scan is read-only -- never writes to your project, lockfile, or `apm_modules/`. Build IDs, CRLF line endings, and BOMs are normalized away so they cannot trigger false positives. If the install cache has not been warmed (e.g. a fresh checkout before the first `apm install`), the drift check is skipped with an informational message and can still exit 0; run `apm install` before relying on drift until cold-cache replay lands. Use `--no-drift` to opt out with reduced coverage; the flag is mutually exclusive with `--strip`/`--file`. Ordinary drift remains advisory in bare audit and fails in `--ci` mode. Remediate `unrecorded` with `apm install`, then commit the regenerated `apm.lock.yaml`. A stale canonical deployment owner is different: `deployment-ledger-owners` is a hard integrity failure in both modes, exits 1, names the owner and path in text/JSON/SARIF, and blocks `--strip`. Remediate it with `apm prune`, then rerun `apm audit`. Drift output is integrated into JSON (top-level `drift` key) and SARIF (rule IDs `apm/drift/` where kind is `modified`/`unintegrated`/`orphaned`/`unrecorded`). `apm audit` runs **drift detection by default** (issue #1071). It replays `apm install` cache-only into a temporary scratch tree and diffs the result against your working tree. It catches four failure modes: (1) `.apm/` source added without re-running `apm install`, (2) hand-edits to deployed files that diverge from canonical source, (3) orphan files left after their source was removed, and (4) `unrecorded` files that install deploys but no lockfile entry claims. The scan is read-only -- never writes to your project, lockfile, or `apm_modules/`. Build IDs, CRLF line endings, and BOMs are normalized away so they cannot trigger false positives. If the install cache has not been warmed (e.g. a fresh checkout before the first `apm install`), the drift check is skipped with an informational message and can still exit 0; run `apm install` before relying on drift until cold-cache replay lands. Use `--no-drift` to opt out with reduced coverage; the flag is mutually exclusive with `--strip`/`--file`. Drift is advisory in bare audit by default unless policy enables `security.audit.fail_on_drift`; `--ci` always gates on drift. Remediate `unrecorded` with `apm install`, then commit the regenerated `apm.lock.yaml`. A stale canonical deployment owner is different: `deployment-ledger-owners` is a hard integrity failure in both modes, exits 1, names the owner and path in text/JSON/SARIF, and blocks `--strip`. Remediate it with `apm prune`, then rerun `apm audit`. Drift output is integrated into JSON (top-level `drift` key) and SARIF (rule IDs `apm/drift/` where kind is `modified`/`unintegrated`/`orphaned`/`unrecorded`). `apm audit` runs **drift detection by default** (issue #1071). It replays `apm install` cache-only into a temporary scratch tree and diffs the result against your working tree. It catches four failure modes: (1) `.apm/` source added without re-running `apm install`, (2) hand-edits to deployed files that diverge from canonical source, (3) orphan files left after their source was removed, and (4) `unrecorded` files that install deploys but no lockfile entry claims. The scan is read-only -- never writes to your project, lockfile, or `apm_modules/`. Build IDs, CRLF line endings, and BOMs are normalized away so they cannot trigger false positives. From a71e8511e07dc5f8340f2f193525a4e6e25eb200 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Mon, 5 Oct 2026 15:13:24 +0200 Subject: [PATCH 07/11] docs: fold redundant self-hydration restatement into existing paragraph Delta-panel finding (doc-writer): the prior fold moved the forward-referencing paragraph after its dependency but left it as a standalone restatement, duplicating the self-hydration sentence it now sits directly beneath. Merge the unique content (no-install-required clarification, cold-cache-caveat disambiguation) into the existing sentence and drop the standalone paragraph. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- packages/apm-guide/.apm/skills/apm-usage/commands.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/apm-guide/.apm/skills/apm-usage/commands.md b/packages/apm-guide/.apm/skills/apm-usage/commands.md index 0ac43d52fc..e3af1826c8 100644 --- a/packages/apm-guide/.apm/skills/apm-usage/commands.md +++ b/packages/apm-guide/.apm/skills/apm-usage/commands.md @@ -309,9 +309,7 @@ rewriting files. Explicit `--file` remains user-directed. |---------|---------|-----------| | `apm audit [PKG]` | Scan installed primitives for hidden Unicode, drift, and lockfile/policy violations | `--file PATH`, `--strip`, `--dry-run`, `-v`, `-f [text\|json\|sarif\|md]`, `-o PATH`, `--ci`, `--policy SOURCE`, `--no-cache`, `--no-fail-fast`, `--no-drift`, `--external NAME` (experimental; ingest a third-party SARIF scanner, e.g. `skillspector`), `--external-sarif PATH`, `--external-llm/--no-external-llm`, `--external-args TEXT` | -`apm audit` runs **drift detection by default** (issue #1071). It replays `apm install` into a temporary scratch tree and diffs the result against your working tree. Catches three failure modes: (1) `.apm/` source added without re-running `apm install`, (2) hand-edits to deployed files that diverge from canonical source, (3) orphan files left after their source was removed. The scan is read-only -- never writes to your project, lockfile, or live `apm_modules/`. Build IDs, CRLF line endings, and BOMs are normalized away so they cannot trigger false positives. Bare `apm audit` still uses the warmed local cache and skips with an informational message when the cache is absent. `apm audit --ci` is stricter: when `apm_modules/` is missing but `apm.lock.yaml` is present, it self-hydrates a lock-pinned scratch install for `skill-subset-consistency`, `config-consistency`, and drift without touching the checkout. That closes the setup-only CI gap for repos that commit deployed files. Repos that gitignore deployed outputs still need those files present on disk for `deployed-files-present`, so keep the full-install CI pattern there. Use `--no-drift` to opt out (e.g. fast inner loops); the flag is mutually exclusive with `--strip`/`--file`. Ordinary drift remains advisory in bare audit and fails only in `--ci` mode or when policy promotes it. A stale canonical deployment owner is different: `deployment-ledger-owners` is a hard integrity failure in both modes, exits 1, names the owner and path in text/JSON/SARIF, and blocks `--strip`. Remediate it with `apm prune`, then rerun `apm audit`. Drift output is integrated into JSON (top-level `drift` key) and SARIF (rule IDs `apm/drift/` where kind is `modified`/`unintegrated`/`orphaned`). - -Because of that self-hydration, no `apm install` is required before `apm audit --ci` even in a fresh checkout -- this is not the "run `apm install` until cold-cache replay lands" caveat below, which is specific to bare (non-CI) `apm audit` drift. Invalid selections, manifest/lock mismatches, deployed-file integrity failures, and drift still fail the audit. +`apm audit` runs **drift detection by default** (issue #1071). It replays `apm install` into a temporary scratch tree and diffs the result against your working tree. Catches three failure modes: (1) `.apm/` source added without re-running `apm install`, (2) hand-edits to deployed files that diverge from canonical source, (3) orphan files left after their source was removed. The scan is read-only -- never writes to your project, lockfile, or live `apm_modules/`. Build IDs, CRLF line endings, and BOMs are normalized away so they cannot trigger false positives. Bare `apm audit` still uses the warmed local cache and skips with an informational message when the cache is absent. `apm audit --ci` is stricter: when `apm_modules/` is missing but `apm.lock.yaml` is present, it self-hydrates a lock-pinned scratch install for `skill-subset-consistency`, `config-consistency`, and drift without touching the checkout -- no `apm install` is required before `apm audit --ci` even in a fresh checkout, and this is not the "run `apm install` until cold-cache replay lands" caveat below, which is specific to bare (non-CI) `apm audit` drift. That closes the setup-only CI gap for repos that commit deployed files. Repos that gitignore deployed outputs still need those files present on disk for `deployed-files-present`, so keep the full-install CI pattern there. Use `--no-drift` to opt out (e.g. fast inner loops); the flag is mutually exclusive with `--strip`/`--file`. Ordinary drift remains advisory in bare audit and fails only in `--ci` mode or when policy promotes it. Invalid selections, manifest/lock mismatches, deployed-file integrity failures, and drift still fail the audit. A stale canonical deployment owner is different: `deployment-ledger-owners` is a hard integrity failure in both modes, exits 1, names the owner and path in text/JSON/SARIF, and blocks `--strip`. Remediate it with `apm prune`, then rerun `apm audit`. Drift output is integrated into JSON (top-level `drift` key) and SARIF (rule IDs `apm/drift/` where kind is `modified`/`unintegrated`/`orphaned`). `apm audit` runs **drift detection by default** (issue #1071). It replays `apm install` cache-only into a temporary scratch tree and diffs the result against your working tree. It catches four failure modes: (1) `.apm/` source added without re-running `apm install`, (2) hand-edits to deployed files that diverge from canonical source, (3) orphan files left after their source was removed, and (4) `unrecorded` files that install deploys but no lockfile entry claims. The scan is read-only -- never writes to your project, lockfile, or `apm_modules/`. Build IDs, CRLF line endings, and BOMs are normalized away so they cannot trigger false positives. If the install cache has not been warmed (e.g. a fresh checkout before the first `apm install`), the drift check is skipped with an informational message and can still exit 0; run `apm install` before relying on drift until cold-cache replay lands. Use `--no-drift` to opt out with reduced coverage; the flag is mutually exclusive with `--strip`/`--file`. Ordinary drift remains advisory in bare audit and fails in `--ci` mode. Remediate `unrecorded` with `apm install`, then commit the regenerated `apm.lock.yaml`. A stale canonical deployment owner is different: `deployment-ledger-owners` is a hard integrity failure in both modes, exits 1, names the owner and path in text/JSON/SARIF, and blocks `--strip`. Remediate it with `apm prune`, then rerun `apm audit`. Drift output is integrated into JSON (top-level `drift` key) and SARIF (rule IDs `apm/drift/` where kind is `modified`/`unintegrated`/`orphaned`/`unrecorded`). `apm audit` runs **drift detection by default** (issue #1071). It replays `apm install` cache-only into a temporary scratch tree and diffs the result against your working tree. It catches four failure modes: (1) `.apm/` source added without re-running `apm install`, (2) hand-edits to deployed files that diverge from canonical source, (3) orphan files left after their source was removed, and (4) `unrecorded` files that install deploys but no lockfile entry claims. The scan is read-only -- never writes to your project, lockfile, or `apm_modules/`. Build IDs, CRLF line endings, and BOMs are normalized away so they cannot trigger false positives. If the install cache has not been warmed (e.g. a fresh checkout before the first `apm install`), the drift check is skipped with an informational message and can still exit 0; run `apm install` before relying on drift until cold-cache replay lands. Use `--no-drift` to opt out with reduced coverage; the flag is mutually exclusive with `--strip`/`--file`. Drift is advisory in bare audit by default unless policy enables `security.audit.fail_on_drift`; `--ci` always gates on drift. Remediate `unrecorded` with `apm install`, then commit the regenerated `apm.lock.yaml`. A stale canonical deployment owner is different: `deployment-ledger-owners` is a hard integrity failure in both modes, exits 1, names the owner and path in text/JSON/SARIF, and blocks `--strip`. Remediate it with `apm prune`, then rerun `apm audit`. Drift output is integrated into JSON (top-level `drift` key) and SARIF (rule IDs `apm/drift/` where kind is `modified`/`unintegrated`/`orphaned`/`unrecorded`). `apm audit` runs **drift detection by default** (issue #1071). It replays `apm install` cache-only into a temporary scratch tree and diffs the result against your working tree. It catches four failure modes: (1) `.apm/` source added without re-running `apm install`, (2) hand-edits to deployed files that diverge from canonical source, (3) orphan files left after their source was removed, and (4) `unrecorded` files that install deploys but no lockfile entry claims. The scan is read-only -- never writes to your project, lockfile, or `apm_modules/`. Build IDs, CRLF line endings, and BOMs are normalized away so they cannot trigger false positives. From c6f05498ba85c832adc08742108d323651093d40 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Mon, 5 Oct 2026 15:42:08 +0200 Subject: [PATCH 08/11] docs: consolidate duplicate apm audit drift-detection paragraphs Four near-duplicate, partially contradictory 'drift detection by default' paragraphs had accumulated in commands.md immediately below this PR's own edited paragraph (CI self-hydration for skill-subset-consistency). Collapse them into one accurate paragraph: correct the failure-mode count to four (including 'unrecorded', which is a real current drift kind per drift.py/drift_render.py), keep the single cold-cache-replay caveat the CI paragraph explicitly references, and keep the precise fail_on_drift policy gating wording. No check or no-checkout semantics changed. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- packages/apm-guide/.apm/skills/apm-usage/commands.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/apm-guide/.apm/skills/apm-usage/commands.md b/packages/apm-guide/.apm/skills/apm-usage/commands.md index e3af1826c8..c217ff1757 100644 --- a/packages/apm-guide/.apm/skills/apm-usage/commands.md +++ b/packages/apm-guide/.apm/skills/apm-usage/commands.md @@ -309,12 +309,8 @@ rewriting files. Explicit `--file` remains user-directed. |---------|---------|-----------| | `apm audit [PKG]` | Scan installed primitives for hidden Unicode, drift, and lockfile/policy violations | `--file PATH`, `--strip`, `--dry-run`, `-v`, `-f [text\|json\|sarif\|md]`, `-o PATH`, `--ci`, `--policy SOURCE`, `--no-cache`, `--no-fail-fast`, `--no-drift`, `--external NAME` (experimental; ingest a third-party SARIF scanner, e.g. `skillspector`), `--external-sarif PATH`, `--external-llm/--no-external-llm`, `--external-args TEXT` | -`apm audit` runs **drift detection by default** (issue #1071). It replays `apm install` into a temporary scratch tree and diffs the result against your working tree. Catches three failure modes: (1) `.apm/` source added without re-running `apm install`, (2) hand-edits to deployed files that diverge from canonical source, (3) orphan files left after their source was removed. The scan is read-only -- never writes to your project, lockfile, or live `apm_modules/`. Build IDs, CRLF line endings, and BOMs are normalized away so they cannot trigger false positives. Bare `apm audit` still uses the warmed local cache and skips with an informational message when the cache is absent. `apm audit --ci` is stricter: when `apm_modules/` is missing but `apm.lock.yaml` is present, it self-hydrates a lock-pinned scratch install for `skill-subset-consistency`, `config-consistency`, and drift without touching the checkout -- no `apm install` is required before `apm audit --ci` even in a fresh checkout, and this is not the "run `apm install` until cold-cache replay lands" caveat below, which is specific to bare (non-CI) `apm audit` drift. That closes the setup-only CI gap for repos that commit deployed files. Repos that gitignore deployed outputs still need those files present on disk for `deployed-files-present`, so keep the full-install CI pattern there. Use `--no-drift` to opt out (e.g. fast inner loops); the flag is mutually exclusive with `--strip`/`--file`. Ordinary drift remains advisory in bare audit and fails only in `--ci` mode or when policy promotes it. Invalid selections, manifest/lock mismatches, deployed-file integrity failures, and drift still fail the audit. A stale canonical deployment owner is different: `deployment-ledger-owners` is a hard integrity failure in both modes, exits 1, names the owner and path in text/JSON/SARIF, and blocks `--strip`. Remediate it with `apm prune`, then rerun `apm audit`. Drift output is integrated into JSON (top-level `drift` key) and SARIF (rule IDs `apm/drift/` where kind is `modified`/`unintegrated`/`orphaned`). -`apm audit` runs **drift detection by default** (issue #1071). It replays `apm install` cache-only into a temporary scratch tree and diffs the result against your working tree. It catches four failure modes: (1) `.apm/` source added without re-running `apm install`, (2) hand-edits to deployed files that diverge from canonical source, (3) orphan files left after their source was removed, and (4) `unrecorded` files that install deploys but no lockfile entry claims. The scan is read-only -- never writes to your project, lockfile, or `apm_modules/`. Build IDs, CRLF line endings, and BOMs are normalized away so they cannot trigger false positives. If the install cache has not been warmed (e.g. a fresh checkout before the first `apm install`), the drift check is skipped with an informational message and can still exit 0; run `apm install` before relying on drift until cold-cache replay lands. Use `--no-drift` to opt out with reduced coverage; the flag is mutually exclusive with `--strip`/`--file`. Ordinary drift remains advisory in bare audit and fails in `--ci` mode. Remediate `unrecorded` with `apm install`, then commit the regenerated `apm.lock.yaml`. A stale canonical deployment owner is different: `deployment-ledger-owners` is a hard integrity failure in both modes, exits 1, names the owner and path in text/JSON/SARIF, and blocks `--strip`. Remediate it with `apm prune`, then rerun `apm audit`. Drift output is integrated into JSON (top-level `drift` key) and SARIF (rule IDs `apm/drift/` where kind is `modified`/`unintegrated`/`orphaned`/`unrecorded`). -`apm audit` runs **drift detection by default** (issue #1071). It replays `apm install` cache-only into a temporary scratch tree and diffs the result against your working tree. It catches four failure modes: (1) `.apm/` source added without re-running `apm install`, (2) hand-edits to deployed files that diverge from canonical source, (3) orphan files left after their source was removed, and (4) `unrecorded` files that install deploys but no lockfile entry claims. The scan is read-only -- never writes to your project, lockfile, or `apm_modules/`. Build IDs, CRLF line endings, and BOMs are normalized away so they cannot trigger false positives. If the install cache has not been warmed (e.g. a fresh checkout before the first `apm install`), the drift check is skipped with an informational message and can still exit 0; run `apm install` before relying on drift until cold-cache replay lands. Use `--no-drift` to opt out with reduced coverage; the flag is mutually exclusive with `--strip`/`--file`. Drift is advisory in bare audit by default unless policy enables `security.audit.fail_on_drift`; `--ci` always gates on drift. Remediate `unrecorded` with `apm install`, then commit the regenerated `apm.lock.yaml`. A stale canonical deployment owner is different: `deployment-ledger-owners` is a hard integrity failure in both modes, exits 1, names the owner and path in text/JSON/SARIF, and blocks `--strip`. Remediate it with `apm prune`, then rerun `apm audit`. Drift output is integrated into JSON (top-level `drift` key) and SARIF (rule IDs `apm/drift/` where kind is `modified`/`unintegrated`/`orphaned`/`unrecorded`). -`apm audit` runs **drift detection by default** (issue #1071). It replays `apm install` cache-only into a temporary scratch tree and diffs the result against your working tree. It catches four failure modes: (1) `.apm/` source added without re-running `apm install`, (2) hand-edits to deployed files that diverge from canonical source, (3) orphan files left after their source was removed, and (4) `unrecorded` files that install deploys but no lockfile entry claims. The scan is read-only -- never writes to your project, lockfile, or `apm_modules/`. Build IDs, CRLF line endings, and BOMs are normalized away so they cannot trigger false positives. +`apm audit` runs **drift detection by default** (issue #1071). It replays `apm install` cache-only into a temporary scratch tree and diffs the result against your working tree. It catches four failure modes: (1) `.apm/` source added without re-running `apm install`, (2) hand-edits to deployed files that diverge from canonical source, (3) orphan files left after their source was removed, and (4) `unrecorded` files that install deploys but no lockfile entry claims. The scan is read-only -- never writes to your project, lockfile, or live `apm_modules/`. Build IDs, CRLF line endings, and BOMs are normalized away so they cannot trigger false positives. Bare `apm audit` still uses the warmed local cache and skips with an informational message when the cache is absent; run `apm install` before relying on drift until cold-cache replay lands. `apm audit --ci` is stricter: when `apm_modules/` is missing but `apm.lock.yaml` is present, it self-hydrates a lock-pinned scratch install for `skill-subset-consistency`, `config-consistency`, and drift without touching the checkout -- no `apm install` is required before `apm audit --ci` even in a fresh checkout, and this is not the bare-audit cache caveat above, which is specific to non-CI drift. That closes the setup-only CI gap for repos that commit deployed files. Repos that gitignore deployed outputs still need those files present on disk for `deployed-files-present`, so keep the full-install CI pattern there. Use `--no-drift` to opt out with reduced coverage (e.g. fast inner loops); the flag is mutually exclusive with `--strip`/`--file`. Drift is advisory in bare audit by default unless policy enables `security.audit.fail_on_drift`; `--ci` always gates on drift. Invalid selections, manifest/lock mismatches, and deployed-file integrity failures still fail the audit in both modes. Remediate `unrecorded` with `apm install`, then commit the regenerated `apm.lock.yaml`. A stale canonical deployment owner is different: `deployment-ledger-owners` is a hard integrity failure in both modes, exits 1, names the owner and path in text/JSON/SARIF, and blocks `--strip`. Remediate it with `apm prune`, then rerun `apm audit`. Drift output is integrated into JSON (top-level `drift` key) and SARIF (rule IDs `apm/drift/` where kind is `modified`/`unintegrated`/`orphaned`/`unrecorded`). -If the install cache has not been warmed (e.g. a fresh checkout before the first `apm install`), the drift check is skipped with an informational message and can still exit 0; run `apm install` before relying on drift until cold-cache replay lands. Use `--no-drift` to opt out with reduced coverage; the flag is mutually exclusive with `--strip`/`--file`. Drift is advisory in bare audit by default unless policy enables `security.audit.fail_on_drift`; `--ci` always gates on drift. Remediate `unrecorded` with `apm install`, then commit the regenerated `apm.lock.yaml`. A stale canonical deployment owner is different: `deployment-ledger-owners` is a hard integrity failure in both modes, exits 1, names the owner and path in text/JSON/SARIF, and blocks `--strip`. Remediate it with `apm prune`, then rerun `apm audit`. Drift output is integrated into JSON (top-level `drift` key) and SARIF (rule IDs `apm/drift/` where kind is `modified`/`unintegrated`/`orphaned`/`unrecorded`). **External scanners (experimental, behind `apm experimental enable external-scanners`).** `--external NAME` runs a third-party SARIF scanner (e.g. `skillspector`) and merges its findings. `--external-llm/--no-external-llm` toggles LLM-powered analysis (default off; sends scanned content to a third-party API, so APM prints a `[!]` egress banner and forwards `OPENAI_API_KEY`/`NVIDIA_INFERENCE_KEY` only when on). `--external-args TEXT` is a single shlex-split string of extra scanner flags, validated against a per-adapter allowlist -- non-allowlisted flags, secret-looking flags, and out-of-cwd paths are rejected fail-closed. `--external-llm`/`--external-args` without `--external` is a usage error (exit 2). Scanner configuration or infrastructure errors (feature disabled, scanner not found, malformed SARIF) exit **3**. Persist defaults with `apm config set external..llm true` and `apm config set external..args -- "--model gpt-4o"`. Precedence: CLI > config > policy floor. From 947e0deee059d51dd2c81f18e16f22d7e075df3e Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Mon, 5 Oct 2026 16:03:55 +0200 Subject: [PATCH 09/11] fix(tests): retarget audit-replay mutation case to actually cover config-consistency The pre-existing 'audit-replay-config-root' mutation case used a bare single-occurrence replace() on 'prepared_replay.modules_root', which textually hit _check_skill_subset_consistency's occurrence first (this PR's new consumer), not _check_config_consistency's occurrence as the name implied. That left _check_config_consistency's own prepared_replay.modules_root fallback completely unmutated/untested, while duplicating coverage already provided by 'audit-replay-subset-checkout-root'. Anchor the mutation on the unique multi-line block around CurrentMcpConfigView.derive(...) so it actually mutates _check_config_consistency, and rename the case to 'audit-replay-config-modules-root' to reflect what it now covers. Folded per the python-architect nit raised in the delta panel review (in-scope per fold-vs-defer rubric: touches a file this PR's diff already modifies). Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../test_architecture_install_compound_mutations.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_architecture_install_compound_mutations.py b/tests/integration/test_architecture_install_compound_mutations.py index f65449ad1f..825975715b 100644 --- a/tests/integration/test_architecture_install_compound_mutations.py +++ b/tests/integration/test_architecture_install_compound_mutations.py @@ -492,10 +492,19 @@ def _replace(old: str, new: str) -> tuple[tuple[str, str], ...]: ), ), CompoundMutation( - "audit-replay-config-root", + "audit-replay-config-modules-root", AUDIT_RULE, "src/apm_cli/policy/ci_checks.py", - _replace("prepared_replay.modules_root", "prepared_replay.project_root"), + _replace( + " prepared_replay.modules_root\n" + " if prepared_replay is not None\n" + " else project_root / APM_MODULES_DIR,\n" + " trust_transitive_self_defined=True,\n", + " prepared_replay.project_root\n" + " if prepared_replay is not None\n" + " else project_root / APM_MODULES_DIR,\n" + " trust_transitive_self_defined=True,\n", + ), ), CompoundMutation( "audit-replay-subset-error-fail-closed", From 5b14b14a099d2d44e99e2d339becbf1624fc3481 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Mon, 5 Oct 2026 16:20:34 +0200 Subject: [PATCH 10/11] Strengthen audit-replay guard: require APM_MODULES_DIR usage in checkout fallback Closes a dual-guardrail gap flagged by a genuine python-architect delta finding: check_audit_replay() verified prepared_replay.modules_root usage in both _check_skill_subset_consistency and _check_config_consistency, but did not verify that their checkout-fallback branches route through the APM_MODULES_DIR constant rather than a hardcoded path. The runtime behavior was already correct in both functions; only the static guard's coverage was incomplete. Adds two mutation-break proofs (audit-replay-subset-fallback-hardcoded, audit-replay-config-fallback-hardcoded) confirming the new guard conjuncts are load-bearing. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../checks/install_frozen_and_audit.py | 2 ++ ..._architecture_install_compound_mutations.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/scripts/architecture_linter/checks/install_frozen_and_audit.py b/scripts/architecture_linter/checks/install_frozen_and_audit.py index d5af5b8773..e1a2286aa8 100644 --- a/scripts/architecture_linter/checks/install_frozen_and_audit.py +++ b/scripts/architecture_linter/checks/install_frozen_and_audit.py @@ -293,6 +293,8 @@ def check_audit_replay(provider: FactsProvider) -> tuple[Violation, ...]: or not _body_has(config_body, "prepared_replay.modules_root") or not _body_has(subset_body, "prepared_replay.modules_root") or not _body_has(subset_body, "prepared_replay_error is not None") + or not _body_has(config_body, "project_root / APM_MODULES_DIR") + or not _body_has(subset_body, "project_root / APM_MODULES_DIR") ): findings.append( _summary( diff --git a/tests/integration/test_architecture_install_compound_mutations.py b/tests/integration/test_architecture_install_compound_mutations.py index 825975715b..15817299c0 100644 --- a/tests/integration/test_architecture_install_compound_mutations.py +++ b/tests/integration/test_architecture_install_compound_mutations.py @@ -506,6 +506,24 @@ def _replace(old: str, new: str) -> tuple[tuple[str, str], ...]: " trust_transitive_self_defined=True,\n", ), ), + CompoundMutation( + "audit-replay-subset-fallback-hardcoded", + AUDIT_RULE, + "src/apm_cli/policy/ci_checks.py", + _replace( + " else project_root / APM_MODULES_DIR\n", + ' else project_root / "apm_modules"\n', + ), + ), + CompoundMutation( + "audit-replay-config-fallback-hardcoded", + AUDIT_RULE, + "src/apm_cli/policy/ci_checks.py", + _replace( + " else project_root / APM_MODULES_DIR,\n", + ' else project_root / "apm_modules",\n', + ), + ), CompoundMutation( "audit-replay-subset-error-fail-closed", AUDIT_RULE, From d095a7d9a86592dc72303533f084dfa384f24a56 Mon Sep 17 00:00:00 2001 From: danielmeppiel Date: Tue, 6 Oct 2026 10:57:28 +0200 Subject: [PATCH 11/11] fix(audit): fold replay guidance and diagnostic review findings Keep the accepted audit-only contract truthful: gitignored outputs are not a pre-install requirement, and CI enforcement must not be attributed to bare audit. Remove duplicated check names in replay errors consistently and retain exact-message and architecture mutation coverage. Addresses all four in-scope CEO follow-ups for #3147. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/src/content/docs/enterprise/enforce-in-ci.md | 6 ++++-- docs/src/content/docs/integrations/ci-cd.md | 1 + docs/src/content/docs/reference/cli/audit.md | 2 +- packages/apm-guide/.apm/skills/apm-usage/commands.md | 8 +++++++- src/apm_cli/policy/ci_checks.py | 4 ++-- .../test_architecture_install_compound_mutations.py | 2 +- tests/unit/policy/test_ci_skill_subset_replay.py | 10 +++++----- 7 files changed, 21 insertions(+), 12 deletions(-) diff --git a/docs/src/content/docs/enterprise/enforce-in-ci.md b/docs/src/content/docs/enterprise/enforce-in-ci.md index 345c61a047..025d230e24 100644 --- a/docs/src/content/docs/enterprise/enforce-in-ci.md +++ b/docs/src/content/docs/enterprise/enforce-in-ci.md @@ -74,8 +74,10 @@ jobs: `microsoft/apm-action@v1` runs `apm install` by default, so by the time `apm audit --ci` runs, the lockfile and deployed files are present. -That remains the right default for repos that gitignore their deployed -outputs, because `deployed-files-present` still expects those files on disk. +Use that default when CI needs to materialize deployed outputs. +Missing gitignored outputs do not fail `deployed-files-present`, so those +repos can also use audit-only CI. Without committed deployed bytes, that +pattern has reduced integrity and drift coverage. Make this job a required status check via [GitHub Rulesets](../github-rulesets/) and a violating PR cannot merge. diff --git a/docs/src/content/docs/integrations/ci-cd.md b/docs/src/content/docs/integrations/ci-cd.md index 1d2f78cb81..3fc4094abf 100644 --- a/docs/src/content/docs/integrations/ci-cd.md +++ b/docs/src/content/docs/integrations/ci-cd.md @@ -97,6 +97,7 @@ checks selected skills against this lock-pinned tree, not the absent checkout dependencies. Invalid selections and manifest/lock mismatches still fail; deployed-file integrity and drift checks still inspect the checkout when outputs are committed. + Repos that gitignore deployed outputs can still use the audit-only pattern: `deployed-files-present` skips gitignored paths automatically, so a fresh checkout of a repo that diff --git a/docs/src/content/docs/reference/cli/audit.md b/docs/src/content/docs/reference/cli/audit.md index bd194f9239..55124ecc4c 100644 --- a/docs/src/content/docs/reference/cli/audit.md +++ b/docs/src/content/docs/reference/cli/audit.md @@ -285,7 +285,7 @@ as metadata repair; see [`apm prune`](../prune/#canonical-deployment-ownership). ### CI checks (`--ci`) -`--ci` runs the baseline lockfile consistency checks defined in `src/apm_cli/policy/ci_checks.py`: lockfile presence, canonical deployment-owner integrity (`deployment-ledger-owners`), ref consistency, deployed-files presence, no orphaned packages, skill-subset consistency, MCP config consistency, content integrity, and an advisory `includes` consent check. A lockfile is required when `apm.yml` declares APM or MCP dependencies. For an MCP-only project, normal [`apm install`](../install/#behavior) creates or repairs the resolved MCP lock state; frozen install fails without writing when that state is missing or stale. Content integrity scans hidden Unicode across the whole-project deployed-file scope and checks SHA-256 drift only where the lockfile provides a baseline. Drift replay runs alongside and contributes to the exit code unless `--no-drift` is set; `--no-drift` never disables hidden-Unicode scanning. On a cold cache, CI mode self-hydrates a scratch install from the lockfile pins instead of reporting a green skip, so setup-only CI can still catch stale committed deployed files without rewriting the checkout. Audit also reports `unrecorded` drift when replay produces governed files that no lockfile entry claims. Repos that gitignore deployed outputs still need those files present on disk for `deployed-files-present`, so the full-install CI pattern remains the right default there. With policy discovery active, declared policy rules are evaluated against the resolved manifest. See [Baseline CI checks](../../baseline-checks/) for the full reference. +`--ci` runs the baseline lockfile consistency checks defined in `src/apm_cli/policy/ci_checks.py`: lockfile presence, canonical deployment-owner integrity (`deployment-ledger-owners`), ref consistency, deployed-files presence, no orphaned packages, skill-subset consistency, MCP config consistency, content integrity, and an advisory `includes` consent check. A lockfile is required when `apm.yml` declares APM or MCP dependencies. For an MCP-only project, normal [`apm install`](../install/#behavior) creates or repairs the resolved MCP lock state; frozen install fails without writing when that state is missing or stale. Content integrity scans hidden Unicode across the whole-project deployed-file scope and checks SHA-256 drift only where the lockfile provides a baseline. Drift replay runs alongside and contributes to the exit code unless `--no-drift` is set; `--no-drift` never disables hidden-Unicode scanning. On a cold cache, CI mode self-hydrates a scratch install from the lockfile pins instead of reporting a green skip, so setup-only CI can still catch stale committed deployed files without rewriting the checkout. Audit also reports `unrecorded` drift when replay produces governed files that no lockfile entry claims. Missing gitignored deployed outputs do not fail `deployed-files-present`; audit-only CI can pass without them, but lacks deployed-byte integrity and drift coverage when no outputs are committed. Use full-install CI when those outputs need to be materialized. With policy discovery active, declared policy rules are evaluated against the resolved manifest. See [Baseline CI checks](../../baseline-checks/) for the full reference. ### Mutual exclusions diff --git a/packages/apm-guide/.apm/skills/apm-usage/commands.md b/packages/apm-guide/.apm/skills/apm-usage/commands.md index c217ff1757..58c7eb3dae 100644 --- a/packages/apm-guide/.apm/skills/apm-usage/commands.md +++ b/packages/apm-guide/.apm/skills/apm-usage/commands.md @@ -309,7 +309,13 @@ rewriting files. Explicit `--file` remains user-directed. |---------|---------|-----------| | `apm audit [PKG]` | Scan installed primitives for hidden Unicode, drift, and lockfile/policy violations | `--file PATH`, `--strip`, `--dry-run`, `-v`, `-f [text\|json\|sarif\|md]`, `-o PATH`, `--ci`, `--policy SOURCE`, `--no-cache`, `--no-fail-fast`, `--no-drift`, `--external NAME` (experimental; ingest a third-party SARIF scanner, e.g. `skillspector`), `--external-sarif PATH`, `--external-llm/--no-external-llm`, `--external-args TEXT` | -`apm audit` runs **drift detection by default** (issue #1071). It replays `apm install` cache-only into a temporary scratch tree and diffs the result against your working tree. It catches four failure modes: (1) `.apm/` source added without re-running `apm install`, (2) hand-edits to deployed files that diverge from canonical source, (3) orphan files left after their source was removed, and (4) `unrecorded` files that install deploys but no lockfile entry claims. The scan is read-only -- never writes to your project, lockfile, or live `apm_modules/`. Build IDs, CRLF line endings, and BOMs are normalized away so they cannot trigger false positives. Bare `apm audit` still uses the warmed local cache and skips with an informational message when the cache is absent; run `apm install` before relying on drift until cold-cache replay lands. `apm audit --ci` is stricter: when `apm_modules/` is missing but `apm.lock.yaml` is present, it self-hydrates a lock-pinned scratch install for `skill-subset-consistency`, `config-consistency`, and drift without touching the checkout -- no `apm install` is required before `apm audit --ci` even in a fresh checkout, and this is not the bare-audit cache caveat above, which is specific to non-CI drift. That closes the setup-only CI gap for repos that commit deployed files. Repos that gitignore deployed outputs still need those files present on disk for `deployed-files-present`, so keep the full-install CI pattern there. Use `--no-drift` to opt out with reduced coverage (e.g. fast inner loops); the flag is mutually exclusive with `--strip`/`--file`. Drift is advisory in bare audit by default unless policy enables `security.audit.fail_on_drift`; `--ci` always gates on drift. Invalid selections, manifest/lock mismatches, and deployed-file integrity failures still fail the audit in both modes. Remediate `unrecorded` with `apm install`, then commit the regenerated `apm.lock.yaml`. A stale canonical deployment owner is different: `deployment-ledger-owners` is a hard integrity failure in both modes, exits 1, names the owner and path in text/JSON/SARIF, and blocks `--strip`. Remediate it with `apm prune`, then rerun `apm audit`. Drift output is integrated into JSON (top-level `drift` key) and SARIF (rule IDs `apm/drift/` where kind is `modified`/`unintegrated`/`orphaned`/`unrecorded`). +`apm audit` runs **drift detection by default** (issue #1071). It replays `apm install` cache-only into a temporary scratch tree and diffs the result against your working tree. It catches four failure modes: (1) `.apm/` source added without re-running `apm install`, (2) hand-edits to deployed files that diverge from canonical source, (3) orphan files left after their source was removed, and (4) `unrecorded` files that install deploys but no lockfile entry claims. The scan is read-only -- never writes to your project, lockfile, or live `apm_modules/`. Build IDs, CRLF line endings, and BOMs are normalized away so they cannot trigger false positives. + +Bare `apm audit` uses the warmed local cache and skips drift with an informational message when the cache is absent; run `apm install` before relying on non-CI drift. `apm audit --ci` is stricter: when `apm_modules/` is missing but `apm.lock.yaml` is present, it self-hydrates a lock-pinned scratch install for `skill-subset-consistency`, `config-consistency`, and drift without touching the checkout. No checkout install is required first. Missing gitignored deployed outputs do not fail `deployed-files-present`, so repos that gitignore them can use audit-only CI too. Without committed deployed bytes, that pattern lacks deployed-byte integrity and drift coverage; use full-install CI when outputs need to be materialized. + +Use `--no-drift` to opt out with reduced coverage (e.g. fast inner loops); the flag is mutually exclusive with `--strip`/`--file`. Drift is advisory in bare audit by default unless policy enables `security.audit.fail_on_drift`; `--ci` always gates on drift. Invalid selections, manifest/lock mismatches, and deployed-file integrity failures still fail `apm audit --ci`. Remediate `unrecorded` with `apm install`, then commit the regenerated `apm.lock.yaml`. + +A stale canonical deployment owner is different: `deployment-ledger-owners` is a hard integrity failure in both modes, exits 1, names the owner and path in text/JSON/SARIF, and blocks `--strip`. Remediate it with `apm prune`, then rerun `apm audit`. Drift output is integrated into JSON (top-level `drift` key) and SARIF (rule IDs `apm/drift/` where kind is `modified`/`unintegrated`/`orphaned`/`unrecorded`). **External scanners (experimental, behind `apm experimental enable external-scanners`).** `--external NAME` runs a third-party SARIF scanner (e.g. `skillspector`) and merges its findings. `--external-llm/--no-external-llm` toggles LLM-powered analysis (default off; sends scanned content to a third-party API, so APM prints a `[!]` egress banner and forwards `OPENAI_API_KEY`/`NVIDIA_INFERENCE_KEY` only when on). `--external-args TEXT` is a single shlex-split string of extra scanner flags, validated against a per-adapter allowlist -- non-allowlisted flags, secret-looking flags, and out-of-cwd paths are rejected fail-closed. `--external-llm`/`--external-args` without `--external` is a usage error (exit 2). Scanner configuration or infrastructure errors (feature disabled, scanner not found, malformed SARIF) exit **3**. Persist defaults with `apm config set external..llm true` and `apm config set external..args -- "--model gpt-4o"`. Precedence: CLI > config > policy floor. diff --git a/src/apm_cli/policy/ci_checks.py b/src/apm_cli/policy/ci_checks.py index 3b9a164b6b..d9689e455c 100644 --- a/src/apm_cli/policy/ci_checks.py +++ b/src/apm_cli/policy/ci_checks.py @@ -354,7 +354,7 @@ def _check_skill_subset_consistency( return CheckResult( name="skill-subset-consistency", passed=False, - message=f"skill-subset-consistency replay failed: {prepared_replay_error}", + message=f"replay failed: {prepared_replay_error}", details=[prepared_replay_error], ) modules_root = ( @@ -451,7 +451,7 @@ def _check_config_consistency( return CheckResult( name="config-consistency", passed=False, - message=f"config-consistency replay failed: {prepared_replay_error}", + message=f"replay failed: {prepared_replay_error}", details=[prepared_replay_error], ) view = CurrentMcpConfigView.derive( diff --git a/tests/integration/test_architecture_install_compound_mutations.py b/tests/integration/test_architecture_install_compound_mutations.py index 15817299c0..f72ab9c4b9 100644 --- a/tests/integration/test_architecture_install_compound_mutations.py +++ b/tests/integration/test_architecture_install_compound_mutations.py @@ -533,7 +533,7 @@ def _replace(old: str, new: str) -> tuple[tuple[str, str], ...]: " return CheckResult(\n" ' name="skill-subset-consistency",\n' " passed=False,\n" - ' message=f"skill-subset-consistency replay failed: ' + ' message=f"replay failed: ' '{prepared_replay_error}",\n' " details=[prepared_replay_error],\n" " )\n" diff --git a/tests/unit/policy/test_ci_skill_subset_replay.py b/tests/unit/policy/test_ci_skill_subset_replay.py index 1c19670146..3f2813b985 100644 --- a/tests/unit/policy/test_ci_skill_subset_replay.py +++ b/tests/unit/policy/test_ci_skill_subset_replay.py @@ -145,11 +145,11 @@ def test_subset_fails_closed_on_prepared_replay_error( prepared_replay_error="scratch materialization failed: disk quota exceeded", ) - check = next(check for check in result.checks if check.name == "skill-subset-consistency") - assert check.passed is False - assert check.details == ["scratch materialization failed: disk quota exceeded"] - assert "skill-subset-consistency replay failed" in check.message - assert "scratch materialization failed: disk quota exceeded" in check.message + for check_name in ("skill-subset-consistency", "config-consistency"): + check = next(check for check in result.checks if check.name == check_name) + assert check.passed is False + assert check.details == ["scratch materialization failed: disk quota exceeded"] + assert check.message == "replay failed: scratch materialization failed: disk quota exceeded" def test_prepared_tree_does_not_override_manifest_lock_subset_mismatch(