From 80f50e1f504a137e67c2cfb9ba29786280df9043 Mon Sep 17 00:00:00 2001 From: Ramesh Padmanabhaiah <22363102+codeforester@users.noreply.github.com> Date: Sun, 18 Oct 2026 17:33:04 +0530 Subject: [PATCH 1/4] fix(workspace): preflight trust before delegated tests --- CHANGELOG.md | 9 + .../basectl/tests/manifest-command-trust.bats | 2 + cli/python/base_projects/engine.py | 2 +- .../tests/test_workspace_test.py | 139 ++++++- cli/python/base_projects/workspace_test.py | 343 ++++++++++++++++-- .../base_setup/manifest_trust_guidance.py | 1 + cli/python/base_trust/tests/test_engine.py | 1 + docs/command-reference.md | 2 +- docs/schemas/workspace-test.json | 22 +- docs/stability-baseline/current.json | 205 ++++++++++- docs/stability-tiers.md | 7 +- docs/workspace-manifest.md | 18 +- tests/contracts/test_workspace_test_schema.py | 13 +- tests/integration/base_workflows.bats | 2 +- 14 files changed, 713 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8e4a283..6a4fd2bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,15 @@ numeric next development line is tracked in `DEVELOPMENT_VERSION`. ## [Unreleased] +### Fixed + +- Make `basectl workspace test` preflight manifest-command trust across the + entire selected set before running any project test, and report explicit + blocked, not-run, start, completion, and dry-run states. +- Stability compatibility: publish workspace-test JSON schema version 2 so + automation can distinguish trust-blocked and not-run projects from tests + that actually executed. + ## [1.10.5] - 2026-10-14 ### Added diff --git a/cli/bash/commands/basectl/tests/manifest-command-trust.bats b/cli/bash/commands/basectl/tests/manifest-command-trust.bats index 0e4cf9dd..2e368f33 100644 --- a/cli/bash/commands/basectl/tests/manifest-command-trust.bats +++ b/cli/bash/commands/basectl/tests/manifest-command-trust.bats @@ -15,6 +15,7 @@ if [[ "${1:-}" == "-m" && "${2:-}" == "base_trust" && "${3:-}" == "require" && " exit 0 fi cat >&2 < int: "--dry-run", is_flag=True, dry_run=True, - help="Show planned clone, pull, or configure work without writing.", + help="Show a workspace test plan or planned clone, pull, or configure work without execution or writing.", ) @base_cli.option("--apply", is_flag=True, help="Apply workspace configure changes after the plan is shown.") @base_cli.option("--yes", is_flag=True, help="Approve workspace setup or configure changes that require confirmation.") diff --git a/cli/python/base_projects/tests/test_workspace_test.py b/cli/python/base_projects/tests/test_workspace_test.py index 8c353740..ae395f88 100644 --- a/cli/python/base_projects/tests/test_workspace_test.py +++ b/cli/python/base_projects/tests/test_workspace_test.py @@ -15,6 +15,9 @@ from base_projects.workspace_manifest import WorkspaceManifestRepo from base_projects.workspace_repo_inspection import WorkspaceRepoInspection from base_projects.workspace_test import workspace_test_manifest_target +from base_setup.manifest import read_manifest +from base_setup.manifest_trust import ManifestCommandTrustStore +from base_setup.manifest_trust import compute_trust_identity def write_workspace_manifest(path: Path, repositories: tuple[str, ...]) -> None: @@ -57,6 +60,15 @@ def write_fake_basectl(base_home: Path) -> Path: return basectl +def allow_workspace_test_manifests(workspace: Path, trusted_projects: tuple[str, ...] | None) -> None: + trust_store = ManifestCommandTrustStore() + for manifest_path in workspace.rglob("base_manifest.yaml"): + manifest = read_manifest(manifest_path) + if trusted_projects is not None and manifest.project_name not in trusted_projects: + continue + trust_store.allow(compute_trust_identity(manifest), base_version="test") + + def write_binary_output_basectl(base_home: Path) -> Path: basectl = base_home / "bin" / "basectl" basectl.parent.mkdir(parents=True, exist_ok=True) @@ -75,6 +87,8 @@ def invoke_workspace_test( args: list[str], base_home: Path, home: Path, + *, + trusted_projects: tuple[str, ...] | None = None, ) -> tuple[int, str, str]: stdout = StringIO() stderr = StringIO() @@ -85,6 +99,8 @@ def invoke_workspace_test( "BASE_PROJECT_MANIFEST": "", } with mock.patch.dict(os.environ, env): + workspace = Path(args[args.index("--workspace") + 1]) + allow_workspace_test_manifests(workspace, trusted_projects) with redirect_stdout(stdout), redirect_stderr(stderr): status = engine.main(args) return status, stdout.getvalue(), stderr.getvalue() @@ -137,7 +153,7 @@ def test_workspace_test_runs_all_projects_and_reports_json_results(self) -> None payload = json.loads(stdout) self.assertEqual(status, 0) self.assertEqual(stderr, "") - self.assertEqual(payload["counts"], {"passed": 2, "failed": 0, "skipped": 0}) + self.assertEqual(payload["counts"], {"passed": 2, "failed": 0, "blocked": 0, "not_run": 0, "skipped": 0}) self.assertEqual(payload["selected_projects"], ["alpha", "beta"]) self.assertEqual([project["status"] for project in payload["projects"]], ["passed", "passed"]) self.assertEqual(run.call_count, 2) @@ -235,7 +251,7 @@ def test_workspace_test_continues_after_a_project_failure(self) -> None: payload = json.loads(stdout) self.assertEqual(status, 1) - self.assertEqual(payload["counts"], {"passed": 2, "failed": 1, "skipped": 0}) + self.assertEqual(payload["counts"], {"passed": 2, "failed": 1, "blocked": 0, "not_run": 0, "skipped": 0}) self.assertEqual(payload["projects"][1]["exit_code"], 7) self.assertEqual(payload["projects"][1]["stderr"], "beta failed\n") self.assertEqual(stderr, "") @@ -276,9 +292,9 @@ def test_workspace_test_fail_fast_skips_remaining_projects(self) -> None: payload = json.loads(stdout) self.assertEqual(status, 1) self.assertEqual(run.call_count, 1) - self.assertEqual(payload["counts"], {"passed": 0, "failed": 1, "skipped": 2}) - self.assertEqual(payload["projects"][1]["status"], "skipped") - self.assertIn("--fail-fast", payload["projects"][1]["detail"]) + self.assertEqual(payload["counts"], {"passed": 0, "failed": 1, "blocked": 0, "not_run": 2, "skipped": 0}) + self.assertEqual(payload["projects"][1]["status"], "not_run") + self.assertIn("execution was stopped", payload["projects"][1]["detail"]) self.assertEqual(stderr, "") def test_workspace_test_projects_filter_runs_only_selected_project(self) -> None: @@ -350,6 +366,117 @@ def test_workspace_test_skips_project_without_test_command(self) -> None: self.assertEqual(status, 0) self.assertEqual(stderr, "") self.assertEqual(run.call_count, 1) - self.assertEqual(payload["counts"], {"passed": 1, "failed": 0, "skipped": 1}) + self.assertEqual(payload["counts"], {"passed": 1, "failed": 0, "blocked": 0, "not_run": 0, "skipped": 1}) self.assertEqual(payload["projects"][1]["status"], "skipped") self.assertIn("does not declare a test command", payload["projects"][1]["detail"]) + + def test_workspace_test_preflights_all_projects_before_running_any_test(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + home = root / "home" + base_home = root / "base" + workspace = root / "workspace" + manifest = root / "workspace.yaml" + home.mkdir() + base_home.mkdir() + write_fake_basectl(base_home) + write_workspace_manifest(manifest, ("trusted", "blocked")) + write_test_manifest(workspace / "trusted", "trusted") + write_test_manifest(workspace / "blocked", "blocked") + + with mock.patch( + "base_projects.workspace_test.run_workspace_subprocess", + return_value=subprocess.CompletedProcess([], 0, "should not run\n", ""), + ) as run: + status, stdout, stderr = invoke_workspace_test( + ["test", "--workspace", str(workspace), "--manifest", str(manifest), "--format", "json"], + base_home, + home, + trusted_projects=("trusted",), + ) + + payload = json.loads(stdout) + self.assertEqual(status, 1) + self.assertEqual(stderr, "") + self.assertEqual(run.call_count, 0) + self.assertEqual(payload["preflight"], "blocked") + self.assertFalse(payload["execution_started"]) + self.assertEqual( + payload["counts"], + {"passed": 0, "failed": 0, "blocked": 1, "not_run": 1, "skipped": 0}, + ) + self.assertEqual([project["status"] for project in payload["projects"]], ["not_run", "blocked"]) + + def test_workspace_test_dry_run_reports_plan_without_running_tests(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + home = root / "home" + base_home = root / "base" + workspace = root / "workspace" + manifest = root / "workspace.yaml" + home.mkdir() + base_home.mkdir() + write_fake_basectl(base_home) + write_workspace_manifest(manifest, ("alpha",)) + write_test_manifest(workspace / "alpha", "alpha") + + with mock.patch("base_projects.workspace_test.run_workspace_subprocess") as run: + status, stdout, stderr = invoke_workspace_test( + [ + "test", + "--workspace", + str(workspace), + "--manifest", + str(manifest), + "--dry-run", + "--format", + "json", + ], + base_home, + home, + ) + + payload = json.loads(stdout) + self.assertEqual(status, 0) + self.assertEqual(stderr, "") + self.assertFalse(run.called) + self.assertTrue(payload["dry_run"]) + self.assertFalse(payload["execution_started"]) + self.assertEqual(payload["projects"][0]["status"], "not_run") + + def test_workspace_test_text_reports_start_and_completion_progress(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + home = root / "home" + base_home = root / "base" + workspace = root / "workspace" + manifest = root / "workspace.yaml" + home.mkdir() + base_home.mkdir() + write_fake_basectl(base_home) + write_workspace_manifest(manifest, ("alpha", "beta")) + write_test_manifest(workspace / "alpha", "alpha") + write_test_manifest(workspace / "beta", "beta") + + with mock.patch( + "base_projects.workspace_test.run_workspace_subprocess", + side_effect=[ + subprocess.CompletedProcess([], 0, "alpha tests\n", ""), + subprocess.CompletedProcess([], 0, "beta tests\n", ""), + ], + ): + status, stdout, stderr = invoke_workspace_test( + ["test", "--workspace", str(workspace), "--manifest", str(manifest)], + base_home, + home, + ) + + self.assertEqual(status, 0) + self.assertEqual(stderr, "") + self.assertIn("Workspace test preflight: 2 selected projects.", stdout) + self.assertIn("[1/2] START project 'alpha' (trust=allowed)", stdout) + self.assertIn("[2/2] START project 'beta' (trust=allowed)", stdout) + self.assertIn("PASS project 'alpha'", stdout) + self.assertIn("PASS project 'beta'", stdout) + self.assertIn("Workspace test completed:", stdout) + self.assertIn("execution_started=yes", stdout) diff --git a/cli/python/base_projects/workspace_test.py b/cli/python/base_projects/workspace_test.py index fdf113fe..658f3760 100644 --- a/cli/python/base_projects/workspace_test.py +++ b/cli/python/base_projects/workspace_test.py @@ -3,6 +3,7 @@ import os import subprocess import sys +import time from dataclasses import dataclass from pathlib import Path from typing import Any, Literal @@ -16,12 +17,17 @@ from base_projects.workspace_repo_inspection import inspect_workspace_repo from base_projects.workspace_scanner import ProjectDiscoveryError from base_projects.workspace_process import run_workspace_subprocess +from base_projects.project_commands import test_command as project_test_command from base_setup.process import decode_subprocess_output from base_setup.process import redact_command_output +from base_setup.manifest import read_manifest +from base_setup.manifest_loader import ManifestError +from base_setup.manifest_trust import ManifestCommandTrustStore +from base_setup.manifest_trust import compute_trust_identity WorkspaceTestAction = Literal["test", "skip"] -WorkspaceTestStatus = Literal["passed", "failed", "skipped"] +WorkspaceTestStatus = Literal["passed", "failed", "blocked", "not_run", "skipped"] WORKSPACE_TEST_TIMEOUT_SECONDS = 1800 @@ -44,12 +50,26 @@ class WorkspaceTestResult: exit_code: int | None = None stdout: str = "" stderr: str = "" + command: str | None = None + trust_status: Literal["allowed", "blocked"] | None = None + executed: bool = False + duration_seconds: float | None = None + + +@dataclass(frozen=True) +class WorkspaceTestPreflight: + status: Literal["allowed", "blocked", "failed", "skipped"] + command: str | None = None + trust_status: Literal["allowed", "blocked"] | None = None + detail: str | None = None @dataclass(frozen=True) class WorkspaceTestCounts: passed: int = 0 failed: int = 0 + blocked: int = 0 + not_run: int = 0 skipped: int = 0 @@ -62,10 +82,6 @@ def workspace_test_from_options(ctx: base_cli.Context, options: Any) -> int: if output_format not in {"text", "json"}: ctx.log.error("Unsupported output format '%s'. Expected: text or json.", options.output_format) return base_cli.ExitCode.USAGE_ERROR - if getattr(options, "dry_run", False): - ctx.log.error("Workspace test does not support --dry-run.") - return base_cli.ExitCode.USAGE_ERROR - try: workspace_root = workspace_context.resolve_workspace_root(ctx, options.workspace) manifest = resolve_workspace_manifest( @@ -96,6 +112,7 @@ def workspace_test_from_options(ctx: base_cli.Context, options: Any) -> int: targets, fail_fast=bool(getattr(options, "fail_fast", False)), output_format=output_format, + dry_run=bool(getattr(options, "dry_run", False)), ) @@ -245,7 +262,7 @@ def validate_distinct_test_manifests(targets: tuple[WorkspaceTestTarget, ...]) - return targets -# pylint: disable=too-many-arguments +# pylint: disable=too-many-arguments,too-many-branches def workspace_test_command( ctx: base_cli.Context, workspace_root: Path, @@ -254,36 +271,77 @@ def workspace_test_command( *, fail_fast: bool, output_format: str, + dry_run: bool, ) -> int: - if ctx.application_home is None: + if ctx.application_home is None and not dry_run: ctx.log.error("BASE_HOME is required to execute workspace tests.") return base_cli.ExitCode.FAILURE - basectl = ctx.application_home / "bin" / "basectl" - if not basectl.is_file() or not os.access(basectl, os.X_OK): + basectl = ctx.application_home / "bin" / "basectl" if ctx.application_home else None + if not dry_run and (basectl is None or not basectl.is_file() or not os.access(basectl, os.X_OK)): ctx.log.error("Base CLI '%s' is missing or is not executable.", basectl) return base_cli.ExitCode.FAILURE + preflights = workspace_test_preflight(targets) + preflight_blocked = any(item.status == "blocked" for item in preflights.values()) + preflight_failed = any(item.status == "failed" for item in preflights.values()) if output_format == "text": - print_workspace_test_header(workspace_root, workspace_manifest, targets) + print_workspace_test_header(workspace_root, workspace_manifest, targets, dry_run=dry_run) + print_workspace_test_preflight(targets, preflights) + + if dry_run or preflight_blocked or preflight_failed: + results = ( + blocked_workspace_test_results(targets, preflights) + if (preflight_blocked or preflight_failed) + else dry_run_workspace_test_results(targets, preflights) + ) + counts = WorkspaceTestCounts() + for _, result in results: + counts = update_workspace_test_counts(counts, result) + if output_format == "text": + if preflight_blocked or preflight_failed: + print_workspace_test_blocked_summary(targets, preflight_blocked, preflight_failed) + else: + print("Workspace test plan complete: no project test commands were executed.") + for target, result in results: + print_workspace_test_result(target, result, show_output=False) + print_workspace_test_summary(counts, execution_started=False) + else: + render_workspace_test_json( + workspace_root, + workspace_manifest, + targets, + results, + counts, + fail_fast=fail_fast, + preflight_status="blocked" if (preflight_blocked or preflight_failed) else "passed", + execution_started=False, + dry_run=dry_run, + ) + return base_cli.ExitCode.FAILURE if (preflight_blocked or preflight_failed) else base_cli.ExitCode.SUCCESS counts = WorkspaceTestCounts() results: list[tuple[WorkspaceTestTarget, WorkspaceTestResult]] = [] stopped = False - for target in targets: + execution_started = False + for index, target in enumerate(targets, start=1): if stopped: - result = WorkspaceTestResult("skipped", "not run because --fail-fast stopped after a failure") + result = WorkspaceTestResult("not_run", "not run because execution was stopped") elif target.action == "skip": status: WorkspaceTestStatus = "failed" if target.fatal else "skipped" result = WorkspaceTestResult(status, target.reason) else: - result = execute_workspace_test_target(ctx, basectl, workspace_root, target) + preflight = preflights[target.name] + if output_format == "text": + print_workspace_test_start(index, len(targets), target, preflight) + execution_started = True + result = execute_workspace_test_target(ctx, basectl, workspace_root, target, preflight) results.append((target, result)) counts = update_workspace_test_counts(counts, result) if output_format == "text": print_workspace_test_result(target, result) - if result.status == "failed" and fail_fast: + if result.status == "blocked" or (result.status == "failed" and fail_fast): stopped = True if output_format == "json": @@ -294,10 +352,117 @@ def workspace_test_command( results, counts, fail_fast=fail_fast, + preflight_status="passed", + execution_started=execution_started, + dry_run=False, ) else: - print_workspace_test_summary(counts) - return base_cli.ExitCode.FAILURE if counts.failed else base_cli.ExitCode.SUCCESS + print_workspace_test_summary(counts, execution_started=execution_started) + return base_cli.ExitCode.FAILURE if counts.failed or counts.blocked else base_cli.ExitCode.SUCCESS + + +def workspace_test_preflight( + targets: tuple[WorkspaceTestTarget, ...], +) -> dict[str, WorkspaceTestPreflight]: + preflights: dict[str, WorkspaceTestPreflight] = {} + trust_store = ManifestCommandTrustStore() + for target in targets: + if target.action != "test": + preflights[target.name] = WorkspaceTestPreflight("skipped", detail=target.reason) + continue + if target.manifest_path is None: + preflights[target.name] = WorkspaceTestPreflight( + "failed", detail="test target is missing its manifest path" + ) + continue + try: + manifest = read_manifest(target.manifest_path) + command_config = project_test_command(manifest.test) if manifest.test is not None else None + if command_config is None: + preflights[target.name] = WorkspaceTestPreflight( + "failed", detail="project does not declare a test command" + ) + continue + identity = compute_trust_identity(manifest) + trust_status = trust_store.status(identity) + except (ManifestError, OSError, ValueError) as exc: + preflights[target.name] = WorkspaceTestPreflight( + "failed", detail=f"could not preflight project test: {exc}" + ) + continue + + command = workspace_test_command_text(command_config.command, command_config.runner) + if trust_status.is_allowed: + preflights[target.name] = WorkspaceTestPreflight( + "allowed", command=command, trust_status="allowed" + ) + else: + preflights[target.name] = WorkspaceTestPreflight( + "blocked", + command=command, + trust_status="blocked", + detail=( + "manifest command trust is blocked " + f"({trust_status.reason})" + ), + ) + return preflights + + +def workspace_test_command_text(command: str, runner: str | None) -> str: + if runner is None: + return command + return f"{runner}: {command}" + + +def blocked_workspace_test_results( + targets: tuple[WorkspaceTestTarget, ...], + preflights: dict[str, WorkspaceTestPreflight], +) -> list[tuple[WorkspaceTestTarget, WorkspaceTestResult]]: + results: list[tuple[WorkspaceTestTarget, WorkspaceTestResult]] = [] + for target in targets: + preflight = preflights[target.name] + if target.action == "skip": + status: WorkspaceTestStatus = "failed" if target.fatal else "skipped" + result = WorkspaceTestResult(status, target.reason) + elif preflight.status == "blocked": + result = WorkspaceTestResult( + "blocked", + preflight.detail, + command=preflight.command, + trust_status="blocked", + ) + elif preflight.status == "allowed": + result = WorkspaceTestResult( + "not_run", + "not run because another selected project's trust preflight blocked execution", + command=preflight.command, + trust_status="allowed", + ) + else: + result = WorkspaceTestResult("failed", preflight.detail) + results.append((target, result)) + return results + + +def dry_run_workspace_test_results( + targets: tuple[WorkspaceTestTarget, ...], + preflights: dict[str, WorkspaceTestPreflight], +) -> list[tuple[WorkspaceTestTarget, WorkspaceTestResult]]: + results: list[tuple[WorkspaceTestTarget, WorkspaceTestResult]] = [] + for target in targets: + preflight = preflights[target.name] + if target.action == "skip": + result = WorkspaceTestResult("skipped", target.reason) + else: + result = WorkspaceTestResult( + "not_run", + "not run because --dry-run only plans execution", + command=preflight.command, + trust_status=preflight.trust_status, + ) + results.append((target, result)) + return results def execute_workspace_test_target( @@ -305,6 +470,7 @@ def execute_workspace_test_target( basectl: Path, workspace_root: Path, target: WorkspaceTestTarget, + preflight: WorkspaceTestPreflight, ) -> WorkspaceTestResult: if target.project_name is None: return WorkspaceTestResult("failed", "test target is missing project routing metadata") @@ -322,6 +488,7 @@ def execute_workspace_test_target( for variable in ("BASE_PROJECT", "BASE_PROJECT_ROOT", "BASE_PROJECT_MANIFEST", "BASE_PROJECT_VENV_DIR"): env.pop(variable, None) + started_at = time.monotonic() try: result = run_workspace_subprocess( command, @@ -333,20 +500,55 @@ def execute_workspace_test_target( return WorkspaceTestResult( "failed", f"timed out after {WORKSPACE_TEST_TIMEOUT_SECONDS} seconds", + command=preflight.command, + trust_status=preflight.trust_status, + executed=True, + duration_seconds=round(time.monotonic() - started_at, 3), ) except OSError as exc: - return WorkspaceTestResult("failed", f"could not run test command: {exc}") + return WorkspaceTestResult( + "failed", + f"could not run test command: {exc}", + command=preflight.command, + trust_status=preflight.trust_status, + executed=True, + duration_seconds=round(time.monotonic() - started_at, 3), + ) stdout = redact_command_output(decode_subprocess_output(result.stdout)) stderr = redact_command_output(decode_subprocess_output(result.stderr)) + duration_seconds = round(time.monotonic() - started_at, 3) if result.returncode == 0: - return WorkspaceTestResult("passed", stdout=stdout, stderr=stderr) + return WorkspaceTestResult( + "passed", + stdout=stdout, + stderr=stderr, + command=preflight.command, + trust_status=preflight.trust_status, + executed=True, + duration_seconds=duration_seconds, + ) + if "Manifest command trust is blocked" in stderr or "Manifest-declared commands are not allowed" in stderr: + return WorkspaceTestResult( + "blocked", + "manifest command trust became blocked before the project test command ran", + exit_code=result.returncode, + stdout=stdout, + stderr=stderr, + command=preflight.command, + trust_status="blocked", + duration_seconds=duration_seconds, + ) return WorkspaceTestResult( "failed", f"test command exited with status {result.returncode}", exit_code=result.returncode, stdout=stdout, stderr=stderr, + command=preflight.command, + trust_status=preflight.trust_status, + executed=True, + duration_seconds=duration_seconds, ) @@ -355,39 +557,107 @@ def update_workspace_test_counts( result: WorkspaceTestResult, ) -> WorkspaceTestCounts: if result.status == "passed": - return WorkspaceTestCounts(counts.passed + 1, counts.failed, counts.skipped) + return WorkspaceTestCounts(counts.passed + 1, counts.failed, counts.blocked, counts.not_run, counts.skipped) if result.status == "failed": - return WorkspaceTestCounts(counts.passed, counts.failed + 1, counts.skipped) - return WorkspaceTestCounts(counts.passed, counts.failed, counts.skipped + 1) + return WorkspaceTestCounts(counts.passed, counts.failed + 1, counts.blocked, counts.not_run, counts.skipped) + if result.status == "blocked": + return WorkspaceTestCounts(counts.passed, counts.failed, counts.blocked + 1, counts.not_run, counts.skipped) + if result.status == "not_run": + return WorkspaceTestCounts(counts.passed, counts.failed, counts.blocked, counts.not_run + 1, counts.skipped) + return WorkspaceTestCounts(counts.passed, counts.failed, counts.blocked, counts.not_run, counts.skipped + 1) def print_workspace_test_header( workspace_root: Path, workspace_manifest: WorkspaceManifest, targets: tuple[WorkspaceTestTarget, ...], + *, + dry_run: bool, ) -> None: print(f"Workspace test: {workspace_root} ({len(targets)} projects)") print(f"Workspace manifest: {workspace_manifest.path} ({workspace_manifest.name})") + print("Execution policy: trust preflight must pass for every selected project before any test starts.") + if dry_run: + print("Mode: dry-run (planning only; no project test commands will execute).") -def print_workspace_test_result(target: WorkspaceTestTarget, result: WorkspaceTestResult) -> None: +def print_workspace_test_preflight( + targets: tuple[WorkspaceTestTarget, ...], + preflights: dict[str, WorkspaceTestPreflight], +) -> None: + print(f"Workspace test preflight: {len(targets)} selected projects.") + for index, target in enumerate(targets, start=1): + project = target.project_name or target.name + preflight = preflights[target.name] + if preflight.status == "allowed": + print(f" [{index}/{len(targets)}] ALLOW project '{project}' (trust=allowed)") + print(f" command: {preflight.command}") + elif preflight.status == "blocked": + print(f" [{index}/{len(targets)}] BLOCK project '{project}' (trust=blocked)") + print(f" command: {preflight.command}") + print(f" reason: {preflight.detail}") + elif preflight.status == "failed": + print(f" [{index}/{len(targets)}] ERROR project '{project}' (preflight failed)") + print(f" reason: {preflight.detail}") + else: + print(f" [{index}/{len(targets)}] SKIP project '{project}': {preflight.detail}") + + +def print_workspace_test_start( + index: int, + total: int, + target: WorkspaceTestTarget, + preflight: WorkspaceTestPreflight, +) -> None: + project = target.project_name or target.name + print(f"[{index}/{total}] START project '{project}' (trust={preflight.trust_status})", flush=True) + print(f" command: {preflight.command}", flush=True) + + +def print_workspace_test_result( + target: WorkspaceTestTarget, + result: WorkspaceTestResult, + *, + show_output: bool = True, +) -> None: project = target.project_name or target.name if result.status == "passed": - print(f"PASS project '{project}' at '{target.root}'.") + print(f"PASS project '{project}' at '{target.root}' ({result.duration_seconds or 0:.3f}s).") elif result.status == "failed": print(f"FAIL project '{project}' at '{target.root}': {result.detail}.") + elif result.status == "blocked": + print(f"BLOCKED project '{project}' at '{target.root}': {result.detail}.") + print(" No project test command was executed.") + elif result.status == "not_run": + print(f"NOT RUN project '{project}' at '{target.root}': {result.detail}.") else: print(f"SKIP project '{project}' at '{target.root}': {result.detail}.") - if result.stdout: + if show_output and result.stdout: print(result.stdout, end="") - if result.stderr: + if show_output and result.stderr: print(result.stderr, end="", file=sys.stderr) -def print_workspace_test_summary(counts: WorkspaceTestCounts) -> None: +def print_workspace_test_blocked_summary( + targets: tuple[WorkspaceTestTarget, ...], + preflight_blocked: bool, + preflight_failed: bool, +) -> None: + test_count = sum(target.action == "test" for target in targets) + reason = "trust is blocked" if preflight_blocked else "trust preflight failed" + print("EXECUTION BLOCKED: No project test command was executed.") + print(f"Workspace test execution blocked: {reason}; 0/{test_count} project test commands started.") + if preflight_failed: + print("Review the preflight errors before retrying.") + + +def print_workspace_test_summary(counts: WorkspaceTestCounts, *, execution_started: bool) -> None: + label = "Workspace test completed" if execution_started else "Workspace test not executed" print( - "Workspace test completed: " - f"passed={counts.passed} failed={counts.failed} skipped={counts.skipped}." + f"{label}: " + f"passed={counts.passed} failed={counts.failed} blocked={counts.blocked} " + f"not_run={counts.not_run} skipped={counts.skipped}; " + f"execution_started={'yes' if execution_started else 'no'}." ) @@ -400,9 +670,12 @@ def render_workspace_test_json( counts: WorkspaceTestCounts, *, fail_fast: bool, + preflight_status: Literal["passed", "blocked"], + execution_started: bool, + dry_run: bool, ) -> None: payload: dict[str, Any] = { - "schema_version": 1, + "schema_version": 2, "workspace": str(workspace_root), "workspace_manifest": { "path": str(workspace_manifest.path), @@ -410,12 +683,17 @@ def render_workspace_test_json( "schema_version": workspace_manifest.schema_version, }, "fail_fast": fail_fast, + "preflight": preflight_status, + "execution_started": execution_started, + "dry_run": dry_run, "selected_projects": [target.project_name or target.name for target in targets], "project_count": len(targets), "projects": [workspace_test_project_to_json(target, result) for target, result in results], "counts": { "passed": counts.passed, "failed": counts.failed, + "blocked": counts.blocked, + "not_run": counts.not_run, "skipped": counts.skipped, }, } @@ -441,4 +719,11 @@ def workspace_test_project_to_json( payload["stdout"] = result.stdout if result.stderr: payload["stderr"] = result.stderr + if result.command is not None: + payload["command"] = result.command + if result.trust_status is not None: + payload["trust_status"] = result.trust_status + payload["executed"] = result.executed + if result.duration_seconds is not None: + payload["duration_seconds"] = result.duration_seconds return payload diff --git a/cli/python/base_setup/manifest_trust_guidance.py b/cli/python/base_setup/manifest_trust_guidance.py index 92084c3a..42881c64 100644 --- a/cli/python/base_setup/manifest_trust_guidance.py +++ b/cli/python/base_setup/manifest_trust_guidance.py @@ -68,6 +68,7 @@ def print_blocked_command_text( *, stream: Any, ) -> None: + print("EXECUTION BLOCKED: No project command was executed.", file=stream) identity = trust_status.identity if trust_status.reason in {"manifest_changed", "test_requirements_changed"}: print( diff --git a/cli/python/base_trust/tests/test_engine.py b/cli/python/base_trust/tests/test_engine.py index 656f241e..c2123568 100644 --- a/cli/python/base_trust/tests/test_engine.py +++ b/cli/python/base_trust/tests/test_engine.py @@ -383,6 +383,7 @@ def test_require_blocks_unapproved_manifest_with_review_guidance(self) -> None: self.assertEqual(status, 1, stderr.getvalue()) self.assertEqual(stdout.getvalue(), "") + self.assertIn("EXECUTION BLOCKED: No project command was executed.", stderr.getvalue()) self.assertIn("Manifest-declared commands are not allowed for project 'demo'", stderr.getvalue()) self.assertIn(f"Manifest SHA-256: {expected_digest}", stderr.getvalue()) self.assertIn("Review first:", stderr.getvalue()) diff --git a/docs/command-reference.md b/docs/command-reference.md index 70008271..e1b32783 100644 --- a/docs/command-reference.md +++ b/docs/command-reference.md @@ -215,7 +215,7 @@ manifest trust. | `basectl workspace init ` | Initialize a workspace from a workspace configuration repository, update local workspace config, and optionally materialize member repositories. | `--owner `, `--path `, `--workspace `, `--manifest `, `--include-optional`, `--dry-run` | | `basectl workspace configure` | Preview the existing `repo configure` repair path by default across discovered Base-managed workspace repositories or an explicit workspace manifest. Use `--apply` to authorize changes; interactive runs prompt unless `--yes` is supplied. Skips missing, non-Base-managed, or non-GitHub repos and continues after per-repo failures. | `--workspace `, `--manifest `, `--dry-run`, `--apply`, `--yes` | | `basectl workspace setup` | Set up eligible repositories from a workspace manifest in manifest order by delegating to each repository's local `basectl setup` command. Skips ineligible repositories, continues after per-repo failures, and reports setup/skipped/failed counts. | `--workspace `, `--manifest `, `--dry-run`, `--yes` | -| `basectl workspace test` | Run declared project test commands serially from each selected repository, without rediscovering unselected siblings. Selection prefers a repository name, then an unambiguous project name; selecting two aliases of the same manifest is rejected. Projects without a declared test command are skipped; failures are reported per project and `--fail-fast` stops after the first failure. JSON uses the stable schema [`docs/schemas/workspace-test.json`](schemas/workspace-test.json). | `--workspace `, `--manifest `, `--projects `, `--fail-fast`, `--format ` | +| `basectl workspace test` | Preflight manifest-command trust for every selected project before starting any delegated test. If any selected test is blocked or its preflight fails, no project test command runs; text output identifies the blocked and not-run projects. Otherwise run declared project test commands serially from each selected repository, without rediscovering unselected siblings. Selection prefers a repository name, then an unambiguous project name; selecting two aliases of the same manifest is rejected. Projects without a declared test command are skipped; runtime failures are reported per project and `--fail-fast` stops after the first runtime failure. JSON uses schema version 2, with explicit `blocked`, `not_run`, `executed`, `preflight`, and `execution_started` state; see [`docs/schemas/workspace-test.json`](schemas/workspace-test.json). | `--workspace `, `--manifest `, `--projects `, `--fail-fast`, `--dry-run`, `--format ` | ## Repository And GitHub Workflow diff --git a/docs/schemas/workspace-test.json b/docs/schemas/workspace-test.json index 8b31480d..0a52ca4a 100644 --- a/docs/schemas/workspace-test.json +++ b/docs/schemas/workspace-test.json @@ -2,23 +2,29 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/basefoundry/base/blob/main/docs/schemas/workspace-test.json", "title": "Base workspace test report", - "description": "Stable schema version 1 for basectl workspace test --format json. Additional properties are allowed for additive, non-breaking fields.", + "description": "Stable schema version 2 for basectl workspace test --format json. Trust preflight and execution state are explicit so blocked and not-run projects cannot be mistaken for executed tests.", "type": "object", "required": [ "schema_version", "workspace", "workspace_manifest", "fail_fast", + "preflight", + "execution_started", + "dry_run", "selected_projects", "project_count", "projects", "counts" ], "properties": { - "schema_version": {"const": 1}, + "schema_version": {"const": 2}, "workspace": {"type": "string"}, "workspace_manifest": {"$ref": "workspace-manifest.json"}, "fail_fast": {"type": "boolean"}, + "preflight": {"enum": ["passed", "blocked"]}, + "execution_started": {"type": "boolean"}, + "dry_run": {"type": "boolean"}, "selected_projects": { "type": "array", "items": {"type": "string"} @@ -39,19 +45,25 @@ "repository": {"type": "string"}, "path": {"type": "string"}, "required": {"type": "boolean"}, - "status": {"enum": ["passed", "failed", "skipped"]}, + "status": {"enum": ["passed", "failed", "blocked", "not_run", "skipped"]}, "detail": {"type": "string"}, "exit_code": {"type": "integer"}, "stdout": {"type": "string"}, - "stderr": {"type": "string"} + "stderr": {"type": "string"}, + "command": {"type": "string"}, + "trust_status": {"enum": ["allowed", "blocked"]}, + "executed": {"type": "boolean"}, + "duration_seconds": {"type": "number", "minimum": 0} } }, "counts": { "type": "object", - "required": ["passed", "failed", "skipped"], + "required": ["passed", "failed", "blocked", "not_run", "skipped"], "properties": { "passed": {"type": "integer", "minimum": 0}, "failed": {"type": "integer", "minimum": 0}, + "blocked": {"type": "integer", "minimum": 0}, + "not_run": {"type": "integer", "minimum": 0}, "skipped": {"type": "integer", "minimum": 0} } } diff --git a/docs/stability-baseline/current.json b/docs/stability-baseline/current.json index 9b310bd9..507f1e1e 100644 --- a/docs/stability-baseline/current.json +++ b/docs/stability-baseline/current.json @@ -1667,6 +1667,9 @@ "--project": { "takes_value": true }, + "--release": { + "takes_value": true + }, "-h": { "takes_value": false }, @@ -2514,13 +2517,179 @@ ], "type": "object" }, + "workspace-status.json": { + "$defs": { + "project": { + "properties": { + "default_branch": { + "type": "string" + }, + "expected": { + "type": "boolean" + }, + "issues": { + "items": { + "type": "string" + }, + "type": "array" + }, + "last_check": { + "properties": { + "checked_at": { + "type": "string" + }, + "status": { + "enum": [ + "error", + "ok", + "warn" + ] + } + }, + "required": [ + "checked_at", + "status" + ], + "type": [ + "null", + "object" + ] + }, + "manifest": { + "type": "string" + }, + "manifest_path": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "python_runtime": { + "$ref": "#/$defs/python_runtime" + }, + "repo": { + "type": "string" + }, + "repository": { + "type": "string" + }, + "required": { + "type": "boolean" + }, + "status": { + "enum": [ + "error", + "ok", + "warn" + ] + }, + "url": { + "type": "string" + }, + "venv": { + "type": "string" + } + }, + "required": [ + "issues", + "last_check", + "manifest", + "manifest_path", + "name", + "path", + "status", + "venv" + ], + "type": "object" + }, + "python_runtime": { + "properties": { + "manager": { + "type": "string" + }, + "pyproject_requires_python": { + "type": "string" + }, + "python": { + "type": "string" + }, + "requires_python": { + "type": "string" + }, + "venv": { + "type": "string" + }, + "version": { + "type": "string" + } + }, + "required": [ + "manager", + "python", + "venv", + "version" + ], + "type": "object" + } + }, + "properties": { + "project_count": { + "type": "integer" + }, + "projects": { + "items": { + "$ref": "#/$defs/project" + }, + "type": "array" + }, + "repository_count": { + "type": "integer" + }, + "schema_version": { + "const": 1 + }, + "status": { + "enum": [ + "error", + "ok", + "warn" + ] + }, + "workspace": { + "type": "string" + }, + "workspace_manifest": { + "$ref": "workspace-manifest.json" + } + }, + "required": [ + "project_count", + "projects", + "schema_version", + "status", + "workspace" + ], + "type": "object" + }, "workspace-test.json": { "$defs": { "counts": { "properties": { + "blocked": { + "type": "integer" + }, "failed": { "type": "integer" }, + "not_run": { + "type": "integer" + }, "passed": { "type": "integer" }, @@ -2529,7 +2698,9 @@ } }, "required": [ + "blocked", "failed", + "not_run", "passed", "skipped" ], @@ -2537,9 +2708,18 @@ }, "project": { "properties": { + "command": { + "type": "string" + }, "detail": { "type": "string" }, + "duration_seconds": { + "type": "number" + }, + "executed": { + "type": "boolean" + }, "exit_code": { "type": "integer" }, @@ -2560,7 +2740,9 @@ }, "status": { "enum": [ + "blocked", "failed", + "not_run", "passed", "skipped" ] @@ -2570,6 +2752,12 @@ }, "stdout": { "type": "string" + }, + "trust_status": { + "enum": [ + "allowed", + "blocked" + ] } }, "required": [ @@ -2586,9 +2774,21 @@ "counts": { "$ref": "#/$defs/counts" }, + "dry_run": { + "type": "boolean" + }, + "execution_started": { + "type": "boolean" + }, "fail_fast": { "type": "boolean" }, + "preflight": { + "enum": [ + "blocked", + "passed" + ] + }, "project_count": { "type": "integer" }, @@ -2599,7 +2799,7 @@ "type": "array" }, "schema_version": { - "const": 1 + "const": 2 }, "selected_projects": { "items": { @@ -2616,7 +2816,10 @@ }, "required": [ "counts", + "dry_run", + "execution_started", "fail_fast", + "preflight", "project_count", "projects", "schema_version", diff --git a/docs/stability-tiers.md b/docs/stability-tiers.md index 45b8849a..3e1e2f83 100644 --- a/docs/stability-tiers.md +++ b/docs/stability-tiers.md @@ -40,8 +40,9 @@ compatibility surfaces, such as Dev Containers and Nix/devenv planning reports, are stable as commands but may keep individual classification details experimental while those ecosystems mature. -`basectl workspace test` is Experimental. Its serial execution, selection, -failure, skip, and JSON-report details may evolve while workspace-wide test +`basectl workspace test` is Experimental. Its workspace-wide trust preflight, +serial execution, selection, failure, skip, blocked/not-run states, dry-run +plan, and JSON-report details may evolve while workspace-wide test orchestration matures. `basectl uninstall |--all` is Stable. It previews by default, applies @@ -78,7 +79,7 @@ Stable JSON contracts include: [`schemas/workspace-agent-brief.json`](schemas/workspace-agent-brief.json). - `basectl workspace update --format json`, whose schema version 1 is published at [`schemas/workspace-update.json`](schemas/workspace-update.json). -- `basectl workspace test --format json`, whose schema version 1 is published +- `basectl workspace test --format json`, whose schema version 2 is published at [`schemas/workspace-test.json`](schemas/workspace-test.json). - `basectl workspace status --format json`, whose schema version 1 is published at [`schemas/workspace-status.json`](schemas/workspace-status.json). diff --git a/docs/workspace-manifest.md b/docs/workspace-manifest.md index 3556a0de..286f61b7 100644 --- a/docs/workspace-manifest.md +++ b/docs/workspace-manifest.md @@ -687,13 +687,21 @@ action also applies the shared repository URL sanitizer defensively. Ordinary command supplies validation, the brief recommends `basectl test`; it does not expose or execute the raw command. -`basectl workspace test --manifest ` runs each selected repository's -declared test command serially. The JSON report's `project_count` includes -every selected repository, and `counts.passed`, `counts.failed`, and +`basectl workspace test --manifest ` first preflights manifest-command +trust for every selected repository. If any selected test is blocked or cannot +be preflighted, no project test command is started; trusted projects are +reported as `not_run` and the blocked project is reported as `blocked`. This +workspace-wide gate prevents a later trusted project from running after an +earlier project displays a trust error. Use `--dry-run` to inspect the same +preflight and planned commands without executing them. + +When preflight passes, each selected repository's declared test command runs +serially. The JSON report's `project_count` includes every selected repository, +and `counts.passed`, `counts.failed`, `counts.blocked`, `counts.not_run`, and `counts.skipped` partition those projects by result; therefore their sum equals `project_count`. A project without a declared test command is skipped, while a -delegated command failure is failed and includes its exit code and redacted -captured output. The published v1 shape is +delegated command failure is failed and includes its exit code, elapsed time, +and redacted captured output. The published schema version 2 shape is [`docs/schemas/workspace-test.json`](schemas/workspace-test.json). `basectl workspace clone --manifest ` clones or validates expected diff --git a/tests/contracts/test_workspace_test_schema.py b/tests/contracts/test_workspace_test_schema.py index 4e3476e9..28cdb6fe 100644 --- a/tests/contracts/test_workspace_test_schema.py +++ b/tests/contracts/test_workspace_test_schema.py @@ -1,10 +1,15 @@ from __future__ import annotations import json +import os from pathlib import Path +from unittest import mock from tests.contracts.workspace_contract_helpers import run_workspace_command from tests.contracts.workspace_contract_helpers import validate_workspace_payload +from base_setup.manifest import read_manifest +from base_setup.manifest_trust import ManifestCommandTrustStore +from base_setup.manifest_trust import compute_trust_identity REPO_ROOT = Path(__file__).resolve().parents[2] @@ -58,6 +63,12 @@ def test_workspace_test_command_matches_published_schema(tmp_path: Path) -> None encoding="utf-8", ) + with mock.patch.dict(os.environ, {"HOME": str(home)}): + trust_store = ManifestCommandTrustStore() + for project in ("pass", "fail"): + project_manifest = read_manifest(workspace / project / "base_manifest.yaml") + trust_store.allow(compute_trust_identity(project_manifest), base_version="test") + result = run_workspace_command( [ "test", @@ -76,7 +87,7 @@ def test_workspace_test_command_matches_published_schema(tmp_path: Path) -> None assert result.stderr == "" payload = json.loads(result.stdout) validate_workspace_payload(payload, SCHEMA_PATH) - assert payload["counts"] == {"passed": 1, "failed": 2, "skipped": 1} + assert payload["counts"] == {"passed": 1, "failed": 2, "blocked": 0, "not_run": 0, "skipped": 1} assert payload["projects"][1]["exit_code"] == 3 assert payload["projects"][1]["stderr"] == "failed\n" assert payload["projects"][3]["project"] is None diff --git a/tests/integration/base_workflows.bats b/tests/integration/base_workflows.bats index ef519e7b..11c8dd38 100755 --- a/tests/integration/base_workflows.bats +++ b/tests/integration/base_workflows.bats @@ -526,7 +526,7 @@ YAML run_basectl workspace test --workspace "$TEST_WORKSPACE" --manifest "$workspace_manifest" --fail-fast --format json [ "$status" -eq 1 ] [[ "$output" == *'"failed": 1'* ]] - [[ "$output" == *'"skipped": 1'* ]] + [[ "$output" == *'"not_run": 1'* ]] [ ! -e "$TEST_STATE_DIR/fake-test.out" ] } From 9bc6e89a94f1a757e8f28635440926b6b08c43f6 Mon Sep 17 00:00:00 2001 From: Ramesh Padmanabhaiah <22363102+codeforester@users.noreply.github.com> Date: Sun, 18 Oct 2026 17:36:00 +0530 Subject: [PATCH 2/4] test(workspace): fix contract import ordering --- tests/contracts/test_workspace_test_schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/contracts/test_workspace_test_schema.py b/tests/contracts/test_workspace_test_schema.py index 28cdb6fe..f58b12e9 100644 --- a/tests/contracts/test_workspace_test_schema.py +++ b/tests/contracts/test_workspace_test_schema.py @@ -5,11 +5,11 @@ from pathlib import Path from unittest import mock -from tests.contracts.workspace_contract_helpers import run_workspace_command -from tests.contracts.workspace_contract_helpers import validate_workspace_payload from base_setup.manifest import read_manifest from base_setup.manifest_trust import ManifestCommandTrustStore from base_setup.manifest_trust import compute_trust_identity +from tests.contracts.workspace_contract_helpers import run_workspace_command +from tests.contracts.workspace_contract_helpers import validate_workspace_payload REPO_ROOT = Path(__file__).resolve().parents[2] From a5d6a31a172930c109fb0d9c9370271c55f7f5d6 Mon Sep 17 00:00:00 2001 From: Ramesh Padmanabhaiah <22363102+codeforester@users.noreply.github.com> Date: Sun, 18 Oct 2026 17:44:25 +0530 Subject: [PATCH 3/4] test(workspace): cover blocked targets in both positions --- .../tests/test_workspace_test.py | 52 +++++++++++++++---- 1 file changed, 41 insertions(+), 11 deletions(-) diff --git a/cli/python/base_projects/tests/test_workspace_test.py b/cli/python/base_projects/tests/test_workspace_test.py index ae395f88..a864b19e 100644 --- a/cli/python/base_projects/tests/test_workspace_test.py +++ b/cli/python/base_projects/tests/test_workspace_test.py @@ -394,18 +394,48 @@ def test_workspace_test_preflights_all_projects_before_running_any_test(self) -> home, trusted_projects=("trusted",), ) + payload = json.loads(stdout) + self.assertEqual(status, 1) + self.assertEqual(stderr, "") + self.assertEqual(run.call_count, 0) + self.assertEqual(payload["preflight"], "blocked") + self.assertFalse(payload["execution_started"]) + self.assertEqual( + payload["counts"], + {"passed": 0, "failed": 0, "blocked": 1, "not_run": 1, "skipped": 0}, + ) + self.assertEqual( + [project["status"] for project in payload["projects"]], ["not_run", "blocked"] + ) - payload = json.loads(stdout) - self.assertEqual(status, 1) - self.assertEqual(stderr, "") - self.assertEqual(run.call_count, 0) - self.assertEqual(payload["preflight"], "blocked") - self.assertFalse(payload["execution_started"]) - self.assertEqual( - payload["counts"], - {"passed": 0, "failed": 0, "blocked": 1, "not_run": 1, "skipped": 0}, - ) - self.assertEqual([project["status"] for project in payload["projects"]], ["not_run", "blocked"]) + manifest.write_text( + "\n".join( + [ + "schema_version: 1", + "workspace:", + " name: test-workspace", + "repos:", + " - name: blocked", + " - name: trusted", + "", + ] + ), + encoding="utf-8", + ) + run.reset_mock() + status, stdout, stderr = invoke_workspace_test( + ["test", "--workspace", str(workspace), "--manifest", str(manifest), "--format", "json"], + base_home, + home, + trusted_projects=("trusted",), + ) + payload = json.loads(stdout) + self.assertEqual(status, 1) + self.assertEqual(stderr, "") + self.assertEqual(run.call_count, 0) + self.assertEqual( + [project["status"] for project in payload["projects"]], ["blocked", "not_run"] + ) def test_workspace_test_dry_run_reports_plan_without_running_tests(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: From ed66295393eb044704315ee90f1418fd9648833c Mon Sep 17 00:00:00 2001 From: Ramesh Padmanabhaiah <22363102+codeforester@users.noreply.github.com> Date: Sun, 18 Oct 2026 21:32:56 +0530 Subject: [PATCH 4/4] fix(workspace): address test preflight review findings --- cli/python/base_projects/test_preflight.py | 18 ++- .../tests/test_workspace_test.py | 35 ++++++ cli/python/base_projects/workspace_test.py | 110 +++++++++++------- 3 files changed, 116 insertions(+), 47 deletions(-) diff --git a/cli/python/base_projects/test_preflight.py b/cli/python/base_projects/test_preflight.py index 0f29e456..d1c97d41 100644 --- a/cli/python/base_projects/test_preflight.py +++ b/cli/python/base_projects/test_preflight.py @@ -9,14 +9,26 @@ from base_setup.manifest_model import BaseManifest from base_setup.test_requirements import check_test_requirements from base_setup.manifest_trust import ( - ManifestCommandTrustStore, compute_trust_identity, manifest_command_surfaces_from_manifest, + ManifestCommandTrustStore, + TrustStatus, + compute_trust_identity, + manifest_command_surfaces_from_manifest, ) from base_setup.manifest_trust_guidance import print_blocked_command_text -def project_test_preflight(ctx: base_cli.Context, manifest: BaseManifest) -> bool: +def project_test_trust_status( + manifest: BaseManifest, + *, + trust_store: ManifestCommandTrustStore | None = None, +) -> TrustStatus: + """Return the trust status used by project and workspace test preflight.""" identity = compute_trust_identity(manifest) - trust_status = ManifestCommandTrustStore().status(identity) + return (trust_store or ManifestCommandTrustStore()).status(identity) + + +def project_test_preflight(ctx: base_cli.Context, manifest: BaseManifest) -> bool: + trust_status = project_test_trust_status(manifest) if not trust_status.is_allowed: print_blocked_command_text( trust_status, manifest_command_surfaces_from_manifest(manifest), stream=sys.stderr, diff --git a/cli/python/base_projects/tests/test_workspace_test.py b/cli/python/base_projects/tests/test_workspace_test.py index a864b19e..bfbb53f4 100644 --- a/cli/python/base_projects/tests/test_workspace_test.py +++ b/cli/python/base_projects/tests/test_workspace_test.py @@ -474,6 +474,41 @@ def test_workspace_test_dry_run_reports_plan_without_running_tests(self) -> None self.assertFalse(payload["execution_started"]) self.assertEqual(payload["projects"][0]["status"], "not_run") + def test_workspace_test_dry_run_fails_for_required_missing_project(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + home = root / "home" + base_home = root / "base" + workspace = root / "workspace" + manifest = root / "workspace.yaml" + home.mkdir() + base_home.mkdir() + workspace.mkdir() + write_workspace_manifest(manifest, ("missing",)) + + status, stdout, stderr = invoke_workspace_test( + [ + "test", + "--workspace", + str(workspace), + "--manifest", + str(manifest), + "--dry-run", + "--format", + "json", + ], + base_home, + home, + ) + + payload = json.loads(stdout) + self.assertEqual(status, 1) + self.assertEqual(stderr, "") + self.assertEqual(payload["counts"], {"passed": 0, "failed": 1, "blocked": 0, "not_run": 0, "skipped": 0}) + self.assertEqual(payload["projects"][0]["status"], "failed") + self.assertIn("repository is missing", payload["projects"][0]["detail"]) + self.assertFalse(payload["execution_started"]) + def test_workspace_test_text_reports_start_and_completion_progress(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir) diff --git a/cli/python/base_projects/workspace_test.py b/cli/python/base_projects/workspace_test.py index 658f3760..14d64744 100644 --- a/cli/python/base_projects/workspace_test.py +++ b/cli/python/base_projects/workspace_test.py @@ -18,12 +18,11 @@ from base_projects.workspace_scanner import ProjectDiscoveryError from base_projects.workspace_process import run_workspace_subprocess from base_projects.project_commands import test_command as project_test_command +from base_projects.test_preflight import project_test_trust_status from base_setup.process import decode_subprocess_output from base_setup.process import redact_command_output -from base_setup.manifest import read_manifest -from base_setup.manifest_loader import ManifestError +from base_setup.manifest_model import BaseManifest from base_setup.manifest_trust import ManifestCommandTrustStore -from base_setup.manifest_trust import compute_trust_identity WorkspaceTestAction = Literal["test", "skip"] @@ -38,6 +37,7 @@ class WorkspaceTestTarget: manifest_path: Path | None project_name: str | None action: WorkspaceTestAction + manifest: BaseManifest | None = None reason: str | None = None required: bool = True fatal: bool = False @@ -194,6 +194,7 @@ def workspace_test_manifest_target( name=repo.name, root=inspection.root, manifest_path=inspection.manifest_path, + manifest=manifest, project_name=manifest.project_name, action="test", required=repo.required, @@ -290,35 +291,17 @@ def workspace_test_command( print_workspace_test_preflight(targets, preflights) if dry_run or preflight_blocked or preflight_failed: - results = ( - blocked_workspace_test_results(targets, preflights) - if (preflight_blocked or preflight_failed) - else dry_run_workspace_test_results(targets, preflights) + return render_workspace_test_plan( + workspace_root, + workspace_manifest, + targets, + preflights, + fail_fast=fail_fast, + output_format=output_format, + dry_run=dry_run, + preflight_blocked=preflight_blocked, + preflight_failed=preflight_failed, ) - counts = WorkspaceTestCounts() - for _, result in results: - counts = update_workspace_test_counts(counts, result) - if output_format == "text": - if preflight_blocked or preflight_failed: - print_workspace_test_blocked_summary(targets, preflight_blocked, preflight_failed) - else: - print("Workspace test plan complete: no project test commands were executed.") - for target, result in results: - print_workspace_test_result(target, result, show_output=False) - print_workspace_test_summary(counts, execution_started=False) - else: - render_workspace_test_json( - workspace_root, - workspace_manifest, - targets, - results, - counts, - fail_fast=fail_fast, - preflight_status="blocked" if (preflight_blocked or preflight_failed) else "passed", - execution_started=False, - dry_run=dry_run, - ) - return base_cli.ExitCode.FAILURE if (preflight_blocked or preflight_failed) else base_cli.ExitCode.SUCCESS counts = WorkspaceTestCounts() results: list[tuple[WorkspaceTestTarget, WorkspaceTestResult]] = [] @@ -361,6 +344,52 @@ def workspace_test_command( return base_cli.ExitCode.FAILURE if counts.failed or counts.blocked else base_cli.ExitCode.SUCCESS +# pylint: disable=too-many-arguments +def render_workspace_test_plan( + workspace_root: Path, + workspace_manifest: WorkspaceManifest, + targets: tuple[WorkspaceTestTarget, ...], + preflights: dict[str, WorkspaceTestPreflight], + *, + fail_fast: bool, + output_format: str, + dry_run: bool, + preflight_blocked: bool, + preflight_failed: bool, +) -> int: + results = ( + blocked_workspace_test_results(targets, preflights) + if (preflight_blocked or preflight_failed) + else dry_run_workspace_test_results(targets, preflights) + ) + counts = WorkspaceTestCounts() + for _, result in results: + counts = update_workspace_test_counts(counts, result) + if output_format == "text": + if preflight_blocked or preflight_failed: + print_workspace_test_blocked_summary(targets, preflight_blocked, preflight_failed) + elif counts.failed or counts.blocked: + print("Workspace test plan failed: a required project selection cannot run.") + else: + print("Workspace test plan complete: no project test commands were executed.") + for target, result in results: + print_workspace_test_result(target, result, show_output=False) + print_workspace_test_summary(counts, execution_started=False) + else: + render_workspace_test_json( + workspace_root, + workspace_manifest, + targets, + results, + counts, + fail_fast=fail_fast, + preflight_status="blocked" if (preflight_blocked or preflight_failed) else "passed", + execution_started=False, + dry_run=dry_run, + ) + return base_cli.ExitCode.FAILURE if counts.failed or counts.blocked else base_cli.ExitCode.SUCCESS + + def workspace_test_preflight( targets: tuple[WorkspaceTestTarget, ...], ) -> dict[str, WorkspaceTestPreflight]: @@ -375,21 +404,13 @@ def workspace_test_preflight( "failed", detail="test target is missing its manifest path" ) continue - try: - manifest = read_manifest(target.manifest_path) - command_config = project_test_command(manifest.test) if manifest.test is not None else None - if command_config is None: - preflights[target.name] = WorkspaceTestPreflight( - "failed", detail="project does not declare a test command" - ) - continue - identity = compute_trust_identity(manifest) - trust_status = trust_store.status(identity) - except (ManifestError, OSError, ValueError) as exc: + if target.manifest is None: preflights[target.name] = WorkspaceTestPreflight( - "failed", detail=f"could not preflight project test: {exc}" + "failed", detail="test target is missing its parsed manifest" ) continue + command_config = project_test_command(target.manifest.test) + trust_status = project_test_trust_status(target.manifest, trust_store=trust_store) command = workspace_test_command_text(command_config.command, command_config.runner) if trust_status.is_allowed: @@ -453,7 +474,8 @@ def dry_run_workspace_test_results( for target in targets: preflight = preflights[target.name] if target.action == "skip": - result = WorkspaceTestResult("skipped", target.reason) + status: WorkspaceTestStatus = "failed" if target.fatal else "skipped" + result = WorkspaceTestResult(status, target.reason) else: result = WorkspaceTestResult( "not_run",