From 1f301864ba39281acc125b28100139289cd2d8af Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:48:07 +0000 Subject: [PATCH 1/6] fix(publish): allowlist two reviewed synthetic test fixture lines The heavy full suite failed test_current_repo_tree_scan_clean and test_bare_selected_copy_has_publisher_runtime_controls: the publish scanner flagged tests/beacon/test_push.py:282 (synthetic path credential, entropy rule) and tests/datasets/test_pin.py:19 (synthetic admin token, credential-value rule). Both lines were added on 2026-10-19 without a reviewed entry. The ".dockerignore" in the second failure is only the first line of list_files output in the assert message; nothing about .dockerignore is checked or broken. Use the existing exact path:sha256(line) allowlist instead of the literal-substring .secretscan-allow, which would exempt the values in every file. A new test proves the entries exempt only those exact lines at those paths: the same bytes in a non-fixture path and a genuine high-entropy secret in the fixture file are still flagged. These top-level tests/test_*.py files are not selected by the PR CI shard glob, which is why only heavy saw the failure. Refs: #19 Co-Authored-By: Claude Sonnet 5.5 --- scripts/publish-allowlist.txt | 2 ++ tests/test_publish_exclusions.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/scripts/publish-allowlist.txt b/scripts/publish-allowlist.txt index e0b0a01..3c04e93 100644 --- a/scripts/publish-allowlist.txt +++ b/scripts/publish-allowlist.txt @@ -175,6 +175,7 @@ src/hypertrain/challenge/store.py:5bc023ca9af1ff8eedaaaa32719611ded1b93836fa6d87 # test fixture value (synthetic, not a credential) tests/auditor/aud_fixtures.py:0d949c3ecc7d9c4044e4279b45bb3f003b569a42beb0ebd5c8fcda1b7b9fb6a5 # L42 +tests/beacon/test_push.py:c6a44b08a042c2fefeb9f4d0dc13aaaa27d6eafba8731c15ad3fd27163c7f336 # L282 tests/challenge/test_admission_v2.py:d6b19e1e09232f110810ee8bd506a4497a0cf7b0f0f6ce9f3d21eecb968c96a8 # L56 tests/challenge/test_contract.py:d8a58e1216b37763ba783aa441e27e1a6d1d4eeb7bebe4cb14d995d1c2072f2b # L37 tests/challenge/test_contract.py:dca6991abfb1e7eb0cf06ef5c6a6389c99d303a012b49e910c44694638086e6f # L77 @@ -185,6 +186,7 @@ tests/challenge/test_service_network_v2.py:39c103316286f9ef21526344b85dca1c86a93 tests/challenge/test_service_network_v2.py:6db3469c33b8debcaeb7597fb238d8a434cce7f3be4fd6c61247d8d5c49c4c03 # L2529 tests/challenge/test_service_network_v2.py:355ba43d75f8d5a922f885c3166b107ca10c387429177317b2a15951abb4f039 # L2536 tests/challenge/test_service_network_v2.py:3d09f1936ce966319dc2d46b6af880c97593e7fe3c35b1b88bd942e907cfa782 # L2551 +tests/datasets/test_pin.py:2e51842f098911cc7c612015b272581ea20449ba26d4d3e04c34429a1f930ebc # L19 tests/e2e/test_service_network_v2_e2e.py:3d09f1936ce966319dc2d46b6af880c97593e7fe3c35b1b88bd942e907cfa782 # L1394 tests/e2e/test_service_network_v2_e2e.py:a98684130926595a80aa7463d4c6632b80945404481c901f15b3dcbe4d98a3d0 # L1441 tests/gpu_ops/conftest.py:57fc89f6e67b8940a8bfc8afe90becae2443a421a57798ec03d1afe3fd8060f7 # L19 diff --git a/tests/test_publish_exclusions.py b/tests/test_publish_exclusions.py index 30c4573..f97a984 100644 --- a/tests/test_publish_exclusions.py +++ b/tests/test_publish_exclusions.py @@ -358,6 +358,21 @@ def test_reviewed_allowlist_is_exact_path_and_line(tmp_path: Path) -> None: ) +def test_reviewed_fixture_lines_do_not_exempt_secrets_elsewhere(tmp_path: Path) -> None: + # The real allowlist exempts these fixture lines only at their exact path and bytes. + source = Path(__file__).resolve().parents[1] + (tmp_path / "scripts").mkdir() + (tmp_path / "scripts/publish-allowlist.txt").write_bytes( + (source / "scripts/publish-allowlist.txt").read_bytes() + ) + for fixture, number in (("tests/beacon/test_push.py", 282), ("tests/datasets/test_pin.py", 19)): + line = (source / fixture).read_text().splitlines()[number - 1] + assert run_scanner(tmp_path, fixture, line + "\n").returncode == 0 + assert run_scanner(tmp_path, "src/hypertrain/leak.py", line + "\n").returncode == 1 + genuine = 'PATH_SECRET = "sk_" + "' + "Zp9Qw2Lm7Xc" + "4Vb8Nk3Rt6Yh1Jd5" + '"' + assert run_scanner(tmp_path, "tests/beacon/test_push.py", genuine + "\n").returncode == 1 + + def test_current_repo_tree_scan_clean() -> None: source = Path(__file__).resolve().parents[1] script = (source / "scripts/publish.sh").read_text() From a0555f30cf884733bad4a94a3080566275bcda4b Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:48:27 +0000 Subject: [PATCH 2/6] test(launch): reach preflight in the launch_cuda_run fail-closed test test_preflight_names_the_first_missing_input_and_spends_nothing has not changed since #8 while the script moved on: #11/#13 added a fail-closed frozen-tree check (exit 1 on stderr) before STAGE=preflight, and #14 moved the default TOOLS from $RD/tools to $REPO/scripts/ops. The test still got rc=1 from the frozen-tree exit, with empty stdout, so its preflight assertion failed in the heavy suite. Stub HT_TREE and pin TOOLS under tmp_path so the run actually reaches preflight; the assertions on the first missing input and on no spend are unchanged. The script is correct; no change there. Refs: #19 Co-Authored-By: Claude Sonnet 5.5 --- tests/test_launch_cuda_run.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/test_launch_cuda_run.py b/tests/test_launch_cuda_run.py index fd65461..de49ca2 100644 --- a/tests/test_launch_cuda_run.py +++ b/tests/test_launch_cuda_run.py @@ -8,7 +8,18 @@ def _run(tmp_path: Path, *args: str) -> subprocess.CompletedProcess[str]: - env = dict(os.environ, RD=str(tmp_path), STATE=str(tmp_path / "state"), PY="python3") + # Stub the frozen tree (checked fail-closed before preflight) and pin TOOLS under tmp_path. + tree = tmp_path / "tree" + (tree / "experiments/gpu_network_v2").mkdir(parents=True) + (tree / "experiments/gpu_network_v2/orchestrate.py").touch() + env = dict( + os.environ, + RD=str(tmp_path), + STATE=str(tmp_path / "state"), + PY="python3", + HT_TREE=str(tree), + TOOLS=str(tmp_path / "tools"), + ) return subprocess.run( ["bash", str(SCRIPT), *args], capture_output=True, text=True, env=env, timeout=60 ) From 0f1840b37e4d5c99374106aac1713975763bc11b Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:49:03 +0000 Subject: [PATCH 3/6] test(capacity): skip system-manager tests when transient units denied Ten capacity and launch-safety tests start real transient units on the system systemd manager, as the product does (island_launch.py never passes --user). The heavy full-suite job runs them as the unprivileged "runner" user, where polkit answers "Interactive authentication required"; PR CI already deselects the same ten ids and the heavy `native` job runs them as root, where they pass. A session fixture probes once with the same kind of call (transient unit, MemoryMax/CPUQuota, --wait) and skips with the probe's stderr as the reason. As root a failing probe is a hard failure, so the native job cannot turn green by skipping. Rejected: xfail, a -k deselect in heavy.yml, and probing `systemd-run --user`, which is not what the product calls. Refs: #19 Co-Authored-By: Claude Sonnet 5.5 --- tests/challenge/test_capacity_outer_v2.py | 2 ++ tests/challenge/test_service_capacity_v2.py | 1 + tests/conftest.py | 37 +++++++++++++++++++++ tests/miner/test_capacity_runtime_v2.py | 2 ++ tests/miner/test_island_launch_v2.py | 1 + tests/relay/test_transport_v2.py | 1 + 6 files changed, 44 insertions(+) diff --git a/tests/challenge/test_capacity_outer_v2.py b/tests/challenge/test_capacity_outer_v2.py index 4e7a604..7d0dd4e 100644 --- a/tests/challenge/test_capacity_outer_v2.py +++ b/tests/challenge/test_capacity_outer_v2.py @@ -193,6 +193,7 @@ def test_request_metadata_rejects_without_tensor_allocation(tiny, tmp_path, monk OuterRequest.parse(data) +@pytest.mark.usefixtures("systemd_manager") def test_actual_bounded_outer_child_matches_original_signed_replay(tiny, tmp_path, monkeypatch): from hypertrain.aggregator.core import OuterState from hypertrain.aggregator.tape_v2 import TapeV2, make_tape, replay_tape @@ -672,6 +673,7 @@ def test_genesis_authority_rejects_before_kernel(genesis_tiny, monkeypatch, atta ).fetchone() +@pytest.mark.usefixtures("systemd_manager") def test_actual_bounded_genesis_preparation_and_duplicate_debit( genesis_tiny, tmp_path, monkeypatch ): diff --git a/tests/challenge/test_service_capacity_v2.py b/tests/challenge/test_service_capacity_v2.py index f3e2a18..ebd7a1a 100644 --- a/tests/challenge/test_service_capacity_v2.py +++ b/tests/challenge/test_service_capacity_v2.py @@ -716,6 +716,7 @@ def prior_bytes(path): assert store._record_v2(run_id, "service-admission", "run") == prior +@pytest.mark.usefixtures("systemd_manager") def test_real_capacity_launch_failure_keeps_debit_and_prevents_restart_rerun( tiny, tmp_path, monkeypatch ): diff --git a/tests/conftest.py b/tests/conftest.py index 1123b67..a14cc3a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,8 @@ import json import os +import subprocess +import uuid from pathlib import Path import numpy as np @@ -110,3 +112,38 @@ def corpus_dir(tmp_path_factory: pytest.TempPathFactory) -> Path: d = build_corpus(tmp_path_factory.mktemp("corpus")) os.environ["HYPERTRAIN_DATA_DIR"] = str(d) return d + + +@pytest.fixture(scope="session") +def systemd_manager() -> None: + """Skip unless this user may start transient units on the system manager, as the product does. + + Root that cannot is a broken host, not an unprivileged one: fail, so the heavy `native` job + (isolated root environment) can never turn green by skipping. + """ + try: + probe = subprocess.run( + [ + "systemd-run", + "--unit=ht-probe-" + uuid.uuid4().hex, + "--wait", + "--collect", + "--quiet", + "--no-ask-password", + "--property=MemoryMax=64M", + "--property=CPUQuota=50%", + "/bin/true", + ], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + ok, reason = probe.returncode == 0, f"rc={probe.returncode} {probe.stderr.strip()[-500:]}" + except (OSError, subprocess.TimeoutExpired) as exc: + ok, reason = False, repr(exc) + if ok: + return + if os.geteuid() == 0: + pytest.fail(f"systemd-run as root failed: {reason}") + pytest.skip(f"system manager transient units unavailable: {reason}") diff --git a/tests/miner/test_capacity_runtime_v2.py b/tests/miner/test_capacity_runtime_v2.py index a9c3021..91ed320 100644 --- a/tests/miner/test_capacity_runtime_v2.py +++ b/tests/miner/test_capacity_runtime_v2.py @@ -23,6 +23,7 @@ def small(monkeypatch, tmp_path): return launch.CapacityAttempt("11" * 32, "22" * 32, tmp_path / "runtime.lock", lambda: True) +@pytest.mark.usefixtures("systemd_manager") @pytest.mark.parametrize("mode", ["success", "oom", "cancel", "deadline"]) def test_actual_small_kernel_attempt_and_cleanup(tmp_path, small, mode): """Subscribe readiness before triggering real bounded command and await pidfd exit.""" @@ -145,6 +146,7 @@ def test_missing_manager_permission_has_no_unbounded_fallback(tmp_path, small, m assert not (tmp_path / "attempt/capacity-observed.json").exists() +@pytest.mark.usefixtures("systemd_manager") def test_real_runner_parent_death_retains_absolute_manager_timer(tmp_path, small): """Fork only a stdlib waiting controller; manager timer survives its SIGKILL.""" path = str(tmp_path / "dead.sock") diff --git a/tests/miner/test_island_launch_v2.py b/tests/miner/test_island_launch_v2.py index 168e042..dcd0fea 100644 --- a/tests/miner/test_island_launch_v2.py +++ b/tests/miner/test_island_launch_v2.py @@ -304,6 +304,7 @@ def test_noisy_failure_retains_only_bounded_error_tail( assert not (tmp_path / "published").exists() +@pytest.mark.usefixtures("systemd_manager") def test_capacity_branch_uses_real_bounded_failure_before_publication(tmp_path, monkeypatch): import os import sys diff --git a/tests/relay/test_transport_v2.py b/tests/relay/test_transport_v2.py index 79f363a..0676969 100644 --- a/tests/relay/test_transport_v2.py +++ b/tests/relay/test_transport_v2.py @@ -1822,6 +1822,7 @@ def test_image_archive_stdin_through_capture_and_native_shim(tmp_path: Path) -> assert json.loads(journal.read_bytes()) == {"cluster": "hypertrain-v2-stream"} +@pytest.mark.usefixtures("systemd_manager") def test_launch_safety_independent_systemd_units(tmp_path: Path) -> None: import json import os From 2ad33beeb4d6d3e107b47ebe029a7a015e01a7f6 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:49:49 +0000 Subject: [PATCH 4/6] ci(heavy): install the kind version pinned in deploy/k8s/versions.json test_provider_fix_pinned_kind_version_and_fail_closed compares the kind on PATH with deploy/k8s/versions.json kind_version (v0.31.0), the same fail-closed check relay_kind_smoke.py makes. No workflow installed kind, so full-suite used the Blacksmith image's kind v0.32.0 and failed. versions.json stays the single source of truth: the step reads kind_version from it, downloads that release, verifies its sha256 and prepends it to PATH. It fails loudly if the pin moves without the checksum. Rejected: bumping the pin to whatever the image ships (it drifts again on the next image refresh and needs the kindest/node image re-validated), and skipping when kind differs (hides the drift the test exists to catch). Refs: #19 Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/heavy.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/heavy.yml b/.github/workflows/heavy.yml index b0c8290..2f9ce83 100644 --- a/.github/workflows/heavy.yml +++ b/.github/workflows/heavy.yml @@ -141,4 +141,23 @@ jobs: enable-cache: true cache-dependency-glob: "uv.lock" - run: uv sync --locked --all-extras --group dev + # The runner image ships its own kind; the od relay tests and relay_kind_smoke.py + # require the kind_version pinned in deploy/k8s/versions.json. + - name: Install pinned kind + env: + KIND_SHA256_V0_31_0: eb244cbafcc157dff60cf68693c14c9a75c4e6e6fedaf9cd71c58117cb93e3fa + run: | + set -euo pipefail + v="$(python3 -c 'import json; print(json.load(open("deploy/k8s/versions.json"))["kind_version"])')" + if [ "$v" != v0.31.0 ]; then + echo "kind_version $v: update KIND_SHA256 in heavy.yml" >&2 + exit 1 + fi + mkdir -p "$RUNNER_TEMP/kind-bin" + curl -fsSLo "$RUNNER_TEMP/kind-bin/kind" \ + "https://github.com/kubernetes-sigs/kind/releases/download/$v/kind-linux-amd64" + echo "$KIND_SHA256_V0_31_0 $RUNNER_TEMP/kind-bin/kind" | sha256sum -c - + chmod 755 "$RUNNER_TEMP/kind-bin/kind" + "$RUNNER_TEMP/kind-bin/kind" version | grep -q "^kind $v " + echo "$RUNNER_TEMP/kind-bin" >> "$GITHUB_PATH" - run: bash scripts/check.sh --ignore=tests/sim --ignore=tests/e2e From 6544a94229f4f086cb04ed80da471d18a4dd1faa Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:03:20 +0000 Subject: [PATCH 5/6] fix(tests): skip systemd probe only when manager unavailable or denied The systemd_manager fixture skipped on any non-zero systemd-run result as non-root, so EAGAIN, a failed transient unit or a timeout silently skipped the 10 capacity tests. A pure _classify_systemd_probe now maps the result to ok/unavailable/error: only a missing systemd-run or the known no-bus/denied stderr markers skip; everything else fails, for root and non-root alike. Root still fails on any non-zero result. Co-Authored-By: Claude Sonnet 5.5 --- tests/conftest.py | 34 ++++++++++++++++++++++++---- tests/test_systemd_probe_classify.py | 30 ++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 tests/test_systemd_probe_classify.py diff --git a/tests/conftest.py b/tests/conftest.py index a14cc3a..2ccb965 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,6 +5,7 @@ import subprocess import uuid from pathlib import Path +from typing import Literal import numpy as np import pytest @@ -114,12 +115,30 @@ def corpus_dir(tmp_path_factory: pytest.TempPathFactory) -> Path: return d +# Messages meaning "this user/host has no usable system manager", not "the probe broke". +_SYSTEMD_UNAVAILABLE = ( + "Interactive authentication required", + "Access denied", + "Failed to connect to bus", + "System has not been booted with systemd", + "Permission denied", +) + + +def _classify_systemd_probe(returncode: int, stderr: str) -> Literal["ok", "unavailable", "error"]: + if returncode == 0: + return "ok" + return "unavailable" if any(m in stderr for m in _SYSTEMD_UNAVAILABLE) else "error" + + @pytest.fixture(scope="session") def systemd_manager() -> None: """Skip unless this user may start transient units on the system manager, as the product does. - Root that cannot is a broken host, not an unprivileged one: fail, so the heavy `native` job - (isolated root environment) can never turn green by skipping. + Non-root skips only when the manager is absent or authorization is denied; any other probe + failure (EAGAIN, failed transient unit, timeout) is a broken host and fails. Root that cannot + is a broken host, not an unprivileged one: fail, so the heavy `native` job (isolated root + environment) can never turn green by skipping. """ try: probe = subprocess.run( @@ -139,11 +158,16 @@ def systemd_manager() -> None: timeout=30, check=False, ) - ok, reason = probe.returncode == 0, f"rc={probe.returncode} {probe.stderr.strip()[-500:]}" + verdict = _classify_systemd_probe(probe.returncode, probe.stderr) + reason = f"rc={probe.returncode} {probe.stderr.strip()[-500:]}" + except FileNotFoundError as exc: + verdict, reason = "unavailable", repr(exc) except (OSError, subprocess.TimeoutExpired) as exc: - ok, reason = False, repr(exc) - if ok: + verdict, reason = "error", repr(exc) + if verdict == "ok": return if os.geteuid() == 0: pytest.fail(f"systemd-run as root failed: {reason}") + if verdict == "error": + pytest.fail(f"systemd-run probe failed: {reason}") pytest.skip(f"system manager transient units unavailable: {reason}") diff --git a/tests/test_systemd_probe_classify.py b/tests/test_systemd_probe_classify.py new file mode 100644 index 0000000..59c8bf3 --- /dev/null +++ b/tests/test_systemd_probe_classify.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import pytest + +from conftest import _classify_systemd_probe + +DENIED = [ + "Failed to start transient service unit: Interactive authentication required.", + "Failed to start transient service unit: Access denied", + "Failed to connect to bus: No such file or directory", + "System has not been booted with systemd as init system (PID 1). Can't operate.", + "Failed to create bus connection: Permission denied", +] + + +def test_ok() -> None: + assert _classify_systemd_probe(0, "") == "ok" + + +@pytest.mark.parametrize("stderr", DENIED) +def test_unavailable(stderr: str) -> None: + assert _classify_systemd_probe(1, stderr) == "unavailable" + + +@pytest.mark.parametrize( + "stderr", + ["Failed to start transient service unit: Resource temporarily unavailable (EAGAIN)", ""], +) +def test_error(stderr: str) -> None: + assert _classify_systemd_probe(1, stderr) == "error" From e10b0540514451c14ae88bd0b3357b6dedebddf3 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:03:20 +0000 Subject: [PATCH 6/6] ci(heavy): print skip reasons in the full-suite summary The non-root runner skips the systemd-gated capacity tests and the summary did not say why. check.sh already forwards its arguments to pytest, so pass -rs there; what heavy runs is unchanged. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/heavy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/heavy.yml b/.github/workflows/heavy.yml index 2f9ce83..b05869a 100644 --- a/.github/workflows/heavy.yml +++ b/.github/workflows/heavy.yml @@ -160,4 +160,4 @@ jobs: chmod 755 "$RUNNER_TEMP/kind-bin/kind" "$RUNNER_TEMP/kind-bin/kind" version | grep -q "^kind $v " echo "$RUNNER_TEMP/kind-bin" >> "$GITHUB_PATH" - - run: bash scripts/check.sh --ignore=tests/sim --ignore=tests/e2e + - run: bash scripts/check.sh -rs --ignore=tests/sim --ignore=tests/e2e