Fix heavy full-suite failures other than the od-tiny fixtures (#19)

.github/workflows/heavy.yml

@@ -141,4 +141,23 @@ jobs:
           enable-cache: true
           cache-dependency-glob: "uv.lock"
       - run: uv sync --locked --all-extras --group dev
-      - run: bash scripts/check.sh --ignore=tests/sim --ignore=tests/e2e
+      # The runner image ships its own kind; the od relay tests and relay_kind_smoke.py
+      # require the kind_version pinned in deploy/k8s/versions.json.
+      - name: Install pinned kind
+        env:
+          KIND_SHA256_V0_31_0: eb244cbafcc157dff60cf68693c14c9a75c4e6e6fedaf9cd71c58117cb93e3fa
+        run: |
+          set -euo pipefail
+          v="$(python3 -c 'import json; print(json.load(open("deploy/k8s/versions.json"))["kind_version"])')"
+          if [ "$v" != v0.31.0 ]; then
+            echo "kind_version $v: update KIND_SHA256 in heavy.yml" >&2
+            exit 1
+          fi
+          mkdir -p "$RUNNER_TEMP/kind-bin"
+          curl -fsSLo "$RUNNER_TEMP/kind-bin/kind" \
+            "https://github.com/kubernetes-sigs/kind/releases/download/$v/kind-linux-amd64"
+          echo "$KIND_SHA256_V0_31_0  $RUNNER_TEMP/kind-bin/kind" | sha256sum -c -
+          chmod 755 "$RUNNER_TEMP/kind-bin/kind"
+          "$RUNNER_TEMP/kind-bin/kind" version | grep -q "^kind $v "
+          echo "$RUNNER_TEMP/kind-bin" >> "$GITHUB_PATH"
+      - run: bash scripts/check.sh -rs --ignore=tests/sim --ignore=tests/e2e

scripts/publish-allowlist.txt

@@ -175,6 +175,7 @@ src/hypertrain/challenge/store.py:5bc023ca9af1ff8eedaaaa32719611ded1b93836fa6d87
 
 # test fixture value (synthetic, not a credential)
 tests/auditor/aud_fixtures.py:0d949c3ecc7d9c4044e4279b45bb3f003b569a42beb0ebd5c8fcda1b7b9fb6a5  # L42
+tests/beacon/test_push.py:c6a44b08a042c2fefeb9f4d0dc13aaaa27d6eafba8731c15ad3fd27163c7f336  # L282
 tests/challenge/test_admission_v2.py:d6b19e1e09232f110810ee8bd506a4497a0cf7b0f0f6ce9f3d21eecb968c96a8  # L56
 tests/challenge/test_contract.py:d8a58e1216b37763ba783aa441e27e1a6d1d4eeb7bebe4cb14d995d1c2072f2b  # L37
 tests/challenge/test_contract.py:dca6991abfb1e7eb0cf06ef5c6a6389c99d303a012b49e910c44694638086e6f  # L77
@@ -185,6 +186,7 @@ tests/challenge/test_service_network_v2.py:39c103316286f9ef21526344b85dca1c86a93
 tests/challenge/test_service_network_v2.py:6db3469c33b8debcaeb7597fb238d8a434cce7f3be4fd6c61247d8d5c49c4c03  # L2529
 tests/challenge/test_service_network_v2.py:355ba43d75f8d5a922f885c3166b107ca10c387429177317b2a15951abb4f039  # L2536
 tests/challenge/test_service_network_v2.py:3d09f1936ce966319dc2d46b6af880c97593e7fe3c35b1b88bd942e907cfa782  # L2551
+tests/datasets/test_pin.py:2e51842f098911cc7c612015b272581ea20449ba26d4d3e04c34429a1f930ebc  # L19
 tests/e2e/test_service_network_v2_e2e.py:3d09f1936ce966319dc2d46b6af880c97593e7fe3c35b1b88bd942e907cfa782  # L1394
 tests/e2e/test_service_network_v2_e2e.py:a98684130926595a80aa7463d4c6632b80945404481c901f15b3dcbe4d98a3d0  # L1441
 tests/gpu_ops/conftest.py:57fc89f6e67b8940a8bfc8afe90becae2443a421a57798ec03d1afe3fd8060f7  # L19

tests/challenge/test_capacity_outer_v2.py

@@ -193,6 +193,7 @@ def test_request_metadata_rejects_without_tensor_allocation(tiny, tmp_path, monk
         OuterRequest.parse(data)
 
 
+@pytest.mark.usefixtures("systemd_manager")
 def test_actual_bounded_outer_child_matches_original_signed_replay(tiny, tmp_path, monkeypatch):
     from hypertrain.aggregator.core import OuterState
     from hypertrain.aggregator.tape_v2 import TapeV2, make_tape, replay_tape
@@ -672,6 +673,7 @@ def test_genesis_authority_rejects_before_kernel(genesis_tiny, monkeypatch, atta
     ).fetchone()
 
 
+@pytest.mark.usefixtures("systemd_manager")
 def test_actual_bounded_genesis_preparation_and_duplicate_debit(
     genesis_tiny, tmp_path, monkeypatch
 ):

tests/challenge/test_service_capacity_v2.py

@@ -716,6 +716,7 @@ def prior_bytes(path):
     assert store._record_v2(run_id, "service-admission", "run") == prior
 
 
+@pytest.mark.usefixtures("systemd_manager")
 def test_real_capacity_launch_failure_keeps_debit_and_prevents_restart_rerun(
     tiny, tmp_path, monkeypatch
 ):

tests/conftest.py

@@ -2,7 +2,10 @@
 
 import json
 import os
+import subprocess
+import uuid
 from pathlib import Path
+from typing import Literal
 
 import numpy as np
 import pytest
@@ -110,3 +113,61 @@ def corpus_dir(tmp_path_factory: pytest.TempPathFactory) -> Path:
     d = build_corpus(tmp_path_factory.mktemp("corpus"))
     os.environ["HYPERTRAIN_DATA_DIR"] = str(d)
     return d
+
+
+# Messages meaning "this user/host has no usable system manager", not "the probe broke".
+_SYSTEMD_UNAVAILABLE = (
+    "Interactive authentication required",
+    "Access denied",
+    "Failed to connect to bus",
+    "System has not been booted with systemd",
+    "Permission denied",
+)
+
+
+def _classify_systemd_probe(returncode: int, stderr: str) -> Literal["ok", "unavailable", "error"]:
+    if returncode == 0:
+        return "ok"
+    return "unavailable" if any(m in stderr for m in _SYSTEMD_UNAVAILABLE) else "error"
+
+
+@pytest.fixture(scope="session")
+def systemd_manager() -> None:
+    """Skip unless this user may start transient units on the system manager, as the product does.
+
+    Non-root skips only when the manager is absent or authorization is denied; any other probe
+    failure (EAGAIN, failed transient unit, timeout) is a broken host and fails. Root that cannot
+    is a broken host, not an unprivileged one: fail, so the heavy `native` job (isolated root
+    environment) can never turn green by skipping.
+    """
+    try:
+        probe = subprocess.run(
+            [
+                "systemd-run",
+                "--unit=ht-probe-" + uuid.uuid4().hex,
+                "--wait",
+                "--collect",
+                "--quiet",
+                "--no-ask-password",
+                "--property=MemoryMax=64M",
+                "--property=CPUQuota=50%",
+                "/bin/true",
+            ],
+            capture_output=True,
+            text=True,
+            timeout=30,
+            check=False,
+        )
+        verdict = _classify_systemd_probe(probe.returncode, probe.stderr)
+        reason = f"rc={probe.returncode} {probe.stderr.strip()[-500:]}"
+    except FileNotFoundError as exc:
+        verdict, reason = "unavailable", repr(exc)
+    except (OSError, subprocess.TimeoutExpired) as exc:
+        verdict, reason = "error", repr(exc)
+    if verdict == "ok":
+        return
+    if os.geteuid() == 0:
+        pytest.fail(f"systemd-run as root failed: {reason}")
+    if verdict == "error":
+        pytest.fail(f"systemd-run probe failed: {reason}")
+    pytest.skip(f"system manager transient units unavailable: {reason}")

tests/miner/test_capacity_runtime_v2.py

@@ -23,6 +23,7 @@ def small(monkeypatch, tmp_path):
     return launch.CapacityAttempt("11" * 32, "22" * 32, tmp_path / "runtime.lock", lambda: True)
 
 
+@pytest.mark.usefixtures("systemd_manager")
 @pytest.mark.parametrize("mode", ["success", "oom", "cancel", "deadline"])
 def test_actual_small_kernel_attempt_and_cleanup(tmp_path, small, mode):
     """Subscribe readiness before triggering real bounded command and await pidfd exit."""
@@ -145,6 +146,7 @@ def test_missing_manager_permission_has_no_unbounded_fallback(tmp_path, small, m
     assert not (tmp_path / "attempt/capacity-observed.json").exists()
 
 
+@pytest.mark.usefixtures("systemd_manager")
 def test_real_runner_parent_death_retains_absolute_manager_timer(tmp_path, small):
     """Fork only a stdlib waiting controller; manager timer survives its SIGKILL."""
     path = str(tmp_path / "dead.sock")

tests/miner/test_island_launch_v2.py

@@ -304,6 +304,7 @@ def test_noisy_failure_retains_only_bounded_error_tail(
     assert not (tmp_path / "published").exists()
 
 
+@pytest.mark.usefixtures("systemd_manager")
 def test_capacity_branch_uses_real_bounded_failure_before_publication(tmp_path, monkeypatch):
     import os
     import sys

tests/relay/test_transport_v2.py

@@ -1822,6 +1822,7 @@ def test_image_archive_stdin_through_capture_and_native_shim(tmp_path: Path) ->
     assert json.loads(journal.read_bytes()) == {"cluster": "hypertrain-v2-stream"}
 
 
+@pytest.mark.usefixtures("systemd_manager")
 def test_launch_safety_independent_systemd_units(tmp_path: Path) -> None:
     import json
     import os

tests/test_launch_cuda_run.py

@@ -8,7 +8,18 @@
 
 
 def _run(tmp_path: Path, *args: str) -> subprocess.CompletedProcess[str]:
-    env = dict(os.environ, RD=str(tmp_path), STATE=str(tmp_path / "state"), PY="python3")
+    # Stub the frozen tree (checked fail-closed before preflight) and pin TOOLS under tmp_path.
+    tree = tmp_path / "tree"
+    (tree / "experiments/gpu_network_v2").mkdir(parents=True)
+    (tree / "experiments/gpu_network_v2/orchestrate.py").touch()
+    env = dict(
+        os.environ,
+        RD=str(tmp_path),
+        STATE=str(tmp_path / "state"),
+        PY="python3",
+        HT_TREE=str(tree),
+        TOOLS=str(tmp_path / "tools"),
+    )
     return subprocess.run(
         ["bash", str(SCRIPT), *args], capture_output=True, text=True, env=env, timeout=60
     )

tests/test_publish_exclusions.py

@@ -358,6 +358,21 @@ def test_reviewed_allowlist_is_exact_path_and_line(tmp_path: Path) -> None:
     )
 
 
+def test_reviewed_fixture_lines_do_not_exempt_secrets_elsewhere(tmp_path: Path) -> None:
+    # The real allowlist exempts these fixture lines only at their exact path and bytes.
+    source = Path(__file__).resolve().parents[1]
+    (tmp_path / "scripts").mkdir()
+    (tmp_path / "scripts/publish-allowlist.txt").write_bytes(
+        (source / "scripts/publish-allowlist.txt").read_bytes()
+    )
+    for fixture, number in (("tests/beacon/test_push.py", 282), ("tests/datasets/test_pin.py", 19)):
+        line = (source / fixture).read_text().splitlines()[number - 1]
+        assert run_scanner(tmp_path, fixture, line + "\n").returncode == 0
+        assert run_scanner(tmp_path, "src/hypertrain/leak.py", line + "\n").returncode == 1
+    genuine = 'PATH_SECRET = "sk_" + "' + "Zp9Qw2Lm7Xc" + "4Vb8Nk3Rt6Yh1Jd5" + '"'
+    assert run_scanner(tmp_path, "tests/beacon/test_push.py", genuine + "\n").returncode == 1
+
+
 def test_current_repo_tree_scan_clean() -> None:
     source = Path(__file__).resolve().parents[1]
     script = (source / "scripts/publish.sh").read_text()

tests/test_systemd_probe_classify.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+import pytest
+
+from conftest import _classify_systemd_probe
+
+DENIED = [
+    "Failed to start transient service unit: Interactive authentication required.",
+    "Failed to start transient service unit: Access denied",
+    "Failed to connect to bus: No such file or directory",
+    "System has not been booted with systemd as init system (PID 1). Can't operate.",
+    "Failed to create bus connection: Permission denied",
+]
+
+
+def test_ok() -> None:
+    assert _classify_systemd_probe(0, "") == "ok"
+
+
+@pytest.mark.parametrize("stderr", DENIED)
+def test_unavailable(stderr: str) -> None:
+    assert _classify_systemd_probe(1, stderr) == "unavailable"
+
+
+@pytest.mark.parametrize(
+    "stderr",
+    ["Failed to start transient service unit: Resource temporarily unavailable (EAGAIN)", ""],
+)
+def test_error(stderr: str) -> None:
+    assert _classify_systemd_probe(1, stderr) == "error"