mutation_test.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Mutation test — the answer to "who tests the tests?"

A test suite is only as good as the bugs it can catch. This script deliberately plants realistic
bugs (mutants) into the framework, one at a time, and re-runs the full suite. Every mutant MUST
make the suite fail; a mutant that survives means the tests have a blind spot. Files are restored
after every run (try/finally), and a final clean run proves the restoration.

Mutants target both the engine AND the guardrail modules (metrics, significance), so the "who
tests the tests" guarantee covers the parts of the framework that actually render verdicts.

Run from the repo root:  python mutation_test.py
"""
import os
import subprocess
import sys

HERE = os.path.dirname(os.path.abspath(__file__))
def _f(name):
    return os.path.join(HERE, "backtest_lab", name)

# (name, target_file, original snippet, mutated snippet, the lie the mutant tells)
MUTANTS = [
    ("cost-sign-flip", _f("engine.py"),
     "- turnover * cost_per_side", "+ turnover * cost_per_side",
     "trading costs ADD to returns (classic sign bug: costs silently inflate performance)"),
    ("free-first-trade", _f("engine.py"),
     "weights.shift(1).fillna(0.0)", "weights.shift(1)",
     "the initial entry is free (NaN row silently drops the first day's turnover)"),
    ("one-day-lookahead", _f("engine.py"),
     "self._prices.index < self.date", "self._prices.index <= self.date",
     "strategies can see the decision day's own price (subtle look-ahead bias)"),
    ("leverage-guard-off", _f("engine.py"),
     "if total > 1.0 + 1e-9:", "if total > 2.0 + 1e-9:",
     "portfolios can quietly run 2x leverage in an 'unlevered' engine"),
    ("nan-weight-guard-off", _f("engine.py"),
     "if nonfinite:", "if nonfinite and False:",
     "a NaN/inf strategy weight passes validation -> fake flat 0%-return curve"),
    ("calmar-sign-flip", _f("metrics.py"),
     "return cagr(returns) / abs(dd) if dd < 0 else float(\"nan\")",
     "return cagr(returns) / abs(dd) if dd <= 0 else float(\"nan\")",
     "Calmar divides by zero drawdown (guardrail metric silently wrong on flat series)"),
    ("bootstrap-pairing-break", _f("significance.py"),
     "metric(pd.Series(av[pos])) - metric(pd.Series(bv[pos]))",
     "metric(pd.Series(av[pos])) - metric(pd.Series(bv))",
     "the bootstrap breaks paired resampling (b not resampled on the same blocks)"),
]


def run_suite() -> bool:
    r = subprocess.run([sys.executable, "-m", "pytest", "tests/", "-q", "-x", "--no-header"],
                       cwd=HERE, capture_output=True, text=True, timeout=600)
    return r.returncode == 0


def main() -> int:
    files = sorted({m[1] for m in MUTANTS})
    pristine = {f: open(f, encoding="utf-8").read() for f in files}
    for name, f, old, _, _ in MUTANTS:
        if old not in pristine[f]:
            print(f"FATAL: snippet for mutant {name!r} not found in {os.path.basename(f)}: {old!r}")
            return 2

    print(f"mutation test: {len(MUTANTS)} planted bugs across "
          f"{len(files)} module(s), each must be caught by the suite\n")
    caught = 0
    try:
        for name, f, old, new, lie in MUTANTS:
            with open(f, "w", encoding="utf-8") as fh:
                fh.write(pristine[f].replace(old, new))
            killed = not run_suite()
            caught += killed
            with open(f, "w", encoding="utf-8") as fh:      # restore immediately (isolate mutants)
                fh.write(pristine[f])
            print(f"  [{'KILLED' if killed else 'SURVIVED'}] {name} ({os.path.basename(f)}): {lie}")
    finally:
        for f in files:
            with open(f, "w", encoding="utf-8") as fh:
                fh.write(pristine[f])

    clean = run_suite()
    print(f"\nmodules restored; clean suite {'PASSES' if clean else 'FAILS (restore problem!)'}")
    print(f"result: {caught}/{len(MUTANTS)} mutants killed")
    if caught == len(MUTANTS) and clean:
        print("VERDICT: the tests catch every planted bug.")
        return 0
    print("VERDICT: FAILURE -- a mutant survived (test blind spot) or restore failed.")
    return 1


if __name__ == "__main__":
    sys.exit(main())