diff --git a/tests/mouse_oracle.py b/tests/mouse_oracle.py index 0fc8de0..22c1ccd 100644 --- a/tests/mouse_oracle.py +++ b/tests/mouse_oracle.py @@ -41,14 +41,14 @@ whether the controls respond at all. The checker is validated by planted faults rather than trusted: run with -`--fault pan` (mousedown never claims the drag) or `--fault pause` (the -tick advances even when paused) and the corresponding check MUST go red. -tests/test_orbit_mouse.sh drives all three runs. +`--fault pan` (mousedown never claims the drag), `--fault pause` (the +tick advances even when paused), or `--fault mode` (the view never changes) +and the corresponding check MUST go red. The wrapper drives all four runs. Assumes an X display (the wrapper uses xvfb-run). Needs the gfx build (EIGENSCRIPT), xdotool, xwd, PIL. """ -import os, struct, subprocess, sys, tempfile, time, shutil +import os, re, struct, subprocess, sys, tempfile, time, shutil from PIL import Image EIGS = os.environ.get("EIGENSCRIPT", "eigenscript") @@ -81,12 +81,51 @@ # damped orbit shrinks toward the fixed point, so late in a session "the # simulation is still advancing" is only a handful of pixels. ADVANCE_MIN = 1 +ACTION_TIMEOUT = 8.0 +POLL_INTERVAL = 0.05 # ---------- X plumbing ---------- -def xdo(*args): - subprocess.run(["xdotool"] + [str(a) for a in args], env=ENV, check=False) +def xdo(*args, timeout=3.0, search=False): + if not search: + print("XDO send %r monotonic=%.6f" % (args, time.monotonic()), flush=True) + result = subprocess.run(["xdotool"] + [str(a) for a in args], env=ENV, + capture_output=True, text=True, timeout=timeout) + # Search rc1 means no matching window yet, not a successful input. + if result.returncode != 0 and not (search and result.returncode == 1): + raise RuntimeError("xdotool %r rc%d: %s" % + (args, result.returncode, result.stderr.strip())) + return result.stdout + + +def app_output(proc): + with open(proc.output_path) as fh: + return fh.read()[-8000:] + + +def ensure_live(proc): + if proc.poll() is not None: + raise RuntimeError("app exited rc%d: %s" % (proc.returncode, app_output(proc))) + + +def start_app(args, cwd, tmp, name): + path = os.path.join(tmp, name + ".log") + with open(path, "w") as log: + proc = subprocess.Popen([EIGS] + args, cwd=cwd, env=ENV, + stdout=log, stderr=subprocess.STDOUT, text=True) + proc.output_path = path + return proc + + +def stop_app(proc): + if proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=5) def xwd_to_image(path): @@ -116,20 +155,21 @@ def has_content(img): def wait_for_window(proc, title): - for _ in range(150): # up to ~30s: window mapping is high-variance - time.sleep(0.2) - r = subprocess.run(["xdotool", "search", "--name", title], - env=ENV, capture_output=True, text=True) - if r.stdout.strip(): - return r.stdout.strip().split("\n")[0] - if proc.poll() is not None: - raise RuntimeError("app exited early: " + (proc.stdout.read() or "")) - raise RuntimeError("window never appeared: " + (proc.stdout.read() or "")) + deadline = time.monotonic() + 30.0 + while time.monotonic() < deadline: + ensure_live(proc) + windows = xdo("search", "--all", "--onlyvisible", "--pid", proc.pid, + "--name", "^" + re.escape(title) + "$", search=True, + timeout=min(3.0, max(0.01, deadline - time.monotonic()))) + if windows.strip(): + return windows.strip().splitlines()[0] + time.sleep(min(POLL_INTERVAL, max(0, deadline - time.monotonic()))) + # Never read a live stdout pipe on timeout: that would defeat the bound. + raise RuntimeError("window deadline expired: " + app_output(proc)) def window_origin(wid): - g = subprocess.run(["xdotool", "getwindowgeometry", wid], env=ENV, - capture_output=True, text=True).stdout + g = xdo("getwindowgeometry", wid) for line in g.splitlines(): if "Position:" in line: xy = line.split("Position:")[1].split("(")[0].strip() @@ -137,17 +177,36 @@ def window_origin(wid): raise RuntimeError("no window position for " + wid) -def grab(wid, tmp): +class CaptureDeadline(RuntimeError): + pass + + +def grab(wid, tmp, proc, deadline=None): + action_deadline = deadline is not None + deadline = deadline if deadline is not None else time.monotonic() + ACTION_TIMEOUT xwd = os.path.join(tmp, "g.xwd") - for _ in range(30): - time.sleep(0.15) - cap = subprocess.run(["xwd", "-id", wid, "-out", xwd], env=ENV, - capture_output=True) + while time.monotonic() < deadline: + ensure_live(proc) + remaining = deadline - time.monotonic() + if remaining <= 0: + break + deadline_limited = action_deadline and remaining < 3.0 + try: + cap = subprocess.run(["xwd", "-id", wid, "-out", xwd], env=ENV, + capture_output=True, timeout=min(3.0, remaining)) + except subprocess.TimeoutExpired: + if deadline_limited and time.monotonic() >= deadline: + raise CaptureDeadline("visible action deadline during capture") + raise if cap.returncode != 0: - continue # transient X error mid-map — retry + raise RuntimeError("xwd rc%d: %s" % + (cap.returncode, cap.stderr.decode(errors="replace"))) img = xwd_to_image(xwd) if has_content(img): return img + time.sleep(min(POLL_INTERVAL, max(0, deadline - time.monotonic()))) + if action_deadline: + raise CaptureDeadline("visible action deadline before capture") raise RuntimeError("could not capture the window") @@ -184,19 +243,16 @@ def build_atlas(tmp): ' gfx_delay of 16\n' ' n is n + 1\n' 'gfx_close of null\n' % (w, CHARSET.replace("\\", "\\\\").replace('"', '\\"'))) - proc = subprocess.Popen([EIGS, app], cwd=REPO, env=ENV, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + proc = start_app([app], REPO, tmp, "atlas") try: wid = wait_for_window(proc, "orbit-atlas") - img = grab(wid, tmp) + img = grab(wid, tmp, proc) px = img.load() atlas = {cell_sig(px, 8 + k * CELL_W, 8): ch for k, ch in enumerate(CHARSET)} atlas[frozenset()] = " " # blank cell — a real space in the label return atlas finally: - proc.terminate() - try: proc.wait(timeout=5) - except Exception: proc.kill() + stop_app(proc) def decode_line(img, atlas, xy, n=22): @@ -241,12 +297,22 @@ def build_tree(tmp, fault): if needle not in src: raise RuntimeError("planted fault %r no longer matches orbit.eigs" % fault) open(path, "w").write(src.replace(needle, repl, 1)) + # Read-only acknowledgement through the existing post-event frame hook. + # This lives ONLY in the disposable app copy. Production input handlers, + # simulation and rendering remain unchanged; a tick is not a pixel verdict. + with open(os.path.join(tree, "orbit.eigs"), "a") as fh: + fh.write('\ndefine mouse_qa_ack(sim) as:\n' + ' write_text of ["mouse-qa-state", ' + 'f"{_app.ticks}|{sim.frame}|{_app.paused}|{_app.mode}|END\\n"]\n' + ' return null\n') with open(os.path.join(tree, "orbit_qa.eigs"), "w") as fh: - fh.write("import orbit\norbit.run_session of [0.0, 0 - 1, \"\"]\n") + fh.write("import orbit\norbit.set_frame_hook of orbit.mouse_qa_ack\n" + "orbit.run_session of [0.0, 0 - 1, \"\"]\n") return tree def main(): + sys.stdout.reconfigure(line_buffering=True) fault = None stop_on_fail = "--stop-on-fail" in sys.argv if "--fault" in sys.argv: @@ -268,16 +334,93 @@ def check(ok, msg): raise Stop() proc = None + infrastructure_error = None + evidence_root = os.environ.get("ORBIT_MOUSE_ARTIFACTS") + evidence = None + if evidence_root: + os.makedirs(evidence_root, exist_ok=True) + evidence = tempfile.mkdtemp(prefix=(fault or "clean") + "-", dir=evidence_root) + print("evidence: " + evidence) try: atlas = build_atlas(tmp) print("atlas: %d glyphs" % len(atlas)) tree = build_tree(tmp, fault) - proc = subprocess.Popen([EIGS, "orbit_qa.eigs"], cwd=tree, env=ENV, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + proc = start_app(["orbit_qa.eigs"], tree, tmp, "orbit") wid = wait_for_window(proc, TITLE) print("window %s at %r" % (wid, window_origin(wid))) + def state(): + ensure_live(proc) + try: + with open(os.path.join(tree, "mouse-qa-state")) as fh: + fields = fh.read().strip().split("|") + if len(fields) != 5 or fields[4] != "END": + return None # write_text may be between truncate/write + ticks, frame, paused = map(int, fields[:3]) + if ticks < 1 or frame < 0 or paused not in (0, 1) or fields[3] not in ("orbit", "bif"): + return None + return dict(ticks=ticks, frame=frame, paused=paused, mode=fields[3]) + except (FileNotFoundError, ValueError): + return None + + def wait_state(name, predicate, duration=0.0): + started = time.monotonic() + deadline = started + ACTION_TIMEOUT + last = None + while time.monotonic() < deadline: + last = state() + if last is not None and predicate(last) and time.monotonic() - started >= duration: + print("ACK %s monotonic=%.6f elapsed=%.3f state=%r" % + (name, time.monotonic(), time.monotonic() - started, last)) + return last + time.sleep(min(POLL_INTERVAL, max(0, deadline - time.monotonic()))) + raise RuntimeError("%s acknowledgement deadline; last=%r" % (name, last)) + + def ticks_after(name, before, duration=0.0): + return wait_state(name, lambda s: s["ticks"] >= before["ticks"] + 3, duration) + + def snapshot(name, deadline=None): + img = grab(wid, tmp, proc, deadline) + print("CAPTURE %s monotonic=%.6f state=%r" % (name, time.monotonic(), state())) + if evidence: + img.save(os.path.join(evidence, name + ".png")) + return img + + def visible(name, predicate, before): + # Poll the postcondition, never resend input. A live newer tick is + # required in addition to pixels; a frozen stale frame cannot pass. + started = time.monotonic() + deadline = started + ACTION_TIMEOUT + last = None + seen = None + captured_state = None + while time.monotonic() < deadline: + seen = state() + if seen is not None and seen["ticks"] > before["ticks"]: + try: + last = grab(wid, tmp, proc, deadline) + except CaptureDeadline: + ensure_live(proc) + break + captured_state = seen + if predicate(last, seen): + print("VISIBLE %s monotonic=%.6f elapsed=%.3f state=%r" % + (name, time.monotonic(), time.monotonic() - started, seen)) + if evidence: + last.save(os.path.join(evidence, name + ".png")) + return last, seen + time.sleep(min(POLL_INTERVAL, max(0, deadline - time.monotonic()))) + print("DEADLINE %s monotonic=%.6f state=%r" % (name, time.monotonic(), seen)) + if last is None: + raise RuntimeError(name + ": no live newer frame before deadline") + if evidence: + last.save(os.path.join(evidence, name + "-deadline.png")) + # Return the failing sample to the existing named pixel assertion. + return last, captured_state + + ready = wait_state("startup", lambda s: s["mode"] == "orbit" and s["paused"] == 0) + # Window-relative pointer moves: `getwindowgeometry` reports the # FRAME position under a window manager, so adding it to a client # coordinate misses every thin control by the title-bar height. @@ -285,35 +428,57 @@ def mv(wx, wy): xdo("mousemove", "--window", wid, wx, wy) def click_at(wx, wy): - mv(wx, wy); xdo("click", "1"); time.sleep(0.25) + mv(wx, wy); xdo("click", "1") + print("INPUT click (%d,%d) monotonic=%.6f" % (wx, wy, time.monotonic())) def drag(path, hold=0.12): + # Gesture pacing keeps native stepped motion; it is never used + # as evidence that an action completed. The visible waiter is. mv(*path[0]); xdo("mousedown", "1"); time.sleep(hold) for wx, wy in path[1:]: mv(wx, wy); time.sleep(hold) - xdo("mouseup", "1"); time.sleep(0.25) + xdo("mouseup", "1") + print("INPUT drag monotonic=%.6f" % time.monotonic()) def key(k): - xdo("key", "--window", wid, k); time.sleep(0.35) + xdo("key", "--window", wid, k) + print("INPUT key %s monotonic=%.6f" % (k, time.monotonic())) + + def pause_to(value): + before = wait_state("before-space", lambda s: True) + if before["paused"] == value: + raise RuntimeError("unexpected pause state before space: %r" % before) + key("space") + ack = wait_state("space-%d" % value, + lambda s: s["ticks"] > before["ticks"] and s["paused"] == value) + # The hook precedes drawing. A following tick witnesses completion + # of the acknowledged tick's render before its pixels are compared. + settled = ticks_after("space-rendered", ack) + if settled["paused"] != value: + raise RuntimeError("pause state changed again before capture") + return settled cx, cy = CANVAS[0] + CANVAS[2] // 2, CANVAS[1] + CANVAS[3] // 2 # 1. running: consecutive frames of the plot differ. Checked first, # while the orbit is still wide — it decays toward the fixed point # from the moment the window opens. - run_a = grab(wid, tmp) - time.sleep(0.4) - run_b = grab(wid, tmp) + run_a = snapshot("running-before") + run_b, running = visible("running", lambda img, s: + s["frame"] > ready["frame"] and + region_diff(run_a, img, CANVAS) >= ADVANCE_MIN, ready) d = region_diff(run_a, run_b, CANVAS) - check(d >= ADVANCE_MIN, "running: the plot advances between frames (%d px differ)" % d) + check(d >= ADVANCE_MIN and running["frame"] > ready["frame"], + "running: the plot advances between frames (%d px differ)" % d) # Park focus on the canvas up front (lib/ui draws a focus ring on # the focused widget, so focus must not move mid-comparison), and # leave the pointer over the plot — the wheel event carries no # cursor position, so the app anchors zoom on the last hover. click_at(cx, cy) + ticks_after("canvas-focus-rendered", running) - start = grab(wid, tmp) + start = snapshot("startup-labels") base_zeta = decode_line(start, atlas, ZETA_LABEL_XY) base_zoom = decode_line(start, atlas, ZOOM_LABEL_XY) print("decoded labels at startup: %r / %r" % (base_zeta, base_zoom)) @@ -321,37 +486,46 @@ def key(k): check(base_zoom == "zoom x1", "render-decode: zoom label reads %r" % base_zoom) # 2. pause: consecutive frames are pixel-identical. - key("space") - paused_a = grab(wid, tmp) - time.sleep(0.4) - paused_b = grab(wid, tmp) + paused = pause_to(1) + paused_a = snapshot("pause-before") + frozen = ticks_after("pause-observation", paused, duration=0.4) + paused_b = snapshot("pause-after") d = region_diff(paused_a, paused_b, CANVAS) - check(d == 0, "pause: advancement freezes — two frames 0.4s apart are identical (%d px differ)" % d) + check(d == 0 and frozen["frame"] == paused["frame"] and frozen["paused"] == 1, + "pause: advancement freezes — live ticks over >=0.4s are identical (%d px differ)" % d) # 3. unpause: the plot advances again (Pause froze it, nothing hung). - key("space") - run_c = grab(wid, tmp) - time.sleep(0.4) - run_d = grab(wid, tmp) + resumed = pause_to(0) + run_c = snapshot("unpause-before") + run_d, advanced = visible("unpause", lambda img, s: + s["frame"] > resumed["frame"] and + region_diff(run_c, img, CANVAS) >= ADVANCE_MIN, resumed) d = region_diff(run_c, run_d, CANVAS) - check(d >= ADVANCE_MIN, "unpause: advancement resumes (%d px differ)" % d) + check(d >= ADVANCE_MIN and advanced["frame"] > resumed["frame"], + "unpause: advancement resumes (%d px differ)" % d) # Freeze again for the view tests: pan/zoom/reset must be judged # against a still simulation. - key("space") - paused_b = grab(wid, tmp) + paused = pause_to(1) + paused_b = snapshot("pre-pan") # 4. drag to pan (real press, stepped motion, release). drag([(cx, cy), (cx - 30, cy - 12), (cx - 60, cy - 24), (cx - 90, cy - 36)]) - panned = grab(wid, tmp) + panned, panned_state = visible("pan", lambda img, s: + region_diff(paused_b, img, CANVAS) > CHANGED_MIN, paused) d = region_diff(paused_b, panned, CANVAS) check(d > CHANGED_MIN, "drag: panning redraws the plot while paused (%d px differ)" % d) # 5. wheel to zoom. - mv(cx, cy); time.sleep(0.2) - xdo("click", "4"); time.sleep(0.2) - xdo("click", "4"); time.sleep(0.35) - zoomed = grab(wid, tmp) + mv(cx, cy) + xdo("click", "4") + xdo("click", "4") + print("INPUT wheel monotonic=%.6f" % time.monotonic()) + zoomed, zoomed_state = visible("zoom", lambda img, s: + region_diff(panned, img, CANVAS) > CHANGED_MIN and + decode_line(img, atlas, ZOOM_LABEL_XY) != base_zoom and + decode_line(img, atlas, ZOOM_LABEL_XY).startswith("zoom x"), + panned_state) d = region_diff(panned, zoomed, CANVAS) check(d > CHANGED_MIN, "wheel: zooming redraws the plot (%d px differ)" % d) zoom_txt = decode_line(zoomed, atlas, ZOOM_LABEL_XY) @@ -361,7 +535,9 @@ def key(k): # 6. reset restores the exact pre-pan view. key("r") - reset = grab(wid, tmp) + reset, reset_state = visible("reset", lambda img, s: + region_diff(paused_b, img, CANVAS) == 0 and + decode_line(img, atlas, ZOOM_LABEL_XY) == base_zoom, zoomed_state) d = region_diff(paused_b, reset, CANVAS) check(d == 0, "reset: `r` restores the pre-pan plot pixel-for-pixel (%d px differ)" % d) check(decode_line(reset, atlas, ZOOM_LABEL_XY) == base_zoom, @@ -370,7 +546,10 @@ def key(k): # 7. slider drag changes zeta (read off the rendered label). sx, sy, sw = SLIDER drag([(sx + 8, sy), (sx + sw // 2, sy), (sx + sw - 20, sy)]) - after_slider = grab(wid, tmp) + after_slider, slider_state = visible("slider", lambda img, s: + decode_line(img, atlas, ZETA_LABEL_XY).startswith("zeta =") and + decode_line(img, atlas, ZETA_LABEL_XY) != base_zeta, + reset_state) zeta_txt = decode_line(after_slider, atlas, ZETA_LABEL_XY) print("decoded zeta label after slider drag: %r" % zeta_txt) check(zeta_txt.startswith("zeta =") and zeta_txt != base_zeta, @@ -384,9 +563,11 @@ def key(k): # pixel-identical is the visible face of "the bifurcation tick # does no per-frame work", which is what the memory gate # (tests/test_bif_mem.sh) measures in RSS. - before_toggle = grab(wid, tmp) + before_toggle = snapshot("pre-mode") click_at(*MODE_BTN) - bif = grab(wid, tmp) + bif, bif_state = visible("mode-bif", lambda img, s: + decode_line(img, atlas, BIF_LABEL_XY) == "logistic map" and + region_diff(before_toggle, img, CANVAS) > CHANGED_MIN, slider_state) bif_txt = decode_line(bif, atlas, BIF_LABEL_XY) print("decoded bifurcation column: %r" % bif_txt) check(bif_txt == "logistic map", @@ -394,8 +575,8 @@ def key(k): d = region_diff(before_toggle, bif, CANVAS) check(d > CHANGED_MIN, "mode: the plot area is redrawn as the bifurcation chart (%d px differ)" % d) - time.sleep(0.4) - bif_b = grab(wid, tmp) + ticks_after("bif-observation", bif_state, duration=0.4) + bif_b = snapshot("bif-static") d = region_diff(bif, bif_b, CANVAS) check(d == 0, "mode: the bifurcation diagram is static between frames (%d px differ)" % d) @@ -403,7 +584,9 @@ def key(k): # 9. toggling back restores the phase portrait — the pre-existing # view must survive a round trip through the new one. click_at(*MODE_BTN) - back = grab(wid, tmp) + back, _ = visible("mode-orbit", lambda img, s: + decode_line(img, atlas, ZETA_LABEL_XY) == zeta_txt and + region_diff(before_toggle, img, CANVAS) == 0, bif_state) back_txt = decode_line(back, atlas, ZETA_LABEL_XY) print("decoded zeta label after toggling back: %r" % back_txt) check(back_txt == zeta_txt, @@ -414,13 +597,21 @@ def key(k): except Stop: print("(stopping at the first failure)") + except (RuntimeError, OSError, subprocess.SubprocessError) as exc: + infrastructure_error = str(exc) + print("ERROR mouse oracle: " + infrastructure_error) finally: if proc is not None: - proc.terminate() - try: proc.wait(timeout=5) - except Exception: proc.kill() + stop_app(proc) + if evidence: + for name in ("orbit.log", "atlas.log"): + path = os.path.join(tmp, name) + if os.path.exists(path): + shutil.copy(path, evidence) shutil.rmtree(tmp, ignore_errors=True) + if infrastructure_error is not None: + sys.exit(2) if fails: print("\n%d mouse-oracle failure(s)" % len(fails)) sys.exit(1) diff --git a/tests/test_orbit_mouse.sh b/tests/test_orbit_mouse.sh index 82037a7..213821e 100755 --- a/tests/test_orbit_mouse.sh +++ b/tests/test_orbit_mouse.sh @@ -8,9 +8,8 @@ # Pause freezes advancement, drag pans, the wheel zooms, and `r` restores # the view exactly. # -# The checker is validated by two planted faults (a drag that never -# claims the pointer, and a tick that advances while paused): each MUST -# be caught, or a green run means nothing. +# The checker is validated by three planted faults (pan, pause, mode): +# each MUST fail its named assertion, or a green run means nothing. # # Exits 2 (= failure in CI, skip locally) when the runtime has no gfx # builtins or the QA tooling is missing, so it can never be silently @@ -24,9 +23,9 @@ cd "$ROOT" TMP="$(mktemp -d)" trap "rm -rf '$TMP'" EXIT -XVFB="" +RUN=() if [ -z "${DISPLAY:-}" ]; then - XVFB="xvfb-run -a" + RUN=(xvfb-run -a) fi echo "--- gfx capability probe ---" @@ -35,8 +34,9 @@ w is gfx_open of [64, 48, "probe"] gfx_close of null print of "GFXOK" EOF -PROBE_OUT=$($XVFB "$EIGS" "$TMP/gfxprobe.eigs" 2>&1 || true) -if ! echo "$PROBE_OUT" | grep -q "GFXOK"; then +PROBE_RC=0 +PROBE_OUT=$("${RUN[@]}" "$EIGS" "$TMP/gfxprobe.eigs" 2>&1) || PROBE_RC=$? +if [ "$PROBE_RC" -ne 0 ] || ! echo "$PROBE_OUT" | grep -qx "GFXOK"; then if echo "$PROBE_OUT" | grep -q "undefined variable 'gfx_open'"; then echo "SKIP: runtime is not gfx-capable (build with 'make gfx') — mouse oracle not run" exit 2 @@ -51,32 +51,48 @@ for tool in xdotool xwd; do done python3 -c "import PIL" 2>/dev/null || { echo "SKIP: python3 PIL is not installed — mouse oracle not run"; exit 2; } -# xvfb-run sometimes fails cleaning up its own tmpdir after the app is -# torn down, which would mask the oracle's verdict — so read the verdict -# from the oracle's own output, not from the wrapper's exit status. +# A wrapper/tool failure cannot stand in for the oracle's own verdict. +# Preserve both the exact exit and the named assertion population. run_oracle() { local log="$1"; shift - set +e - $XVFB python3 tests/mouse_oracle.py "$@" > "$log" 2>&1 - set -e + ORACLE_RC=0 + "${RUN[@]}" python3 tests/mouse_oracle.py "$@" > "$log" 2>&1 || ORACLE_RC=$? cat "$log" } echo "--- mouse + render-decode oracle (real window, real input) ---" run_oracle "$TMP/clean.log" -grep -q "^all mouse + render-decode checks passed$" "$TMP/clean.log" || { - echo "FAIL: the orbit-lab controls did not respond as specified" +UNIQUE_PASSES=$(grep '^PASS ' "$TMP/clean.log" | sort -u | wc -l) || { + echo "FAIL: could not enumerate distinct mouse checks" exit 1 } +if [ "$ORACLE_RC" -ne 0 ] || + [ "$(grep -c '^PASS ' "$TMP/clean.log" || true)" -ne 16 ] || + grep -q '^FAIL \|^ERROR \|^DEADLINE ' "$TMP/clean.log" || + [ "$(grep -c '^all mouse + render-decode checks passed$' "$TMP/clean.log" || true)" -ne 1 ] || + [ "$UNIQUE_PASSES" -ne 16 ]; then + echo "FAIL: the orbit-lab controls did not respond as specified" + exit 1 +fi for fault in pause pan mode; do echo "--- planted fault: $fault ---" run_oracle "$TMP/fault_$fault.log" --fault "$fault" --stop-on-fail - grep -q "mouse-oracle failure" "$TMP/fault_$fault.log" || { - echo "FAIL: planted fault '$fault' was NOT caught — the mouse oracle can't discriminate" + case "$fault" in + pause) killed='pause: advancement freezes'; passes=3 ;; + pan) killed='drag: panning redraws'; passes=5 ;; + mode) killed='mode: the control column reads'; passes=11 ;; + esac + if [ "$ORACLE_RC" -ne 1 ] || + [ "$(grep -c '^PASS ' "$TMP/fault_$fault.log" || true)" -ne "$passes" ] || + [ "$(grep -c '^FAIL ' "$TMP/fault_$fault.log" || true)" -ne 1 ] || + grep -q '^ERROR ' "$TMP/fault_$fault.log" || + ! grep -q "^FAIL $killed" "$TMP/fault_$fault.log" || + [ "$(grep -c '^1 mouse-oracle failure(s)$' "$TMP/fault_$fault.log" || true)" -ne 1 ]; then + echo "FAIL: planted fault '$fault' did not fail its named assertion with rc1" exit 1 - } + fi echo "PASS: planted fault '$fault' is caught by the oracle" done -echo "PASS: mouse + render-decode oracle, and both planted faults caught" +echo "PASS: mouse + render-decode oracle (16 checks), and all three planted faults caught"