diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d12c67..5ecce60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,8 +33,19 @@ jobs: run: bash setup/pre-publish-check.sh # Build + import the MCP server in a clean env โ€” catches "works in the repo, broken on install". + # + # ๐Ÿ”ด Windows is in this matrix on purpose. The 2026-08-24 field report that produced the + # `_run` fixes came from Windows, and the rest of CI is ubuntu+macos only โ€” so the + # ORIGIN ENVIRONMENT OF THE DEFECT WAS NOT COVERED BY THE GREEN. Install+import alone + # also never exercised the subprocess contract those fixes changed, so the contract + # tests run here too. The gate suite is bash and stays on the unix matrix above; this + # job is the Python layer, which is where both reported P0s actually lived. mcp: - runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -43,6 +54,9 @@ jobs: - name: Install yeoul-mcp from source run: pip install ./mcp - name: Import the server + confirm the tool surface + # `shell: bash` is required now that this job also runs on Windows: the default shell + # there is PowerShell, which has no heredoc โ€” `python - <<'PY'` is a parse error. + shell: bash run: | python - <<'PY' from yeoul_mcp import server @@ -53,3 +67,5 @@ jobs: assert len(tools) == 12, tools print("yeoul-mcp OK โ€” 12 tools") PY + - name: _run subprocess contract (stdin never inherited, UTF-8 pinned) + run: python mcp/tests/test_run_contract.py diff --git a/bin/arc-close b/bin/arc-close index a33cb25..a015702 100755 --- a/bin/arc-close +++ b/bin/arc-close @@ -113,26 +113,48 @@ emit_seal_check() { # non-KILL close with a seal linked โ€” cross-check it regar # Trivial-evasion check. Both paths share ONE bar โ€” kept apart, they drift (proved 2026-08-05: # the same answer sealed internally and was refused by this file). +# โ˜…2026-08-24 redesign: the substance check used to be a single `[ ${#vans} -ge 6 ]` (>= 6 chars). +# Measured: 28 of 28 evasive answers sealed straight through (`aaaaaa`, `yes yes`, `yes ok`, ...). +# The Windows/CP949 field report was ONE CASE of that class, not the class. +# The judgement now lives in substance_check.py โ€” a file kept **byte-identical** with the internal +# copy, emitting machine codes only. Prose is attached here, so the two copies stay in step by +# construction rather than by discipline. +SUBSTANCE="$(dirname "${BASH_SOURCE[0]}")/substance_check.py" +TAB="$(printf '\t')" # NOTE: ${v%%\t*} is a literal `t` in shell patterns โ€” this must be a real tab + check_answers() { # check_answers
+ # โ˜… The positive control runs at the HEAD OF THE GATE'S EXECUTION PATH, not in a test file. + # If the checker cannot separate planted violations from planted genuine answers, we do NOT + # interpret the real answers โ€” we die. An instrument is not trusted because it is green; + # it is trusted because it just proved it can still say no. + if ! st="$(python3 "$SUBSTANCE" --selftest 2>&1)"; then + echo "โ›” seal refused: the substance checker failed its own positive control โ€” the instrument is broken, so the real verdict is not interpreted." + printf '%s\n' "$st" | sed 's/^/ /' + exit 6 + fi + echo " ๐Ÿงช substance-checker positive control: ${st##*: }" while IFS= read -r vline; do case "$vline" in "- **"*"**:"*) ;; *) continue ;; esac vlabel="$(printf '%s' "$vline" | sed -E 's/^- \*\*([^*]+)\*\*.*/\1/')" - # answer extraction via python (Unicode-safe โ€” sed's multibyte โ† match is locale-fragile) - vans="$(printf '%s' "$vline" | python3 -c 'import sys,re; s=sys.stdin.read(); s=re.sub(r"^- \*\*[^*]+\*\*:\s*","",s); s=re.sub(r"\s*โ†.*$","",s); sys.stdout.write(s.strip())')" - vlow="$(printf '%s' "$vans" | tr '[:upper:]' '[:lower:]')" - case "$vlow" in ""|y|yes|ok|okay|na|n/a|done|.|-|x|pass|true|good|yep|sure|n) - echo "โ›” seal refused: '$vlabel' answer too trivial: '$vans'"; exit 5 ;; - esac - case "$vlabel" in - *[Cc]atalog*) # a catalog id, or the literal 'none' - case "$vlow" in none) : ;; *) printf '%s' "$vans" | grep -qE '[A-Za-z0-9_]{3,}' \ - || { echo "โ›” seal refused: catalog cross-check needs a catalog id or 'none': '$vans'"; exit 5; } ;; esac ;; - *[Aa]nchor*) # a number, or a seal/reproduction reference - printf '%s' "$vlow" | grep -qE '[0-9]|seal|anchor|reproduc|converg|hash' \ - || { echo "โ›” seal refused: anchor answer needs a number or a seal/reproduction reference: '$vans'"; exit 5; } ;; - *) # otherwise require a little substance - [ "${#vans}" -ge 6 ] || { echo "โ›” seal refused: '$vlabel' answer too short: '$vans'"; exit 5; } ;; + # ๐Ÿ”ด Read the verdict from the emitted CODE, not the exit status. If the checker cannot run at + # all the output is empty โ€” which is not OK โ€” so this **fails closed**. A false default would + # turn "never measured" into "measured and fine". + vout="$(printf '%s' "$vline" | python3 "$SUBSTANCE" --label "$vlabel" 2>/dev/null || true)" + vcode="${vout%%"$TAB"*}"; vans="${vout#*"$TAB"}" + [ "$vcode" = "OK" ] && continue + case "$vcode" in + DECODE_FAILED) echo "โ›” seal refused: '$vlabel' answer was not readable as UTF-8 (encoding damage) โ€” could not judge" ;; + TRIVIAL_VOCAB) echo "โ›” seal refused: '$vlabel' answer is trivial vocabulary only: '$vans'" ;; + DEFERRAL) echo "โ›” seal refused: '$vlabel' answer defers ('don't know' / 'TODO') โ€” honest, but it cannot ground a seal: '$vans'" ;; + REPEATED_UNIT) echo "โ›” seal refused: '$vlabel' answer is one unit repeated: '$vans'" ;; + LOW_DIVERSITY) echo "โ›” seal refused: '$vlabel' answer has almost no character variety: '$vans'" ;; + THIN_CONTENT) echo "โ›” seal refused: '$vlabel' answer carries no substance (too few content tokens): '$vans'" ;; + NEED_CATALOG) echo "โ›” seal refused: catalog cross-check needs a real catalog id or 'none': '$vans'" ;; + NEED_ANCHOR) echo "โ›” seal refused: anchor answer needs a number or a seal/reproduction reference: '$vans'" ;; + "") echo "โ›” seal refused: could not run the substance checker for '$vlabel' โ€” an unmeasured field is not a passing field" ;; + *) echo "โ›” seal refused: '$vlabel' failed the substance check ($vcode): '$vans'" ;; esac + exit 5 done < <(sed -n "/$1/,\$p" "$SUMMARY") } diff --git a/bin/substance_check.py b/bin/substance_check.py new file mode 100755 index 0000000..7e69718 --- /dev/null +++ b/bin/substance_check.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""substance_check.py โ€” the one place an arc close decides whether an answer has substance. + +This file is kept BYTE-IDENTICAL between the internal copy (apps/nacc/scripts/) and the OSS +copy (bin/). It emits machine CODES only; each shell renders its own prose. Drift between the +two copies is prevented by construction rather than by discipline โ€” a previous fix went into +both copies while its regression test went into only one. + +Why the length test was dropped +------------------------------- +The check used to be a single `[ ${#vans} -ge 6 ]` โ€” six characters or more and the answer +sealed. Measured 2026-08-24 against the real binary: **28 of 28 evasive answers sealed, 0 +caught** (`aaaaaa`, `yes yes`, `yes ok`, ...). Length is a proxy, and the set of non-answers +that satisfy a proxy is unbounded. + +Raising the bar fails in the other direction: a genuine answer can be short +(`d=0.05 < 0.2, under the sealed bar`). So this judges CONTENT, and the repair was measured +both ways โ€” evasions caught AND genuine answers not refused. + +Korean here is functional data, not decoration +---------------------------------------------- +Yeoul is used in Korean as well as English, so the trivial/deferral vocabularies and the field +labels include Korean. This repo's publish guard treats raw Hangul as a personalization leak +(`setup/pre-publish-check.sh`), so every Korean literal below is written as an escape carrying +a romanization and an English gloss. The guard uses that same technique on itself. The escapes +were generated, not typed, and round-tripped back to the source words before being committed. + +Rejection codes (the shell attaches the wording) +------------------------------------------------ + DECODE_FAILED not readable as UTF-8 โ€” "could not judge", which is NOT "refused on the + merits"; merging the two disguises one as the other + TRIVIAL_VOCAB nothing but trivial vocabulary: `yes` `ok` `n/a` `.` `-` + DEFERRAL an honest non-answer: "don't know", "TODO", "not measured" โ€” not a lie, but + it cannot ground a seal + REPEATED_UNIT one unit repeated: `yesyesyes`, `abcabcabc` + LOW_DIVERSITY almost no character variety: `aaaaaab` + THIN_CONTENT too few content tokens to carry a claim: `qwerty`, `123456` + NEED_CATALOG catalog field without a real catalog id (or the literal "none") + NEED_ANCHOR anchor field without a number or a seal/reproduction reference + +๐Ÿ”ด These rules NARROW the class; they do not close it. They catch evasions we thought of. +That is exactly why the load-bearing part of this file is `--selftest`: the gate runs planted +specimens through this judge at the HEAD OF ITS EXECUTION PATH, and if the judge cannot +separate them it refuses to interpret the real answers at all. +""" +import re +import sys +import unicodedata + +# โ”€โ”€ Korean vocabulary, escaped so this file carries no raw Hangul โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +_KO = { + "YES1": "\uc608", # ye โ€” "yes" + "YES2": "\ub124", # ne โ€” "yes" + "NO_": "\uc544\ub2c8\uc624", # anio โ€” "no" + "UHUH": "\uc751", # eung โ€” "uh-huh" + "DID": "\ud568", # ham โ€” "did it" + "DONE_": "\ub428", # doem โ€” "done" + "NONE_CH": "\ubb34", # mu โ€” "none" + "CONFIRM": "\ud655\uc778", # hwagin โ€” "confirmed" + "COMPLETE": "\uc644\ub8cc", # wallyo โ€” "complete" + "PASSED": "\ud1b5\uacfc", # tonggwa โ€” "passed" + "CORRECT": "\ub9de\uc74c", # majeum โ€” "correct" + "ABSENT": "\uc5c6\uc74c", # eopseum โ€” "none / absent" + "NA_KO": "\ud574\ub2f9\uc5c6\uc74c", # haedang-eopseum โ€” "not applicable" + "SO": "\uadf8\ub807\ub2e4", # geureota โ€” "that is so" + "NOT_SO": "\uc544\ub2c8\ub2e4", # anida โ€” "that is not so" + "APPLIED": "\uc801\uc6a9", # jeogyong โ€” "applied" + "NORMAL": "\uc815\uc0c1", # jeongsang โ€” "normal" + "DUNNO1": "\ubaa8\ub984", # moreum โ€” "don't know" + "DUNNO2": "\ubaa8\ub974\uaca0", # moreugess โ€” "don't know (stem)" + "UNCONF": "\ubbf8\ud655\uc778", # mihwagin โ€” "unconfirmed" + "UNMEAS": "\ubbf8\uce21\uc815", # micheukjeong โ€” "unmeasured" + "LATER1": "\ub098\uc911\uc5d0", # najunge โ€” "later" + "LATER2": "\ucd94\ud6c4", # chuhu โ€” "later" + "LATER3": "\ucc28\ud6c4", # chahu โ€” "later" + "HOLD": "\ubcf4\ub958", # boryu โ€” "on hold" + "NOTMEAS1": "\uc548\u0020\uc7c0", # an jaess โ€” "did not measure" + "NOTMEAS2": "\uc548\uc7c0", # anjaess โ€” "did not measure" + "NOTMEAS3": "\ubabb\u0020\uc7c0", # mot jaess โ€” "could not measure" + "NOTMEAS4": "\ubabb\uc7c0", # motjaess โ€” "could not measure" + "NOTDONE1": "\uc548\u0020\ud568", # an ham โ€” "did not do" + "NOTDONE2": "\uc548\ud568", # anham โ€” "did not do" + "CATALOG": "\ub3c4\uac10", # dogam โ€” "catalog" + "ANCHOR": "\uc575\ucee4", # aengkeo โ€” "anchor" + "REPRO": "\uc7ac\ud604", # jaehyeon โ€” "reproduce" + "CONVERGE": "\uc218\ub834", # suryeom โ€” "converge" +} + +# โ”€โ”€ Trivial vocabulary: tokens that assert nothing on their own โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +TRIVIAL = { + "", "y", "n", "yes", "no", "ok", "okay", "na", "n/a", "nil", "none", "null", + "pass", "fail", "true", "false", "done", "good", "yep", "yup", "sure", "fine", + "x", "o", ".", "-", "--", "?", "!", "test", "tbd", +} | {_KO[k] for k in ( + "YES1", "YES2", "NO_", "UHUH", "DID", "DONE_", "NONE_CH", "CONFIRM", "COMPLETE", + "PASSED", "CORRECT", "ABSENT", "NA_KO", "SO", "NOT_SO", "APPLIED", "NORMAL", +)} + +# โ”€โ”€ Deferral vocabulary: honest non-answers. True, but they cannot ground a seal โ”€โ”€ +DEFERRAL = [ + "todo", "tbd", "later", "unknown", "unclear", "not sure", "dunno", "pending", + "not run", "not measured", +] + [_KO[k] for k in ( + "DUNNO1", "DUNNO2", "UNCONF", "UNMEAS", "LATER1", "LATER2", "LATER3", "HOLD", + "NOTMEAS1", "NOTMEAS2", "NOTMEAS3", "NOTMEAS4", "NOTDONE1", "NOTDONE2", +)] + +MIN_DEFERRAL_CONTENT = 3 # below this, a deferral IS the answer rather than a clause in it +MIN_CONTENT_TOKENS = 2 # a real answer says at least two things +MIN_DISTINCT_CHARS = 10 # a single long token can still carry substance (unspaced Korean) +# ๐Ÿ”ด Do NOT measure character variety as a RATIO. Alphabets are finite, so a longer answer +# scores lower โ€” that ruler rejects answers for being long, which is backwards. Measured: +# an 85-char English sentence has 26 distinct chars = 0.31 and was refused. A Korean-heavy +# corpus cannot expose this (thousands of syllables keep the ratio high), so the sealed +# measurement went green and an OSS regression test caught it instead. +MIN_DISTINCT_FLOOR = 5 # length-independent absolute floor + +_PUNCT = re.compile(r"[\s,./ยท;:()\[\]{}<>\"'`~!?*_=+|\\@#$%^&โ€”โ€“โ€ฆ]+") + + +def extract(raw_bytes): + """One gate line (raw bytes) -> (code, answer). Everything after the `โ†` hint is dropped. + + โ˜… Extraction lives in this file on purpose. The 2026-08-24 defect was IN THE EXTRACTOR: + under a CP949 mis-decode the `โ†` was not stripped, so the hint became the body of the + answer and a trivial "yes" arrived long enough to clear the checks. Testing the judge + but leaving the extractor outside would put that hole back outside the tests. + """ + body = raw_bytes.split("โ†".encode("utf-8"))[0] + s = body.decode("utf-8", "replace") + if "๏ฟฝ" in s: + return "DECODE_FAILED", "" + s = re.sub(r"^-\s*\*\*[^*]+\*\*:\s*", "", s) + return "OK", s.strip() + + +def _normalize(ans): + s = unicodedata.normalize("NFKC", ans) + s = re.sub(r"\*\*|__|`", "", s) # markdown emphasis is not content + return s.strip() + + +def _tokens(norm): + return [t for t in _PUNCT.split(norm.lower()) if t] + + +def _repeated_unit(s): + """Is the whole string one unit repeated (`abcabc`, `yesyesyes`)?""" + t = re.sub(r"\s+", "", s) + n = len(t) + if n < 4: + return False + for unit in range(1, n // 2 + 1): + if n % unit == 0 and t[:unit] * (n // unit) == t: + return True + return False + + +def judge(label, ans): + """(code, detail). 'OK' passes. Field-specific branches are handled here too.""" + norm = _normalize(ans) + low = norm.lower() + + # Catalog field: an id, or an explicit "not applicable" + if _KO["CATALOG"] in label or re.search(r"catalog", label, re.I): + if low in (_KO["NA_KO"], "none"): + return "OK", "exempt" + # An id-SHAPE test alone lets `zzz` through โ€” the same kind of proxy as the length bar. + # Content-token count cannot be used (a real id like `vacuous_pass` is one token), so + # repetition and character variety are what apply. + if not re.search(r"[A-Za-z0-9_]{3,}", norm): + return "NEED_CATALOG", norm + _bare = re.sub(r"\s+", "", norm) + if _repeated_unit(norm) or (_bare and len(set(_bare)) < MIN_DISTINCT_FLOOR): + return "NEED_CATALOG", norm + return "OK", "catalog-id" + + # Anchor field: a number, or a seal/reproduction reference. Falls through to the general + # test afterwards, so a bare `123456` cannot satisfy it. + if _KO["ANCHOR"] in label or re.search(r"anchor", label, re.I): + pat = "[0-9]|seal|anchor|reproduc|converg|hash|%s|%s|%s" % ( + _KO["REPRO"], _KO["CONVERGE"], _KO["ANCHOR"]) + if not re.search(pat, low): + return "NEED_ANCHOR", norm + + toks = _tokens(norm) + content = [t for t in toks if t not in TRIVIAL and not t.isspace()] + + if not content: + return "TRIVIAL_VOCAB", norm + # ๐Ÿ”ด A deferral only counts when it IS the whole answer. A deferral CLAUSE inside a + # reasoned answer does not remove its substance โ€” measured: 1 of 32 real answers + # already sealed by this gate was wrongly refused by the naive form of this rule + # ("not run โ€” the face arm is absent, so no anchor could be formed"). + if len(set(content)) < MIN_DEFERRAL_CONTENT and any(d in low for d in DEFERRAL): + return "DEFERRAL", norm + if _repeated_unit(norm): + return "REPEATED_UNIT", norm + + bare = re.sub(r"\s+", "", norm) + if bare and len(set(bare)) < MIN_DISTINCT_FLOOR: + return "LOW_DIVERSITY", norm + + if len(set(content)) < MIN_CONTENT_TOKENS and len(set(bare)) < MIN_DISTINCT_CHARS: + return "THIN_CONTENT", norm + + return "OK", norm + + +# โ”€โ”€ Planted specimens. The gate runs these at the head of its execution path โ”€โ”€โ”€โ”€โ”€ +# ๐Ÿ”ด BIDIRECTIONAL ON PURPOSE. Plant only violations and a checker that rejects everything +# scores full marks. Measured: sabotaging the judge to always return OK scores a PARTIAL, +# not a zero, precisely because the genuine specimens still have to pass. +# Korean specimens are built FROM the vocabulary table above, so this file needs no raw Hangul +# and the Korean paths still get exercised on both copies. +_L_IND = "Independent angles converged" +_L_IMP = "Implementation defect ruled out" +_L_KILL = "Kill wording match" +_L_CAT_EN = "Catalog cross-check" +_L_ANC_EN = "Anchor (positive control) reproduced" + +_PLANT_VIOLATIONS = [ + (_L_IND, "aaaaaa", "REPEATED_UNIT"), + (_L_IND, "yes yes", "TRIVIAL_VOCAB"), + (_L_IND, "yes ok", "TRIVIAL_VOCAB"), + (_L_IMP, "aaaaaab", "LOW_DIVERSITY"), + (_L_IMP, "qwerty", "THIN_CONTENT"), + (_L_KILL, "TODO later", "DEFERRAL"), + (_L_KILL, "......", "TRIVIAL_VOCAB"), + (_L_CAT_EN, "zzz", "NEED_CATALOG"), + (_L_CAT_EN, "aaaa", "NEED_CATALOG"), + (_L_ANC_EN, "it just went fine", "NEED_ANCHOR"), + # Korean paths: trivial vocabulary, one unit repeated, a bare deferral โ€” and the two + # Korean FIELD LABELS, which must route to the catalog/anchor branches just as the + # English ones do (that routing is a plain substring test and would fail silently). + (_L_IMP, _KO["CONFIRM"] + " " + _KO["COMPLETE"], "TRIVIAL_VOCAB"), + (_L_IMP, _KO["NA_KO"] * 2, "REPEATED_UNIT"), + (_L_KILL, _KO["DUNNO2"], "DEFERRAL"), + (_KO["CATALOG"], "zzz", "NEED_CATALOG"), + (_KO["ANCHOR"], "it just went fine", "NEED_ANCHOR"), +] +_PLANT_GENUINE = [ + (_L_IND, "two independent angles agreed from different evidence"), + (_L_IMP, "code compared verbatim, the mechanism is not at fault"), + (_L_KILL, "d=0.05 < 0.2, hits the sealed wording with no post-hoc widening"), + (_L_CAT_EN, "vacuous_pass"), + (_L_CAT_EN, "none"), + (_L_ANC_EN, "anchor reproduced 3x (seal 84007e65)"), + # ๐Ÿ”ด A genuine answer carrying a deferral CLAUSE must survive. The naive rule refused this + # shape, and it was a REAL answer this gate had already sealed. + (_L_ANC_EN, "not run - the face arm is absent so no anchor could be formed; verdict-void"), + # Korean genuine answers must still seal + (_L_CAT_EN, _KO["NA_KO"]), + (_L_ANC_EN, _KO["REPRO"] + " 3x (seal 84007e65)"), + (_L_IND, _KO["CONVERGE"] + " over 2 seeds, code cited"), +] +# Raw-line specimens cover EXTRACTION too โ€” a `โ†` hint, and a CP949-damaged line. +_HINT = _KO["REPRO"] + " " + _KO["CONVERGE"] +_RAW_OK = ("- **%s**: yes โ† %s" % (_L_IND, _HINT)).encode("utf-8") +_PLANT_RAW = [ + (_RAW_OK, _L_IND, "TRIVIAL_VOCAB"), + (_RAW_OK.decode("cp949", "replace").encode("utf-8"), _L_IND, "DECODE_FAILED"), +] + + +def selftest(verbose=False): + """Run the planted specimens through this judge. Returns (passed, total, failures).""" + fails = [] + total = 0 + for label, ans, expect in _PLANT_VIOLATIONS: + total += 1 + code, _ = judge(label, ans) + if code != expect: + fails.append("violation not caught/misclassified: [%s] %r -> %s (expected %s)" + % (label, ans, code, expect)) + for label, ans in _PLANT_GENUINE: + total += 1 + code, _ = judge(label, ans) + if code != "OK": + fails.append("genuine answer wrongly refused: [%s] %r -> %s (expected OK)" + % (label, ans, code)) + for raw, label, expect in _PLANT_RAW: + total += 1 + code, ans = extract(raw) + if code == "OK": + code, _ = judge(label, ans) + if code != expect: + fails.append("raw line misclassified: %r -> %s (expected %s)" % (raw[:40], code, expect)) + if verbose: + for f in fails: + print(" x " + f) + return total - len(fails), total, fails + + +def main(argv): + if "--selftest" in argv: + ok, total, fails = selftest(verbose=True) + # ๐Ÿ”ด Print the DENOMINATOR. A checker that measured nothing also prints green. + print("substance_check selftest: %d/%d (violations %d - genuine %d - raw %d)" + % (ok, total, len(_PLANT_VIOLATIONS), len(_PLANT_GENUINE), len(_PLANT_RAW))) + return 0 if not fails else 9 + label = "" + if "--label" in argv: + label = argv[argv.index("--label") + 1] + code, ans = extract(sys.stdin.buffer.read()) + if code == "OK": + code, _ = judge(label, ans) + sys.stdout.write(code + "\t" + ans) + return 0 if code == "OK" else 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/mcp/tests/test_run_contract.py b/mcp/tests/test_run_contract.py new file mode 100644 index 0000000..6ab56cf --- /dev/null +++ b/mcp/tests/test_run_contract.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Contract tests for `server._run` โ€” the two field-report P0s, pinned. + +Why this file exists +-------------------- +Both defects fixed for the 2026-08-24 Windows/Codex field report live in `_run`, and NOTHING +exercised them: the `mcp` CI job installed the package and imported it, which proves the module +loads, not that the subprocess contract holds. A documented fix with no test is a promise. + +The two contracts: + + 1. **stdin is never inherited.** On an MCP STDIO server the parent's stdin IS the protocol + pipe. A child that inherits it steals protocol bytes and the tool hangs until timeout. + 2. **UTF-8 is pinned regardless of the ambient locale.** Under a non-UTF-8 default (CP949 on + the reporter's machine) the gate's `โ†` hint strip failed, so a trivial "yes" arrived long + enough to clear the substance checks. That is gate integrity, not display. + +Contract 2 is the *mechanism* that was reported on Windows, reproduced portably: we run the +child under a deliberately non-UTF-8 ambient locale and require the round-trip to survive. +Run this on Windows too โ€” the origin environment is the one the rest of CI does not cover. +""" +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from yeoul_mcp import server # noqa: E402 + +FAIL = [] + + +def check(desc, cond, detail=""): + print((" ok " if cond else " FAIL ") + desc + ((" -- " + detail) if detail and not cond else "")) + if not cond: + FAIL.append(desc) + + +def _install(tmp, name, body): + """Put a throwaway script where _run looks for it (server.BIN), and return its name.""" + p = Path(server.BIN) / name + p.write_text(body, encoding="utf-8") + return name, p + + +def test_stdin_not_inherited(tmp): + """The child must see EOF on stdin, not the parent's protocol bytes.""" + # A .py helper, not .sh: _run dispatches .py through sys.executable, so this test runs + # identically on Windows โ€” the very platform the field report came from. A bash helper + # would quietly skip there, and a skip that reads as a pass is how this hole got in. + name, p = _install(tmp, "_t_stdin.py", + 'import sys\nsys.stdout.write(sys.stdin.read())\nprint("CHILD_DONE")\n') + try: + # Drive it through a parent whose OWN stdin carries sentinel bytes. If _run leaks the + # parent's stdin to the child, `cat` echoes the sentinel โ€” that is the protocol theft. + driver = ( + "import sys, os; sys.path.insert(0, %r);\n" + "from yeoul_mcp import server\n" + "r = server._run(%r)\n" + "sys.stdout.write('RC=%%s|OUT=%%s' %% (r['exit_code'], r['stdout'].replace(chr(10), ' ')))\n" + % (str(Path(server.__file__).resolve().parents[1]), name) + ) + d = subprocess.run( + [sys.executable, "-c", driver], + input="SENTINEL_PROTOCOL_BYTES\n", capture_output=True, text=True, timeout=60, + ) + out = d.stdout + check("child does not inherit the parent's stdin", "SENTINEL_PROTOCOL_BYTES" not in out, out[:200]) + check("child still ran to completion", "CHILD_DONE" in out, out[:200]) + check("no hang / timeout", "RC=0" in out, out[:200]) + finally: + p.unlink(missing_ok=True) + + +def test_utf8_pinned_under_hostile_locale(tmp): + """Non-ASCII must survive even when the ambient locale is not UTF-8 (the CP949 mechanism).""" + # The gate strips everything after `โ†`; if the decode is wrong that marker survives and the + # answer arrives wearing the hint as its body. + name, p = _install(tmp, "_t_enc.py", + 'import sys\nsys.stdout.buffer.write(b"ANSWER \\xe2\\x86\\x90 HINT\\n")\n') + try: + # ๐Ÿ”ด `LC_ALL=C` ALONE IS NOT HOSTILE on modern Linux โ€” PEP 538/540 coerce the C + # locale back to UTF-8, so the default encoding stays UTF-8 and this test would + # pass with the fix REMOVED. Verified: it did. PYTHONCOERCECLOCALE=0 plus + # PYTHONUTF8=0 is what actually yields an ASCII default, which is the portable + # stand-in for the reporter's CP949 machine. + hostile = {k: v for k, v in os.environ.items() + if k not in ("PYTHONUTF8", "PYTHONIOENCODING")} + hostile.update({"LC_ALL": "C", "LANG": "C", + "PYTHONCOERCECLOCALE": "0", "PYTHONUTF8": "0"}) + d = subprocess.run( + [sys.executable, "-c", + "import sys; sys.path.insert(0, %r)\n" + "from yeoul_mcp import server\n" + "r = server._run(%r)\n" + "sys.stdout.buffer.write(r['stdout'].encode('utf-8'))\n" + % (str(Path(server.__file__).resolve().parents[1]), name)], + capture_output=True, env=hostile, timeout=60, + ) + got = d.stdout.decode("utf-8", "replace") + check("hostile-locale run succeeded", d.returncode == 0, d.stderr.decode("utf-8", "replace")[:200]) + check("the arrow round-trips under a non-UTF-8 ambient locale", "โ†" in got, repr(got)[:200]) + # ๐Ÿ”ด Guard this one against passing on EMPTY output: if the run died, "" contains no + # replacement chars and this check would go green for the wrong reason (observed). + check("no replacement characters (nothing was mis-decoded)", + bool(got.strip()) and "๏ฟฝ" not in got, repr(got)[:200]) + finally: + p.unlink(missing_ok=True) + + +def test_missing_script_is_not_a_pass(tmp): + """A script that isn't there must report 127, not a silent success.""" + r = server._run("_t_does_not_exist.py") + check("missing script reports 127", r["exit_code"] == 127, str(r)[:200]) + check("missing script is not reported as OK", r["exit_code"] != 0) + + +def main(): + print("mcp _run contract tests") + with tempfile.TemporaryDirectory() as tmp: + test_stdin_not_inherited(tmp) + test_utf8_pinned_under_hostile_locale(tmp) + test_missing_script_is_not_a_pass(tmp) + print("\n%s (%d failed)" % ("all _run contract tests passed" if not FAIL else "CONTRACT TESTS FAILED", len(FAIL))) + return 1 if FAIL else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/mcp/yeoul_mcp/server.py b/mcp/yeoul_mcp/server.py index 8ff7c0b..11e460a 100644 --- a/mcp/yeoul_mcp/server.py +++ b/mcp/yeoul_mcp/server.py @@ -13,6 +13,7 @@ import os import shlex import subprocess +import sys from pathlib import Path from mcp.server.fastmcp import FastMCP @@ -34,11 +35,20 @@ def _run(script: str, *args: str, cwd: str | None = None, stdin: str | None = No path = BIN / script if not path.exists(): return {"exit_code": 127, "stdout": "", "stderr": f"script not found: {path}"} - cmd = ["bash", str(path), *args] if not script.endswith(".py") else ["python3", str(path), *args] + interp = [os.environ.get("YEOUL_BASH", "bash")] if not script.endswith(".py") else [sys.executable] + cmd = [*interp, str(path), *args] + # ๐Ÿ”ด stdin: never inherit the parent's. On an MCP STDIO server the parent's stdin IS the + # protocol pipe, and a child that inherits it steals protocol bytes โ€” the tool then + # hangs until timeout (field report, Windows/Codex, 2026-08-24). + # ๐Ÿ”ด encoding: pin UTF-8. The gate strips a `โ†` hint before judging an answer; under a + # non-UTF-8 default (CP949) the strip fails and a trivial "yes" arrives long enough to + # clear the substance checks. That is a gate-integrity bug, not a display bug. + env = {**os.environ, "PYTHONUTF8": "1", "PYTHONIOENCODING": "utf-8"} try: p = subprocess.run( - cmd, cwd=cwd or os.getcwd(), input=stdin, - capture_output=True, text=True, timeout=120, + cmd, cwd=cwd or os.getcwd(), + input=stdin if stdin is not None else "", + capture_output=True, text=True, encoding="utf-8", env=env, timeout=120, ) return {"exit_code": p.returncode, "stdout": p.stdout, "stderr": p.stderr} except subprocess.TimeoutExpired: diff --git a/tests/test_gates.sh b/tests/test_gates.sh index 64dd114..122c662 100755 --- a/tests/test_gates.sh +++ b/tests/test_gates.sh @@ -165,6 +165,121 @@ printf '# close\n- **Closed**: 2026-01-01\n- **stop_reason**: converged\n- **Ver || { echo " โœ— index-append failed silently"; FAIL=1; } "$BIN/index-append" "$IDX_ARC" >/dev/null 2>&1; assert "index-append never blocks the close" 0 $? +# --- encoding damage must FAIL CLOSED (regression: 2026-08-24 Windows/CP949 field report) --- +# What happened: on Windows the UTF-8 bytes were decoded as CP949, so the `โ†` hint marker +# survived the strip. The answer "yes" then arrived as a 43-char string wearing the hint as +# its body โ€” it matched no entry in the trivial list AND cleared the >=6 length check, so the +# arc SEALED. The block above already keeps a `โ†` hint to exercise extraction, but it runs in +# a UTF-8 locale, so it could never fire. We inject the damage directly: this now fails on any +# platform, not only on the one where it was found. +"$BIN/arc-open" enc --topic="encoding gate" --arcs-dir="$WS/arcs" >/dev/null 2>&1 +EARC="$(ls -d "$WS"/arcs/*_enc)" +"$BIN/arc-close" "$EARC" "KILL โ€” enc" --stop=falsified >/dev/null 2>&1 +ESUM="$(ls "$EARC"/_SUMMARY_*.md)" +sedi 's/- (fill in)/- concrete conclusion here/' "$ESUM" +sedi 's/(unfilled)/yes/g' "$ESUM" +python3 - "$ESUM" <<'PYDAMAGE' +import sys +p = sys.argv[1] +out = [] +for line in open(p, encoding="utf-8"): + # only the answer lines get mis-decoded; ASCII structure survives CP949 either way + if line.startswith("- **"): + line = line.encode("utf-8").decode("cp949", "replace") + out.append(line) +open(p, "w", encoding="utf-8").write("".join(out)) +PYDAMAGE +# the damage must actually have landed โ€” otherwise every check below passes vacuously +grep -q "$(printf '\357\277\275')" "$ESUM" \ + && echo " โœ“ damage injected (replacement chars present)" \ + || { echo " โœ— damage did NOT land โ€” the checks below would pass for the wrong reason"; FAIL=1; } +# run the gate ONCE and judge both the code and the reason from the same run +ENCOUT="$("$BIN/arc-close" "$EARC" "KILL โ€” enc" --stop=falsified 2>&1)"; ENCRC=$? +assert "encoding damage fails closed (never seals)" 5 "$ENCRC" +ls -d "$WS/arcs/_archive"/*_enc >/dev/null 2>&1 \ + && { echo " โœ— SEALED despite unreadable answers"; FAIL=1; } \ + || echo " โœ“ damaged arc not archived" +# and the refusal must say WHY โ€” "could not judge" is not "refused on the merits" +case "$ENCOUT" in + *"not readable as UTF-8"*) echo " โœ“ refusal names the encoding damage" ;; + *) echo " โœ— refused for another reason: $(printf '%s' "$ENCOUT" | tail -1)"; FAIL=1 ;; +esac + +# --- substance check: the CLASS, not just the CP949 case (2026-08-24) --- +# The block above pinned one *case* (encoding damage). The class is wider: the substance check +# was `[ ${#vans} -ge 6 ]`, so ANY >=6-char non-answer sealed. Measured before the repair: +# 28 of 28 evasive answers sealed. These assertions pin the class in BOTH directions โ€” a +# repair that only tightened would pass the top half and quietly reject real answers. +SUBSTANCE="$BIN/substance_check.py" +python3 "$SUBSTANCE" --selftest >/dev/null 2>&1; assert "substance checker passes its own positive control" 0 $? +echo " $(python3 "$SUBSTANCE" --selftest 2>&1 | tail -1)" # ๐Ÿ”ด print the denominator, not just green + +subst_case() { # subst_case + "$BIN/arc-open" sc --topic="substance" --arcs-dir="$WS/arcs" >/dev/null 2>&1 + local A; A="$(ls -d "$WS"/arcs/*_sc)" + "$BIN/arc-close" "$A" "KILL โ€” sc" --stop=falsified >/dev/null 2>&1 + local S; S="$(ls "$A"/_SUMMARY_*.md)" + sedi 's/- (fill in)/- concrete conclusion here/' "$S" + # anchor/catalog have their own branches; drive the general branch with the candidate + python3 - "$S" "$2" <<'PYFILL' +import sys, re +p, ans = sys.argv[1], sys.argv[2] +out = [] +for line in open(p, encoding="utf-8"): + if "(unfilled)" in line and line.startswith("- **"): + label = re.match(r"^- \*\*([^*]+)\*\*", line).group(1) + hint = " โ†" + line.split("โ†", 1)[1].rstrip("\n") if "โ†" in line else "" + if "Anchor" in label: body = "anchor reproduced 3x (seal 84007e65)" + elif "Catalog" in label: body = "vacuous_pass" + else: body = ans + out.append(f"- **{label}**: {body}{hint}\n"); continue + out.append(line) +open(p, "w", encoding="utf-8").write("".join(out)) +PYFILL + "$BIN/arc-close" "$A" "KILL โ€” sc" --stop=falsified >/dev/null 2>&1 + assert "$1" "$3" $? + rm -rf "$A" "$WS/arcs/_archive/$(basename "$A")" 2>/dev/null +} +# must be REFUSED (5) โ€” each cleared the old >=6-char bar and sealed +subst_case "evasive 'aaaaaa' refused" 'aaaaaa' 5 +subst_case "evasive 'yes yes' refused" 'yes yes' 5 +subst_case "evasive 'yes ok' refused" 'yes ok' 5 +subst_case "evasive 'qwerty' refused" 'qwerty' 5 +subst_case "evasive '......' refused" '......' 5 +subst_case "deferral 'TODO later' refused" 'TODO later' 5 +# must still SEAL (0) โ€” the counter-accident: raising a length bar would reject these +subst_case "short genuine answer still seals" 'd=0.05 < 0.2, under the sealed bar' 0 +subst_case "genuine answer carrying a deferral CLAUSE still seals" \ + 'not run โ€” the face arm is absent so no anchor could be formed; recorded as verdict-void' 0 + +# --- the positive control must be LOAD-BEARING, not decorative --- +# A documented fallback is untested code. Sabotage the checker so it can never say no, then +# assert the gate REFUSES TO INTERPRET (exit 6) instead of sealing an evasive answer. +SBIN="$(mktemp -d)"; cp "$BIN"/* "$SBIN/" 2>/dev/null +python3 - "$SBIN/substance_check.py" <<'PYSAB' +import sys +p = sys.argv[1]; s = open(p, encoding="utf-8").read() +i = s.index("def judge(label, ans):") +s = s[:i] + 'def judge(label, ans):\n return "OK", ans\n\ndef _judge_disabled(label, ans):\n' + s[i + len("def judge(label, ans):\n"):] +open(p, "w", encoding="utf-8").write(s) +PYSAB +# the sabotage must actually have landed, or every assertion below passes vacuously +python3 "$SBIN/substance_check.py" --selftest >/dev/null 2>&1 \ + && { echo " โœ— sabotage did NOT land โ€” the checks below would pass for the wrong reason"; FAIL=1; } \ + || echo " โœ“ sabotage landed (checker can no longer fail a planted violation)" +"$SBIN/arc-open" sb --topic="sabotage" --arcs-dir="$WS/arcs" >/dev/null 2>&1 +SBARC="$(ls -d "$WS"/arcs/*_sb)" +"$SBIN/arc-close" "$SBARC" "KILL โ€” sb" --stop=falsified >/dev/null 2>&1 +SBSUM="$(ls "$SBARC"/_SUMMARY_*.md)" +sedi 's/- (fill in)/- concrete conclusion here/' "$SBSUM" +sedi 's/(unfilled)/yes ok/g' "$SBSUM" +"$SBIN/arc-close" "$SBARC" "KILL โ€” sb" --stop=falsified >/dev/null 2>&1 +assert "broken checker => gate refuses to interpret (does not seal)" 6 $? +ls -d "$WS/arcs/_archive"/*_sb >/dev/null 2>&1 \ + && { echo " โœ— SEALED while the checker was broken"; FAIL=1; } \ + || echo " โœ“ nothing archived while the instrument was broken" +rm -rf "$SBIN" + echo if [ "$FAIL" -eq 0 ]; then echo "โœ… all gate tests passed"; else echo "โ›” gate tests FAILED"; fi exit "$FAIL"