From 2526b86a74ffd6c74a0ad8a57a7fe02942b0ee81 Mon Sep 17 00:00:00 2001 From: Swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Sun, 27 Sep 2026 23:02:12 -0700 Subject: [PATCH] engine: generate the Souffle program once and hash that file for the engine id (#1593) The engine id was computed from a second rendering of the program, made through printf pipelines whose writes were never checked. Under load a bash 3.2 pipe write interrupted by a signal lost a line with exit 0, the id changed, and a warm rebuild became a full C++ compile. The program is now assembled in the shell with no pipe, written once with a checked write, read back, verified against an independent derivation from the maps, and renamed into place. The id hashes that exact file and its includes (same definition, so every id is unchanged). A compiled engine is verified against the id before the atomic rename into the cache. --- .github/RELEASING.md | 5 +- graph/pipeline/run-souffle.sh | 278 +++++++++++++++---- graph/test/java/run-tests.sh | 7 + graph/test/tools/engine-id-lost-line-test.sh | 65 +++++ 4 files changed, 298 insertions(+), 57 deletions(-) create mode 100644 graph/test/tools/engine-id-lost-line-test.sh diff --git a/.github/RELEASING.md b/.github/RELEASING.md index da8e1b5e1..67f7f6025 100644 --- a/.github/RELEASING.md +++ b/.github/RELEASING.md @@ -66,4 +66,7 @@ from Actions at any time (it publishes only if you tick `publish`). Compiled engines are cached by ENGINE_ID, the hash of a language's rules and the Soufflé version, so a change that touches no rules reuses every engine and a rule change recompiles only its own -language. The nightly and every real publish build fresh. +language. Precisely: sha256 over the pinned Soufflé version, the bytes of the program file that is +compiled, and every file it includes, in include order. The program is generated once, verified and +then hashed, so the id is path-independent (the same from any checkout) and rule-sensitive (any rule, +map or manifest change moves it), and it never comes from a second rendering of the program (#1593). The nightly and every real publish build fresh. diff --git a/graph/pipeline/run-souffle.sh b/graph/pipeline/run-souffle.sh index a7655b4e8..5cf84aacf 100755 --- a/graph/pipeline/run-souffle.sh +++ b/graph/pipeline/run-souffle.sh @@ -112,23 +112,68 @@ done # read an import map (relationcsv-basename per line), skipping comments (#) and blanks read_map(){ grep -vE '^[[:space:]]*(#|$)' "$1"; } -# ── THE PROGRAM, as a pure function of the repository ──────────────────────────────────── +# ── THE PROGRAM, generated ONCE; the engine id is the hash of exactly those bytes ───────── # Written so that the SAME text comes out of every checkout and of CI: includes are relative # to graph/ (souffle resolves them through -I "$SRC"), and the .input list is derived from the -# maps rather than from a listing of the staged facts dir — so it needs no client IR, and a +# maps rather than from a listing of the staged facts dir, so it needs no client IR, and a # machine that cannot stage (CI) still produces the text the binary was built from. The run # path asserts below that staging created a facts file for every .input it declares. -# The list of input relations is: every client relation, the lib signature relations, the lib -# body relations (filled per iteration), and the four knob facts. -input_relations(){ - while IFS=$'\t' read -r rel csv; do printf '%s\n' "$rel"; done < <(read_map "$TPL/client-ir.map") - while IFS=$'\t' read -r rel csv; do - case " $LIB_SIG " in *" ${rel#lib_} "*) printf '%s\n' "$rel";; esac - done < <(read_map "$TPL/lib.map") - for r in $LIB_BODY; do printf '%s\n' "$r"; done - printf '%s\n' jdk_max_depth lib_max_depth taint_gating dispatch_cap +# The input relations are: every client relation, the lib signature relations, the lib body +# relations (filled per iteration), and the four knob facts. +# +# NO PIPES, AND EVERY WRITE CHECKED (#1593). The text used to be produced by `printf` loops +# feeding `sort` through pipes, and produced TWICE: once into the program souffle compiles, +# once more inside engine_id() just to hash it. bash 3.2 (macOS /bin/bash) does not restart a +# pipe write interrupted by a signal, and nothing checked the status of a write inside a +# pipeline, so under load a line was sometimes lost with exit 0 ("printf: write error: +# Interrupted system call"). Lost from the hashed copy, it gave a different id and a warm +# rebuild became a full C++ compile (measured: 2183 s instead of 11 s, 3 in ~460 builds). +# Lost from the compiled copy, it would have cached a program missing an .input under the +# correct id. So now: +# 1. the text is assembled in memory, in this shell (string appends, globs and `read` from +# the map files; no subprocess, no pipe), and sorted here too; +# 2. it is written to a temp file with ONE checked write, read back and compared byte for +# byte, and checked against an independent derivation from the maps (verify_program); +# only then renamed onto the destination, so no reader ever sees a partial program; +# 3. the engine id (engine_id_of) hashes THAT FILE, the one souffle compiles, never a +# second rendering of it. +# A failed attempt is retried (program_file); three failures abort the run loudly rather +# than continue with a partial program. + +# sort the array _SU in place and drop duplicates. Byte order: every caller pins LC_ALL=C. +# In-shell (an insertion sort over ~100 names) so that no line passes through a pipe. +sort_unique_su(){ + local i j x n=${#_SU[@]} out=() + for ((i=1; i "$x" ]]; do _SU[j+1]="${_SU[j]}"; j=$((j-1)); done + _SU[j+1]="$x" + done + for ((i=0; i&2; return 1; } + while IFS= read -r line || [ -n "$line" ]; do + rel="${line#"${line%%[![:space:]]*}"}" + case "$rel" in ""|"#"*) continue;; esac + line="${line#"${line%%[!$'\t']*}"}"; rel="${line%%$'\t'*}" + if [ "${2:-}" = sig ]; then case " $LIB_SIG " in *" ${rel#lib_} "*) ;; *) continue;; esac; fi + _SU+=("$rel") + done < "$1" } -write_program(){ # $1 = destination file +# write_program DEST: ONE attempt. Builds the text in memory, writes it to DEST.tmp.$$ with a +# single checked write, verifies, then renames onto DEST. Returns 1 (DEST untouched) on any +# failure; program_file retries. +write_program(){ # COLLATION IS PART OF THE PROGRAM TEXT, so it is pinned here rather than inherited. The # #include lines below come from shell globs, and bash orders a glob by LC_COLLATE, not by # byte value. A UTF-8 collation ignores punctuation when comparing, so call-site.dl and @@ -140,54 +185,157 @@ write_program(){ # $1 = destination file # so the mismatch is the common case. Measured: java and python ids differ between macOS # and MSYS2, and between LC_ALL=C and en_US.UTF-8 on glibc; typescript and javascript # agree only because no pair of their filenames collides. Invisible on macOS, whose - # collation matches C either way, which is why it survived. The sorts below were already - # forced to C for this reason; the globs were not. + # collation matches C either way, which is why it survived. The same pin orders the + # in-shell sorts below. # See issue #895 and graph/test/tools/engine-id-locale-test.sh. local LC_ALL=C LC_COLLATE=C - { - echo "#include \"$LANG_ARG/souffle/decls_base.dl\""; echo "#include \"$LANG_ARG/souffle/decls_all.dl\"" - # rfc4180=true: the IR is CSV, not TSV. The parser quotes any field containing a - # quote, tab or newline and doubles the inner quotes, so reading it as plain TSV hands - # the rules the ESCAPED text. Souffle parses RFC4180 itself, so this costs one flag - # rather than a re-encode of GB-scale input. - # LC_ALL=C sort: the order is part of the program text, so it must not depend on locale. - input_relations | LC_ALL=C sort -u | while read -r r; do printf '.input %s(IO=file, filename="%s.facts", delimiter="\\t", rfc4180=true)\n' "$r" "$r"; done - for d in projections containment resolution config-resolution expression-resolution call-edge-generation framework-behavior; do - # [ -f ] guard: a phase directory that is empty (or absent for a language that has - # not implemented that layer yet) leaves the glob unexpanded, and souffle's C - # preprocessor then fails on a literal '*.dl' include. - for f in "$ENG/$d/"*.dl; do [ -f "$f" ] && echo "#include \"${f#"$SRC/"}\""; done - done - # engine-ii: the first→third forward-chain engine (mirrors engine/, lib-seeded). Same solve, - # included AFTER engine/ so it reads engine/'s relations (client_calls_lib seed). Glob its - # phase subfolders (both nesting levels; globs are space-safe, the repo path has spaces). - # export/ is doc-only (like engine/export) — skip it. - if [ "$ENGINE_II_MODE" = "on" ]; then + local dest="$1" tmp="$1.tmp.$$" nl=$'\n' t="" r f d line pred file got + _SU=() + t+="#include \"$LANG_ARG/souffle/decls_base.dl\"$nl#include \"$LANG_ARG/souffle/decls_all.dl\"$nl" + map_rels_su "$TPL/client-ir.map" || return 1 + map_rels_su "$TPL/lib.map" sig || return 1 + for r in $LIB_BODY; do _SU+=("$r"); done + _SU+=(jdk_max_depth lib_max_depth taint_gating dispatch_cap) + sort_unique_su + PROGRAM_INPUTS=(${_SU[@]+"${_SU[@]}"}) + # rfc4180=true: the IR is CSV, not TSV. The parser quotes any field containing a + # quote, tab or newline and doubles the inner quotes, so reading it as plain TSV hands + # the rules the ESCAPED text. Souffle parses RFC4180 itself, so this costs one flag + # rather than a re-encode of GB-scale input. + for r in ${_SU[@]+"${_SU[@]}"}; do + t+=".input $r(IO=file, filename=\"$r.facts\", delimiter=\"\\t\", rfc4180=true)$nl" + done + for d in projections containment resolution config-resolution expression-resolution call-edge-generation framework-behavior; do + # [ -f ] guard: a phase directory that is empty (or absent for a language that has + # not implemented that layer yet) leaves the glob unexpanded, and souffle's C + # preprocessor then fails on a literal '*.dl' include. + for f in "$ENG/$d/"*.dl; do [ -f "$f" ] && t+="#include \"${f#"$SRC/"}\"$nl"; done + done + # engine-ii: the first→third forward-chain engine (mirrors engine/, lib-seeded). Same solve, + # included AFTER engine/ so it reads engine/'s relations (client_calls_lib seed). Glob its + # phase subfolders (both nesting levels; globs are space-safe, the repo path has spaces). + # export/ is doc-only (like engine/export) — skip it. + if [ "$ENGINE_II_MODE" = "on" ]; then for f in "$ENG2/"*/*.dl "$ENG2/"*/*/*.dl; do case "$f" in */export/*) continue;; esac - [ -f "$f" ] && echo "#include \"${f#"$SRC/"}\"" + [ -f "$f" ] && t+="#include \"${f#"$SRC/"}\"$nl" done - fi - # Relative output filenames — the -D at run time supplies the directory. Keeping $OUT out - # of the program makes the compiled binary independent of the output path (better reuse). - while IFS=$'\t' read -r pred file; do [ -n "$pred" ] && printf '.output %s(IO=file, filename="%s", delimiter="\\t")\n' "$pred" "$file"; done < <(LC_ALL=C sort -u "$DL/export_manifest.tsv") - } > "$1" + fi + # Relative output filenames — the -D at run time supplies the directory. Keeping $OUT out + # of the program makes the compiled binary independent of the output path (better reuse). + # Whole manifest lines, sorted and deduplicated, then split as `IFS=$'\t' read pred file`. + _SU=() + [ -r "$DL/export_manifest.tsv" ] || { echo " ! cannot read $DL/export_manifest.tsv" >&2; return 1; } + while IFS= read -r line || [ -n "$line" ]; do _SU+=("$line"); done < "$DL/export_manifest.tsv" + sort_unique_su + for line in ${_SU[@]+"${_SU[@]}"}; do + line="${line#"${line%%[!$'\t']*}"}"; pred="${line%%$'\t'*}"; file="" + case "$line" in *$'\t'*) file="${line#*$'\t'}"; file="${file#"${file%%[!$'\t']*}"}"; file="${file%"${file##*[!$'\t']}"}";; esac + [ -n "$pred" ] && t+=".output $pred(IO=file, filename=\"$file\", delimiter=\"\\t\")$nl" + done + # ONE write, checked; read back and compared; independently verified; then renamed. + rm -f "$tmp" + printf '%s' "$t" > "$tmp" || { echo " ! writing the program to $tmp failed" >&2; rm -f "$tmp"; return 1; } + got=""; IFS= read -r -d '' got < "$tmp" || true + [ "$got" = "$t" ] || { echo " ! short write: $tmp holds ${#got} of ${#t} bytes" >&2; rm -f "$tmp"; return 1; } + verify_program "$tmp" || { rm -f "$tmp"; return 1; } + mv -f "$tmp" "$dest" || { echo " ! could not rename $tmp onto $dest" >&2; rm -f "$tmp"; return 1; } +} +# verify_program FILE: an INDEPENDENT derivation of what the program must declare, read by awk +# straight from the maps and the manifest (not from anything write_program produced), so a line +# lost on either side shows up as a difference. Every .input and .output line must be exactly +# the expected one, each exactly once, with nothing extra. Exit status only: no pipe to lose. +verify_program(){ + awk -v cmap="$TPL/client-ir.map" -v lmap="$TPL/lib.map" -v man="$DL/export_manifest.tsv" \ + -v libsig=" $LIB_SIG " -v extra="$LIB_BODY jdk_max_depth lib_max_depth taint_gating dispatch_cap" ' + function want(line) { if (!(line in need)) { need[line] = 1; n++ } } + function inp(r) { want(".input " r "(IO=file, filename=\"" r ".facts\", delimiter=\"\\t\", rfc4180=true)") } + function maprels(path, sigonly, l, rc, r, k) { + while ((rc = (getline l < path)) > 0) { + if (l ~ /^[[:space:]]*(#|$)/) continue + sub(/^\t+/, "", l); r = l; k = index(r, "\t"); if (k) r = substr(r, 1, k - 1) + if (sigonly) { s = r; sub(/^lib_/, "", s); if (!index(libsig, " " s " ")) continue } + inp(r) + } + if (rc < 0) { print " ! verify: cannot read " path > "/dev/stderr"; bad = 1 } + close(path) + } + BEGIN { + maprels(cmap, 0); maprels(lmap, 1) + m = split(extra, e, " "); for (i = 1; i <= m; i++) if (e[i] != "") inp(e[i]) + while ((rc = (getline l < man)) > 0) { + sub(/^\t+/, "", l); k = index(l, "\t"); p = l; f = "" + if (k) { p = substr(l, 1, k - 1); f = substr(l, k + 1); sub(/^\t+/, "", f); sub(/\t+$/, "", f) } + if (p != "") want(".output " p "(IO=file, filename=\"" f "\", delimiter=\"\\t\")") + } + if (rc < 0) { print " ! verify: cannot read " man > "/dev/stderr"; bad = 1 } + close(man) + } + /^\.(input|output) / { + if (!($0 in need)) { print " ! verify: unexpected line in the program: " $0 > "/dev/stderr"; bad = 1 } + else if (seen[$0]++) { print " ! verify: duplicated line in the program: " $0 > "/dev/stderr"; bad = 1 } + } + END { + for (x in need) if (!(x in seen)) { print " ! verify: the program lacks: " x > "/dev/stderr"; bad = 1 } + if (n < 1) { print " ! verify: nothing expected (empty maps?)" > "/dev/stderr"; bad = 1 } + exit bad + }' "$1" } -# The engine id: sha256 over the pinned code-generator version, the program text, and every -# file it includes, in include order. A function of the repository alone — the same from any -# path, on any machine, with or without souffle — and different for any rule change. It names -# the local cache entry AND the engine package CI publishes, which is what lets a machine -# without souffle know which binary is its own. -engine_id(){ - local prog; prog="$(mktemp)"; write_program "$prog" - { printf 'souffle=%s\n' "$SOUFFLE_VERSION"; cat "$prog" - sed -n 's/^#include "\(.*\)"$/\1/p' "$prog" | while read -r inc; do cat "$SRC/$inc"; done - } | sha256_stdin - rm -f "$prog" +# program_file DEST: write_program, retried. Three failed attempts abort with a loud error; +# the caller never continues with a partial or unverified program. +program_file(){ + local try + for try in 1 2 3; do + write_program "$1" && return 0 + echo " ! generating the Soufflé program failed (attempt $try of 3)" >&2 + done + echo "❌ could not write the Soufflé program intact after 3 attempts; refusing to continue" >&2 + return 1 +} +# The engine id: sha256 over the pinned code-generator version, the program FILE's bytes, and +# every file it includes, in include order. A function of the repository alone: the same from +# any path, on any machine, with or without souffle, and different for any rule change. It +# names the local cache entry AND the engine package CI publishes, which is what lets a +# machine without souffle know which binary is its own. +# engine_id_of PROGRAM sets ENGINE_ID. It hashes the file it is given (the one souffle +# compiles), never a regenerated copy. The hash input is assembled in a temp file with +# checked writes and its size checked, the digest is read from a file argument (no pipe into +# the hasher), and the result must be 64 hex digits, or it returns 1 and ENGINE_ID is empty. +engine_id_of(){ + local prog="$1" hin line inc incs=() want have h + ENGINE_ID="" + [ -n "$_SHA256_CMD" ] || { echo "❌ neither sha256sum nor shasum is on PATH" >&2; return 1; } + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in '#include "'*'"') inc="${line#'#include "'}"; incs+=("$SRC/${inc%'"'}");; esac + done < "$prog" + hin="$(mktemp "${TMPDIR:-/tmp}/axiom-engine-id.XXXXXX")" && [ -f "$hin" ] \ + || { echo "❌ engine id: mktemp failed" >&2; return 1; } + if ! printf 'souffle=%s\n' "$SOUFFLE_VERSION" > "$hin" || ! cat "$prog" ${incs[@]+"${incs[@]}"} >> "$hin"; then + echo "❌ engine id: writing the hash input failed" >&2; rm -f "$hin"; return 1 + fi + # The expected size comes from the SOURCE files (wc's last line is their total), not from a + # second read through cat, so a cat that loses bytes cannot agree with itself. + want="$(wc -c "$prog" ${incs[@]+"${incs[@]}"})"; have="$(wc -c < "$hin")" + want="${want##*$'\n'}"; want="${want#"${want%%[![:space:]]*}"}"; want="${want%% *}" + case "$want" in ""|*[!0-9]*) want=-1;; *) want=$(( want + ${#SOUFFLE_VERSION} + 9 ));; esac # 9 = "souffle=" + "\n" + if [ "${have//[[:space:]]/}" != "$want" ]; then + echo "❌ engine id: the hash input is ${have//[[:space:]]/} bytes, expected $want (short write)" >&2; rm -f "$hin"; return 1 + fi + h="$($_SHA256_CMD "$hin")"; rm -f "$hin" + h="${h%% *}"; h="${h#\\}" + if [ "${#h}" -ne 64 ] || case "$h" in *[!0-9a-f]*) true;; *) false;; esac; then + echo "❌ engine id: the digest is not 64 hex digits (got '$h')" >&2; return 1 + fi + ENGINE_ID="$h" } case "$MODE" in - print-engine-id) engine_id; exit 0;; - emit-program) write_program "$EMIT"; exit 0;; + print-engine-id) + _pd="$(mktemp -d "${TMPDIR:-/tmp}/axiom-program.XXXXXX")" && [ -d "$_pd" ] || { echo "❌ mktemp failed" >&2; exit 1; } + if program_file "$_pd/program.dl" && { engine_id_of "$_pd/program.dl" || engine_id_of "$_pd/program.dl" || engine_id_of "$_pd/program.dl"; }; then rm -rf "$_pd" + else rm -rf "$_pd"; exit 1; fi + printf '%s\n' "$ENGINE_ID" || exit 1 + exit 0;; + emit-program) program_file "$EMIT" || exit 1; exit 0;; esac [ -n "${CLIENT:-}" ] && [ -n "${INT:-}" ] && [ -n "${OUT:-}" ] || { echo "usage: run-souffle.sh --client-ir DIR --library DIR --intermediate DIR --output DIR [--language L]" >&2; exit 1; } @@ -361,15 +509,19 @@ echo "▶ dispatch cap = $( [ -s "$FACTS/dispatch_cap.facts" ] && echo "$(cat "$ echo "▶ engine-ii = $( [ "$ENGINE_II_MODE" = "on" ] && echo 'ON (lib frontier included)' || echo 'OFF (client-only — engine-i)' )" # --- the program, and the binary for it: from npm, or compiled here --- +# Generated ONCE (program_file), and the id is the hash of that file (engine_id_of): the id is +# a function of the bytes souffle compiles, not of a second rendering of them. PROG="$INT/souffle-program.dl" -write_program "$PROG" +program_file "$PROG" || exit 1 # Every declared input must have been staged, or souffle would fail on a missing file after # the (possibly long) library staging. The program lists inputs from the maps; staging # created them from the same maps, so a mismatch is a bug in this script, and says so. -for r in $(sed -n 's/^\.input \([A-Za-z0-9_]*\)(.*/\1/p' "$PROG"); do +[ "${#PROGRAM_INPUTS[@]}" -gt 0 ] || { echo "❌ the program declares no inputs" >&2; exit 1; } +for r in "${PROGRAM_INPUTS[@]}"; do [ -f "$FACTS/$r.facts" ] || { echo "❌ program declares input $r but staging created no $r.facts" >&2; exit 1; } done -ENGINE_ID="$(engine_id)" +engine_id_of "$PROG" || engine_id_of "$PROG" || engine_id_of "$PROG" \ + || { echo "❌ could not compute the engine id of $PROG; refusing to guess a cache entry" >&2; exit 1; } echo "▶ engine id = $ENGINE_ID (rules + souffle $SOUFFLE_VERSION)" # What we cache is OUR engine compiled to a native binary (souffle -g turns the .dl rules @@ -451,9 +603,23 @@ elif command -v souffle >/dev/null 2>&1; then cat "$INT/.souffle-gen.log" >&2; exit 1 fi awk '/No rules\/facts defined/{skip=2;next} skip>0{skip--;next} {print}' "$INT/.souffle-gen.log" >&2 + [ -s "$INT/souffle-program.cpp" ] || { echo "❌ souffle wrote no C++ for $PROG" >&2; exit 1; } CXX_PLATFORM="" case "$(uname -s)" in CYGWIN*) CXX_PLATFORM="-Wa,-mbig-obj";; esac - c++ -std=c++17 -O3 -march=native -w $CXX_PLATFORM -I "$INNER" "$INT/souffle-program.cpp" -o "$BIN.tmp.$$" + if ! c++ -std=c++17 -O3 -march=native -w $CXX_PLATFORM -I "$INNER" "$INT/souffle-program.cpp" -o "$BIN.tmp.$$"; then + rm -f "$BIN.tmp.$$"; echo "❌ compiling the engine failed" >&2; exit 1 + fi + # VERIFY, THEN PUBLISH. The cache entry is trusted by name alone from now on, so nothing may + # land under $ENGINE_ID unless it is a whole binary built from the program that id names: + # the temp binary must be a non-empty executable, and the program must still hash to the id + # (a program or rule file that changed during the compile would otherwise be cached under + # the old id). Only then the atomic rename. + _built_id="$ENGINE_ID" + if [ ! -s "$BIN.tmp.$$" ] || [ ! -x "$BIN.tmp.$$" ] || ! engine_id_of "$PROG" || [ "$ENGINE_ID" != "$_built_id" ]; then + rm -f "$BIN.tmp.$$" + echo "❌ the compiled engine did not verify (program now hashes to ${ENGINE_ID:-nothing}, built as $_built_id); not caching it" >&2 + exit 1 + fi mv -f "$BIN.tmp.$$" "$BIN" else echo "❌ no engine for $LANG_ARG@${ENGINE_ID:0:12}… on this machine. Either:" >&2 diff --git a/graph/test/java/run-tests.sh b/graph/test/java/run-tests.sh index 4a2c433dd..53541cb2a 100755 --- a/graph/test/java/run-tests.sh +++ b/graph/test/java/run-tests.sh @@ -139,6 +139,13 @@ if ! bash "$ROOT/graph/test/tools/engine-id-test.sh"; then echo "aborting: the engine id is not a function of the rules alone" exit 1 fi +# A line lost while the program is generated or hashed (an interrupted pipe write under load) +# must be refused or harmless, never a different id with exit 0: that id misses the cached +# binary and a warm rebuild becomes a full compile. +if ! bash "$ROOT/graph/test/tools/engine-id-lost-line-test.sh"; then + echo "aborting: a lost line changes the engine id silently" + exit 1 +fi if ! bash "$ROOT/graph/test/tools/engine-package-test.sh"; then echo "aborting: the packaged-engine path does not check what it runs" exit 1 diff --git a/graph/test/tools/engine-id-lost-line-test.sh b/graph/test/tools/engine-id-lost-line-test.sh new file mode 100644 index 000000000..b1d62db97 --- /dev/null +++ b/graph/test/tools/engine-id-lost-line-test.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# ───────────────────────────────────────────────────────────────────────────── +# A line lost while the program is generated or hashed must never yield a DIFFERENT engine id +# with exit 0. That id names the cached binary, so a wrong one turns a warm rebuild into a full +# C++ compile (measured: 2183 s instead of 11 s under load), and a program missing a line under +# the RIGHT id would cache a wrong engine. The loss was a bash 3.2 pipe write interrupted by a +# signal ("printf: write error: Interrupted system call"), unchecked inside a pipeline (#1593). +# +# Each case puts one tool on PATH that silently drops a line of its output (exit 0), the way +# the interrupted write did, and asks for the id. The contract: the id is unchanged, or the +# run fails loudly. Controls: the shims are exercised (a shim that is never called would make +# every case pass on nothing), and the intact id is a digest. +# ───────────────────────────────────────────────────────────────────────────── +set -u +ROOT="$(d="$(cd "$(dirname "$0")" && pwd)"; while [ "$d" != / ] && { [ ! -f "$d/package.json" ] || [ ! -d "$d/graph" ]; }; do d="$(dirname "$d")"; done; echo "$d")" +RUN="$ROOT/graph/pipeline/run-souffle.sh" +fail=0; checks=0 +bad(){ echo " ✗ $*"; fail=$((fail+1)); } +ok(){ checks=$((checks+1)); } +W="$(mktemp -d)"; trap 'rm -rf "$W"' EXIT + +# shim NAME AWK-FILTER: a NAME on PATH that runs the real one and filters its output through +# the awk program, logging each call so the test can prove the shim was reached. +shim(){ + local name="$1" filter="$2" real; real="$(command -v "$name")" + mkdir -p "$W/$name" + cat > "$W/$name/$name" <> "$W/$name.calls" +"$real" "\$@" | awk '$filter' +exit 0 +SH + chmod +x "$W/$name/$name" +} +# sort: drop one relation name from any list it sorts (the observed loss, simulated) +shim sort '$0 != "dispatch_cap" && $0 != "jdk_max_depth"' +# cat: drop the last line of everything it prints +shim cat '{ if (NR > 1) print prev; prev = $0 }' + +for lang in python java; do + good="$(bash "$RUN" --language "$lang" --print-engine-id 2>/dev/null)"; rc=$? + case "$good" in [0-9a-f]*) [ "${#good}" -eq 64 ] && [ "$rc" -eq 0 ] && ok || bad "$lang: intact id is not a 64-digit digest ('$good', exit $rc)";; + *) bad "$lang: intact id is not a digest ('$good', exit $rc)";; esac + for t in sort cat; do + : > "$W/$t.calls" + got="$(PATH="$W/$t:$PATH" bash "$RUN" --language "$lang" --print-engine-id 2>"$W/err")"; rc=$? + if [ "$rc" -eq 0 ] && [ "$got" != "$good" ]; then + bad "$lang: a $t that loses a line gave a different id with exit 0 ($got vs $good)" + else ok; fi + # the program must come out intact too, or not at all + rm -f "$W/p.dl" + PATH="$W/$t:$PATH" bash "$RUN" --language "$lang" --emit-program "$W/p.dl" 2>/dev/null; rc=$? + bash "$RUN" --language "$lang" --emit-program "$W/ref.dl" 2>/dev/null + if [ "$rc" -eq 0 ] && ! cmp -s "$W/p.dl" "$W/ref.dl"; then + bad "$lang: a $t that loses a line emitted a different program with exit 0" + else ok; fi + done +done +# control: the cat shim must actually have been on the id path (the sort shim no longer is, +# which is the fix: the program is generated without a pipe through sort) +[ -s "$W/cat.calls" ] && ok || bad "control: the cat shim was never called; the cat cases measured nothing" + +[ "$checks" -ge 1 ] || { echo " ✗ this test asserted nothing"; fail=$((fail+1)); } +if [ "$fail" -eq 0 ]; then echo "engine-id-lost-line: ok ($checks checks: a lost line is refused or harmless)" +else echo "engine-id-lost-line: $fail failure(s)"; exit 1; fi