diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2394eac6..a0bf9e35 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -395,6 +395,21 @@ jobs: - name: JIT vs interpreter differential run: bash tools/jit_diff.sh + # #1665 follow-up: the JIT-tier sections against `make jit-checked`, whose + # emitter calls a slot witness before every direct env-slot access and + # aborts with the read site on an out-of-range slot or a stale cached + # values base (#1661 was such a read). The plan line pins all six + # sections as executed, not merely selected. + - name: JIT-tier sections under make jit-checked + env: + EIGS_SUITE_SECTIONS: 82 100 129 130 0fd 0fe + run: | + set -o pipefail + make jit-checked + bash tests/run_all_tests.sh 2>&1 | tee "$RUNNER_TEMP/jit-checked.log" + grep -q 'PLAN: sections=82,100,129,130,0fd,0fe bearing=6 chunks=6$' "$RUNNER_TEMP/jit-checked.log" + grep -qE '^ RESULTS: [0-9]+/[0-9]+ passed, 0 failed, 0 skipped$' "$RUNNER_TEMP/jit-checked.log" + - name: Require compiler guard activity if: always() run: bash tools/cc_guard.sh --report diff --git a/Makefile b/Makefile index 048a6658..a1bc7024 100644 --- a/Makefile +++ b/Makefile @@ -77,7 +77,7 @@ define AUX_REFRESH done endef -.PHONY: db-params-test all build slot-rehearsal server server-db full http net gfx zlib lib amalgamation tsan test test-changed precheck sandbox-intern-test install install-gfx clean coverage coverage-clean fuzz fuzz-run lsp lsp-asan dap jit-smoke embed-smoke embed-smoke-asan embed-smoke-asan-server embed-smoke-gfx embed-concurrent asan asan-server valgrind pgo poison freestanding-check freestanding-libc-diff asan-http asan-gfx tsan-server tsan-http nativefn-test arming-mt-test embed-roads print-% sigpipe-contract-test sigpipe-partial-test ui-sdl-input-gfx pool-off asan-pool-off pool-off-num pool-off-callenv +.PHONY: db-params-test all build slot-rehearsal server server-db full http net gfx zlib lib amalgamation tsan test test-changed precheck sandbox-intern-test install install-gfx clean coverage coverage-clean fuzz fuzz-run lsp lsp-asan dap jit-smoke embed-smoke embed-smoke-asan embed-smoke-asan-server embed-smoke-gfx embed-concurrent asan asan-server valgrind pgo poison freestanding-check freestanding-libc-diff asan-http asan-gfx tsan-server tsan-http nativefn-test arming-mt-test embed-roads print-% sigpipe-contract-test sigpipe-partial-test ui-sdl-input-gfx pool-off asan-pool-off pool-off-num pool-off-callenv jit-checked # ---- Per-variant objdir engine (#740) ------------------------------------- # The engine's rules are defined before `all`, so pin the default goal. @@ -178,8 +178,16 @@ SRC_V_pool-off-callenv := $(HOSTED_SOURCES) FLAGS_pool-off-callenv := $(CFLAGS) $(DEFS_OFF) -DEIGENSCRIPT_EXT_GFX=1 -DEIGS_POOL_OFF_CALLENV $(VERDEF) LIBS_pool-off-callenv := $(LDFLAGS) -ldl +# #1665 follow-up: release build whose JIT emits a slot witness before every +# direct env-slot access (GET_LOCAL/SET_LOCAL/LOCAL_DOT_*, the inline EnvIC +# hit paths): a slot out of range of the env, or a stale cached values base, +# aborts with the read site (jit.h EIGS_JIT_CHECKED). The default emitter is +# unchanged; CI runs the JIT-tier sections against this build. +SRC_V_jit-checked := $(HOSTED_SOURCES) +FLAGS_jit-checked := $(CFLAGS) $(DEFS_OFF) -DEIGENSCRIPT_EXT_GFX=1 -DEIGS_JIT_CHECKED $(VERDEF) +LIBS_jit-checked := $(LDFLAGS) -ldl -VARIANTS := release slot-rehearsal server server-db zlib asan asan-server tsan tsan-server valgrind poison pool-off asan-pool-off pool-off-num pool-off-callenv +VARIANTS := release slot-rehearsal server server-db zlib asan asan-server tsan tsan-server valgrind poison pool-off asan-pool-off pool-off-num pool-off-callenv jit-checked # Objects depend on Makefile+VERSION so a flag or version-string change # rebuilds; header edits are covered by the generated .d files. @@ -370,6 +378,16 @@ build/$(INTERN_OWNER_VARIANT)/test_intern_owners: tests/test_intern_owners.c $(I intern-owner-test: build/$(INTERN_OWNER_VARIANT)/test_intern_owners @echo "Intern owner structural oracle built: $<" +# #1665 follow-up: planted stale reads of the NUM freelist and the parked call +# env, against the owning (ASan) variant; no CLI relink. Suite [0fg]. +POOL_POISON_VARIANT ?= asan +POOL_POISON_OBJ := $(filter-out build/$(POOL_POISON_VARIANT)/main.o,$(OBJ_$(POOL_POISON_VARIANT))) +build/$(POOL_POISON_VARIANT)/test_pool_poison: tests/test_pool_poison.c $(POOL_POISON_OBJ) $(wildcard $(SRC_DIR)/*.h) Makefile tools/werror_flags.txt + $(CC) $(FLAGS_$(POOL_POISON_VARIANT)) -I$(SRC_DIR) -o $@ $< $(POOL_POISON_OBJ) $(LIBS_$(POOL_POISON_VARIANT)) +.PHONY: pool-poison-test +pool-poison-test: build/$(POOL_POISON_VARIANT)/test_pool_poison + @echo "Pool poison plants built: $<" + # #1056: use the same variant as the CLI under test, without relinking it. ROAD_VARIANT ?= release EMBED_ROADS_OBJ := $(filter-out build/$(ROAD_VARIANT)/main.o,$(OBJ_$(ROAD_VARIANT))) @@ -661,6 +679,12 @@ pool-off-callenv: build/pool-off-callenv/eigenscript $(call RELINK,pool-off-callenv) @echo "EigenScript $(VERSION) (pool-off C3: call-env recycling off) built. Binary: $(BINARY)" +# JIT slot witnesses (see FLAGS_jit-checked). Run the JIT-tier sections: +# make jit-checked && cd tests && EIGS_SUITE_SECTIONS='82 100 129 130 0fd 0fe' bash run_all_tests.sh +jit-checked: build/jit-checked/eigenscript + $(call RELINK,jit-checked) + @echo "EigenScript $(VERSION) (jit-checked: emitted slot witnesses) built. Binary: $(BINARY)" + # Profile-guided optimization. Builds an instrumented binary, runs the # DMG cpu_instrs workload to collect branch/edge counters, then rebuilds diff --git a/changes/added/1665-pool-stale-read-detectors.md b/changes/added/1665-pool-stale-read-detectors.md new file mode 100644 index 00000000..62c9c614 --- /dev/null +++ b/changes/added/1665-pool-stale-read-detectors.md @@ -0,0 +1,8 @@ +- **Stale-read detectors for the two kept recycling pools (#1665).** ASan + builds now poison a NUM parked on the per-thread freelist (all but its link + word) and a call env parked on its chunk (its slot, name, count and hash + arrays and the Env struct, all but `parent`), so a read of a recycled + object reports `use-after-poison` instead of passing silently (#1661 was + such a read). `make jit-checked` builds a JIT that checks every emitted + direct env-slot access against the env and aborts with the read site; CI + runs the JIT-tier sections against it. Release builds are unchanged. diff --git a/src/chunk.c b/src/chunk.c index 10935dc6..e52e0f08 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -95,6 +95,7 @@ void chunk_decref(EigsChunk *chunk) { /* Stage 5i: release the parked call env (not captured, all slots * already null; the chunk holds its single ref — env_decref destroys * it and drops its owned parent ref). */ + if (chunk->env_cache) eigs_env_park_unpoison(chunk->env_cache); env_decref(chunk->env_cache); free(chunk->lines); free(chunk->cols); diff --git a/src/eigenscript.c b/src/eigenscript.c index 6f32f5d9..c4b2bd39 100644 --- a/src/eigenscript.c +++ b/src/eigenscript.c @@ -1547,12 +1547,12 @@ int observer_slot_from_trajectory(ObserverSlot *out, Value *dict) { void eigs_thread_drain_caches(EigsThread *th) { if (!th) return; - /* num_freelist: each entry's first sizeof(Value*) bytes of data are - * overlaid with the next-pointer. Save next, free node, advance. */ + /* num_freelist: each entry's link word (EIGS_NUM_LINK, inside data) holds + * the next-pointer. Save next, free node, advance. */ Value *vn = th->num_freelist; while (vn) { Value *next; - memcpy(&next, &vn->data, sizeof(Value *)); + memcpy(&next, EIGS_NUM_LINK(vn), sizeof(Value *)); free(vn); vn = next; } @@ -1625,10 +1625,11 @@ void free_value(Value *v) { #ifndef EIGS_POOL_OFF_NUM /* Route freed NUMs to freelist for reuse by make_num */ if (g_num_freelist_count < NUM_FREELIST_CAP) { - memcpy(&v->data, &g_num_freelist, sizeof(Value *)); + memcpy(EIGS_NUM_LINK(v), &g_num_freelist, sizeof(Value *)); g_num_freelist = v; g_num_freelist_count++; EIGS_VG_NOACCESS(&v->refcount, sizeof(v->refcount)); /* #297/#298 */ + eigs_num_park_poison(v); /* ASan: all but the link word */ return; } #endif /* EIGS_POOL_OFF_NUM (#1665): never park — a refcount-0 NUM is free()d so @@ -1718,9 +1719,10 @@ Value* make_num(double n) { #ifndef EIGS_POOL_OFF_NUM if (g_num_freelist) { v = g_num_freelist; - memcpy(&g_num_freelist, &v->data, sizeof(Value *)); + memcpy(&g_num_freelist, EIGS_NUM_LINK(v), sizeof(Value *)); g_num_freelist_count--; EIGS_VG_DEFINED(&v->refcount, sizeof(v->refcount)); /* un-poison before reuse */ + eigs_num_park_unpoison(v); memset(v, 0, sizeof(Value)); } else #endif /* EIGS_POOL_OFF_NUM (#1665): never satisfy a make_num from the freelist — @@ -1747,10 +1749,11 @@ void recycle_intermediate(Value *v) { free(v); return; } - memcpy(&v->data, &g_num_freelist, sizeof(Value *)); + memcpy(EIGS_NUM_LINK(v), &g_num_freelist, sizeof(Value *)); g_num_freelist = v; g_num_freelist_count++; EIGS_VG_NOACCESS(&v->refcount, sizeof(v->refcount)); /* #297/#298 */ + eigs_num_park_poison(v); /* ASan: all but the link word */ #endif } @@ -4378,6 +4381,21 @@ static int gc_next_threshold(int live, int last_universe) { return (int)t; } +#ifdef EIGS_ASAN_POOL_POISON +/* Re-poison every parked call env the collector unpoisoned during discovery. + * Only SURVIVING chunks still own one (a garbage chunk's env_cache was cleared + * and freed with it), so after a completed collection only marked chunks are + * visited; after an aborted one the graph is unchanged and all are. */ +static void gc_repoison_parked(GcU *u, int survivors_only) { + for (int n = 0; n < u->count; n++) { + if (u->kind[n] != GC_KIND_CHUNK || (survivors_only && !u->mark[n])) + continue; + Env *parked = ((EigsChunk *)u->objs[n])->env_cache; + if (parked) eigs_env_park_poison(parked); + } +} +#endif + static void gc_collect_impl(Value **seeds, int seed_count, int include_captured_envs) { if (g_in_gc || g_vm_multithreaded) return; @@ -4408,6 +4426,13 @@ static void gc_collect_impl(Value **seeds, int seed_count, for (int n = 0; n < u.count; n++) { GC_FOR_EACH_CHILD(&u, n, child, child_kind, { gcu_add(&u, child, child_kind); +#ifdef EIGS_ASAN_POOL_POISON + /* A chunk's only Env edge is its parked env_cache: the collector + * reads it from here on, so lift the park poison (eigenscript.h); + * gc_repoison_parked restores it before returning. */ + if (child_kind == GC_KIND_ENV && u.kind[n] == GC_KIND_CHUNK) + eigs_env_park_unpoison((Env *)child); +#endif /* gcu_add may reallocate the arrays; retain indices, not pointers * into them. The target is now present even on a forward edge. */ int ci = gcu_find(&u, child); @@ -4440,6 +4465,9 @@ static void gc_collect_impl(Value **seeds, int seed_count, uint64_t discovery_work = 0; for (int n = 0; n < u.count; n++) discovery_work += gc_node_work(u.objs[n], u.kind[n]); +#ifdef EIGS_ASAN_POOL_POISON + gc_repoison_parked(&u, 0); +#endif free(stack); free(u.table); free(u.objs); free(u.kind); free(u.internal); free(u.pinned); free(u.mark); @@ -4491,6 +4519,9 @@ static void gc_collect_impl(Value **seeds, int seed_count, if (eigs_env_flag("EIGS_GC_DEBUG")) fprintf(stderr, "[gc] universe %d, freed %d, live captured %d\n", u.count, garbage, g_gc_captured_live); +#ifdef EIGS_ASAN_POOL_POISON + gc_repoison_parked(&u, 1); +#endif free(u.table); free(u.objs); free(u.kind); free(u.internal); free(u.pinned); free(u.mark); diff --git a/src/eigenscript.h b/src/eigenscript.h index 08498c8e..4464784d 100644 --- a/src/eigenscript.h +++ b/src/eigenscript.h @@ -486,6 +486,98 @@ _Static_assert(sizeof(((Value *)0)->data.str) == sizeof(((Value *)0)->data.strv. _Static_assert(sizeof(((Value *)0)->data.strv) <= sizeof(((Value *)0)->data.fn), "cached string length must fit in the union without growing it past `fn` on any pointer width — #1183"); +/* #1665 follow-up: ASan stale-read detectors for the two recycling pools the + * per-layer rule kept (the Value NUM freelist and call-env recycling). A + * recycled object stays "allocated" to ASan, so a read of a parked object was + * invisible (#1661 was one, tripped only by the JIT). Under AddressSanitizer + * the parked object is POISONED while it sits in the pool and unpoisoned when + * the pool hands it back out, so a stale read reports + * use-after-poison at the reading instruction. Compiled only when the + * compiler instruments for ASan; every other build expands these to nothing, + * so release objects are byte-identical. + * + * Parked NUM (free_value / recycle_intermediate -> make_num): the whole Value + * is poisoned EXCEPT the link word, the sizeof(Value *) bytes the freelist + * threads its next pointer through (EIGS_NUM_LINK). Only the pool's own push, + * pop (make_num) and drain (eigs_thread_drain_caches) touch those bytes. In + * every other build the link overlays the number itself (offset + * offsetof(Value, data)), which would leave a stale NUMBER read -- the + * commonest stale read of a recycled NUM -- undetectable; so under ASan the + * link moves one double further into the union, to bytes + * [offsetof(Value, data) + 8, offsetof(Value, data) + 16) (16..24 on LP64, + * the strv.len / list.count+capacity word, meaningless for a NUM). `type`, + * the number, `refcount`, the rest of the union and the tail flags are all + * poisoned, so a stale v->type check, number read, or incref/decref on a + * recycled NUM is reported. Release keeps the old offset: same bytes. + * + * Parked call env (vm_park_call_env -> vm_take_call_env, chunk->env_cache): + * values[], names[], assign_counts[] (capacity entries each), the three hash + * arrays (mask + 1 entries each) and the Env struct are poisoned, EXCEPT the + * `parent` field, which vm_take_call_env reads to decide whether the parked + * env matches the callee's closure. The pool's other legitimate readers -- + * the cycle collector (it walks chunk -> env_cache as an owned edge) and + * chunk_free (it drops the parked ref) -- unpoison before they touch it. */ +#if defined(__SANITIZE_ADDRESS__) +# define EIGS_ASAN_POOL_POISON 1 +#elif defined(__has_feature) +# if __has_feature(address_sanitizer) +# define EIGS_ASAN_POOL_POISON 1 +# endif +#endif +#ifdef EIGS_ASAN_POOL_POISON +# define EIGS_NUM_LINK_OFF (offsetof(Value, data) + sizeof(double)) +#else +# define EIGS_NUM_LINK_OFF offsetof(Value, data) +#endif +/* Address of a parked NUM's freelist link (see above). */ +#define EIGS_NUM_LINK(v) ((void *)((char *)(v) + EIGS_NUM_LINK_OFF)) +_Static_assert(EIGS_NUM_LINK_OFF + sizeof(Value *) <= + offsetof(Value, data) + sizeof(((Value *)0)->data), + "the NUM freelist link must lie inside the data union"); +#ifdef EIGS_ASAN_POOL_POISON +#include +_Static_assert(EIGS_NUM_LINK_OFF % 8 == 0 && sizeof(Value) % 8 == 0, + "NUM park poisoning assumes 8-byte ASan granules around the link word"); +static inline void eigs_num_park_poison(Value *v) { + size_t link = EIGS_NUM_LINK_OFF, link_end = link + sizeof(Value *); + ASAN_POISON_MEMORY_REGION(v, link); + ASAN_POISON_MEMORY_REGION((char *)v + link_end, sizeof(Value) - link_end); +} +static inline void eigs_num_park_unpoison(Value *v) { + ASAN_UNPOISON_MEMORY_REGION(v, sizeof(Value)); +} +_Static_assert(offsetof(Env, parent) % 8 == 0, + "call-env park poisoning keeps `parent` addressable on its own granule"); +static inline void eigs_env_park_mark(Env *e, int poison) { + /* Sizes are read from the struct, so poison it LAST and unpoison it + * FIRST. */ + if (!poison) ASAN_UNPOISON_MEMORY_REGION(e, sizeof(Env)); + size_t cap = e->capacity > 0 ? (size_t)e->capacity : 0; + size_t hcap = e->hash.hashes ? (size_t)e->hash.mask + 1 : 0; + void *blk[6] = { e->values, e->names, e->assign_counts, + e->hash.hashes, e->hash.indices, e->hash.generations }; + size_t len[6] = { cap * sizeof(EigsSlot), cap * sizeof(char *), cap * sizeof(int), + hcap * sizeof(uint32_t), hcap * sizeof(int), hcap * sizeof(uint32_t) }; + for (int i = 0; i < 6; i++) { + if (!blk[i] || !len[i]) continue; + if (poison) ASAN_POISON_MEMORY_REGION(blk[i], len[i]); + else ASAN_UNPOISON_MEMORY_REGION(blk[i], len[i]); + } + if (poison) { + size_t p = offsetof(Env, parent), p_end = p + sizeof(Env *); + ASAN_POISON_MEMORY_REGION(e, p); + ASAN_POISON_MEMORY_REGION((char *)e + p_end, sizeof(Env) - p_end); + } +} +# define eigs_env_park_poison(e) eigs_env_park_mark((e), 1) +# define eigs_env_park_unpoison(e) eigs_env_park_mark((e), 0) +#else +# define eigs_num_park_poison(v) ((void)0) +# define eigs_num_park_unpoison(v) ((void)0) +# define eigs_env_park_poison(e) ((void)0) +# define eigs_env_park_unpoison(e) ((void)0) +#endif + /* Install a VAL_STR / VAL_JSON_RAW payload. `s` is adopted (the Value frees * it) and `n` MUST equal strlen(s). This is the ONLY way to write the payload * — `v->data.str` is const, so the compiler refuses the alternative. */ diff --git a/src/jit.c b/src/jit.c index f35687d1..67e1af3a 100644 --- a/src/jit.c +++ b/src/jit.c @@ -512,6 +512,16 @@ static void ensure_layout(void) { g_layout_inited = 1; } +/* `make jit-checked` (EIGS_JIT_CHECKED): each op whose emitted code indexes + * an env's values[] directly gets a slot-witness call (emit_checked_local / + * emit_checked_env_slot below); charge its bytes to the size estimate. */ +#ifdef EIGS_JIT_CHECKED +#define JIT_CHECKED_BYTES 48 +#define JIT_CHECKED_SCAN_EXTRA() (*extra_size += JIT_CHECKED_BYTES) +#else +#define JIT_CHECKED_SCAN_EXTRA() ((void)0) +#endif + /* Scan a chunk-start prefix of opcodes we can natively translate. * Returns prefix length in *bytes* (0 = nothing compilable). Sets * *needs_env_cache when the prefix touches fn_env->values[]; sets @@ -598,6 +608,7 @@ static int jit_supported_prefix(const struct EigsChunk *chunk, } i += 3; ops++; non_line_ops++; *needs_env_cache = 1; + JIT_CHECKED_SCAN_EXTRA(); } else if (op == OP_GET_NAME) { if (i + 3 > chunk->code_len) { *stop_op = op; *stop_offset = i; break; } uint16_t idx = (uint16_t)(chunk->code[i + 1] | @@ -614,6 +625,7 @@ static int jit_supported_prefix(const struct EigsChunk *chunk, /* Stage 5b: inline EnvIC fast path reads frame->env via the * %r15 frame cache; helper fallback still needs %r14=chunk. */ *needs_frame_cache = 1; + JIT_CHECKED_SCAN_EXTRA(); /* Result could be any slot type (whatever the binding holds). * Subsequent OP_POP cannot use the immediate peephole. */ } else if (op == OP_SET_NAME || op == OP_SET_NAME_LOCAL || @@ -636,6 +648,7 @@ static int jit_supported_prefix(const struct EigsChunk *chunk, i += 3; ops++; non_line_ops++; *has_bail_op = 1; *needs_frame_cache = 1; /* Stage 5b inline IC fast path */ + JIT_CHECKED_SCAN_EXTRA(); } else if (op == OP_LOCAL_IDX_GET) { /* 5-byte op: [op][slot:16][idx:16]. Helper handles VAL_BUFFER/ * VAL_LIST/VAL_STR dispatch and reports a latched error. @@ -658,6 +671,7 @@ static int jit_supported_prefix(const struct EigsChunk *chunk, i += 5; ops++; non_line_ops++; *has_bail_op = 1; *needs_env_cache = 1; + JIT_CHECKED_SCAN_EXTRA(); } else if (op == OP_LOCAL_IDX_DOT_GET) { /* Stage 4v: 7-byte op [op][slot:16][list_idx:16][name_idx:16]. * Helper needs chunk pointer (const_interns/const_hashes) — @@ -712,6 +726,7 @@ static int jit_supported_prefix(const struct EigsChunk *chunk, i += 5; ops++; non_line_ops++; *has_bail_op = 1; *needs_env_cache = 1; /* Stage 5d inline fast path */ + JIT_CHECKED_SCAN_EXTRA(); } else if (op == OP_SET_LOCAL) { if (i + 3 > chunk->code_len) { *stop_op = op; *stop_offset = i; break; } /* #348: same bounds rule as OP_GET_LOCAL above. */ @@ -723,6 +738,7 @@ static int jit_supported_prefix(const struct EigsChunk *chunk, i += 3; ops++; non_line_ops++; *needs_env_cache = 1; *has_bail_op = 1; /* #1063: %r14=chunk for the traced slow path */ + JIT_CHECKED_SCAN_EXTRA(); } else if (op == OP_DUP) { i += 1; ops++; non_line_ops++; } else if (op == OP_DUP2) { @@ -2201,6 +2217,57 @@ static uint8_t *emit_jb_rel32(uint8_t *w, uint8_t **patch) { static uint8_t *emit_mov_rdx_rdi(uint8_t *w) { *w++ = 0x48; *w++ = 0x89; *w++ = 0xD7; return w; } + +#ifdef EIGS_JIT_CHECKED +/* `make jit-checked`: a call to a slot witness (jit.h) before every emitted + * direct env-slot access. Thunk bodies run at %rsp == 8 (mod 16), so the + * pushes stay odd in number to keep the call aligned. Nothing here exists + * in the default build: the emitter is unchanged there (byte-identical + * jit.o). JIT_CHECKED_SITE packs (opcode << 24) | bytecode offset. */ +#define JIT_CHECKED_SITE(op, off) (((uint32_t)(op) << 24) | ((uint32_t)(off) & 0xFFFFFFu)) +/* JIT_CHECKED_BYTES (scanner, above) bounds either sequence below (37 and + * 42 bytes). */ +_Static_assert(JIT_CHECKED_BYTES >= 42, "checked-slot sequence size bound"); +/* movabs $imm64, %rcx (10 bytes) */ +static uint8_t *emit_checked_movabs_rcx(uint8_t *w, uint64_t imm) { + *w++ = 0x48; *w++ = 0xB9; return emit_u64(w, imm); +} +/* %r12-based ops: jit_checked_local(%r12, slot, site, chunk). 37 bytes: + * push %rcx; mov %r12,%rdi; mov $slot,%esi; mov $site,%edx; + * movabs $chunk,%rcx; movabs $helper,%rax; call *%rax; pop %rcx. */ +static uint8_t *emit_checked_local(uint8_t *w, uint32_t slot, uint32_t site, + const struct EigsChunk *chunk) { + w = emit_push_rcx(w); + *w++ = 0x4C; *w++ = 0x89; *w++ = 0xE7; /* mov %r12, %rdi */ + w = emit_mov_imm32_esi(w, slot); + w = emit_mov_imm32_edx(w, site); + w = emit_checked_movabs_rcx(w, (uint64_t)(uintptr_t)chunk); + w = emit_movabs_rax(w, (uint64_t)(uintptr_t)&jit_checked_local); + w = emit_call_rax(w); + w = emit_pop_rcx(w); + return w; +} +/* Inline EnvIC hit (%rdx = target env, %rax = ic): + * jit_checked_env_slot(target, ic->slot_idx, site, chunk), preserving + * %rax/%rdx for the hit path. 42 bytes. */ +static uint8_t *emit_checked_env_slot(uint8_t *w, uint32_t site, + const struct EigsChunk *chunk) { + w = emit_push_rcx(w); + *w++ = 0x50; /* push %rax */ + *w++ = 0x52; /* push %rdx */ + w = emit_mov_rdx_rdi(w); + *w++ = 0x8B; *w++ = 0xB0; /* mov disp32(%rax), %esi */ + w = emit_u32(w, (uint32_t)offsetof(EnvIC, slot_idx)); + w = emit_mov_imm32_edx(w, site); + w = emit_checked_movabs_rcx(w, (uint64_t)(uintptr_t)chunk); + w = emit_movabs_rax(w, (uint64_t)(uintptr_t)&jit_checked_env_slot); + w = emit_call_rax(w); + *w++ = 0x5A; /* pop %rdx */ + *w++ = 0x58; /* pop %rax */ + w = emit_pop_rcx(w); + return w; +} +#endif /* EIGS_JIT_CHECKED */ /* or %rsi, %rdx (3 bytes) — rdx |= rsi. */ static uint8_t *emit_or_rsi_rdx(uint8_t *w) { *w++ = 0x48; *w++ = 0x09; *w++ = 0xF2; return w; @@ -2746,6 +2813,9 @@ static void jit_compile_to_thunk(struct EigsChunk *chunk, } else if (op == OP_GET_LOCAL) { uint16_t slot = (uint16_t)(chunk->code[i + 1] | ((uint16_t)chunk->code[i + 2] << 8)); +#ifdef EIGS_JIT_CHECKED + w = emit_checked_local(w, slot, JIT_CHECKED_SITE(op, i), chunk); +#endif /* %rax = values[slot] (8-byte slot at slot*8 offset) */ w = emit_mov_disp32_r12_to_rax(w, (int32_t)slot * 8); int bail = 0; @@ -2797,6 +2867,9 @@ static void jit_compile_to_thunk(struct EigsChunk *chunk, w = emit_mov_disp32_rdx_to_rsi(w, (int32_t)offsetof(Env, binding_version)); w = emit_cmp_rsi_disp32_rax(w, (int32_t)offsetof(EnvIC, target_ver)); w = emit_jne_rel32(w, &slow_p[slow_n]); slow_n++; +#ifdef EIGS_JIT_CHECKED + w = emit_checked_env_slot(w, JIT_CHECKED_SITE(op, i), chunk); +#endif /* Hit: load slot, incref, push. */ w = emit_mov_disp32_rax_to_esi(w, (int32_t)offsetof(EnvIC, slot_idx)); w = emit_mov_disp32_rdx_to_rdx(w, (int32_t)offsetof(Env, values)); @@ -2912,6 +2985,9 @@ static void jit_compile_to_thunk(struct EigsChunk *chunk, w = emit_cmp_rsi_disp32_rax(w, (int32_t)offsetof(EnvIC, target_ver)); w = emit_jne_rel32(w, &slow_p[slow_n]); slow_n++; /* Hit. %rdx = target env, %rax = ic. */ +#ifdef EIGS_JIT_CHECKED + w = emit_checked_env_slot(w, JIT_CHECKED_SITE(op, i), chunk); +#endif w = emit_mov_disp32_rax_to_r9d(w, (int32_t)offsetof(EnvIC, slot_idx)); w = emit_load_stack_to_r8(w, g_layout.off_stack - 8); /* s */ /* slot_incref(s): immediates skip, pointers get the lock-incref. */ @@ -3027,6 +3103,9 @@ static void jit_compile_to_thunk(struct EigsChunk *chunk, } uint8_t *slow_p[10]; int slow_n = 0; +#ifdef EIGS_JIT_CHECKED + w = emit_checked_local(w, slot, JIT_CHECKED_SITE(op, i), chunk); +#endif uint8_t *done_p = NULL; /* Defensive: the shl $4 in the probe hard-codes the entry * size; fall back to helper-only emission if it ever moves. */ @@ -3158,6 +3237,9 @@ static void jit_compile_to_thunk(struct EigsChunk *chunk, uint8_t *slow_p[13]; int slow_n = 0; uint8_t *done_p = NULL; +#ifdef EIGS_JIT_CHECKED + w = emit_checked_local(w, slot, JIT_CHECKED_SITE(op, i), chunk); +#endif if (g_layout.sizeof_dcache_entry == 16 && g_layout.dcache_ways == 2) { w = emit_dict_cache_probe(w, slot, h, key, slow_p, &slow_n); /* existing field: exclusive num. */ @@ -3328,6 +3410,9 @@ static void jit_compile_to_thunk(struct EigsChunk *chunk, int32_t off = (int32_t)slot * 8; uint8_t *swap_p[5], *plain_p[2], *done_p[2]; int swap_n = 0, plain_n = 0, done_n = 0; +#ifdef EIGS_JIT_CHECKED + w = emit_checked_local(w, slot, JIT_CHECKED_SITE(op, i), chunk); +#endif /* #1063: g_trace_hist armed -> out-of-line helper, exactly as * the SET_NAME* arms do. The inline path below cannot record * history, and a slot-bound interrogated name (a parameter, or diff --git a/src/jit.h b/src/jit.h index c544a21d..a30300fc 100644 --- a/src/jit.h +++ b/src/jit.h @@ -270,4 +270,24 @@ void jit_helper_return(void); * pattern as OP_RETURN. */ void jit_helper_return_null(void); +#ifdef EIGS_JIT_CHECKED +/* #1665 follow-up: `make jit-checked` (-DEIGS_JIT_CHECKED). Every emitted + * direct env-slot access calls one of these first; a violation prints the + * read site and aborts. Never emitted, declared or defined in any other + * build, so the default emitter is unchanged (byte-identical jit.o). + * site = (opcode << 24) | bytecode offset of the accessing op. + * jit_checked_local: the %r12-based ops (GET_LOCAL, SET_LOCAL, + * LOCAL_DOT_GET/SET). The cached values base must still be the current + * frame's fn_env->values, and slot < fn_env->count -- the interpreter's + * CASE(GET_LOCAL)/CASE(SET_LOCAL) range check. + * jit_checked_env_slot: the inline EnvIC hit paths (GET_NAME, SET_NAME*), + * which index target->values[ic->slot_idx]; slot_idx < target->count. */ +struct EigsChunk; +struct Env; +void jit_checked_local(uint64_t *cached_values, uint32_t slot, + uint32_t site, struct EigsChunk *chunk); +void jit_checked_env_slot(struct Env *target, uint32_t slot, + uint32_t site, struct EigsChunk *chunk); +#endif + #endif /* EIGS_JIT_H */ diff --git a/src/vm.c b/src/vm.c index de6dc4bd..8fa552c5 100644 --- a/src/vm.c +++ b/src/vm.c @@ -518,6 +518,7 @@ static inline int vm_park_call_env(EigsChunk *chunk, Env *env) { int obs_retain_cap = expected <= INT_MAX - 7 ? expected + 7 : -1; observer_slot_reset_bounded(env, obs_retain_cap); chunk->env_cache = env; + eigs_env_park_poison(env); /* ASan: parked until take (eigenscript.h) */ return 1; #endif /* EIGS_POOL_OFF_CALLENV */ } @@ -532,6 +533,7 @@ static inline Env *vm_take_call_env(EigsChunk *fn_chunk, Env *closure, if (e && !g_vm_multithreaded && e->parent == closure && (param_count <= 1 || argc >= param_count)) { fn_chunk->env_cache = NULL; + eigs_env_park_unpoison(e); return e; } #else /* EIGS_POOL_OFF_CALLENV */ @@ -2833,6 +2835,40 @@ void jit_helper_return_null(void) { g_vm.stack[g_vm.sp++] = slot_null(); } +#ifdef EIGS_JIT_CHECKED +/* `make jit-checked` slot witnesses (jit.h). Print the read site, abort. */ +static void jit_checked_fail(EigsChunk *chunk, uint32_t site, + const char *what, long a, long b) { + uint32_t off = site & 0xFFFFFFu; + int line = (chunk && chunk->lines && (int)off < chunk->lines_len) + ? chunk->lines[off] : 0; + fprintf(stderr, + "EIGS_JIT_CHECKED: %s (%ld vs %ld) at %s offset %u in chunk '%s' line %d\n", + what, a, b, op_name((uint8_t)(site >> 24)), off, + (chunk && chunk->name) ? chunk->name : "", line); + fflush(stderr); + abort(); +} + +void jit_checked_local(uint64_t *cached_values, uint32_t slot, + uint32_t site, EigsChunk *chunk) { + Env *e = g_vm.frames[g_vm.frame_count - 1].fn_env; + if ((void *)cached_values != (void *)e->values) + jit_checked_fail(chunk, site, "stale %r12 values base (cached != fn_env->values)", + (long)(intptr_t)cached_values, (long)(intptr_t)e->values); + if ((int)slot >= e->count) + jit_checked_fail(chunk, site, "local slot out of range (slot vs fn_env->count)", + (long)slot, (long)e->count); +} + +void jit_checked_env_slot(Env *target, uint32_t slot, uint32_t site, + EigsChunk *chunk) { + if ((int)slot >= target->count) + jit_checked_fail(chunk, site, "IC slot out of range (slot_idx vs target->count)", + (long)slot, (long)target->count); +} +#endif + /* Helper called by the JIT prologue on platforms where the JIT can't * encode TLS access directly — Darwin/Mach-O uses TLV descriptors * (per-variable callable thunks), not the fixed-offset %fs:tpoff that diff --git a/tests/sections/0fg-pool-poison.sh b/tests/sections/0fg-pool-poison.sh new file mode 100644 index 00000000..4f6d18d7 --- /dev/null +++ b/tests/sections/0fg-pool-poison.sh @@ -0,0 +1,67 @@ +echo "[0fg] Pool stale-read poisoning (#1665)" +# The two recycling pools #1665 kept (the Value NUM freelist and call-env +# recycling on chunk->env_cache) keep a parked object "allocated" to ASan, so a +# stale read of one was invisible (#1661). On ASan builds the parked object is +# now poisoned (eigenscript.h, EIGS_ASAN_POOL_POISON). tests/test_pool_poison.c +# plants one stale read per pool; each must be reported as use-after-poison AT +# THE PLANTED ADDRESS (the child prints it; no symbolizer needed): a parked +# NUM's number, a parked env's values[0], and a field of the parked Env. On an ASan +# build without the poisoning every plant prints UNDETECTED and exit 0, which +# fails here. The control runs the pools' own reuse paths (freelist pop, +# call-env take, a cycle collection that walks chunk -> parked env) and must +# stay sanitizer-clean with leaks on. Release builds compile no poisoning: +# this section announces a skip there. +check_binary_fingerprint +__pp_root="$TESTS_DIR/.." +__pp_variant= +for __pp_c in "$__pp_root"/build/*/eigenscript; do + if [ "$__pp_root/src/eigenscript" -ef "$__pp_c" ]; then + __pp_variant=$(basename "$(dirname "$__pp_c")"); break + fi +done +__pp_probe=$(ASAN_OPTIONS=help=1 ./eigenscript --version 2>&1) +if ! grep -q 'AddressSanitizer' <<< "$__pp_probe"; then + section_skip "[0fg] not an ASan build -- pool poisoning is compiled only under AddressSanitizer" +else + case "$__pp_variant" in + "") + TOTAL=$((TOTAL + 1)); FAIL=$((FAIL + 1)) + echo " FAIL: [0fg] src/eigenscript is not a hard link to any build//eigenscript" ;; + *pool-off*) + section_skip "[0fg] variant $__pp_variant compiles both pools out -- nothing is parked to poison" ;; + *) + TOTAL=$((TOTAL + 1)) + __pp_build=$(make --no-print-directory -C "$__pp_root" pool-poison-test "POOL_POISON_VARIANT=$__pp_variant" 2>&1); __pp_rc=$? + __pp_bin="$__pp_root/build/$__pp_variant/test_pool_poison" + if [ "$__pp_rc" -ne 0 ] || [ ! -x "$__pp_bin" ]; then + FAIL=$((FAIL + 1)); echo " FAIL: [0fg] building the plants for $__pp_variant (rc=$__pp_rc)" + printf '%s\n' "$__pp_build" | sed 's/^/ /' + else + PASS=$((PASS + 1)); echo " PASS: [0fg] plants built against the $__pp_variant objects" + for __pp_mode in num env envfield; do + TOTAL=$((TOTAL + 1)) + __pp_out=$(ASAN_OPTIONS=detect_leaks=0 $EIGS_TMO "$__pp_bin" "$__pp_mode" &1); __pp_rc=$? + __pp_addr=$(sed -n "s/^ARMED $__pp_mode: .*, reading \(0x[0-9a-f]*\)\$/\1/p" <<< "$__pp_out") + if [ "$__pp_rc" -ne 0 ] && [ "$__pp_rc" -ne 124 ] && [ -n "$__pp_addr" ] && + grep -q "ERROR: AddressSanitizer: use-after-poison on address $__pp_addr " <<< "$__pp_out" && + ! grep -q '^UNDETECTED' <<< "$__pp_out"; then + PASS=$((PASS + 1)); echo " PASS: [0fg] stale read of a parked $__pp_mode is use-after-poison at $__pp_addr" + else + FAIL=$((FAIL + 1)); echo " FAIL: [0fg] stale read of a parked $__pp_mode not reported at the planted address (rc=$__pp_rc, addr='${__pp_addr}')" + printf '%s\n' "$__pp_out" | sed 's/^/ /' + fi + done + TOTAL=$((TOTAL + 1)) + __pp_out=$(ASAN_OPTIONS=detect_leaks=1 $EIGS_TMO "$__pp_bin" control &1); __pp_rc=$? + if [ "$__pp_rc" -eq 0 ] && grep -q '^control: all clean$' <<< "$__pp_out" && + ! grep -qE 'AddressSanitizer|LeakSanitizer|runtime error:' <<< "$__pp_out"; then + PASS=$((PASS + 1)); echo " PASS: [0fg] pool reuse paths (pop, take, collector walk) are poison-clean" + else + FAIL=$((FAIL + 1)); echo " FAIL: [0fg] pool reuse paths reported or misbehaved (rc=$__pp_rc)" + printf '%s\n' "$__pp_out" | sed 's/^/ /' + fi + fi ;; + esac +fi +unset __pp_root __pp_variant __pp_c __pp_probe __pp_build __pp_rc __pp_bin __pp_mode __pp_out __pp_addr +echo "" diff --git a/tests/test_pool_poison.c b/tests/test_pool_poison.c new file mode 100644 index 00000000..21b83437 --- /dev/null +++ b/tests/test_pool_poison.c @@ -0,0 +1,185 @@ +/* #1665 follow-up: planted stale reads of the two kept recycling pools. + * + * test_pool_poison num read a NUM parked on the per-thread freelist + * test_pool_poison env read values[0] of a call env parked on its chunk + * test_pool_poison envfield read a field (count) of that parked Env struct + * test_pool_poison control the pools' own reuse paths (pop, take, a cycle + * collection over a parked env) -- must stay clean + * + * A fault mode prints "ARMED : ... reading
" just before its + * stale read and "UNDETECTED " after it. On an ASan build with pool poisoning the read + * is reported as use-after-poison and the process dies between the two lines; + * on a build without it (release, or an ASan build that predates the + * poisoning) the read is an ordinary in-bounds load of still-allocated memory + * and the process prints UNDETECTED and exits 0. Exit 3 = the plant could not + * be armed (no parked object to read), never a verdict. */ +#include +#include + +#include "eigs_embed.h" +#include "eigenscript.h" +#include "vm.h" + +static const char *PROGRAM = + "define pool_poison_fn(a) as:\n" + " b is a + 1\n" + " return b\n" + "pool_poison_r is pool_poison_fn of [41]\n"; + +/* The function's chunk, whose env_cache holds the env parked on return. */ +static EigsChunk *fn_chunk(void) { + Value *fn = eigs_get_global("pool_poison_fn"); + if (!fn || fn->type != VAL_FN || fn->data.fn.body_count != -1) { + if (fn) eigs_value_release(fn); + return NULL; + } + EigsChunk *c = (EigsChunk *)fn->data.fn.body; + eigs_value_release(fn); /* the global binding keeps it alive */ + return c; +} + +static int run_program(void) { + Value *r = eigs_eval_string(PROGRAM); + if (!r) { + fprintf(stderr, "SETUP FAIL: eval: %s\n", + eigs_last_error_message() ? eigs_last_error_message() : "?"); + return 0; + } + eigs_value_release(r); + return 1; +} + +static int plant_num(void) { + Value *v = make_num(42.5); + val_decref(v); /* refcount 0 -> parked on the freelist */ + if (g_num_freelist != v) { puts("NOT-ARMED num: value not parked"); return 3; } + printf("ARMED num: parked NUM %p, reading %p\n", (void *)v, + (void *)&VAL_NUM_RAW(v)); + fflush(stdout); + volatile double stale = VAL_NUM_RAW(v); /* the planted stale read */ + printf("UNDETECTED num: read %g from a parked NUM\n", stale); + return 0; +} + +/* The values[] base of a parked env, read WITHOUT instrumentation: the Env + * struct is poisoned too, so an instrumented `parked->values` would report at + * the struct field before the planted array read could run. */ +#if defined(__SANITIZE_ADDRESS__) || defined(__clang__) +__attribute__((no_sanitize_address)) +#endif +static EigsSlot *values_base_unchecked(Env *e) { + return *(EigsSlot *volatile *)&e->values; +} + +static Env *parked_env(const char *mode) { + if (!run_program()) return NULL; + EigsChunk *c = fn_chunk(); + Env *parked = c ? c->env_cache : NULL; + if (!parked) printf("NOT-ARMED %s: no parked call env\n", mode); + return parked; +} + +/* env: the parked env's values[0] (its slot array). */ +static int plant_env(void) { + Env *parked = parked_env("env"); + if (!parked) return 3; + EigsSlot *vals = values_base_unchecked(parked); + printf("ARMED env: parked call env %p, reading %p\n", (void *)parked, + (void *)&vals[0]); + fflush(stdout); + volatile uint64_t stale = vals[0].u; /* the planted stale read */ + printf("UNDETECTED env: read %#llx from a parked env's values[0]\n", + (unsigned long long)stale); + return 0; +} + +/* envfield: a field of the parked Env struct itself (count). */ +static int plant_envfield(void) { + Env *parked = parked_env("envfield"); + if (!parked) return 3; + printf("ARMED envfield: parked call env %p, reading %p\n", (void *)parked, + (void *)&parked->count); + fflush(stdout); + volatile int stale = parked->count; /* the planted stale read */ + printf("UNDETECTED envfield: read count %d from a parked env\n", stale); + return 0; +} + +static int control(void) { + int bad = 0; + /* NUM pop: make_num hands the parked Value back, readable again. */ + Value *v = make_num(1.5); + val_decref(v); + Value *again = make_num(2.5); + if (again != v) { puts("NOT-ARMED control: freelist did not reuse"); return 3; } + if (VAL_NUM_RAW(again) != 2.5 || again->refcount != 1) bad = 1; + printf("control num reuse: %s\n", bad ? "WRONG" : "ok"); + val_decref(again); + + /* Env take: the second call takes the parked env (same struct). */ + if (!run_program()) return 3; + EigsChunk *c = fn_chunk(); + Env *first = c ? c->env_cache : NULL; + if (!first) { puts("NOT-ARMED control: no parked call env"); return 3; } + /* A cycle collection walks chunk -> env_cache; it must lift and then + * restore the poison without reporting. A captured env binding the + * function puts fn -> chunk -> parked env in the collector's universe. */ + Value *r = eigs_eval_string( + "define pool_poison_mk() as:\n" + " f is pool_poison_fn\n" + " return () => f\n" + "pool_poison_c is pool_poison_mk of []\n"); + if (r) eigs_value_release(r); + gc_collect_cycles(); +#if defined(EIGS_ASAN_POOL_POISON) + /* The collector lifts the park poison to walk the env; it must restore + * it for a chunk that survives. */ + if (!__asan_address_is_poisoned(&first->count)) { + puts("control gc: parked env left UNPOISONED after a collection"); + bad = 1; + } +#endif + r = eigs_eval_string("pool_poison_r is pool_poison_fn of [7]\n"); + if (!r) { puts("control take: eval failed"); return 1; } + eigs_value_release(r); + Value *res = eigs_get_global("pool_poison_r"); + int ok = res && res->type == VAL_NUM && VAL_NUM_RAW(res) == 8.0; + if (res) eigs_value_release(res); + int same = c->env_cache == first; + printf("control env take: %s (re-parked same env: %s)\n", + ok ? "ok" : "WRONG", same ? "yes" : "no"); + if (!ok || !same) bad = 1; + /* chunk_free drops a parked env: rebinding the only owner of a function + * frees its chunk (the defining eval's module chunk is already gone) + * while its call env is parked. */ + r = eigs_eval_string("define pool_poison_drop(a) as:\n" + " b is a + 1\n" + " return b\n" + "pool_poison_d is pool_poison_drop of [3]\n"); + if (r) eigs_value_release(r); + Value *dfn = eigs_get_global("pool_poison_drop"); + EigsChunk *dc = (dfn && dfn->type == VAL_FN && dfn->data.fn.body_count == -1) + ? (EigsChunk *)dfn->data.fn.body : NULL; + int drop_armed = dc && dc->env_cache && dc->refcount == 1; + if (dfn) eigs_value_release(dfn); + if (!drop_armed) { printf("NOT-ARMED control: drop chunk %p env_cache %p refcount %d\n", (void *)dc, dc ? (void *)dc->env_cache : NULL, dc ? dc->refcount : -1); return 3; } + r = eigs_eval_string("pool_poison_drop is null\n"); + if (r) eigs_value_release(r); + puts("control chunk_free of a parked env: ok"); + printf("control: %s\n", bad ? "FAIL" : "all clean"); + return bad; +} + +int main(int argc, char **argv) { + const char *mode = argc > 1 ? argv[1] : ""; + EigsState *st = eigs_open(); + if (!st) { fputs("SETUP FAIL: eigs_open\n", stderr); return 2; } + int rc; + if (!strcmp(mode, "num")) rc = plant_num(); + else if (!strcmp(mode, "env")) rc = plant_env(); + else if (!strcmp(mode, "envfield")) rc = plant_envfield(); + else if (!strcmp(mode, "control")) rc = control(); + else { fprintf(stderr, "usage: %s num|env|envfield|control\n", argv[0]); rc = 2; } + eigs_close(st); + return rc; +}