From 8ef16de6229fd6dee78dc013f8ae4c5b31d7e2fc Mon Sep 17 00:00:00 2001 From: Taylor Holliday Date: Fri, 28 Aug 2026 17:09:32 -0700 Subject: [PATCH 1/2] Give the stack VM real f32x4 SIMD instead of scalarizing it (#67) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stack VM was the only backend with no vector support for f32x4. Every vector op was lowered to four per-lane load/op/store trips through the float window, materialized into a fresh 16-byte temp, and copied back into the destination — 69 dispatches for `acc = acc * k + one`, which made f32x4 5x slower than writing the same math with four scalar f32s. Add fourteen f32x4 opcodes in two families. The plain forms name their destination frame slot in the immediate and push its address, for use in expression position. The `*Store` forms pop the destination address off the int window and push nothing, so an assignment, a `var` initializer, a `return`, or an expression-bodied lambda computes into its destination with no temp slot and no memory.copy 16. The handlers do the arithmetic on a `vector_size(16)` type, going through memcpy in both directions: alloc_memory rounds frame allocations to 8-byte slots, so a 16-byte f32x4 is only 8-byte aligned and the loads have to be unaligned. Clang emits `ldr q / ldr q / fmul.4s / str q`. Operands are read into registers before the store, so a destination that aliases an operand (`v = v * v`) is fine. 30M iterations of `acc = acc * k + one` on the stack VM: 1.58s -> 0.237s, with the loop body down from 69 dispatches to 11. The same math written as four scalar f32s takes 0.30s, so f32x4 is now the faster way to write it rather than the slower one. The vector TOS window (item 4 of the issue) is left alone: it needs the f64 window shrunk or overlaid on the f32 one first. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01XSicxo3USpAaDFu8o74mdX --- src/stack_codegen.rs | 211 +++++++++++++++------ src/stack_depth.rs | 21 ++ src/stack_interp.c | 163 ++++++++++++++++ src/stack_interp_bridge.rs | 39 ++++ src/stack_ir.rs | 53 ++++++ src/stack_rebase_lm.rs | 9 + tests/cases/simd/f32x4_store_forms.lyte | 71 +++++++ tests/cases/stack_ir/f32x4_vector_ops.lyte | 32 ++++ 8 files changed, 538 insertions(+), 61 deletions(-) create mode 100644 tests/cases/simd/f32x4_store_forms.lyte create mode 100644 tests/cases/stack_ir/f32x4_vector_ops.lyte diff --git a/src/stack_codegen.rs b/src/stack_codegen.rs index cbaee8a7..599aed0b 100644 --- a/src/stack_codegen.rs +++ b/src/stack_codegen.rs @@ -556,7 +556,20 @@ impl<'a> FunctionTranslator<'a> { // the last expression's value is left on the stack for Return. let returns_void_no_sret = !has_sret && matches!(&*self.decl.ret, Type::Void); if let Some(body) = self.decl.body { - if returns_void_no_sret { + // A body that is itself an f32x4 computation — a lambda's + // expression body, say — writes into the sret buffer directly, + // with no result temp and no 16-byte copy. + let sret_vector_body = match self.output_ptr_slot { + Some(slot) => self.f32x4_store_op(body).map(|op| (slot, op)), + None => None, + }; + if let Some((sret_slot, store_op)) = sret_vector_body { + self.emit_f32x4_operands(body, func); + func.emit(StackOp::LocalGet(sret_slot)); + func.emit(store_op); + func.emit(StackOp::ReturnVoid); + self.has_returned = true; + } else if returns_void_no_sret { self.translate_void(body, func); } else { self.translate_expr(body, func); @@ -626,8 +639,15 @@ impl<'a> FunctionTranslator<'a> { func.emit(StackOp::Drop); } } - // Assignment lowering already consults void_ctx and suppresses - // result materialization when the assigned value is dead. + // An f32x4 assignment in statement position: void context is + // what lets translate_assign send the vector ops straight at + // the destination instead of computing into a temp and copying + // 16 bytes over. Nothing is left on the stack to drop. + Expr::Binop(Binop::Assign, lhs_id, _) + if matches!(&*self.expr_type(*lhs_id), Type::Float32x4) => + { + self.translate_expr_inner(expr, func, true); + } // Block: recurse with void context for every expression, // including the last. Using translate_void for the last // expression lets value-producing constructs (If, For, While, @@ -921,6 +941,20 @@ impl<'a> FunctionTranslator<'a> { let size = self.vm_type_size(&ty); let mem_slot = self.alloc_memory(size); if let Some(init_id) = init { + // An f32x4 initializer computes into the variable's + // own storage — no temp, no 16-byte copy. + if let Some(store_op) = self.f32x4_store_op(init_id) { + self.emit_f32x4_operands(init_id, func); + func.emit(StackOp::LocalAddr(mem_slot)); + func.emit(store_op); + self.shadow_outer_binding(&name); + self.variables.insert(name, LocalKind::Memory(mem_slot)); + self.variable_types.insert(name, ty); + if !self.void_ctx { + func.emit(StackOp::I64Const(0)); + } + return; + } self.translate_expr(init_id, func); self.emit_wrap_for_expected_slice(ty, init_id, func); let tmp = self.alloc_scalar(); @@ -993,6 +1027,19 @@ impl<'a> FunctionTranslator<'a> { Expr::Return(expr_id) => { let expr_id = *expr_id; let ret_ty = self.expr_type(expr_id); + + // An f32x4 result computes straight into the sret buffer. + if let Some(sret_slot) = self.output_ptr_slot { + if let Some(store_op) = self.f32x4_store_op(expr_id) { + self.emit_f32x4_operands(expr_id, func); + func.emit(StackOp::LocalGet(sret_slot)); + func.emit(store_op); + func.emit(StackOp::ReturnVoid); + self.has_returned = true; + return; + } + } + self.translate_expr(expr_id, func); if returns_via_pointer(ret_ty) { @@ -1233,6 +1280,78 @@ impl<'a> FunctionTranslator<'a> { func.emit(StackOp::I64Const(0)); } + /// The store-form vector op for an f32x4-producing expression, or + /// `None` if this isn't an expression the f32x4 ops can compute + /// straight into a caller-supplied destination. + /// + /// Callers pair this with [`Self::emit_f32x4_operands`]: emit the + /// operands, push the destination address, then emit this op. That + /// writes the result into the destination directly, skipping the + /// temporary frame slot and the 16-byte copy the generic + /// pointer-represented path would otherwise need. + fn f32x4_store_op(&self, expr: ExprID) -> Option { + if !matches!(&*self.expr_type(expr), Type::Float32x4) { + return None; + } + match &self.decl.arena.exprs[expr] { + Expr::Binop(op, lhs_id, _) => { + if !matches!(&*self.expr_type(*lhs_id), Type::Float32x4) { + return None; + } + match op { + Binop::Plus => Some(StackOp::F32x4AddStore), + Binop::Minus => Some(StackOp::F32x4SubStore), + Binop::Mult => Some(StackOp::F32x4MulStore), + Binop::Div => Some(StackOp::F32x4DivStore), + _ => None, + } + } + Expr::Unop(Unop::Neg, arg_id) => { + if matches!(&*self.expr_type(*arg_id), Type::Float32x4) { + Some(StackOp::F32x4NegStore) + } else { + None + } + } + Expr::Call(fn_id, arg_ids) => { + if self.holds_fat_pointer(*fn_id) { + return None; + } + let Expr::Id(name) = &self.decl.arena.exprs[*fn_id] else { + return None; + }; + match (name.as_str(), arg_ids.len()) { + ("f32x4", 4) => Some(StackOp::F32x4BuildStore), + ("f32x4_splat", 1) => Some(StackOp::F32x4SplatStore), + _ => None, + } + } + _ => None, + } + } + + /// Push the operands of an expression [`Self::f32x4_store_op`] + /// accepted, leaving the destination address to the caller. + fn emit_f32x4_operands(&mut self, expr: ExprID, func: &mut StackFunction) { + match &self.decl.arena.exprs[expr] { + Expr::Binop(_, lhs_id, rhs_id) => { + let (lhs_id, rhs_id) = (*lhs_id, *rhs_id); + self.translate_expr(lhs_id, func); + self.translate_expr(rhs_id, func); + } + Expr::Unop(_, arg_id) => { + let arg_id = *arg_id; + self.translate_expr(arg_id, func); + } + Expr::Call(_, arg_ids) => { + for arg_id in arg_ids.clone() { + self.translate_expr(arg_id, func); + } + } + _ => unreachable!("emit_f32x4_operands on a non-vector expression"), + } + } + /// Translate a binary operation. fn translate_binop( &mut self, @@ -1250,43 +1369,19 @@ impl<'a> FunctionTranslator<'a> { let ty = self.expr_type(lhs_id); - // f32x4 SIMD ops — emit element-wise using the F-window. + // f32x4 SIMD ops — one vector instruction, result in a fresh + // 16-byte frame slot whose address is left on the stack. if matches!(&*ty, Type::Float32x4) { self.translate_expr(lhs_id, func); self.translate_expr(rhs_id, func); let mem_slot = self.alloc_memory(16); - let lhs_local = self.alloc_scalar(); - let rhs_local = self.alloc_scalar(); - func.emit(StackOp::LocalSet(rhs_local)); - func.emit(StackOp::LocalSet(lhs_local)); - let fop = match op { - Binop::Plus => StackOp::FAddF, - Binop::Minus => StackOp::FSubF, - Binop::Mult => StackOp::FMulF, - Binop::Div => StackOp::FDivF, + func.emit(match op { + Binop::Plus => StackOp::F32x4Add(mem_slot), + Binop::Minus => StackOp::F32x4Sub(mem_slot), + Binop::Mult => StackOp::F32x4Mul(mem_slot), + Binop::Div => StackOp::F32x4Div(mem_slot), _ => panic!("unsupported f32x4 binop: {:?}", op), - }; - for lane in 0..4i32 { - let off = lane * 4; - func.emit(StackOp::LocalGet(lhs_local)); - func.emit(StackOp::LoadF32OffF(off)); - func.emit(StackOp::LocalGet(rhs_local)); - func.emit(StackOp::LoadF32OffF(off)); - func.emit(fop.clone()); - // Store the f32 result through the F-window to - // locals[mem_slot + off] — the StoreF32OffF handler - // pops an address from the int TOS and a float from - // the F TOS, so push the address first. - func.emit(StackOp::LocalAddr(mem_slot)); - // We need the address BELOW the value for StoreF32OffF. - // The F-window value is on top of f-window; we pushed - // the address AFTER the arithmetic, which means the - // order is (int addr, f-window value). StoreF32OffF - // pops both independently from their windows, so - // order doesn't matter across windows. - func.emit(StackOp::StoreF32OffF(off)); - } - func.emit(StackOp::LocalAddr(mem_slot)); + }); return; } @@ -1515,6 +1610,18 @@ impl<'a> FunctionTranslator<'a> { } } + // f32x4 assignment: compute the vector straight into the + // destination, skipping the temp slot and the 16-byte copy the + // generic path below would emit. + if self.void_ctx { + if let Some(store_op) = self.f32x4_store_op(rhs_id) { + self.emit_f32x4_operands(rhs_id, func); + self.translate_lvalue(lhs_id, func); + func.emit(store_op); + return; + } + } + // General assignment: compute rhs, compute lvalue address, store. // Optimization: if RHS is a simple local variable, reuse it directly // instead of creating a temp (avoids get_set + get pattern). @@ -1730,21 +1837,11 @@ impl<'a> FunctionTranslator<'a> { fn translate_unop(&mut self, op: Unop, arg_id: ExprID, func: &mut StackFunction) { let ty = self.expr_type(arg_id); - // f32x4 negation — element-wise via the F-window. + // f32x4 negation — one vector instruction. if op == Unop::Neg && matches!(&*ty, Type::Float32x4) { self.translate_expr(arg_id, func); - let src_local = self.alloc_scalar(); - func.emit(StackOp::LocalSet(src_local)); let mem_slot = self.alloc_memory(16); - for lane in 0..4i32 { - let off = lane * 4; - func.emit(StackOp::LocalGet(src_local)); - func.emit(StackOp::LoadF32OffF(off)); - func.emit(StackOp::FNegF); - func.emit(StackOp::LocalAddr(mem_slot)); - func.emit(StackOp::StoreF32OffF(off)); - } - func.emit(StackOp::LocalAddr(mem_slot)); + func.emit(StackOp::F32x4Neg(mem_slot)); return; } @@ -1814,30 +1911,22 @@ impl<'a> FunctionTranslator<'a> { return; } - // f32x4 constructor. Each arg is an f32 in the float window. + // f32x4 constructor. The four lanes are pushed onto the float + // window in order, and one op packs them into a frame slot. if *name == "f32x4" && arg_ids.len() == 4 { - let mem_slot = self.alloc_memory(16); - for (i, arg_id) in arg_ids.iter().enumerate() { - func.emit(StackOp::LocalAddr(mem_slot)); + for arg_id in arg_ids.iter() { self.translate_expr(*arg_id, func); - func.emit(StackOp::StoreF32OffF((i * 4) as i32)); } - func.emit(StackOp::LocalAddr(mem_slot)); + let mem_slot = self.alloc_memory(16); + func.emit(StackOp::F32x4Build(mem_slot)); return; } // f32x4_splat. Arg is an f32 in the float window. if *name == "f32x4_splat" && arg_ids.len() == 1 { self.translate_expr(arg_ids[0], func); - let val_local = self.alloc_scalar(); - func.emit(StackOp::LocalSetF(val_local)); let mem_slot = self.alloc_memory(16); - for i in 0..4 { - func.emit(StackOp::LocalAddr(mem_slot)); - func.emit(StackOp::LocalGetF(val_local)); - func.emit(StackOp::StoreF32OffF((i * 4) as i32)); - } - func.emit(StackOp::LocalAddr(mem_slot)); + func.emit(StackOp::F32x4Splat(mem_slot)); return; } diff --git a/src/stack_depth.rs b/src/stack_depth.rs index 62bc8d18..2de27ab5 100644 --- a/src/stack_depth.rs +++ b/src/stack_depth.rs @@ -400,6 +400,23 @@ pub fn stack_delta(op: &StackOp) -> i32 { | StackOp::FusedGetSet8D(_) | StackOp::FusedF64ConstDGtJumpIfZeroD(_, _) | StackOp::FusedGetF64ConstDGtJumpIfZeroD(_, _, _) => 0, + + // f32x4: the plain forms pop their operand addresses and push the + // destination slot's address; the constructors take their lanes + // from the float window, so they only push. The `*Store` forms + // pop a destination address too and push nothing. + StackOp::F32x4Add(_) + | StackOp::F32x4Sub(_) + | StackOp::F32x4Mul(_) + | StackOp::F32x4Div(_) => -1, + StackOp::F32x4Neg(_) => 0, + StackOp::F32x4Build(_) | StackOp::F32x4Splat(_) => 1, + StackOp::F32x4AddStore + | StackOp::F32x4SubStore + | StackOp::F32x4MulStore + | StackOp::F32x4DivStore => -3, + StackOp::F32x4NegStore => -2, + StackOp::F32x4BuildStore | StackOp::F32x4SplatStore => -1, } } @@ -622,6 +639,10 @@ pub fn float_stack_delta(op: &StackOp) -> i32 { // f-window loads push f0 StackOp::LoadF32F | StackOp::LoadF32OffF(_) => 1, + // f32x4 constructors take their lanes off the float window. + StackOp::F32x4Build(_) | StackOp::F32x4BuildStore => -4, + StackOp::F32x4Splat(_) | StackOp::F32x4SplatStore => -1, + _ => 0, } } diff --git a/src/stack_interp.c b/src/stack_interp.c index 08bb3767..f0316008 100644 --- a/src/stack_interp.c +++ b/src/stack_interp.c @@ -113,6 +113,22 @@ static inline void store_f64_unaligned(void* p, double v) { memcpy(p, &v, sizeof(v)); } +// f32x4 lives in 16 bytes of frame memory. alloc_memory rounds frame +// allocations to 8-byte slots, so those 16 bytes are only 8-byte aligned: +// go through memcpy so the compiler emits an unaligned vector load/store +// (`ldur q` on aarch64, `movups` on x86-64) rather than assuming 16. +typedef float v4f __attribute__((vector_size(16))); + +static inline v4f load_v4f(const void* p) { + v4f v; + memcpy(&v, p, sizeof(v)); + return v; +} + +static inline void store_v4f(void* p, v4f v) { + memcpy(p, &v, sizeof(v)); +} + // ============================================================================ // Integer power // ============================================================================ @@ -219,6 +235,13 @@ static int64_t ipow(int64_t base, uint32_t exp) { } while(0) #define FDROP1() do { f0 = f1; f1 = f2; f2 = f3; f3 = *--fsp; } while(0) + +// Drop 4 floats (the whole window) — the f32x4 constructors consume all +// four lanes at once, so the window refills entirely from the spill area. +#define FDROP4() do { \ + f0 = *(fsp - 1); f1 = *(fsp - 2); f2 = *(fsp - 3); f3 = *(fsp - 4); \ + fsp -= 4; \ +} while(0) #define FBINOP_SHIFT() do { f1 = f2; f2 = f3; f3 = *--fsp; } while(0) // f64 TOS window push/pop — exact mirror of the f32 window above, but @@ -2318,6 +2341,146 @@ HANDLER(op_fused_get_f64const_dgt_jiz_d) { NEXT(); } +// ============================================================================ +// f32x4 SIMD +// ============================================================================ +// +// An f32x4 travels as an address in the int window, like every other +// pointer-represented type. Each op loads whole 16-byte vectors, does the +// arithmetic on a `v4f`, and stores the result — one SIMD instruction per +// operation instead of the four per-lane load/op/store trips through the +// float window that the scalarized path emitted. +// +// The plain forms name their destination frame slot in imm[0] and push its +// address (expression position); the `*Store` forms take the destination +// address off the top of the int window and push nothing, so an assignment +// writes straight into its destination with no temp and no 16-byte copy. +// The operands are read into registers before the store, so a destination +// aliasing either operand (`v = v * k`) is fine. + +HANDLER(op_f32x4_add) { + v4f a = load_v4f((const void*)t1); + v4f b = load_v4f((const void*)t0); + void* dst = (void*)(locals + pc->imm[0]); + store_v4f(dst, a + b); + t0 = (uint64_t)dst; + BINOP_SHIFT(); + NEXT(); +} + +HANDLER(op_f32x4_sub) { + v4f a = load_v4f((const void*)t1); + v4f b = load_v4f((const void*)t0); + void* dst = (void*)(locals + pc->imm[0]); + store_v4f(dst, a - b); + t0 = (uint64_t)dst; + BINOP_SHIFT(); + NEXT(); +} + +HANDLER(op_f32x4_mul) { + v4f a = load_v4f((const void*)t1); + v4f b = load_v4f((const void*)t0); + void* dst = (void*)(locals + pc->imm[0]); + store_v4f(dst, a * b); + t0 = (uint64_t)dst; + BINOP_SHIFT(); + NEXT(); +} + +HANDLER(op_f32x4_div) { + v4f a = load_v4f((const void*)t1); + v4f b = load_v4f((const void*)t0); + void* dst = (void*)(locals + pc->imm[0]); + store_v4f(dst, a / b); + t0 = (uint64_t)dst; + BINOP_SHIFT(); + NEXT(); +} + +HANDLER(op_f32x4_neg) { + v4f a = load_v4f((const void*)t0); + void* dst = (void*)(locals + pc->imm[0]); + store_v4f(dst, -a); + t0 = (uint64_t)dst; + NEXT(); +} + +// Lanes are pushed 0,1,2,3, so f0 holds lane 3 and f3 holds lane 0. +HANDLER(op_f32x4_build) { + v4f v = { f3, f2, f1, f0 }; + void* dst = (void*)(locals + pc->imm[0]); + store_v4f(dst, v); + FDROP4(); + PUSH((uint64_t)dst); + NEXT(); +} + +HANDLER(op_f32x4_splat) { + v4f v = { f0, f0, f0, f0 }; + void* dst = (void*)(locals + pc->imm[0]); + store_v4f(dst, v); + FDROP1(); + PUSH((uint64_t)dst); + NEXT(); +} + +// Store forms: t0 = destination address, t1 = b, t2 = a. +HANDLER(op_f32x4_add_store) { + v4f a = load_v4f((const void*)t2); + v4f b = load_v4f((const void*)t1); + store_v4f((void*)t0, a + b); + DROP3(); + NEXT(); +} + +HANDLER(op_f32x4_sub_store) { + v4f a = load_v4f((const void*)t2); + v4f b = load_v4f((const void*)t1); + store_v4f((void*)t0, a - b); + DROP3(); + NEXT(); +} + +HANDLER(op_f32x4_mul_store) { + v4f a = load_v4f((const void*)t2); + v4f b = load_v4f((const void*)t1); + store_v4f((void*)t0, a * b); + DROP3(); + NEXT(); +} + +HANDLER(op_f32x4_div_store) { + v4f a = load_v4f((const void*)t2); + v4f b = load_v4f((const void*)t1); + store_v4f((void*)t0, a / b); + DROP3(); + NEXT(); +} + +HANDLER(op_f32x4_neg_store) { + v4f a = load_v4f((const void*)t1); + store_v4f((void*)t0, -a); + DROP2(); + NEXT(); +} + +HANDLER(op_f32x4_build_store) { + v4f v = { f3, f2, f1, f0 }; + store_v4f((void*)t0, v); + FDROP4(); + DROP1(); + NEXT(); +} + +HANDLER(op_f32x4_splat_store) { + v4f v = { f0, f0, f0, f0 }; + store_v4f((void*)t0, v); + FDROP1(); + DROP1(); + NEXT(); +} + // ============================================================================ // Entry point // ============================================================================ diff --git a/src/stack_interp_bridge.rs b/src/stack_interp_bridge.rs index 726441e5..04d157e3 100644 --- a/src/stack_interp_bridge.rs +++ b/src/stack_interp_bridge.rs @@ -364,6 +364,20 @@ extern "C" { fn op_fused_get_set8_d(); fn op_fused_f64const_dgt_jiz_d(); fn op_fused_get_f64const_dgt_jiz_d(); + fn op_f32x4_add(); + fn op_f32x4_sub(); + fn op_f32x4_mul(); + fn op_f32x4_div(); + fn op_f32x4_neg(); + fn op_f32x4_build(); + fn op_f32x4_splat(); + fn op_f32x4_add_store(); + fn op_f32x4_sub_store(); + fn op_f32x4_mul_store(); + fn op_f32x4_div_store(); + fn op_f32x4_neg_store(); + fn op_f32x4_build_store(); + fn op_f32x4_splat_store(); } /// Get the C handler function pointer for a StackOp. @@ -678,6 +692,22 @@ fn handler_for(op: &StackOp) -> *const () { StackOp::FusedGetF64ConstDGtJumpIfZeroD(_, _, _) => { op_fused_get_f64const_dgt_jiz_d as *const () } + + // === f32x4 SIMD ops === + StackOp::F32x4Add(_) => op_f32x4_add as *const (), + StackOp::F32x4Sub(_) => op_f32x4_sub as *const (), + StackOp::F32x4Mul(_) => op_f32x4_mul as *const (), + StackOp::F32x4Div(_) => op_f32x4_div as *const (), + StackOp::F32x4Neg(_) => op_f32x4_neg as *const (), + StackOp::F32x4Build(_) => op_f32x4_build as *const (), + StackOp::F32x4Splat(_) => op_f32x4_splat as *const (), + StackOp::F32x4AddStore => op_f32x4_add_store as *const (), + StackOp::F32x4SubStore => op_f32x4_sub_store as *const (), + StackOp::F32x4MulStore => op_f32x4_mul_store as *const (), + StackOp::F32x4DivStore => op_f32x4_div_store as *const (), + StackOp::F32x4NegStore => op_f32x4_neg_store as *const (), + StackOp::F32x4BuildStore => op_f32x4_build_store as *const (), + StackOp::F32x4SplatStore => op_f32x4_splat_store as *const (), } } @@ -911,6 +941,15 @@ fn encode_imm(op: &StackOp, func_idx: u32) -> [u64; 3] { StackOp::FusedGetF64ConstDGtJumpIfZeroD(n, v, off) => { [(*n as u64) * 8, f64::to_bits(*v), *off as i64 as u64] } + // f32x4 ops name their destination frame slot. + StackOp::F32x4Add(d) + | StackOp::F32x4Sub(d) + | StackOp::F32x4Mul(d) + | StackOp::F32x4Div(d) + | StackOp::F32x4Neg(d) + | StackOp::F32x4Build(d) + | StackOp::F32x4Splat(d) => [*d as u64, 0, 0], + _ => [0, 0, 0], } } diff --git a/src/stack_ir.rs b/src/stack_ir.rs index 7081566e..d294fde5 100644 --- a/src/stack_ir.rs +++ b/src/stack_ir.rs @@ -598,6 +598,45 @@ pub enum StackOp { /// if !(locals[n] > const) jump. Pop 0, conditionally jump. FusedGetF64ConstDGtJumpIfZeroD(u16, f64, i32), + // === f32x4 SIMD ops === + // + // An f32x4 value is 16 bytes of frame memory referenced by its address + // in the int window, the same as any other pointer-represented type. + // These ops load whole vectors, do the arithmetic with a C vector type + // (one SIMD instruction), and store the result — replacing the four + // per-lane load/op/store sequences the F-window path emitted. + // + // The plain forms name a destination frame slot in the immediate and + // push its address, for use in expression position. The `*Store` forms + // pop the destination address off the int window and push nothing, so + // an assignment lands in its destination without a temp + 16-byte copy. + /// locals[dst] = a b, where b and a are addresses popped from the + /// int window. Pushes the address of `dst`. Pop 2, push 1. + F32x4Add(u16), + F32x4Sub(u16), + F32x4Mul(u16), + F32x4Div(u16), + /// locals[dst] = -a. Pop 1, push 1. + F32x4Neg(u16), + /// locals[dst] = the four lanes on the float window (lane 0 pushed + /// first, so f0 holds lane 3). Pops 4 from the float window, pushes + /// the destination address. + F32x4Build(u16), + /// locals[dst] = splat(f0). Pops 1 from the float window, pushes the + /// destination address. + F32x4Splat(u16), + /// Pop dst, b, a (dst on top); *dst = a b. Pop 3, push 0. + F32x4AddStore, + F32x4SubStore, + F32x4MulStore, + F32x4DivStore, + /// Pop dst, a; *dst = -a. Pop 2, push 0. + F32x4NegStore, + /// Pop dst from the int window and four lanes from the float window. + F32x4BuildStore, + /// Pop dst from the int window and one lane from the float window. + F32x4SplatStore, + Halt, Nop, } @@ -1296,6 +1335,20 @@ impl fmt::Display for StackOp { StackOp::FusedGetF64ConstDGtJumpIfZeroD(n, v, o) => { write!(f, "dw.fused.get_f64const_dgt_jiz {} {} {}", n, v, o) } + StackOp::F32x4Add(d) => write!(f, "f32x4.add {}", d), + StackOp::F32x4Sub(d) => write!(f, "f32x4.sub {}", d), + StackOp::F32x4Mul(d) => write!(f, "f32x4.mul {}", d), + StackOp::F32x4Div(d) => write!(f, "f32x4.div {}", d), + StackOp::F32x4Neg(d) => write!(f, "f32x4.neg {}", d), + StackOp::F32x4Build(d) => write!(f, "f32x4.build {}", d), + StackOp::F32x4Splat(d) => write!(f, "f32x4.splat {}", d), + StackOp::F32x4AddStore => write!(f, "f32x4.add_store"), + StackOp::F32x4SubStore => write!(f, "f32x4.sub_store"), + StackOp::F32x4MulStore => write!(f, "f32x4.mul_store"), + StackOp::F32x4DivStore => write!(f, "f32x4.div_store"), + StackOp::F32x4NegStore => write!(f, "f32x4.neg_store"), + StackOp::F32x4BuildStore => write!(f, "f32x4.build_store"), + StackOp::F32x4SplatStore => write!(f, "f32x4.splat_store"), StackOp::Halt => write!(f, "halt"), StackOp::Nop => write!(f, "nop"), } diff --git a/src/stack_rebase_lm.rs b/src/stack_rebase_lm.rs index 2a438bde..37218236 100644 --- a/src/stack_rebase_lm.rs +++ b/src/stack_rebase_lm.rs @@ -68,6 +68,15 @@ pub fn rebase(func: &mut StackFunction) { StackOp::FusedLocalArrayStore32F(s, _) => *s += lc, StackOp::FusedGetAddrFMulFAddF(_, s, _) => *s += lc, StackOp::FusedGetAddrFMulFSubF(_, s, _) => *s += lc, + + // f32x4 ops name their destination frame slot in the immediate. + StackOp::F32x4Add(s) + | StackOp::F32x4Sub(s) + | StackOp::F32x4Mul(s) + | StackOp::F32x4Div(s) + | StackOp::F32x4Neg(s) + | StackOp::F32x4Build(s) + | StackOp::F32x4Splat(s) => *s += lc, _ => {} } } diff --git a/tests/cases/simd/f32x4_store_forms.lyte b/tests/cases/simd/f32x4_store_forms.lyte new file mode 100644 index 00000000..17ed15c5 --- /dev/null +++ b/tests/cases/simd/f32x4_store_forms.lyte @@ -0,0 +1,71 @@ +// The VM-family backends compute an f32x4 straight into its destination +// rather than into a temp that is then copied over. That destination can +// alias an operand, and the four constructor lanes are all live at once — +// on the stack VM they fill the whole float window and spill. + +// expected stdout: +// compilation successful +// 9 +// 16 +// 25 +// 36 +// 4 +// 6 +// 1 +// 1 +// 6 +// 12 +// 18 +// 24 +// 5 +// 5 +// 2 +// 4 +// 25 + +scale(v: f32x4, k: f32x4) -> f32x4 { + return v * k +} + +apply(f: f32x4 -> f32x4, v: f32x4) -> f32x4 { f(v) } + +main { + // A destination that aliases both operands. + var v = f32x4(3.0, 4.0, 5.0, 6.0) + v = v * v + print(v[0] as i32) + print(v[1] as i32) + print(v[2] as i32) + print(v[3] as i32) + + // Lanes built from expressions, not constants. + var x = 2.0 + var w = f32x4(x + 2.0, x * 3.0, x - 1.0, x / 2.0) + print(w[0] as i32) + print(w[1] as i32) + print(w[2] as i32) + print(w[3] as i32) + + // Division, and a result returned through the sret pointer. + var q = scale(f32x4(12.0, 24.0, 36.0, 48.0), f32x4_splat(2.0)) / f32x4_splat(4.0) + print(q[0] as i32) + print(q[1] as i32) + print(q[2] as i32) + print(q[3] as i32) + + // Negation into a destination that aliases its operand. + var n = f32x4_splat(0.0 - 5.0) + n = -n + print(n[0] as i32) + print(n[3] as i32) + + // Splat of a computed value. + var s = f32x4_splat(x) + print(s[2] as i32) + + // A lambda body is the whole function body, so its vector result goes + // straight into the sret buffer with no fall-through copy. + var p = apply(| y | y * y, f32x4(2.0, 3.0, 4.0, 5.0)) + print(p[0] as i32) + print(p[3] as i32) +} diff --git a/tests/cases/stack_ir/f32x4_vector_ops.lyte b/tests/cases/stack_ir/f32x4_vector_ops.lyte new file mode 100644 index 00000000..8adc7887 --- /dev/null +++ b/tests/cases/stack_ir/f32x4_vector_ops.lyte @@ -0,0 +1,32 @@ +// Stack-IR check that f32x4 lowers to whole-vector ops rather than four +// per-lane trips through the float window, and that an assignment computes +// into its destination instead of a temp slot plus a 16-byte copy. +// args: --stack-ir +// expected stdout: +// fn main (params: 0, locals: 0, memory: 64 bytes): +// 0: fw.f32.const 1 +// 1: fw.f32.const 2 +// 2: fw.f32.const 3 +// 3: fw.f32.const 4 +// 4: local.addr 0 +// 5: f32x4.build_store +// 6: fw.f32.const 0.5 +// 7: local.addr 2 +// 8: f32x4.splat_store +// 9: local.addr 0 +// 10: local.addr 4 +// 11: f32x4.neg_store +// 12: local.addr 0 +// 13: local.addr 2 +// 14: f32x4.mul 6 +// 15: local.addr 4 +// 16: local.addr 0 +// 17: f32x4.add_store +// 18: return_void + +main { + var a = f32x4(1.0, 2.0, 3.0, 4.0) + var k = f32x4_splat(0.5) + var b = -a + a = a * k + b +} From 64a1b6dfca7cfa2c0d719d396e4768523a2503f6 Mon Sep 17 00:00:00 2001 From: Taylor Holliday Date: Fri, 28 Aug 2026 18:03:41 -0700 Subject: [PATCH 2/2] Fuse f32x4 into three-address and multiply-accumulate ops MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The vector ops added in the previous commit pass f32x4 values by address, so `acc = acc * k + one` cost six dispatches: two local.addr pushes and a multiply into a temp slot, then two more pushes and an add. But the stack VM is dispatch-bound — a micro-interpreter modelling its exact dispatch (same 18-arg preserve_none signature, same musttail chain) puts a vector load/store at well under the cost of reaching the next handler. So when every operand and the destination is a 16-byte frame slot, drop the addresses entirely and compute between the slots: F32x4{Add,Sub,Mul, Div}3 and F32x4Neg2 take their operands and destination as immediates and touch neither operand stack. Codegen resolves an f32x4 expression tree to slots, spilling any operand that isn't already a local into a temporary, so nested subtrees still work. Everything admitted is a pure read of a frame slot, which is what makes it safe to evaluate operands in any order and to let the destination alias any of them. On top of that, `a * b + c` (either operand order) and `a * b - c` fold into F32x4MulAddSet / F32x4MulSubSet. 30M iterations of `acc = acc * k + one`: 0.237s -> 0.148s, the loop body down from 15 dispatches to 10, with the whole vector computation in one instruction. Against the 1.58s this started at, 10.7x. The same math written as four scalar f32s takes 0.267s, so f32x4 is now the faster way to write it by a factor of 1.8. No movement on the scalar benchmarks. Also pin -ffp-contract=off on the interpreter. Left to clang's default, `a * b + c` contracts to a single fmla, rounding once where Cranelift rounds twice, so the stack VM returned different numbers than the JIT for the same source — and the stack VM is the backend shipped on iOS, so that is desktop-vs-device audio, not just a test concern. The stack VM also disagreed with itself: `a * b + c` diverged while `t = a * b; t + c` did not. Whether it happened at all depended on the host compiler's default, so the same source could behave differently depending on who built it. Backend agreement is worth more than the last ulp here: the cost is ~6% on f32x4 multiply-accumulate (0.140s -> 0.148s) and nothing measurable elsewhere, and tests/cases/simd/f32x4_fma_rounding.lyte fails without the flag. Scalar f32 is a separate, pre-existing story and is left alone: both the LLVM backend and vm_arm64.S (which uses fmadd/fmsub by hand) still contract `(c + 0.0) + a * b`, where Cranelift and the register VM do not. The new test covers only f32x4, where all five backends agree, so it needs no skip directives. This is also why the vector TOS register window stays unbuilt. In the same model, a register window without fusion is worth 0.8%, and with fusion a further 8% — against making f32x4 a window value throughout codegen. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01XSicxo3USpAaDFu8o74mdX --- build.rs | 13 +++ src/stack_codegen.rs | 128 +++++++++++++++++++++ src/stack_depth.rs | 9 ++ src/stack_interp.c | 58 ++++++++++ src/stack_interp_bridge.rs | 23 ++++ src/stack_ir.rs | 28 +++++ src/stack_rebase_lm.rs | 18 +++ tests/cases/simd/f32x4_fma_rounding.lyte | 40 +++++++ tests/cases/simd/f32x4_three_address.lyte | 99 ++++++++++++++++ tests/cases/stack_ir/f32x4_vector_ops.lyte | 20 ++-- 10 files changed, 423 insertions(+), 13 deletions(-) create mode 100644 tests/cases/simd/f32x4_fma_rounding.lyte create mode 100644 tests/cases/simd/f32x4_three_address.lyte diff --git a/build.rs b/build.rs index c77562d3..286d984b 100644 --- a/build.rs +++ b/build.rs @@ -43,6 +43,19 @@ fn main() { // with a clang that doesn't understand preserve_none, the // build fails loudly instead of producing a subtly broken // interpreter. + // Round the multiply before the add, the way Cranelift and + // LLVM do. Left to clang's default, the fused arithmetic + // handlers (op_f32x4_muladd_set, op_fused_get_get_fmul_fadd_f, + // ...) contract into fmla/fmadd, which rounds once instead of + // twice — so the stack VM returns different numbers than the + // JIT for the same source, and the stack VM is the shipped iOS + // backend (see src/ffi.rs). Worse, whether it happens at all + // depends on the host compiler's default, so the same source + // can behave differently depending on who built it. Backend + // agreement is worth more here than the last ulp; the cost is + // ~6% on f32x4 multiply-accumulate and nothing measurable + // elsewhere. tests/cases/simd/f32x4_fma_rounding.lyte guards it. + .flag("-ffp-contract=off") .flag("-Werror=unknown-attributes") .compile("stack_interp"); println!("cargo:rustc-cfg=has_stack_interp"); diff --git a/src/stack_codegen.rs b/src/stack_codegen.rs index 599aed0b..438d8810 100644 --- a/src/stack_codegen.rs +++ b/src/stack_codegen.rs @@ -943,6 +943,16 @@ impl<'a> FunctionTranslator<'a> { if let Some(init_id) = init { // An f32x4 initializer computes into the variable's // own storage — no temp, no 16-byte copy. + if matches!(&*ty, Type::Float32x4) && self.f32x4_slot_form(init_id) { + self.emit_f32x4_into_slot(init_id, mem_slot, func); + self.shadow_outer_binding(&name); + self.variables.insert(name, LocalKind::Memory(mem_slot)); + self.variable_types.insert(name, ty); + if !self.void_ctx { + func.emit(StackOp::I64Const(0)); + } + return; + } if let Some(store_op) = self.f32x4_store_op(init_id) { self.emit_f32x4_operands(init_id, func); func.emit(StackOp::LocalAddr(mem_slot)); @@ -1330,6 +1340,116 @@ impl<'a> FunctionTranslator<'a> { } } + /// True if `expr` is an f32x4 computation the three-address vector ops + /// can evaluate entirely between frame slots: a memory-backed local, or + /// an arithmetic node whose operands are themselves in that shape. + /// + /// Everything admitted here is a pure read of a frame slot, so the + /// operands can be evaluated in any order and a destination that + /// aliases an operand is safe — the final op is the only write. + fn f32x4_slot_form(&self, expr: ExprID) -> bool { + if !matches!(&*self.expr_type(expr), Type::Float32x4) { + return false; + } + if self.get_memory_slot(expr).is_some() { + return true; + } + match &self.decl.arena.exprs[expr] { + Expr::Binop(op, lhs_id, rhs_id) => { + matches!(op, Binop::Plus | Binop::Minus | Binop::Mult | Binop::Div) + && self.f32x4_slot_form(*lhs_id) + && self.f32x4_slot_form(*rhs_id) + } + Expr::Unop(Unop::Neg, arg_id) => self.f32x4_slot_form(*arg_id), + _ => false, + } + } + + /// The operands of `expr` if it is an f32x4 multiplication. + fn f32x4_mul_operands(&self, expr: ExprID) -> Option<(ExprID, ExprID)> { + match &self.decl.arena.exprs[expr] { + Expr::Binop(Binop::Mult, lhs_id, rhs_id) + if matches!(&*self.expr_type(expr), Type::Float32x4) => + { + Some((*lhs_id, *rhs_id)) + } + _ => None, + } + } + + /// The frame slot holding `expr`, computing it into a fresh temp slot + /// first when it isn't already a memory-backed local. Only valid when + /// [`Self::f32x4_slot_form`] holds. + fn f32x4_operand_slot(&mut self, expr: ExprID, func: &mut StackFunction) -> u16 { + if let Some(slot) = self.get_memory_slot(expr) { + return slot; + } + let tmp = self.alloc_memory(16); + self.emit_f32x4_into_slot(expr, tmp, func); + tmp + } + + /// Emit `expr` computed into 16-byte frame slot `dst` using the + /// three-address vector ops. Only valid when [`Self::f32x4_slot_form`] + /// holds for `expr`. + fn emit_f32x4_into_slot(&mut self, expr: ExprID, dst: u16, func: &mut StackFunction) { + // A bare local: the caller wanted the value in `dst`, so copy it. + if let Some(src) = self.get_memory_slot(expr) { + if src != dst { + func.emit(StackOp::LocalAddr(dst)); + func.emit(StackOp::LocalAddr(src)); + func.emit(StackOp::MemCopy(16)); + } + return; + } + match &self.decl.arena.exprs[expr] { + Expr::Binop(op, lhs_id, rhs_id) => { + let (op, lhs_id, rhs_id) = (*op, *lhs_id, *rhs_id); + // `a * b + c`, `c + a * b` and `a * b - c` each collapse to + // one multiply-accumulate. + let mul_add = match op { + Binop::Plus => self + .f32x4_mul_operands(lhs_id) + .map(|(a, b)| (a, b, rhs_id, false)) + .or_else(|| { + self.f32x4_mul_operands(rhs_id) + .map(|(a, b)| (a, b, lhs_id, false)) + }), + Binop::Minus => self + .f32x4_mul_operands(lhs_id) + .map(|(a, b)| (a, b, rhs_id, true)), + _ => None, + }; + if let Some((a_id, b_id, c_id, is_sub)) = mul_add { + let a = self.f32x4_operand_slot(a_id, func); + let b = self.f32x4_operand_slot(b_id, func); + let c = self.f32x4_operand_slot(c_id, func); + func.emit(if is_sub { + StackOp::F32x4MulSubSet(a, b, c, dst) + } else { + StackOp::F32x4MulAddSet(a, b, c, dst) + }); + return; + } + let a = self.f32x4_operand_slot(lhs_id, func); + let b = self.f32x4_operand_slot(rhs_id, func); + func.emit(match op { + Binop::Plus => StackOp::F32x4Add3(a, b, dst), + Binop::Minus => StackOp::F32x4Sub3(a, b, dst), + Binop::Mult => StackOp::F32x4Mul3(a, b, dst), + Binop::Div => StackOp::F32x4Div3(a, b, dst), + _ => unreachable!("f32x4_slot_form admitted a non-arithmetic binop"), + }); + } + Expr::Unop(Unop::Neg, arg_id) => { + let arg_id = *arg_id; + let a = self.f32x4_operand_slot(arg_id, func); + func.emit(StackOp::F32x4Neg2(a, dst)); + } + _ => unreachable!("emit_f32x4_into_slot on a non-slot-form expression"), + } + } + /// Push the operands of an expression [`Self::f32x4_store_op`] /// accepted, leaving the destination address to the caller. fn emit_f32x4_operands(&mut self, expr: ExprID, func: &mut StackFunction) { @@ -1614,6 +1734,14 @@ impl<'a> FunctionTranslator<'a> { // destination, skipping the temp slot and the 16-byte copy the // generic path below would emit. if self.void_ctx { + // Destination and operands all in frame slots: the three-address + // ops compute between slots with nothing on the operand stack. + if let Some(dst) = self.get_memory_slot(lhs_id) { + if self.f32x4_slot_form(rhs_id) { + self.emit_f32x4_into_slot(rhs_id, dst, func); + return; + } + } if let Some(store_op) = self.f32x4_store_op(rhs_id) { self.emit_f32x4_operands(rhs_id, func); self.translate_lvalue(lhs_id, func); diff --git a/src/stack_depth.rs b/src/stack_depth.rs index 2de27ab5..1a345f5c 100644 --- a/src/stack_depth.rs +++ b/src/stack_depth.rs @@ -417,6 +417,15 @@ pub fn stack_delta(op: &StackOp) -> i32 { | StackOp::F32x4DivStore => -3, StackOp::F32x4NegStore => -2, StackOp::F32x4BuildStore | StackOp::F32x4SplatStore => -1, + + // Three-address forms read and write frame slots only. + StackOp::F32x4Add3(_, _, _) + | StackOp::F32x4Sub3(_, _, _) + | StackOp::F32x4Mul3(_, _, _) + | StackOp::F32x4Div3(_, _, _) + | StackOp::F32x4Neg2(_, _) + | StackOp::F32x4MulAddSet(_, _, _, _) + | StackOp::F32x4MulSubSet(_, _, _, _) => 0, } } diff --git a/src/stack_interp.c b/src/stack_interp.c index f0316008..dfa09dbe 100644 --- a/src/stack_interp.c +++ b/src/stack_interp.c @@ -2481,6 +2481,64 @@ HANDLER(op_f32x4_splat_store) { NEXT(); } +// Three-address forms: every operand and the destination is a frame slot, +// so nothing touches the operand stack. imm[2] of the multiply-accumulate +// ops packs c in the low half and dst in the high half. +// +// `a * b + c` contracts to a single fmla.4s. That makes these ops round +// once where a separate multiply and add would round twice — the same +// trade the scalar op_fused_get_get_fmul_fadd_f already makes. + +HANDLER(op_f32x4_add3) { + v4f a = load_v4f(locals + pc->imm[0]); + v4f b = load_v4f(locals + pc->imm[1]); + store_v4f(locals + pc->imm[2], a + b); + NEXT(); +} + +HANDLER(op_f32x4_sub3) { + v4f a = load_v4f(locals + pc->imm[0]); + v4f b = load_v4f(locals + pc->imm[1]); + store_v4f(locals + pc->imm[2], a - b); + NEXT(); +} + +HANDLER(op_f32x4_mul3) { + v4f a = load_v4f(locals + pc->imm[0]); + v4f b = load_v4f(locals + pc->imm[1]); + store_v4f(locals + pc->imm[2], a * b); + NEXT(); +} + +HANDLER(op_f32x4_div3) { + v4f a = load_v4f(locals + pc->imm[0]); + v4f b = load_v4f(locals + pc->imm[1]); + store_v4f(locals + pc->imm[2], a / b); + NEXT(); +} + +HANDLER(op_f32x4_neg2) { + v4f a = load_v4f(locals + pc->imm[0]); + store_v4f(locals + pc->imm[1], -a); + NEXT(); +} + +HANDLER(op_f32x4_muladd_set) { + v4f a = load_v4f(locals + pc->imm[0]); + v4f b = load_v4f(locals + pc->imm[1]); + v4f c = load_v4f(locals + (pc->imm[2] & 0xFFFFu)); + store_v4f(locals + (pc->imm[2] >> 16), a * b + c); + NEXT(); +} + +HANDLER(op_f32x4_mulsub_set) { + v4f a = load_v4f(locals + pc->imm[0]); + v4f b = load_v4f(locals + pc->imm[1]); + v4f c = load_v4f(locals + (pc->imm[2] & 0xFFFFu)); + store_v4f(locals + (pc->imm[2] >> 16), a * b - c); + NEXT(); +} + // ============================================================================ // Entry point // ============================================================================ diff --git a/src/stack_interp_bridge.rs b/src/stack_interp_bridge.rs index 04d157e3..e94acd24 100644 --- a/src/stack_interp_bridge.rs +++ b/src/stack_interp_bridge.rs @@ -378,6 +378,13 @@ extern "C" { fn op_f32x4_neg_store(); fn op_f32x4_build_store(); fn op_f32x4_splat_store(); + fn op_f32x4_add3(); + fn op_f32x4_sub3(); + fn op_f32x4_mul3(); + fn op_f32x4_div3(); + fn op_f32x4_neg2(); + fn op_f32x4_muladd_set(); + fn op_f32x4_mulsub_set(); } /// Get the C handler function pointer for a StackOp. @@ -708,6 +715,13 @@ fn handler_for(op: &StackOp) -> *const () { StackOp::F32x4NegStore => op_f32x4_neg_store as *const (), StackOp::F32x4BuildStore => op_f32x4_build_store as *const (), StackOp::F32x4SplatStore => op_f32x4_splat_store as *const (), + StackOp::F32x4Add3(_, _, _) => op_f32x4_add3 as *const (), + StackOp::F32x4Sub3(_, _, _) => op_f32x4_sub3 as *const (), + StackOp::F32x4Mul3(_, _, _) => op_f32x4_mul3 as *const (), + StackOp::F32x4Div3(_, _, _) => op_f32x4_div3 as *const (), + StackOp::F32x4Neg2(_, _) => op_f32x4_neg2 as *const (), + StackOp::F32x4MulAddSet(_, _, _, _) => op_f32x4_muladd_set as *const (), + StackOp::F32x4MulSubSet(_, _, _, _) => op_f32x4_mulsub_set as *const (), } } @@ -949,6 +963,15 @@ fn encode_imm(op: &StackOp, func_idx: u32) -> [u64; 3] { | StackOp::F32x4Neg(d) | StackOp::F32x4Build(d) | StackOp::F32x4Splat(d) => [*d as u64, 0, 0], + StackOp::F32x4Add3(a, b, d) + | StackOp::F32x4Sub3(a, b, d) + | StackOp::F32x4Mul3(a, b, d) + | StackOp::F32x4Div3(a, b, d) => [*a as u64, *b as u64, *d as u64], + StackOp::F32x4Neg2(a, d) => [*a as u64, *d as u64, 0], + // imm[2] packs c in the low half, dst in the high half. + StackOp::F32x4MulAddSet(a, b, c, d) | StackOp::F32x4MulSubSet(a, b, c, d) => { + [*a as u64, *b as u64, (*c as u64) | ((*d as u64) << 16)] + } _ => [0, 0, 0], } diff --git a/src/stack_ir.rs b/src/stack_ir.rs index d294fde5..22dc0c7c 100644 --- a/src/stack_ir.rs +++ b/src/stack_ir.rs @@ -637,6 +637,23 @@ pub enum StackOp { /// Pop dst from the int window and one lane from the float window. F32x4SplatStore, + // Three-address forms. When every operand and the destination is a + // 16-byte frame slot, the whole computation needs no addresses on the + // stack at all: these read and write `locals` directly and are the + // f32x4 analogue of FusedGetGetFMulSet and friends. Pop 0, push 0. + /// locals[dst] = locals[a] locals[b]. + F32x4Add3(u16, u16, u16), + F32x4Sub3(u16, u16, u16), + F32x4Mul3(u16, u16, u16), + F32x4Div3(u16, u16, u16), + /// locals[dst] = -locals[a]. + F32x4Neg2(u16, u16), + /// locals[dst] = locals[a] * locals[b] + locals[c] — the DSP workhorse, + /// and one instruction (`fmla.4s`) once the C compiler contracts it. + F32x4MulAddSet(u16, u16, u16, u16), + /// locals[dst] = locals[a] * locals[b] - locals[c]. + F32x4MulSubSet(u16, u16, u16, u16), + Halt, Nop, } @@ -1349,6 +1366,17 @@ impl fmt::Display for StackOp { StackOp::F32x4NegStore => write!(f, "f32x4.neg_store"), StackOp::F32x4BuildStore => write!(f, "f32x4.build_store"), StackOp::F32x4SplatStore => write!(f, "f32x4.splat_store"), + StackOp::F32x4Add3(a, b, d) => write!(f, "f32x4.add3 {} {} {}", a, b, d), + StackOp::F32x4Sub3(a, b, d) => write!(f, "f32x4.sub3 {} {} {}", a, b, d), + StackOp::F32x4Mul3(a, b, d) => write!(f, "f32x4.mul3 {} {} {}", a, b, d), + StackOp::F32x4Div3(a, b, d) => write!(f, "f32x4.div3 {} {} {}", a, b, d), + StackOp::F32x4Neg2(a, d) => write!(f, "f32x4.neg2 {} {}", a, d), + StackOp::F32x4MulAddSet(a, b, c, d) => { + write!(f, "f32x4.muladd_set {} {} {} {}", a, b, c, d) + } + StackOp::F32x4MulSubSet(a, b, c, d) => { + write!(f, "f32x4.mulsub_set {} {} {} {}", a, b, c, d) + } StackOp::Halt => write!(f, "halt"), StackOp::Nop => write!(f, "nop"), } diff --git a/src/stack_rebase_lm.rs b/src/stack_rebase_lm.rs index 37218236..e3feaa79 100644 --- a/src/stack_rebase_lm.rs +++ b/src/stack_rebase_lm.rs @@ -77,6 +77,24 @@ pub fn rebase(func: &mut StackFunction) { | StackOp::F32x4Neg(s) | StackOp::F32x4Build(s) | StackOp::F32x4Splat(s) => *s += lc, + StackOp::F32x4Add3(a, b, d) + | StackOp::F32x4Sub3(a, b, d) + | StackOp::F32x4Mul3(a, b, d) + | StackOp::F32x4Div3(a, b, d) => { + *a += lc; + *b += lc; + *d += lc; + } + StackOp::F32x4Neg2(a, d) => { + *a += lc; + *d += lc; + } + StackOp::F32x4MulAddSet(a, b, c, d) | StackOp::F32x4MulSubSet(a, b, c, d) => { + *a += lc; + *b += lc; + *c += lc; + *d += lc; + } _ => {} } } diff --git a/tests/cases/simd/f32x4_fma_rounding.lyte b/tests/cases/simd/f32x4_fma_rounding.lyte new file mode 100644 index 00000000..5e020f1e --- /dev/null +++ b/tests/cases/simd/f32x4_fma_rounding.lyte @@ -0,0 +1,40 @@ +// `a * b + c` must round the multiply before the add on every backend. If +// the C interpreter is built without -ffp-contract=off, clang contracts +// F32x4MulAddSet into a single fmla, which rounds once instead of twice — +// and the stack VM is the backend shipped on iOS, so an Audulus node would +// produce different audio there than on the desktop JIT. +// +// c cancels the *rounded* product exactly, so every lane is 0 only if the +// multiply rounded before the add. Under contraction the first case reads +// back the multiply's lost low bits instead. The second computes the same +// value without the multiply-accumulate op, and the two must agree. +// +// Scalar f32 is deliberately not covered here: the LLVM and asm backends +// both still contract op_fused_get_get_fmul_fadd_f's equivalent, which is +// a pre-existing divergence tracked separately. Every backend agrees on +// the f32x4 shapes below, so this test needs no skip directives. + +// expected stdout: +// compilation successful +// 0 +// 0 +// 0 +// 0 + +main { + var a = f32x4_splat(12345.678) + var b = f32x4_splat(98765.43) + var p = a * b + var c = f32x4_splat(0.0) - p + + // Contracted, this is exact(a*b) - round(a*b) rather than 0. + var fused = a * b + c + print(fused[0] as i32) + print(fused[3] as i32) + + // The same value computed without the multiply-accumulate op. + var t = a * b + var split = t + c + print(split[0] as i32) + print(split[3] as i32) +} diff --git a/tests/cases/simd/f32x4_three_address.lyte b/tests/cases/simd/f32x4_three_address.lyte new file mode 100644 index 00000000..0ff77cf7 --- /dev/null +++ b/tests/cases/simd/f32x4_three_address.lyte @@ -0,0 +1,99 @@ +// When an f32x4 computation's operands and destination are all frame slots, +// the VM-family backends evaluate it between those slots with nothing on the +// operand stack, and contract `a * b + c` into one multiply-accumulate. The +// destination is free to alias any operand, and nested subtrees land in +// temporaries first. + +// expected stdout: +// compilation successful +// 15 +// 45 +// 95 +// 165 +// 15 +// 45 +// 95 +// 165 +// 5 +// 35 +// 85 +// 155 +// 10 +// 20 +// 30 +// 40 +// 6 +// 6 +// 6 +// 6 +// 9 +// 18 +// 27 +// 36 +// 10 +// 10 +// 10 +// 10 +// -1 +// -2 +// -3 +// -4 +// 30 +// 105 +// 200 +// 315 +// 30 +// 105 +// 200 +// 315 + +main { + var a = f32x4(1.0, 2.0, 3.0, 4.0) + var b = f32x4(10.0, 20.0, 30.0, 40.0) + var c = f32x4(5.0, 5.0, 5.0, 5.0) + + // Multiply-accumulate, with the product on either side of the `+`. + var d = a * b + c + show(d) + var e = c + a * b + show(e) + + // Multiply-subtract. + var f = a * b - c + show(f) + + // A destination aliasing the accumulator, run twice. + var acc = f32x4_splat(0.0) + acc = a * c + acc + acc = a * c + acc + show(acc) + + // Destinations aliasing a multiplicand. + var g = f32x4(2.0, 2.0, 2.0, 2.0) + g = g * g + g + show(g) + + // Plain three-address forms: sub, div, neg. + var h = b - a + show(h) + var i = b / a + show(i) + var j = -a + show(j) + + // Nested subtrees on both sides need temporaries. + var k = (a + c) * (b - c) + show(k) + + // A whole-vector copy between locals. + var m = f32x4_splat(0.0) + m = k + show(m) +} + +show(v: f32x4) { + print(v[0] as i32) + print(v[1] as i32) + print(v[2] as i32) + print(v[3] as i32) +} diff --git a/tests/cases/stack_ir/f32x4_vector_ops.lyte b/tests/cases/stack_ir/f32x4_vector_ops.lyte index 8adc7887..27428b23 100644 --- a/tests/cases/stack_ir/f32x4_vector_ops.lyte +++ b/tests/cases/stack_ir/f32x4_vector_ops.lyte @@ -1,9 +1,10 @@ // Stack-IR check that f32x4 lowers to whole-vector ops rather than four -// per-lane trips through the float window, and that an assignment computes -// into its destination instead of a temp slot plus a 16-byte copy. +// per-lane trips through the float window. Operands that are all frame +// slots use the three-address forms, so nothing reaches the operand +// stack, and `a * k + b` contracts to a single multiply-accumulate. // args: --stack-ir // expected stdout: -// fn main (params: 0, locals: 0, memory: 64 bytes): +// fn main (params: 0, locals: 0, memory: 48 bytes): // 0: fw.f32.const 1 // 1: fw.f32.const 2 // 2: fw.f32.const 3 @@ -13,16 +14,9 @@ // 6: fw.f32.const 0.5 // 7: local.addr 2 // 8: f32x4.splat_store -// 9: local.addr 0 -// 10: local.addr 4 -// 11: f32x4.neg_store -// 12: local.addr 0 -// 13: local.addr 2 -// 14: f32x4.mul 6 -// 15: local.addr 4 -// 16: local.addr 0 -// 17: f32x4.add_store -// 18: return_void +// 9: f32x4.neg2 0 4 +// 10: f32x4.muladd_set 0 2 4 0 +// 11: return_void main { var a = f32x4(1.0, 2.0, 3.0, 4.0)