diff --git a/build.rs b/build.rs index c77562d3..286d984b 100644 --- a/build.rs +++ b/build.rs @@ -43,6 +43,19 @@ fn main() { // with a clang that doesn't understand preserve_none, the // build fails loudly instead of producing a subtly broken // interpreter. + // Round the multiply before the add, the way Cranelift and + // LLVM do. Left to clang's default, the fused arithmetic + // handlers (op_f32x4_muladd_set, op_fused_get_get_fmul_fadd_f, + // ...) contract into fmla/fmadd, which rounds once instead of + // twice — so the stack VM returns different numbers than the + // JIT for the same source, and the stack VM is the shipped iOS + // backend (see src/ffi.rs). Worse, whether it happens at all + // depends on the host compiler's default, so the same source + // can behave differently depending on who built it. Backend + // agreement is worth more here than the last ulp; the cost is + // ~6% on f32x4 multiply-accumulate and nothing measurable + // elsewhere. tests/cases/simd/f32x4_fma_rounding.lyte guards it. + .flag("-ffp-contract=off") .flag("-Werror=unknown-attributes") .compile("stack_interp"); println!("cargo:rustc-cfg=has_stack_interp"); diff --git a/src/stack_codegen.rs b/src/stack_codegen.rs index cbaee8a7..438d8810 100644 --- a/src/stack_codegen.rs +++ b/src/stack_codegen.rs @@ -556,7 +556,20 @@ impl<'a> FunctionTranslator<'a> { // the last expression's value is left on the stack for Return. let returns_void_no_sret = !has_sret && matches!(&*self.decl.ret, Type::Void); if let Some(body) = self.decl.body { - if returns_void_no_sret { + // A body that is itself an f32x4 computation — a lambda's + // expression body, say — writes into the sret buffer directly, + // with no result temp and no 16-byte copy. + let sret_vector_body = match self.output_ptr_slot { + Some(slot) => self.f32x4_store_op(body).map(|op| (slot, op)), + None => None, + }; + if let Some((sret_slot, store_op)) = sret_vector_body { + self.emit_f32x4_operands(body, func); + func.emit(StackOp::LocalGet(sret_slot)); + func.emit(store_op); + func.emit(StackOp::ReturnVoid); + self.has_returned = true; + } else if returns_void_no_sret { self.translate_void(body, func); } else { self.translate_expr(body, func); @@ -626,8 +639,15 @@ impl<'a> FunctionTranslator<'a> { func.emit(StackOp::Drop); } } - // Assignment lowering already consults void_ctx and suppresses - // result materialization when the assigned value is dead. + // An f32x4 assignment in statement position: void context is + // what lets translate_assign send the vector ops straight at + // the destination instead of computing into a temp and copying + // 16 bytes over. Nothing is left on the stack to drop. + Expr::Binop(Binop::Assign, lhs_id, _) + if matches!(&*self.expr_type(*lhs_id), Type::Float32x4) => + { + self.translate_expr_inner(expr, func, true); + } // Block: recurse with void context for every expression, // including the last. Using translate_void for the last // expression lets value-producing constructs (If, For, While, @@ -921,6 +941,30 @@ impl<'a> FunctionTranslator<'a> { let size = self.vm_type_size(&ty); let mem_slot = self.alloc_memory(size); if let Some(init_id) = init { + // An f32x4 initializer computes into the variable's + // own storage — no temp, no 16-byte copy. + if matches!(&*ty, Type::Float32x4) && self.f32x4_slot_form(init_id) { + self.emit_f32x4_into_slot(init_id, mem_slot, func); + self.shadow_outer_binding(&name); + self.variables.insert(name, LocalKind::Memory(mem_slot)); + self.variable_types.insert(name, ty); + if !self.void_ctx { + func.emit(StackOp::I64Const(0)); + } + return; + } + if let Some(store_op) = self.f32x4_store_op(init_id) { + self.emit_f32x4_operands(init_id, func); + func.emit(StackOp::LocalAddr(mem_slot)); + func.emit(store_op); + self.shadow_outer_binding(&name); + self.variables.insert(name, LocalKind::Memory(mem_slot)); + self.variable_types.insert(name, ty); + if !self.void_ctx { + func.emit(StackOp::I64Const(0)); + } + return; + } self.translate_expr(init_id, func); self.emit_wrap_for_expected_slice(ty, init_id, func); let tmp = self.alloc_scalar(); @@ -993,6 +1037,19 @@ impl<'a> FunctionTranslator<'a> { Expr::Return(expr_id) => { let expr_id = *expr_id; let ret_ty = self.expr_type(expr_id); + + // An f32x4 result computes straight into the sret buffer. + if let Some(sret_slot) = self.output_ptr_slot { + if let Some(store_op) = self.f32x4_store_op(expr_id) { + self.emit_f32x4_operands(expr_id, func); + func.emit(StackOp::LocalGet(sret_slot)); + func.emit(store_op); + func.emit(StackOp::ReturnVoid); + self.has_returned = true; + return; + } + } + self.translate_expr(expr_id, func); if returns_via_pointer(ret_ty) { @@ -1233,6 +1290,188 @@ impl<'a> FunctionTranslator<'a> { func.emit(StackOp::I64Const(0)); } + /// The store-form vector op for an f32x4-producing expression, or + /// `None` if this isn't an expression the f32x4 ops can compute + /// straight into a caller-supplied destination. + /// + /// Callers pair this with [`Self::emit_f32x4_operands`]: emit the + /// operands, push the destination address, then emit this op. That + /// writes the result into the destination directly, skipping the + /// temporary frame slot and the 16-byte copy the generic + /// pointer-represented path would otherwise need. + fn f32x4_store_op(&self, expr: ExprID) -> Option { + if !matches!(&*self.expr_type(expr), Type::Float32x4) { + return None; + } + match &self.decl.arena.exprs[expr] { + Expr::Binop(op, lhs_id, _) => { + if !matches!(&*self.expr_type(*lhs_id), Type::Float32x4) { + return None; + } + match op { + Binop::Plus => Some(StackOp::F32x4AddStore), + Binop::Minus => Some(StackOp::F32x4SubStore), + Binop::Mult => Some(StackOp::F32x4MulStore), + Binop::Div => Some(StackOp::F32x4DivStore), + _ => None, + } + } + Expr::Unop(Unop::Neg, arg_id) => { + if matches!(&*self.expr_type(*arg_id), Type::Float32x4) { + Some(StackOp::F32x4NegStore) + } else { + None + } + } + Expr::Call(fn_id, arg_ids) => { + if self.holds_fat_pointer(*fn_id) { + return None; + } + let Expr::Id(name) = &self.decl.arena.exprs[*fn_id] else { + return None; + }; + match (name.as_str(), arg_ids.len()) { + ("f32x4", 4) => Some(StackOp::F32x4BuildStore), + ("f32x4_splat", 1) => Some(StackOp::F32x4SplatStore), + _ => None, + } + } + _ => None, + } + } + + /// True if `expr` is an f32x4 computation the three-address vector ops + /// can evaluate entirely between frame slots: a memory-backed local, or + /// an arithmetic node whose operands are themselves in that shape. + /// + /// Everything admitted here is a pure read of a frame slot, so the + /// operands can be evaluated in any order and a destination that + /// aliases an operand is safe — the final op is the only write. + fn f32x4_slot_form(&self, expr: ExprID) -> bool { + if !matches!(&*self.expr_type(expr), Type::Float32x4) { + return false; + } + if self.get_memory_slot(expr).is_some() { + return true; + } + match &self.decl.arena.exprs[expr] { + Expr::Binop(op, lhs_id, rhs_id) => { + matches!(op, Binop::Plus | Binop::Minus | Binop::Mult | Binop::Div) + && self.f32x4_slot_form(*lhs_id) + && self.f32x4_slot_form(*rhs_id) + } + Expr::Unop(Unop::Neg, arg_id) => self.f32x4_slot_form(*arg_id), + _ => false, + } + } + + /// The operands of `expr` if it is an f32x4 multiplication. + fn f32x4_mul_operands(&self, expr: ExprID) -> Option<(ExprID, ExprID)> { + match &self.decl.arena.exprs[expr] { + Expr::Binop(Binop::Mult, lhs_id, rhs_id) + if matches!(&*self.expr_type(expr), Type::Float32x4) => + { + Some((*lhs_id, *rhs_id)) + } + _ => None, + } + } + + /// The frame slot holding `expr`, computing it into a fresh temp slot + /// first when it isn't already a memory-backed local. Only valid when + /// [`Self::f32x4_slot_form`] holds. + fn f32x4_operand_slot(&mut self, expr: ExprID, func: &mut StackFunction) -> u16 { + if let Some(slot) = self.get_memory_slot(expr) { + return slot; + } + let tmp = self.alloc_memory(16); + self.emit_f32x4_into_slot(expr, tmp, func); + tmp + } + + /// Emit `expr` computed into 16-byte frame slot `dst` using the + /// three-address vector ops. Only valid when [`Self::f32x4_slot_form`] + /// holds for `expr`. + fn emit_f32x4_into_slot(&mut self, expr: ExprID, dst: u16, func: &mut StackFunction) { + // A bare local: the caller wanted the value in `dst`, so copy it. + if let Some(src) = self.get_memory_slot(expr) { + if src != dst { + func.emit(StackOp::LocalAddr(dst)); + func.emit(StackOp::LocalAddr(src)); + func.emit(StackOp::MemCopy(16)); + } + return; + } + match &self.decl.arena.exprs[expr] { + Expr::Binop(op, lhs_id, rhs_id) => { + let (op, lhs_id, rhs_id) = (*op, *lhs_id, *rhs_id); + // `a * b + c`, `c + a * b` and `a * b - c` each collapse to + // one multiply-accumulate. + let mul_add = match op { + Binop::Plus => self + .f32x4_mul_operands(lhs_id) + .map(|(a, b)| (a, b, rhs_id, false)) + .or_else(|| { + self.f32x4_mul_operands(rhs_id) + .map(|(a, b)| (a, b, lhs_id, false)) + }), + Binop::Minus => self + .f32x4_mul_operands(lhs_id) + .map(|(a, b)| (a, b, rhs_id, true)), + _ => None, + }; + if let Some((a_id, b_id, c_id, is_sub)) = mul_add { + let a = self.f32x4_operand_slot(a_id, func); + let b = self.f32x4_operand_slot(b_id, func); + let c = self.f32x4_operand_slot(c_id, func); + func.emit(if is_sub { + StackOp::F32x4MulSubSet(a, b, c, dst) + } else { + StackOp::F32x4MulAddSet(a, b, c, dst) + }); + return; + } + let a = self.f32x4_operand_slot(lhs_id, func); + let b = self.f32x4_operand_slot(rhs_id, func); + func.emit(match op { + Binop::Plus => StackOp::F32x4Add3(a, b, dst), + Binop::Minus => StackOp::F32x4Sub3(a, b, dst), + Binop::Mult => StackOp::F32x4Mul3(a, b, dst), + Binop::Div => StackOp::F32x4Div3(a, b, dst), + _ => unreachable!("f32x4_slot_form admitted a non-arithmetic binop"), + }); + } + Expr::Unop(Unop::Neg, arg_id) => { + let arg_id = *arg_id; + let a = self.f32x4_operand_slot(arg_id, func); + func.emit(StackOp::F32x4Neg2(a, dst)); + } + _ => unreachable!("emit_f32x4_into_slot on a non-slot-form expression"), + } + } + + /// Push the operands of an expression [`Self::f32x4_store_op`] + /// accepted, leaving the destination address to the caller. + fn emit_f32x4_operands(&mut self, expr: ExprID, func: &mut StackFunction) { + match &self.decl.arena.exprs[expr] { + Expr::Binop(_, lhs_id, rhs_id) => { + let (lhs_id, rhs_id) = (*lhs_id, *rhs_id); + self.translate_expr(lhs_id, func); + self.translate_expr(rhs_id, func); + } + Expr::Unop(_, arg_id) => { + let arg_id = *arg_id; + self.translate_expr(arg_id, func); + } + Expr::Call(_, arg_ids) => { + for arg_id in arg_ids.clone() { + self.translate_expr(arg_id, func); + } + } + _ => unreachable!("emit_f32x4_operands on a non-vector expression"), + } + } + /// Translate a binary operation. fn translate_binop( &mut self, @@ -1250,43 +1489,19 @@ impl<'a> FunctionTranslator<'a> { let ty = self.expr_type(lhs_id); - // f32x4 SIMD ops — emit element-wise using the F-window. + // f32x4 SIMD ops — one vector instruction, result in a fresh + // 16-byte frame slot whose address is left on the stack. if matches!(&*ty, Type::Float32x4) { self.translate_expr(lhs_id, func); self.translate_expr(rhs_id, func); let mem_slot = self.alloc_memory(16); - let lhs_local = self.alloc_scalar(); - let rhs_local = self.alloc_scalar(); - func.emit(StackOp::LocalSet(rhs_local)); - func.emit(StackOp::LocalSet(lhs_local)); - let fop = match op { - Binop::Plus => StackOp::FAddF, - Binop::Minus => StackOp::FSubF, - Binop::Mult => StackOp::FMulF, - Binop::Div => StackOp::FDivF, + func.emit(match op { + Binop::Plus => StackOp::F32x4Add(mem_slot), + Binop::Minus => StackOp::F32x4Sub(mem_slot), + Binop::Mult => StackOp::F32x4Mul(mem_slot), + Binop::Div => StackOp::F32x4Div(mem_slot), _ => panic!("unsupported f32x4 binop: {:?}", op), - }; - for lane in 0..4i32 { - let off = lane * 4; - func.emit(StackOp::LocalGet(lhs_local)); - func.emit(StackOp::LoadF32OffF(off)); - func.emit(StackOp::LocalGet(rhs_local)); - func.emit(StackOp::LoadF32OffF(off)); - func.emit(fop.clone()); - // Store the f32 result through the F-window to - // locals[mem_slot + off] — the StoreF32OffF handler - // pops an address from the int TOS and a float from - // the F TOS, so push the address first. - func.emit(StackOp::LocalAddr(mem_slot)); - // We need the address BELOW the value for StoreF32OffF. - // The F-window value is on top of f-window; we pushed - // the address AFTER the arithmetic, which means the - // order is (int addr, f-window value). StoreF32OffF - // pops both independently from their windows, so - // order doesn't matter across windows. - func.emit(StackOp::StoreF32OffF(off)); - } - func.emit(StackOp::LocalAddr(mem_slot)); + }); return; } @@ -1515,6 +1730,26 @@ impl<'a> FunctionTranslator<'a> { } } + // f32x4 assignment: compute the vector straight into the + // destination, skipping the temp slot and the 16-byte copy the + // generic path below would emit. + if self.void_ctx { + // Destination and operands all in frame slots: the three-address + // ops compute between slots with nothing on the operand stack. + if let Some(dst) = self.get_memory_slot(lhs_id) { + if self.f32x4_slot_form(rhs_id) { + self.emit_f32x4_into_slot(rhs_id, dst, func); + return; + } + } + if let Some(store_op) = self.f32x4_store_op(rhs_id) { + self.emit_f32x4_operands(rhs_id, func); + self.translate_lvalue(lhs_id, func); + func.emit(store_op); + return; + } + } + // General assignment: compute rhs, compute lvalue address, store. // Optimization: if RHS is a simple local variable, reuse it directly // instead of creating a temp (avoids get_set + get pattern). @@ -1730,21 +1965,11 @@ impl<'a> FunctionTranslator<'a> { fn translate_unop(&mut self, op: Unop, arg_id: ExprID, func: &mut StackFunction) { let ty = self.expr_type(arg_id); - // f32x4 negation — element-wise via the F-window. + // f32x4 negation — one vector instruction. if op == Unop::Neg && matches!(&*ty, Type::Float32x4) { self.translate_expr(arg_id, func); - let src_local = self.alloc_scalar(); - func.emit(StackOp::LocalSet(src_local)); let mem_slot = self.alloc_memory(16); - for lane in 0..4i32 { - let off = lane * 4; - func.emit(StackOp::LocalGet(src_local)); - func.emit(StackOp::LoadF32OffF(off)); - func.emit(StackOp::FNegF); - func.emit(StackOp::LocalAddr(mem_slot)); - func.emit(StackOp::StoreF32OffF(off)); - } - func.emit(StackOp::LocalAddr(mem_slot)); + func.emit(StackOp::F32x4Neg(mem_slot)); return; } @@ -1814,30 +2039,22 @@ impl<'a> FunctionTranslator<'a> { return; } - // f32x4 constructor. Each arg is an f32 in the float window. + // f32x4 constructor. The four lanes are pushed onto the float + // window in order, and one op packs them into a frame slot. if *name == "f32x4" && arg_ids.len() == 4 { - let mem_slot = self.alloc_memory(16); - for (i, arg_id) in arg_ids.iter().enumerate() { - func.emit(StackOp::LocalAddr(mem_slot)); + for arg_id in arg_ids.iter() { self.translate_expr(*arg_id, func); - func.emit(StackOp::StoreF32OffF((i * 4) as i32)); } - func.emit(StackOp::LocalAddr(mem_slot)); + let mem_slot = self.alloc_memory(16); + func.emit(StackOp::F32x4Build(mem_slot)); return; } // f32x4_splat. Arg is an f32 in the float window. if *name == "f32x4_splat" && arg_ids.len() == 1 { self.translate_expr(arg_ids[0], func); - let val_local = self.alloc_scalar(); - func.emit(StackOp::LocalSetF(val_local)); let mem_slot = self.alloc_memory(16); - for i in 0..4 { - func.emit(StackOp::LocalAddr(mem_slot)); - func.emit(StackOp::LocalGetF(val_local)); - func.emit(StackOp::StoreF32OffF((i * 4) as i32)); - } - func.emit(StackOp::LocalAddr(mem_slot)); + func.emit(StackOp::F32x4Splat(mem_slot)); return; } diff --git a/src/stack_depth.rs b/src/stack_depth.rs index 62bc8d18..1a345f5c 100644 --- a/src/stack_depth.rs +++ b/src/stack_depth.rs @@ -400,6 +400,32 @@ pub fn stack_delta(op: &StackOp) -> i32 { | StackOp::FusedGetSet8D(_) | StackOp::FusedF64ConstDGtJumpIfZeroD(_, _) | StackOp::FusedGetF64ConstDGtJumpIfZeroD(_, _, _) => 0, + + // f32x4: the plain forms pop their operand addresses and push the + // destination slot's address; the constructors take their lanes + // from the float window, so they only push. The `*Store` forms + // pop a destination address too and push nothing. + StackOp::F32x4Add(_) + | StackOp::F32x4Sub(_) + | StackOp::F32x4Mul(_) + | StackOp::F32x4Div(_) => -1, + StackOp::F32x4Neg(_) => 0, + StackOp::F32x4Build(_) | StackOp::F32x4Splat(_) => 1, + StackOp::F32x4AddStore + | StackOp::F32x4SubStore + | StackOp::F32x4MulStore + | StackOp::F32x4DivStore => -3, + StackOp::F32x4NegStore => -2, + StackOp::F32x4BuildStore | StackOp::F32x4SplatStore => -1, + + // Three-address forms read and write frame slots only. + StackOp::F32x4Add3(_, _, _) + | StackOp::F32x4Sub3(_, _, _) + | StackOp::F32x4Mul3(_, _, _) + | StackOp::F32x4Div3(_, _, _) + | StackOp::F32x4Neg2(_, _) + | StackOp::F32x4MulAddSet(_, _, _, _) + | StackOp::F32x4MulSubSet(_, _, _, _) => 0, } } @@ -622,6 +648,10 @@ pub fn float_stack_delta(op: &StackOp) -> i32 { // f-window loads push f0 StackOp::LoadF32F | StackOp::LoadF32OffF(_) => 1, + // f32x4 constructors take their lanes off the float window. + StackOp::F32x4Build(_) | StackOp::F32x4BuildStore => -4, + StackOp::F32x4Splat(_) | StackOp::F32x4SplatStore => -1, + _ => 0, } } diff --git a/src/stack_interp.c b/src/stack_interp.c index 08bb3767..dfa09dbe 100644 --- a/src/stack_interp.c +++ b/src/stack_interp.c @@ -113,6 +113,22 @@ static inline void store_f64_unaligned(void* p, double v) { memcpy(p, &v, sizeof(v)); } +// f32x4 lives in 16 bytes of frame memory. alloc_memory rounds frame +// allocations to 8-byte slots, so those 16 bytes are only 8-byte aligned: +// go through memcpy so the compiler emits an unaligned vector load/store +// (`ldur q` on aarch64, `movups` on x86-64) rather than assuming 16. +typedef float v4f __attribute__((vector_size(16))); + +static inline v4f load_v4f(const void* p) { + v4f v; + memcpy(&v, p, sizeof(v)); + return v; +} + +static inline void store_v4f(void* p, v4f v) { + memcpy(p, &v, sizeof(v)); +} + // ============================================================================ // Integer power // ============================================================================ @@ -219,6 +235,13 @@ static int64_t ipow(int64_t base, uint32_t exp) { } while(0) #define FDROP1() do { f0 = f1; f1 = f2; f2 = f3; f3 = *--fsp; } while(0) + +// Drop 4 floats (the whole window) — the f32x4 constructors consume all +// four lanes at once, so the window refills entirely from the spill area. +#define FDROP4() do { \ + f0 = *(fsp - 1); f1 = *(fsp - 2); f2 = *(fsp - 3); f3 = *(fsp - 4); \ + fsp -= 4; \ +} while(0) #define FBINOP_SHIFT() do { f1 = f2; f2 = f3; f3 = *--fsp; } while(0) // f64 TOS window push/pop — exact mirror of the f32 window above, but @@ -2318,6 +2341,204 @@ HANDLER(op_fused_get_f64const_dgt_jiz_d) { NEXT(); } +// ============================================================================ +// f32x4 SIMD +// ============================================================================ +// +// An f32x4 travels as an address in the int window, like every other +// pointer-represented type. Each op loads whole 16-byte vectors, does the +// arithmetic on a `v4f`, and stores the result — one SIMD instruction per +// operation instead of the four per-lane load/op/store trips through the +// float window that the scalarized path emitted. +// +// The plain forms name their destination frame slot in imm[0] and push its +// address (expression position); the `*Store` forms take the destination +// address off the top of the int window and push nothing, so an assignment +// writes straight into its destination with no temp and no 16-byte copy. +// The operands are read into registers before the store, so a destination +// aliasing either operand (`v = v * k`) is fine. + +HANDLER(op_f32x4_add) { + v4f a = load_v4f((const void*)t1); + v4f b = load_v4f((const void*)t0); + void* dst = (void*)(locals + pc->imm[0]); + store_v4f(dst, a + b); + t0 = (uint64_t)dst; + BINOP_SHIFT(); + NEXT(); +} + +HANDLER(op_f32x4_sub) { + v4f a = load_v4f((const void*)t1); + v4f b = load_v4f((const void*)t0); + void* dst = (void*)(locals + pc->imm[0]); + store_v4f(dst, a - b); + t0 = (uint64_t)dst; + BINOP_SHIFT(); + NEXT(); +} + +HANDLER(op_f32x4_mul) { + v4f a = load_v4f((const void*)t1); + v4f b = load_v4f((const void*)t0); + void* dst = (void*)(locals + pc->imm[0]); + store_v4f(dst, a * b); + t0 = (uint64_t)dst; + BINOP_SHIFT(); + NEXT(); +} + +HANDLER(op_f32x4_div) { + v4f a = load_v4f((const void*)t1); + v4f b = load_v4f((const void*)t0); + void* dst = (void*)(locals + pc->imm[0]); + store_v4f(dst, a / b); + t0 = (uint64_t)dst; + BINOP_SHIFT(); + NEXT(); +} + +HANDLER(op_f32x4_neg) { + v4f a = load_v4f((const void*)t0); + void* dst = (void*)(locals + pc->imm[0]); + store_v4f(dst, -a); + t0 = (uint64_t)dst; + NEXT(); +} + +// Lanes are pushed 0,1,2,3, so f0 holds lane 3 and f3 holds lane 0. +HANDLER(op_f32x4_build) { + v4f v = { f3, f2, f1, f0 }; + void* dst = (void*)(locals + pc->imm[0]); + store_v4f(dst, v); + FDROP4(); + PUSH((uint64_t)dst); + NEXT(); +} + +HANDLER(op_f32x4_splat) { + v4f v = { f0, f0, f0, f0 }; + void* dst = (void*)(locals + pc->imm[0]); + store_v4f(dst, v); + FDROP1(); + PUSH((uint64_t)dst); + NEXT(); +} + +// Store forms: t0 = destination address, t1 = b, t2 = a. +HANDLER(op_f32x4_add_store) { + v4f a = load_v4f((const void*)t2); + v4f b = load_v4f((const void*)t1); + store_v4f((void*)t0, a + b); + DROP3(); + NEXT(); +} + +HANDLER(op_f32x4_sub_store) { + v4f a = load_v4f((const void*)t2); + v4f b = load_v4f((const void*)t1); + store_v4f((void*)t0, a - b); + DROP3(); + NEXT(); +} + +HANDLER(op_f32x4_mul_store) { + v4f a = load_v4f((const void*)t2); + v4f b = load_v4f((const void*)t1); + store_v4f((void*)t0, a * b); + DROP3(); + NEXT(); +} + +HANDLER(op_f32x4_div_store) { + v4f a = load_v4f((const void*)t2); + v4f b = load_v4f((const void*)t1); + store_v4f((void*)t0, a / b); + DROP3(); + NEXT(); +} + +HANDLER(op_f32x4_neg_store) { + v4f a = load_v4f((const void*)t1); + store_v4f((void*)t0, -a); + DROP2(); + NEXT(); +} + +HANDLER(op_f32x4_build_store) { + v4f v = { f3, f2, f1, f0 }; + store_v4f((void*)t0, v); + FDROP4(); + DROP1(); + NEXT(); +} + +HANDLER(op_f32x4_splat_store) { + v4f v = { f0, f0, f0, f0 }; + store_v4f((void*)t0, v); + FDROP1(); + DROP1(); + NEXT(); +} + +// Three-address forms: every operand and the destination is a frame slot, +// so nothing touches the operand stack. imm[2] of the multiply-accumulate +// ops packs c in the low half and dst in the high half. +// +// `a * b + c` contracts to a single fmla.4s. That makes these ops round +// once where a separate multiply and add would round twice — the same +// trade the scalar op_fused_get_get_fmul_fadd_f already makes. + +HANDLER(op_f32x4_add3) { + v4f a = load_v4f(locals + pc->imm[0]); + v4f b = load_v4f(locals + pc->imm[1]); + store_v4f(locals + pc->imm[2], a + b); + NEXT(); +} + +HANDLER(op_f32x4_sub3) { + v4f a = load_v4f(locals + pc->imm[0]); + v4f b = load_v4f(locals + pc->imm[1]); + store_v4f(locals + pc->imm[2], a - b); + NEXT(); +} + +HANDLER(op_f32x4_mul3) { + v4f a = load_v4f(locals + pc->imm[0]); + v4f b = load_v4f(locals + pc->imm[1]); + store_v4f(locals + pc->imm[2], a * b); + NEXT(); +} + +HANDLER(op_f32x4_div3) { + v4f a = load_v4f(locals + pc->imm[0]); + v4f b = load_v4f(locals + pc->imm[1]); + store_v4f(locals + pc->imm[2], a / b); + NEXT(); +} + +HANDLER(op_f32x4_neg2) { + v4f a = load_v4f(locals + pc->imm[0]); + store_v4f(locals + pc->imm[1], -a); + NEXT(); +} + +HANDLER(op_f32x4_muladd_set) { + v4f a = load_v4f(locals + pc->imm[0]); + v4f b = load_v4f(locals + pc->imm[1]); + v4f c = load_v4f(locals + (pc->imm[2] & 0xFFFFu)); + store_v4f(locals + (pc->imm[2] >> 16), a * b + c); + NEXT(); +} + +HANDLER(op_f32x4_mulsub_set) { + v4f a = load_v4f(locals + pc->imm[0]); + v4f b = load_v4f(locals + pc->imm[1]); + v4f c = load_v4f(locals + (pc->imm[2] & 0xFFFFu)); + store_v4f(locals + (pc->imm[2] >> 16), a * b - c); + NEXT(); +} + // ============================================================================ // Entry point // ============================================================================ diff --git a/src/stack_interp_bridge.rs b/src/stack_interp_bridge.rs index 726441e5..e94acd24 100644 --- a/src/stack_interp_bridge.rs +++ b/src/stack_interp_bridge.rs @@ -364,6 +364,27 @@ extern "C" { fn op_fused_get_set8_d(); fn op_fused_f64const_dgt_jiz_d(); fn op_fused_get_f64const_dgt_jiz_d(); + fn op_f32x4_add(); + fn op_f32x4_sub(); + fn op_f32x4_mul(); + fn op_f32x4_div(); + fn op_f32x4_neg(); + fn op_f32x4_build(); + fn op_f32x4_splat(); + fn op_f32x4_add_store(); + fn op_f32x4_sub_store(); + fn op_f32x4_mul_store(); + fn op_f32x4_div_store(); + fn op_f32x4_neg_store(); + fn op_f32x4_build_store(); + fn op_f32x4_splat_store(); + fn op_f32x4_add3(); + fn op_f32x4_sub3(); + fn op_f32x4_mul3(); + fn op_f32x4_div3(); + fn op_f32x4_neg2(); + fn op_f32x4_muladd_set(); + fn op_f32x4_mulsub_set(); } /// Get the C handler function pointer for a StackOp. @@ -678,6 +699,29 @@ fn handler_for(op: &StackOp) -> *const () { StackOp::FusedGetF64ConstDGtJumpIfZeroD(_, _, _) => { op_fused_get_f64const_dgt_jiz_d as *const () } + + // === f32x4 SIMD ops === + StackOp::F32x4Add(_) => op_f32x4_add as *const (), + StackOp::F32x4Sub(_) => op_f32x4_sub as *const (), + StackOp::F32x4Mul(_) => op_f32x4_mul as *const (), + StackOp::F32x4Div(_) => op_f32x4_div as *const (), + StackOp::F32x4Neg(_) => op_f32x4_neg as *const (), + StackOp::F32x4Build(_) => op_f32x4_build as *const (), + StackOp::F32x4Splat(_) => op_f32x4_splat as *const (), + StackOp::F32x4AddStore => op_f32x4_add_store as *const (), + StackOp::F32x4SubStore => op_f32x4_sub_store as *const (), + StackOp::F32x4MulStore => op_f32x4_mul_store as *const (), + StackOp::F32x4DivStore => op_f32x4_div_store as *const (), + StackOp::F32x4NegStore => op_f32x4_neg_store as *const (), + StackOp::F32x4BuildStore => op_f32x4_build_store as *const (), + StackOp::F32x4SplatStore => op_f32x4_splat_store as *const (), + StackOp::F32x4Add3(_, _, _) => op_f32x4_add3 as *const (), + StackOp::F32x4Sub3(_, _, _) => op_f32x4_sub3 as *const (), + StackOp::F32x4Mul3(_, _, _) => op_f32x4_mul3 as *const (), + StackOp::F32x4Div3(_, _, _) => op_f32x4_div3 as *const (), + StackOp::F32x4Neg2(_, _) => op_f32x4_neg2 as *const (), + StackOp::F32x4MulAddSet(_, _, _, _) => op_f32x4_muladd_set as *const (), + StackOp::F32x4MulSubSet(_, _, _, _) => op_f32x4_mulsub_set as *const (), } } @@ -911,6 +955,24 @@ fn encode_imm(op: &StackOp, func_idx: u32) -> [u64; 3] { StackOp::FusedGetF64ConstDGtJumpIfZeroD(n, v, off) => { [(*n as u64) * 8, f64::to_bits(*v), *off as i64 as u64] } + // f32x4 ops name their destination frame slot. + StackOp::F32x4Add(d) + | StackOp::F32x4Sub(d) + | StackOp::F32x4Mul(d) + | StackOp::F32x4Div(d) + | StackOp::F32x4Neg(d) + | StackOp::F32x4Build(d) + | StackOp::F32x4Splat(d) => [*d as u64, 0, 0], + StackOp::F32x4Add3(a, b, d) + | StackOp::F32x4Sub3(a, b, d) + | StackOp::F32x4Mul3(a, b, d) + | StackOp::F32x4Div3(a, b, d) => [*a as u64, *b as u64, *d as u64], + StackOp::F32x4Neg2(a, d) => [*a as u64, *d as u64, 0], + // imm[2] packs c in the low half, dst in the high half. + StackOp::F32x4MulAddSet(a, b, c, d) | StackOp::F32x4MulSubSet(a, b, c, d) => { + [*a as u64, *b as u64, (*c as u64) | ((*d as u64) << 16)] + } + _ => [0, 0, 0], } } diff --git a/src/stack_ir.rs b/src/stack_ir.rs index 7081566e..22dc0c7c 100644 --- a/src/stack_ir.rs +++ b/src/stack_ir.rs @@ -598,6 +598,62 @@ pub enum StackOp { /// if !(locals[n] > const) jump. Pop 0, conditionally jump. FusedGetF64ConstDGtJumpIfZeroD(u16, f64, i32), + // === f32x4 SIMD ops === + // + // An f32x4 value is 16 bytes of frame memory referenced by its address + // in the int window, the same as any other pointer-represented type. + // These ops load whole vectors, do the arithmetic with a C vector type + // (one SIMD instruction), and store the result — replacing the four + // per-lane load/op/store sequences the F-window path emitted. + // + // The plain forms name a destination frame slot in the immediate and + // push its address, for use in expression position. The `*Store` forms + // pop the destination address off the int window and push nothing, so + // an assignment lands in its destination without a temp + 16-byte copy. + /// locals[dst] = a b, where b and a are addresses popped from the + /// int window. Pushes the address of `dst`. Pop 2, push 1. + F32x4Add(u16), + F32x4Sub(u16), + F32x4Mul(u16), + F32x4Div(u16), + /// locals[dst] = -a. Pop 1, push 1. + F32x4Neg(u16), + /// locals[dst] = the four lanes on the float window (lane 0 pushed + /// first, so f0 holds lane 3). Pops 4 from the float window, pushes + /// the destination address. + F32x4Build(u16), + /// locals[dst] = splat(f0). Pops 1 from the float window, pushes the + /// destination address. + F32x4Splat(u16), + /// Pop dst, b, a (dst on top); *dst = a b. Pop 3, push 0. + F32x4AddStore, + F32x4SubStore, + F32x4MulStore, + F32x4DivStore, + /// Pop dst, a; *dst = -a. Pop 2, push 0. + F32x4NegStore, + /// Pop dst from the int window and four lanes from the float window. + F32x4BuildStore, + /// Pop dst from the int window and one lane from the float window. + F32x4SplatStore, + + // Three-address forms. When every operand and the destination is a + // 16-byte frame slot, the whole computation needs no addresses on the + // stack at all: these read and write `locals` directly and are the + // f32x4 analogue of FusedGetGetFMulSet and friends. Pop 0, push 0. + /// locals[dst] = locals[a] locals[b]. + F32x4Add3(u16, u16, u16), + F32x4Sub3(u16, u16, u16), + F32x4Mul3(u16, u16, u16), + F32x4Div3(u16, u16, u16), + /// locals[dst] = -locals[a]. + F32x4Neg2(u16, u16), + /// locals[dst] = locals[a] * locals[b] + locals[c] — the DSP workhorse, + /// and one instruction (`fmla.4s`) once the C compiler contracts it. + F32x4MulAddSet(u16, u16, u16, u16), + /// locals[dst] = locals[a] * locals[b] - locals[c]. + F32x4MulSubSet(u16, u16, u16, u16), + Halt, Nop, } @@ -1296,6 +1352,31 @@ impl fmt::Display for StackOp { StackOp::FusedGetF64ConstDGtJumpIfZeroD(n, v, o) => { write!(f, "dw.fused.get_f64const_dgt_jiz {} {} {}", n, v, o) } + StackOp::F32x4Add(d) => write!(f, "f32x4.add {}", d), + StackOp::F32x4Sub(d) => write!(f, "f32x4.sub {}", d), + StackOp::F32x4Mul(d) => write!(f, "f32x4.mul {}", d), + StackOp::F32x4Div(d) => write!(f, "f32x4.div {}", d), + StackOp::F32x4Neg(d) => write!(f, "f32x4.neg {}", d), + StackOp::F32x4Build(d) => write!(f, "f32x4.build {}", d), + StackOp::F32x4Splat(d) => write!(f, "f32x4.splat {}", d), + StackOp::F32x4AddStore => write!(f, "f32x4.add_store"), + StackOp::F32x4SubStore => write!(f, "f32x4.sub_store"), + StackOp::F32x4MulStore => write!(f, "f32x4.mul_store"), + StackOp::F32x4DivStore => write!(f, "f32x4.div_store"), + StackOp::F32x4NegStore => write!(f, "f32x4.neg_store"), + StackOp::F32x4BuildStore => write!(f, "f32x4.build_store"), + StackOp::F32x4SplatStore => write!(f, "f32x4.splat_store"), + StackOp::F32x4Add3(a, b, d) => write!(f, "f32x4.add3 {} {} {}", a, b, d), + StackOp::F32x4Sub3(a, b, d) => write!(f, "f32x4.sub3 {} {} {}", a, b, d), + StackOp::F32x4Mul3(a, b, d) => write!(f, "f32x4.mul3 {} {} {}", a, b, d), + StackOp::F32x4Div3(a, b, d) => write!(f, "f32x4.div3 {} {} {}", a, b, d), + StackOp::F32x4Neg2(a, d) => write!(f, "f32x4.neg2 {} {}", a, d), + StackOp::F32x4MulAddSet(a, b, c, d) => { + write!(f, "f32x4.muladd_set {} {} {} {}", a, b, c, d) + } + StackOp::F32x4MulSubSet(a, b, c, d) => { + write!(f, "f32x4.mulsub_set {} {} {} {}", a, b, c, d) + } StackOp::Halt => write!(f, "halt"), StackOp::Nop => write!(f, "nop"), } diff --git a/src/stack_rebase_lm.rs b/src/stack_rebase_lm.rs index 2a438bde..e3feaa79 100644 --- a/src/stack_rebase_lm.rs +++ b/src/stack_rebase_lm.rs @@ -68,6 +68,33 @@ pub fn rebase(func: &mut StackFunction) { StackOp::FusedLocalArrayStore32F(s, _) => *s += lc, StackOp::FusedGetAddrFMulFAddF(_, s, _) => *s += lc, StackOp::FusedGetAddrFMulFSubF(_, s, _) => *s += lc, + + // f32x4 ops name their destination frame slot in the immediate. + StackOp::F32x4Add(s) + | StackOp::F32x4Sub(s) + | StackOp::F32x4Mul(s) + | StackOp::F32x4Div(s) + | StackOp::F32x4Neg(s) + | StackOp::F32x4Build(s) + | StackOp::F32x4Splat(s) => *s += lc, + StackOp::F32x4Add3(a, b, d) + | StackOp::F32x4Sub3(a, b, d) + | StackOp::F32x4Mul3(a, b, d) + | StackOp::F32x4Div3(a, b, d) => { + *a += lc; + *b += lc; + *d += lc; + } + StackOp::F32x4Neg2(a, d) => { + *a += lc; + *d += lc; + } + StackOp::F32x4MulAddSet(a, b, c, d) | StackOp::F32x4MulSubSet(a, b, c, d) => { + *a += lc; + *b += lc; + *c += lc; + *d += lc; + } _ => {} } } diff --git a/tests/cases/simd/f32x4_fma_rounding.lyte b/tests/cases/simd/f32x4_fma_rounding.lyte new file mode 100644 index 00000000..5e020f1e --- /dev/null +++ b/tests/cases/simd/f32x4_fma_rounding.lyte @@ -0,0 +1,40 @@ +// `a * b + c` must round the multiply before the add on every backend. If +// the C interpreter is built without -ffp-contract=off, clang contracts +// F32x4MulAddSet into a single fmla, which rounds once instead of twice — +// and the stack VM is the backend shipped on iOS, so an Audulus node would +// produce different audio there than on the desktop JIT. +// +// c cancels the *rounded* product exactly, so every lane is 0 only if the +// multiply rounded before the add. Under contraction the first case reads +// back the multiply's lost low bits instead. The second computes the same +// value without the multiply-accumulate op, and the two must agree. +// +// Scalar f32 is deliberately not covered here: the LLVM and asm backends +// both still contract op_fused_get_get_fmul_fadd_f's equivalent, which is +// a pre-existing divergence tracked separately. Every backend agrees on +// the f32x4 shapes below, so this test needs no skip directives. + +// expected stdout: +// compilation successful +// 0 +// 0 +// 0 +// 0 + +main { + var a = f32x4_splat(12345.678) + var b = f32x4_splat(98765.43) + var p = a * b + var c = f32x4_splat(0.0) - p + + // Contracted, this is exact(a*b) - round(a*b) rather than 0. + var fused = a * b + c + print(fused[0] as i32) + print(fused[3] as i32) + + // The same value computed without the multiply-accumulate op. + var t = a * b + var split = t + c + print(split[0] as i32) + print(split[3] as i32) +} diff --git a/tests/cases/simd/f32x4_store_forms.lyte b/tests/cases/simd/f32x4_store_forms.lyte new file mode 100644 index 00000000..17ed15c5 --- /dev/null +++ b/tests/cases/simd/f32x4_store_forms.lyte @@ -0,0 +1,71 @@ +// The VM-family backends compute an f32x4 straight into its destination +// rather than into a temp that is then copied over. That destination can +// alias an operand, and the four constructor lanes are all live at once — +// on the stack VM they fill the whole float window and spill. + +// expected stdout: +// compilation successful +// 9 +// 16 +// 25 +// 36 +// 4 +// 6 +// 1 +// 1 +// 6 +// 12 +// 18 +// 24 +// 5 +// 5 +// 2 +// 4 +// 25 + +scale(v: f32x4, k: f32x4) -> f32x4 { + return v * k +} + +apply(f: f32x4 -> f32x4, v: f32x4) -> f32x4 { f(v) } + +main { + // A destination that aliases both operands. + var v = f32x4(3.0, 4.0, 5.0, 6.0) + v = v * v + print(v[0] as i32) + print(v[1] as i32) + print(v[2] as i32) + print(v[3] as i32) + + // Lanes built from expressions, not constants. + var x = 2.0 + var w = f32x4(x + 2.0, x * 3.0, x - 1.0, x / 2.0) + print(w[0] as i32) + print(w[1] as i32) + print(w[2] as i32) + print(w[3] as i32) + + // Division, and a result returned through the sret pointer. + var q = scale(f32x4(12.0, 24.0, 36.0, 48.0), f32x4_splat(2.0)) / f32x4_splat(4.0) + print(q[0] as i32) + print(q[1] as i32) + print(q[2] as i32) + print(q[3] as i32) + + // Negation into a destination that aliases its operand. + var n = f32x4_splat(0.0 - 5.0) + n = -n + print(n[0] as i32) + print(n[3] as i32) + + // Splat of a computed value. + var s = f32x4_splat(x) + print(s[2] as i32) + + // A lambda body is the whole function body, so its vector result goes + // straight into the sret buffer with no fall-through copy. + var p = apply(| y | y * y, f32x4(2.0, 3.0, 4.0, 5.0)) + print(p[0] as i32) + print(p[3] as i32) +} diff --git a/tests/cases/simd/f32x4_three_address.lyte b/tests/cases/simd/f32x4_three_address.lyte new file mode 100644 index 00000000..0ff77cf7 --- /dev/null +++ b/tests/cases/simd/f32x4_three_address.lyte @@ -0,0 +1,99 @@ +// When an f32x4 computation's operands and destination are all frame slots, +// the VM-family backends evaluate it between those slots with nothing on the +// operand stack, and contract `a * b + c` into one multiply-accumulate. The +// destination is free to alias any operand, and nested subtrees land in +// temporaries first. + +// expected stdout: +// compilation successful +// 15 +// 45 +// 95 +// 165 +// 15 +// 45 +// 95 +// 165 +// 5 +// 35 +// 85 +// 155 +// 10 +// 20 +// 30 +// 40 +// 6 +// 6 +// 6 +// 6 +// 9 +// 18 +// 27 +// 36 +// 10 +// 10 +// 10 +// 10 +// -1 +// -2 +// -3 +// -4 +// 30 +// 105 +// 200 +// 315 +// 30 +// 105 +// 200 +// 315 + +main { + var a = f32x4(1.0, 2.0, 3.0, 4.0) + var b = f32x4(10.0, 20.0, 30.0, 40.0) + var c = f32x4(5.0, 5.0, 5.0, 5.0) + + // Multiply-accumulate, with the product on either side of the `+`. + var d = a * b + c + show(d) + var e = c + a * b + show(e) + + // Multiply-subtract. + var f = a * b - c + show(f) + + // A destination aliasing the accumulator, run twice. + var acc = f32x4_splat(0.0) + acc = a * c + acc + acc = a * c + acc + show(acc) + + // Destinations aliasing a multiplicand. + var g = f32x4(2.0, 2.0, 2.0, 2.0) + g = g * g + g + show(g) + + // Plain three-address forms: sub, div, neg. + var h = b - a + show(h) + var i = b / a + show(i) + var j = -a + show(j) + + // Nested subtrees on both sides need temporaries. + var k = (a + c) * (b - c) + show(k) + + // A whole-vector copy between locals. + var m = f32x4_splat(0.0) + m = k + show(m) +} + +show(v: f32x4) { + print(v[0] as i32) + print(v[1] as i32) + print(v[2] as i32) + print(v[3] as i32) +} diff --git a/tests/cases/stack_ir/f32x4_vector_ops.lyte b/tests/cases/stack_ir/f32x4_vector_ops.lyte new file mode 100644 index 00000000..27428b23 --- /dev/null +++ b/tests/cases/stack_ir/f32x4_vector_ops.lyte @@ -0,0 +1,26 @@ +// Stack-IR check that f32x4 lowers to whole-vector ops rather than four +// per-lane trips through the float window. Operands that are all frame +// slots use the three-address forms, so nothing reaches the operand +// stack, and `a * k + b` contracts to a single multiply-accumulate. +// args: --stack-ir +// expected stdout: +// fn main (params: 0, locals: 0, memory: 48 bytes): +// 0: fw.f32.const 1 +// 1: fw.f32.const 2 +// 2: fw.f32.const 3 +// 3: fw.f32.const 4 +// 4: local.addr 0 +// 5: f32x4.build_store +// 6: fw.f32.const 0.5 +// 7: local.addr 2 +// 8: f32x4.splat_store +// 9: f32x4.neg2 0 4 +// 10: f32x4.muladd_set 0 2 4 0 +// 11: return_void + +main { + var a = f32x4(1.0, 2.0, 3.0, 4.0) + var k = f32x4_splat(0.5) + var b = -a + a = a * k + b +}