From f84857acfb0e3fdcebe1aacb519c5f5c5565ce33 Mon Sep 17 00:00:00 2001 From: Izaak Branderhorst Date: Wed, 9 Sep 2026 21:10:31 +0200 Subject: [PATCH 1/3] Add checked native Stack maps and recurrences Recognize eligible concrete checked loops through immutable values and ordered effects, and generate precompiled arithmetic from the same recipes. Add an opt-in Stack compiler path for pointwise maps, one-pole and biquad loops. Retain ordinary bytecode behind a shared guarded execution boundary, including state publication and cancellation fallback. Consolidate bridge case tables, retain only consumed analysis metadata, and format all pointwise recipes alike. --- build.rs | 24 +- build/pointwise.rs | 112 +++ build/recurrence.rs | 113 +++ src/compiler.rs | 14 + src/lib.rs | 8 + src/loop_recipe.rs | 182 +++++ src/native_loop_tests.rs | 514 +++++++++++++ src/pointwise.rs | 112 +++ src/recurrence.rs | 98 +++ src/stack_codegen.rs | 286 ++++++- src/stack_depth.rs | 2 + src/stack_inline.rs | 4 + src/stack_interp.c | 267 ++++++- src/stack_interp_bridge.rs | 125 ++- src/stack_interp_bridge/native_loop_tests.rs | 714 ++++++++++++++++++ src/stack_ir.rs | 185 +++++ src/stack_optimize.rs | 27 +- src/stack_vm.rs | 3 + src/value_loops.rs | 553 ++++++++++++++ src/value_loops/recipes.rs | 161 ++++ src/value_loops/tests.rs | 275 +++++++ tests/cases/arith/packed_sum_signed_zero.lyte | 31 + 22 files changed, 3767 insertions(+), 43 deletions(-) create mode 100644 build/pointwise.rs create mode 100644 build/recurrence.rs create mode 100644 src/loop_recipe.rs create mode 100644 src/native_loop_tests.rs create mode 100644 src/pointwise.rs create mode 100644 src/recurrence.rs create mode 100644 src/stack_interp_bridge/native_loop_tests.rs create mode 100644 src/value_loops.rs create mode 100644 src/value_loops/recipes.rs create mode 100644 src/value_loops/tests.rs create mode 100644 tests/cases/arith/packed_sum_signed_zero.lyte diff --git a/build.rs b/build.rs index 286d984b..37441824 100644 --- a/build.rs +++ b/build.rs @@ -1,7 +1,29 @@ use std::io::Write; use std::process::Command; +#[allow(dead_code)] +#[path = "src/loop_recipe.rs"] +mod loop_recipe; +#[allow(dead_code)] +#[path = "src/pointwise.rs"] +mod pointwise; +#[path = "build/pointwise.rs"] +mod pointwise_build; +#[allow(dead_code)] +#[path = "src/recurrence.rs"] +mod recurrence; +#[path = "build/recurrence.rs"] +mod recurrence_build; + fn main() { + let out_dir = std::env::var("OUT_DIR").unwrap(); + pointwise_build::generate(std::path::Path::new(&out_dir)); + recurrence_build::generate(std::path::Path::new(&out_dir)); + println!("cargo:rerun-if-changed=src/loop_recipe.rs"); + println!("cargo:rerun-if-changed=src/pointwise.rs"); + println!("cargo:rerun-if-changed=build/pointwise.rs"); + println!("cargo:rerun-if-changed=src/recurrence.rs"); + println!("cargo:rerun-if-changed=build/recurrence.rs"); // Compile the C stack interpreter with preserve_none + musttail. // Clang-only feature (skip on GCC). The handler signature keeps // the TOS window (t0..t3), the int stack pointer, the float @@ -17,6 +39,7 @@ fn main() { cc::Build::new() .file("src/stack_interp.c") .include("src") + .include(&out_dir) .opt_level(3) .flag("-std=c11") .flag("-Wno-unused-parameter") @@ -63,7 +86,6 @@ fn main() { println!("cargo:rerun-if-changed=src/stack_interp.c"); println!("cargo:rerun-if-changed=src/stack_interp.h"); // Determine sizeof(jmp_buf) on this platform by compiling and running a C snippet. - let out_dir = std::env::var("OUT_DIR").unwrap(); let c_path = format!("{}/jmpbuf_size.c", out_dir); let bin_path = format!("{}/jmpbuf_size", out_dir); diff --git a/build/pointwise.rs b/build/pointwise.rs new file mode 100644 index 00000000..26b3c0cb --- /dev/null +++ b/build/pointwise.rs @@ -0,0 +1,112 @@ +//! Build-time expansion of the shared pointwise recipes into native C bodies. +//! +//! Every recipe produces typed SSA statements in its declared order. Handler +//! selection happens while the backend is constructed; constant kernel kinds +//! remove the generated switch before the interpreter is compiled. + +use crate::pointwise::{self, BinaryOp, Node, Recipe}; +use std::fmt::Write; +use std::path::Path; + +pub fn generate(out_dir: &Path) { + pointwise::validate().expect("invalid native pointwise recipe catalog"); + // The compact native encoding has exactly four coefficient slots. + assert_eq!(pointwise::MAX_COEFFICIENTS, 4); + + let mut catalog = String::from("// Generated from src/pointwise.rs; do not edit.\n"); + for (id, recipe) in pointwise::recipes() { + writeln!( + catalog, + "POINTWISE_RECIPE({}, {}, {}, {})", + id.index(), + recipe.name.to_ascii_uppercase(), + recipe.name, + recipe.coefficient_count(), + ) + .unwrap(); + } + std::fs::write(out_dir.join("pointwise_catalog.inc"), catalog) + .expect("failed to write native pointwise catalog"); + + let mut chunks = String::from("// Generated from src/pointwise.rs; do not edit.\n"); + for (suffix, scalar, load) in [ + ("f32", "float", "load_f32_unaligned"), + ("f64", "double", "load_f64_unaligned"), + ] { + writeln!( + chunks, + "static inline __attribute__((always_inline)) void native_pointwise_{suffix}(\n\ + NativeKernelKind kind, const uint8_t* input, uint8_t* output,\n\ + int64_t count, const {scalar}* coefficients\n\ + ) {{\n switch (kind) {{" + ) + .unwrap(); + for (_, recipe) in pointwise::recipes() { + writeln!( + chunks, + " case NATIVE_POINTWISE_{}: {{", + recipe.name.to_ascii_uppercase() + ) + .unwrap(); + for (index, node) in recipe.nodes.iter().enumerate() { + if let Node::Parameter(parameter) = node { + writeln!( + chunks, + " {scalar} v{index} = coefficients[{parameter}];" + ) + .unwrap(); + } + } + // The one-pointer spelling lets LLVM prove exact in-place access + // safe for SIMD without making a restrict promise about host spans. + chunks.push_str(" if (input == output) {\n"); + emit_loop(&mut chunks, recipe, scalar, load, "output"); + chunks.push_str(" } else {\n"); + emit_loop(&mut chunks, recipe, scalar, load, "input"); + chunks.push_str(" }\n return;\n }\n"); + } + chunks.push_str(" default: __builtin_unreachable();\n }\n}\n"); + } + std::fs::write(out_dir.join("pointwise_chunks.inc"), chunks) + .expect("failed to write native pointwise arithmetic"); +} + +fn emit_loop(code: &mut String, recipe: &Recipe, scalar: &str, load: &str, input: &str) { + writeln!( + code, + " for (int64_t item = 0; item < count; ++item) {{" + ) + .unwrap(); + writeln!( + code, + " size_t offset = (size_t)item * sizeof({scalar});" + ) + .unwrap(); + for (index, node) in recipe.nodes.iter().enumerate() { + let expression = match node { + Node::Read => format!("{load}({input} + offset)"), + Node::Parameter(_) => continue, + Node::Carry(_) => unreachable!("validated pointwise recipes have no carries"), + Node::Binary(operator, left, right) => { + let operator = match operator { + BinaryOp::Add => '+', + BinaryOp::Sub => '-', + BinaryOp::Mul => '*', + }; + format!("v{left} {operator} v{right}") + } + }; + writeln!( + code, + " {scalar} v{index} = {expression};" + ) + .unwrap(); + } + writeln!( + code, + " __builtin_memcpy(output + offset, &v{}, sizeof(v{}));", + recipe.result, recipe.result, + ) + .unwrap(); + code.push_str(" }\n"); +} diff --git a/build/recurrence.rs b/build/recurrence.rs new file mode 100644 index 00000000..2314e9a2 --- /dev/null +++ b/build/recurrence.rs @@ -0,0 +1,113 @@ +//! Expand the checked recurrence recipes into statically specialized C loops. +//! +//! Carry nodes name iteration-entry values. The next-carry map is published only +//! after every node has been evaluated, so history shifts do not depend on the +//! order in which mutable C carry variables are assigned. + +use crate::loop_recipe::{BinaryOp, Node}; +use crate::recurrence; +use std::fmt::Write; +use std::path::Path; + +pub fn generate(out_dir: &Path) { + recurrence::validate().expect("invalid native recurrence recipe catalog"); + // Preserve the existing native boundary and biquad descriptor capacities. + assert_eq!(recurrence::MAX_COEFFICIENTS, 5); + assert_eq!(recurrence::MAX_CARRIES, 4); + + let mut catalog = String::from("// Generated from src/recurrence.rs; do not edit.\n"); + for (id, recipe) in recurrence::recipes() { + writeln!( + catalog, + "RECURRENCE_RECIPE({}, {}, {}, {}, {})", + id.index(), + recipe.name.to_ascii_uppercase(), + recipe.name, + recipe.coefficient_count(), + recipe.carry_count(), + ) + .unwrap(); + } + std::fs::write(out_dir.join("recurrence_catalog.inc"), catalog) + .expect("failed to write native recurrence catalog"); + + let mut chunks = String::from("// Generated from src/recurrence.rs; do not edit.\n"); + for (suffix, scalar, load) in [ + ("f32", "float", "load_f32_unaligned"), + ("f64", "double", "load_f64_unaligned"), + ] { + writeln!( + chunks, + "static inline __attribute__((always_inline)) void native_recurrence_{suffix}(\n\ + NativeKernelKind kind, const uint8_t* input, uint8_t* output,\n\ + int64_t count, const {scalar}* coefficients, {scalar}* carried\n\ + ) {{\n switch (kind) {{" + ) + .unwrap(); + for (_, recipe) in recurrence::recipes() { + writeln!( + chunks, + " case NATIVE_{}: {{", + recipe.name.to_ascii_uppercase() + ) + .unwrap(); + for (index, node) in recipe.nodes.iter().enumerate() { + if let Node::Parameter(parameter) = node { + writeln!( + chunks, + " {scalar} v{index} = coefficients[{parameter}];" + ) + .unwrap(); + } + } + for carry in 0..recipe.carry_count() { + writeln!(chunks, " {scalar} c{carry} = carried[{carry}];").unwrap(); + } + writeln!( + chunks, + " for (int64_t item = 0; item < count; ++item) {{" + ) + .unwrap(); + writeln!( + chunks, + " size_t offset = (size_t)item * sizeof({scalar});" + ) + .unwrap(); + for (index, node) in recipe.nodes.iter().enumerate() { + let expression = match node { + Node::Read => format!("{load}(input + offset)"), + Node::Parameter(_) => continue, + Node::Carry(carry) => format!("c{carry}"), + Node::Binary(operator, left, right) => { + let operator = match operator { + BinaryOp::Add => '+', + BinaryOp::Sub => '-', + BinaryOp::Mul => '*', + }; + format!("v{left} {operator} v{right}") + } + }; + writeln!(chunks, " {scalar} v{index} = {expression};").unwrap(); + } + // Every root is an SSA value from the completed iteration. Updating + // c0 cannot change a later root which refers to the old c0 snapshot. + for (carry, next) in recipe.next_carries.iter().enumerate() { + writeln!(chunks, " c{carry} = v{next};").unwrap(); + } + writeln!( + chunks, + " __builtin_memcpy(output + offset, &v{}, sizeof(v{}));", + recipe.result, recipe.result, + ) + .unwrap(); + chunks.push_str(" }\n"); + for carry in 0..recipe.carry_count() { + writeln!(chunks, " carried[{carry}] = c{carry};").unwrap(); + } + chunks.push_str(" return;\n }\n"); + } + chunks.push_str(" default: __builtin_unreachable();\n }\n}\n"); + } + std::fs::write(out_dir.join("recurrence_chunks.inc"), chunks) + .expect("failed to write native recurrence arithmetic"); +} diff --git a/src/compiler.rs b/src/compiler.rs index 5f35c5c8..060732c8 100644 --- a/src/compiler.rs +++ b/src/compiler.rs @@ -1097,7 +1097,21 @@ impl Compiler { /// Compile to stack-based IR (for Silverfir-nano-style interpreters). pub fn compile_stack(&self) -> Result { + self.compile_stack_with_native_loops(false) + } + + /// Experimental consumer of concrete checked loops. Unmatched loops and + /// native runtime guard failures retain ordinary Stack execution. + pub fn compile_stack_native_loops(&self) -> Result { + self.compile_stack_with_native_loops(true) + } + + fn compile_stack_with_native_loops( + &self, + native_loops: bool, + ) -> Result { let mut codegen = crate::stack_codegen::StackCodegen::new(); + codegen.native_loops = native_loops; let entry_points = self.effective_entry_points(); let mut program = codegen.compile_multi(self.specialized_program()?, &entry_points)?; // Inline trivial leaf functions (like cmp(a, b) -> a - b) so their diff --git a/src/lib.rs b/src/lib.rs index ebd91dd0..9d554962 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -87,6 +87,14 @@ pub mod stack_ir; pub mod stack_codegen; +pub mod loop_recipe; +pub mod pointwise; +pub mod recurrence; +pub mod value_loops; + +#[cfg(all(test, has_stack_interp))] +mod native_loop_tests; + pub mod stack_optimize; pub mod stack_depth; diff --git a/src/loop_recipe.rs b/src/loop_recipe.rs new file mode 100644 index 00000000..370041cf --- /dev/null +++ b/src/loop_recipe.rs @@ -0,0 +1,182 @@ +//! Compiler-independent ordered values shared by native loop matching and C +//! generation. Carry updates are simultaneous next-state roots, not assignments +//! to execute in role order. The output is the single terminal memory effect. + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum BinaryOp { + Add, + Sub, + Mul, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Node { + Read, + Parameter(u8), + Carry(u8), + Binary(BinaryOp, usize, usize), +} + +#[derive(Clone, Copy, Debug)] +pub struct Recipe { + pub name: &'static str, + pub nodes: &'static [Node], + /// Value written by the terminal output effect, after all ordered work. + pub result: usize, + /// Final value for each incoming carry role. All roots refer to the same + /// iteration's immutable values, including its incoming carry snapshots. + pub next_carries: &'static [usize], +} + +impl Recipe { + pub fn coefficient_count(&self) -> usize { + self.nodes + .iter() + .filter_map(|node| match node { + Node::Parameter(role) => Some(usize::from(*role) + 1), + _ => None, + }) + .max() + .unwrap_or(0) + } + + pub fn carry_count(&self) -> usize { + self.next_carries.len() + } +} + +/// Validate semantic recipes before either consumer uses them. Physical operand +/// encodings supply their own limits. No work node may be unused: native C could +/// remove it while the matcher promises coverage of the complete ordered tape. +pub fn validate_catalog( + catalog: &[Recipe], + max_coefficients: usize, + max_carries: usize, +) -> Result<(), String> { + use Node::*; + if catalog.len() > usize::from(u8::MAX) + 1 { + return Err("native recipe catalog exceeds identity range".into()); + } + let mut names = std::collections::HashSet::new(); + for recipe in catalog { + let valid_name = recipe + .name + .bytes() + .all(|c| c.is_ascii_lowercase() || c == b'_'); + if recipe.name.is_empty() || !valid_name || !names.insert(recipe.name) { + return Err(format!("invalid or duplicate recipe name: {}", recipe.name)); + } + if recipe.result >= recipe.nodes.len() + || recipe + .next_carries + .iter() + .any(|&next| next >= recipe.nodes.len()) + || recipe.carry_count() > max_carries + { + return Err(format!("{}: invalid output or carry roots", recipe.name)); + } + let mut reads = 0; + let mut parameters = vec![false; max_coefficients]; + let mut carries = vec![false; recipe.carry_count()]; + for (index, node) in recipe.nodes.iter().enumerate() { + let (roles, role, label) = match *node { + Read => { + reads += 1; + continue; + } + Parameter(role) => (&mut parameters, usize::from(role), "parameter"), + Carry(role) => (&mut carries, usize::from(role), "carry"), + Binary(_, left, right) => { + if left >= index || right >= index { + return Err(format!("{}: operand is not an earlier value", recipe.name)); + } + continue; + } + }; + if role >= roles.len() || roles[role] { + return Err(format!("{}: invalid {label} role", recipe.name)); + } + roles[role] = true; + } + if reads != 1 + || parameters[..recipe.coefficient_count()].contains(&false) + || carries.contains(&false) + { + return Err(format!( + "{}: expected one read and complete parameter/carry roles", + recipe.name + )); + } + let mut live = vec![false; recipe.nodes.len()]; + live[recipe.result] = true; + for &next in recipe.next_carries { + live[next] = true; + } + for index in (0..recipe.nodes.len()).rev() { + if live[index] { + if let Binary(_, left, right) = recipe.nodes[index] { + live[left] = true; + live[right] = true; + } + } + } + if live.contains(&false) { + return Err(format!( + "{}: unused node would violate work coverage", + recipe.name + )); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use Node::*; + + #[test] + fn output_and_next_state_are_independent_roots_with_complete_roles() { + let recipe = Recipe { + name: "state_roots", + nodes: &[ + Read, + Carry(0), + Carry(1), + Binary(BinaryOp::Add, 0, 1), + Binary(BinaryOp::Mul, 2, 3), + ], + result: 0, + next_carries: &[3, 4], + }; + assert!(validate_catalog(&[recipe], 0, 2).is_ok()); + for invalid in [ + Recipe { + next_carries: &[3, 5], + ..recipe + }, + Recipe { + next_carries: &[3], + ..recipe + }, + Recipe { + next_carries: &[3, 2], + ..recipe + }, + Recipe { + nodes: &[Read, Carry(0), Carry(0)], + result: 0, + next_carries: &[1, 2], + ..recipe + }, + Recipe { + nodes: &[Read, Carry(0), Binary(BinaryOp::Add, 0, 3)], + result: 0, + next_carries: &[2], + ..recipe + }, + ] { + assert!(validate_catalog(&[invalid], 0, 2).is_err()); + } + } +} diff --git a/src/native_loop_tests.rs b/src/native_loop_tests.rs new file mode 100644 index 00000000..44a4ab34 --- /dev/null +++ b/src/native_loop_tests.rs @@ -0,0 +1,514 @@ +//! End-to-end consumers of checked identities and the optional native-loop path. +//! Raw alias/cancellation/frame-boundary contracts are covered by bridge tests. + +use crate::stack_interp_bridge::StackBackend; +use crate::stack_ir::{NativeStreamBody, StackOp, StackProgram}; +use crate::*; + +fn checked(source: &str, entries: &[&str]) -> Compiler { + let mut compiler = Compiler::new(); + compiler.quiet = true; + compiler.set_entry_points(entries); + assert!( + compiler.parse(source, "native_loop_tests.lyte"), + "{:?}", + compiler.last_errors + ); + assert!(compiler.check(), "{:?}", compiler.last_errors); + compiler.specialize().unwrap(); + compiler +} + +fn native_bodies(program: &StackProgram) -> Vec<&NativeStreamBody> { + program + .functions + .iter() + .flat_map(|function| &function.ops) + .filter_map(|op| { + if let StackOp::NativeLoop(kernel) = op { + Some(&kernel.body) + } else { + None + } + }) + .collect() +} + +fn variants(compiler: &Compiler) -> (StackProgram, StackProgram) { + let ordinary = compiler.compile_stack().unwrap(); + assert!( + native_bodies(&ordinary).is_empty(), + "ordinary compilation must remain opt-out" + ); + let selected = compiler.compile_stack_native_loops().unwrap(); + (ordinary, selected) +} + +fn execute(program: &StackProgram, entries: &[&str]) -> Vec<(i64, Vec)> { + let mut backend = Box::new(StackBackend::new(program)); + let mut globals = vec![0u8; program.globals_size.max(1)]; + entries + .iter() + .map(|entry| { + let function = program.entry_points[&Name::str(entry)]; + let result = backend.call_entry(function, globals.as_mut_ptr()); + assert!(!backend.cancelled()); + assert_eq!(backend.trap_reason(), crate::cancel::TRAP_NONE); + (result, globals.clone()) + }) + .collect() +} + +// Replace the retained loop entry with a trap without changing jump coordinates. +// Valid, non-polling examples must execute native work to reach their results. +pub(crate) fn forbid_fallback(program: &mut StackProgram) { + let mut gateways = 0; + for function in &mut program.functions { + for index in 0..function.ops.len() { + if let StackOp::NativeLoop(kernel) = &function.ops[index] { + assert!(kernel.done >= 2); + function.ops[index + 1] = StackOp::I64Const(0); + function.ops[index + 2] = StackOp::Assert; + gateways += 1; + } + } + } + assert!(gateways > 0, "expected executable native work"); +} + +fn execute_native(mut program: StackProgram, entries: &[&str]) -> Vec<(i64, Vec)> { + let expected = execute(&program, entries); + forbid_fallback(&mut program); + assert_eq!(execute(&program, entries), expected); + expected +} + +#[test] +fn affine_slices_publish_output_and_reload_headers_after_callback() { + struct Redirect { + headers: [*mut u8; 2], + replacements: [*const u8; 2], + output: *const u8, + size: usize, + observed: Vec, + } + unsafe extern "C" fn redirect(data: *mut u8) -> bool { + let probe = &mut *data.cast::(); + probe.observed = std::slice::from_raw_parts(probe.output, probe.size).to_vec(); + for (header, replacement) in probe.headers.iter().copied().zip(probe.replacements) { + std::ptr::write_unaligned(header.cast::<*const u8>(), replacement); + } + false + } + + for scalar in ["f32", "f64"] { + let compiler = checked( + &format!( + "var input: [{scalar}] + var output: [{scalar}] + process() -> i32 {{ + var prefix = 0 + for j in 0 .. 1022 {{ prefix = prefix + 1 }} + if input.len >= 4 && output.len >= 4 {{ + for i in 0 .. 4 {{ output[i] = input[i]*0.5{scalar}+1.0{scalar} }} + }} + prefix + }}" + ), + &["process"], + ); + let info = compiler.globals_info_with_offset(crate::cancel::CANCEL_FLAG_RESERVED as usize); + let offsets = + ["input", "output"].map(|name| info.iter().find(|entry| entry.0 == name).unwrap().1); + let bytes = |values: &[f64]| -> Vec { + values + .iter() + .flat_map(|&value| { + if scalar == "f64" { + value.to_le_bytes().to_vec() + } else { + (value as f32).to_le_bytes().to_vec() + } + }) + .collect() + }; + for callback in [false, true] { + let (ordinary, mut native) = variants(&compiler); + if !callback { + forbid_fallback(&mut native); + } + for program in [ordinary, native] { + let input = bytes(&[2.0, 4.0, 6.0, 8.0]); + let replacement = bytes(&[10.0, 12.0, 14.0, 16.0]); + let mut output = bytes(&[0.0; 4]); + let mut redirected = output.clone(); + let mut globals = vec![0u8; program.globals_size]; + let mut probe = Redirect { + headers: offsets.map(|offset| unsafe { globals.as_mut_ptr().add(offset) }), + replacements: [replacement.as_ptr(), redirected.as_mut_ptr()], + output: output.as_ptr(), + size: output.len(), + observed: Vec::new(), + }; + for (offset, pointer) in offsets.iter().zip([input.as_ptr(), output.as_mut_ptr()]) { + globals[*offset..*offset + 8].copy_from_slice(&(pointer as u64).to_le_bytes()); + globals[*offset + 8..*offset + 12].copy_from_slice(&4i32.to_le_bytes()); + } + let mut backend = StackBackend::new(&program); + if callback { + backend + .set_cancel_callback(Some(redirect), (&mut probe as *mut Redirect).cast()); + } + assert_eq!( + backend.call_entry(program.entry, globals.as_mut_ptr()), + 1022 + ); + assert_eq!(backend.trap_reason(), crate::cancel::TRAP_NONE); + if callback { + assert_eq!(probe.observed, bytes(&[2.0, 3.0, 0.0, 0.0])); + assert_eq!(output, probe.observed); + assert_eq!(redirected, bytes(&[0.0, 0.0, 8.0, 9.0])); + } else { + assert_eq!(output, bytes(&[2.0, 3.0, 4.0, 5.0])); + } + } + } + } +} + +#[test] +fn captured_outer_and_uncaptured_shadow_select_affine_and_preserve_closure() { + let compiler = checked( + r#" + kernel(input: &[f32; 4], output: &[f32; 4]) -> f32 { + let gain = 0.25 + let read = || { gain } + { + let gain = 0.5 + for i in 0 .. 4 { output[i] = input[i]*gain+1.0 } + } + read() + } + main() -> i32 { + var input = [2.0, 4.0, 6.0, 8.0] + var output: [f32; 4] + let captured = kernel(input, output) + ((captured + output[0]+output[1]+output[2]+output[3])*100.0) as i32 + } + "#, + &["main"], + ); + let (ordinary, selected) = variants(&compiler); + assert!(matches!( + native_bodies(&selected).as_slice(), + [NativeStreamBody::Pointwise { .. }] + )); + let expected = execute(&ordinary, &["main"]); + assert_eq!(expected[0].0, 1425); + assert_eq!(execute_native(selected, &["main"]), expected); + + let mut renamed = compiler.specialized_program().unwrap().clone(); + for decl in &mut renamed.decls.decls { + let Decl::Func(function) = decl else { continue }; + for local in &mut function.arena.locals { + local.name = Name::str("same"); + } + for id in function.arena.ids() { + let mut expr = function.arena[id].clone(); + match &mut expr { + Expr::Id(name) + | Expr::TypeApp(name, _) + | Expr::Let(name, ..) + | Expr::Var(name, ..) + | Expr::For { var: name, .. } => *name = Name::str("same"), + Expr::Lambda { params, .. } => { + for param in params { + param.name = Name::str("same"); + } + } + _ => {} + } + function.arena.replace(id, expr, function.arena.ty(id)); + } + } + renamed.validate().unwrap(); + let mut codegen = crate::stack_codegen::StackCodegen::new(); + codegen.native_loops = true; + let mut program = codegen.compile(&renamed).unwrap(); + crate::stack_inline::inline_trivial(&mut program); + for function in &mut program.functions { + crate::stack_rebase_lm::rebase(function); + crate::stack_optimize::optimize(function); + crate::stack_rebase_lm::patch_call_preserve(function); + } + assert_eq!(execute_native(program, &["main"]), expected); +} + +#[test] +fn one_pole_snapshots_and_state_survive_repeated_entry_calls() { + for scalar in ["f32", "f64"] { + let compiler = checked( + &format!( + r#" + var input: [{scalar}; 4] + var output: [{scalar}; 4] + var history: {scalar} + init() {{ + input[0] = 2.0{scalar}; input[1] = 4.0{scalar} + input[2] = 6.0{scalar}; input[3] = 8.0{scalar} + history = 1.0{scalar} + }} + process() -> i32 {{ + var state = history + let gain = 0.5{scalar} + let feedback = 0.25{scalar} + for i in 0 .. 4 {{ + let before = state + var x = input[i] + var feed = x*gain + let decay = before*feedback + state = feed+decay + output[i] = state + }} + history = state + (state*1024.0{scalar}) as i32 + }} + "# + ), + &["init", "process"], + ); + let (ordinary, selected) = variants(&compiler); + assert!(matches!( + native_bodies(&selected).as_slice(), + [NativeStreamBody::OnePole { .. }] + )); + let entries = ["init", "process", "process", "process", "init", "process"]; + let expected = execute(&ordinary, &entries); + assert_eq!(expected[1].0, 5012); + assert_ne!(expected[1].0, expected[2].0); + assert_eq!(expected[1], expected[5]); + assert_eq!(execute_native(selected, &entries), expected, "{scalar}"); + } +} + +#[test] +fn biquad_staged_history_updates_preserve_every_carried_state() { + for scalar in ["f32", "f64"] { + let compiler = checked( + &format!( + r#" + var input: [{scalar}; 4] + var output: [{scalar}; 4] + var history_x1: {scalar} + var history_x2: {scalar} + var history_y1: {scalar} + var history_y2: {scalar} + init() {{ + input[0] = 2.0{scalar}; input[1] = 4.0{scalar} + input[2] = 6.0{scalar}; input[3] = 8.0{scalar} + history_x1 = 0.5{scalar}; history_x2 = 1.0{scalar} + history_y1 = 0.75{scalar}; history_y2 = 0.25{scalar} + }} + process() -> i32 {{ + var y2 = history_y2 + var x1 = history_x1 + var y1 = history_y1 + var x2 = history_x2 + let b0 = 0.5{scalar}; let b1 = 0.25{scalar}; let b2 = 0.125{scalar} + let a1 = 0.0625{scalar}; let a2 = 0.03125{scalar} + for i in 0 .. 4 {{ + let x = input[i] + var y = b0*x + y = y+b1*x1 + y = y+b2*x2 + y = y-a1*y1 + y = y-a2*y2 + let old_input = x1 + let old_output = y1 + y1 = y; x1 = x; x2 = old_input; y2 = old_output + output[i] = y + }} + history_x1 = x1; history_x2 = x2 + history_y1 = y1; history_y2 = y2 + ((x1+x2+y1+y2)*1024.0{scalar}) as i32 + }} + "# + ), + &["init", "process"], + ); + let (ordinary, selected) = variants(&compiler); + assert!(matches!( + native_bodies(&selected).as_slice(), + [NativeStreamBody::Biquad { .. }] + )); + let entries = ["init", "process", "process", "process", "init", "process"]; + let expected = execute(&ordinary, &entries); + assert_ne!(expected[1].0, expected[2].0); + assert_eq!(expected[1], expected[5]); + assert_eq!(execute_native(selected, &entries), expected, "{scalar}"); + } +} + +#[test] +fn same_typed_generic_globals_select_distinct_native_streams() { + let compiler = checked( + r#" + var input: [f32; 4] + var output: [f32; 4] + main() -> i32 { + input⟨i32⟩[0] = 2.0 + input⟨f32⟩[0] = 8.0 + for i in 0 .. 4 { output⟨i32⟩[i] = input⟨i32⟩[i]*0.5+1.0 } + for i in 0 .. 4 { output⟨f32⟩[i] = input⟨f32⟩[i]*0.5+1.0 } + (output⟨i32⟩[0]*100.0 + output⟨f32⟩[0]) as i32 + } + "#, + &["main"], + ); + let program = compiler.specialized_program().unwrap(); + let inputs: Vec<_> = program + .globals() + .filter(|(id, _)| { + let source = program.instances[id.index()].definition; + program.globals().any(|(other, _)| { + other != *id && program.instances[other.index()].definition == source + }) + }) + .collect(); + assert_eq!( + inputs.len(), + 4, + "two concrete instances of each same-typed global" + ); + let (ordinary, selected) = variants(&compiler); + assert_eq!(native_bodies(&selected).len(), 2); + assert!(native_bodies(&selected) + .iter() + .all(|body| matches!(body, NativeStreamBody::Pointwise { .. }))); + let expected = execute(&ordinary, &["main", "main"]); + assert_eq!(expected[0].0, 205); + assert_eq!(execute_native(selected, &["main", "main"]), expected); +} + +#[test] +fn unsupported_indexing_calls_and_captured_inputs_remain_ordinary() { + for (body, captured) in [ + ("output[i] = input[i+1]*gain+1.0", false), + ("output[i] = twice(input[i])", false), + ("output[i] = input[i]*gain+1.0", true), + ] { + let capture = if captured { + "let read = || { gain }" + } else { + "" + }; + let result = if captured { "read()" } else { "gain" }; + let compiler = checked( + &format!( + r#" + twice(x: f32) -> f32 {{ x+x }} + kernel(input: &[f32; 4], output: &[f32; 4]) -> f32 {{ + let gain = 0.5 + {capture} + for i in 0 .. 3 {{ {body} }} + {result} + }} + main() -> i32 {{ + var input = [2.0, 4.0, 6.0, 8.0] + var output: [f32; 4] + let value = kernel(input, output) + ((value+output[0]+output[1]+output[2])*100.0) as i32 + }} + "# + ), + &["main"], + ); + let (ordinary, selected) = variants(&compiler); + assert!( + native_bodies(&selected).is_empty(), + "unexpected native selection for {}", + body + ); + assert_eq!(execute(&selected, &["main"]), execute(&ordinary, &["main"])); + } + + let compiler = checked( + "main() -> i32 { + var input = [2.0, 4.0, 6.0, 8.0] + var output: [f32; 4] + let map = |gain: f32| { + for i in 0 .. 4 { output[i] = input[i]*gain+1.0 } + } + map(0.5) + (output[0]+output[1]+output[2]+output[3]) as i32 + }", + &["main"], + ); + let (ordinary, selected) = variants(&compiler); + assert!(native_bodies(&selected).is_empty()); + let expected = execute(&ordinary, &["main"]); + assert_eq!(expected[0].0, 14); + assert_eq!(execute(&selected, &["main"]), expected); +} + +#[test] +fn native_setup_frame_overflow_retries_the_exact_ordinary_function() { + fn source(padding: usize, padding_first: bool) -> String { + let allocation = format!( + "var padding: [f32; {padding}]; padding[{}] = gain;\n\ + var marker: [f32; 1]; marker[0] = padding[{}];", + padding - 1, + padding - 1, + ); + let map = "for i in 0 .. 4 { output[i] = input[i]*gain+bias }"; + let body = if padding_first { + format!("{allocation}\n{map}") + } else { + format!("{map}\n{allocation}") + }; + format!( + r#" + helper(x: f32) -> f32 {{ x }} + kernel(input: &[f32; 4], output: &[f32; 4], gain: f32, bias: f32) -> f32 {{ + {body} + let finish = |x| x + helper(finish(marker[0]+output[3])) + }} + main() -> i32 {{ + var input: [f32; 4] + var output: [f32; 4] + kernel(input, output, 2.0, 0.5) as i32 + }} + "# + ) + } + + for padding_first in [true, false] { + let small = checked(&source(16, padding_first), &["main"]); + let (ordinary, selected) = variants(&small); + assert_eq!(native_bodies(&selected).len(), 1); + let kernel = ordinary + .functions + .iter() + .find(|f| f.name == "kernel") + .unwrap(); + let frame_slots = usize::from(kernel.local_count) + kernel.local_memory as usize / 8; + // Add only array storage; the rest of this function's slot demand stays + // unchanged. Its ordinary frame fits, but native setup needs extra slots. + let padding = 16 + 2 * (usize::from(u16::MAX) - frame_slots); + let compiler = checked(&source(padding, padding_first), &["main"]); + let (ordinary, selected) = variants(&compiler); + assert!(native_bodies(&selected).is_empty()); + assert_eq!(ordinary.entry_points, selected.entry_points); + assert_eq!(ordinary.functions.len(), selected.functions.len()); + for (prior, retried) in ordinary.functions.iter().zip(&selected.functions) { + assert_eq!(prior.name, retried.name); + assert_eq!(prior.local_count, retried.local_count); + assert_eq!(prior.local_memory, retried.local_memory); + assert_eq!(prior.ops, retried.ops); + } + let expected = execute(&ordinary, &["main"]); + assert_eq!(expected[0].0, 2); + assert_eq!(execute(&selected, &["main"]), expected); + } +} diff --git a/src/pointwise.rs b/src/pointwise.rs new file mode 100644 index 00000000..8fee0851 --- /dev/null +++ b/src/pointwise.rs @@ -0,0 +1,112 @@ +//! Ordered pointwise recipes shared by checked-loop matching and C generation. +//! +//! This module is deliberately independent of compiler types: build.rs reads the +//! same graph that runtime selection matches. Nodes are SSA values in evaluation +//! order, not algebraic expressions to reassociate. The finite catalog bounds +//! precompiled code growth; it is not interpreted during sample processing. + +pub const MAX_COEFFICIENTS: usize = 4; + +pub use crate::loop_recipe::{BinaryOp, Node, Recipe}; + +/// A catalog identity; construction only occurs through the shared catalog. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RecipeId(u8); + +impl RecipeId { + pub fn index(self) -> usize { + usize::from(self.0) + } + + pub fn recipe(self) -> &'static Recipe { + &CATALOG[self.index()] + } +} + +/// The affine map recipe `input * gain + bias`. +pub const AFFINE: RecipeId = RecipeId(0); + +use BinaryOp::{Add, Mul, Sub}; +use Node::{Binary, Parameter, Read}; + +const CATALOG: &[Recipe] = &[ + Recipe { + name: "affine_map", + // x*g+b + nodes: &[ + Read, + Parameter(0), + Binary(Mul, 0, 1), + Parameter(1), + Binary(Add, 2, 3), + ], + result: 4, + next_carries: &[], + }, + Recipe { + name: "offset_scale", + // (x-b)*g: subtraction precedes multiplication. + nodes: &[ + Read, + Parameter(0), + Binary(Sub, 0, 1), + Parameter(1), + Binary(Mul, 2, 3), + ], + result: 4, + next_carries: &[], + }, + Recipe { + name: "cubic", + // x-k*((x*x)*x), with one read reused by every occurrence of x. + nodes: &[ + Read, + Parameter(0), + Binary(Mul, 0, 0), + Binary(Mul, 2, 0), + Binary(Mul, 1, 3), + Binary(Sub, 0, 4), + ], + result: 5, + next_carries: &[], + }, + Recipe { + name: "horner", + // ((a*x+b)*x+c)*x+d, preserving each multiply/add rounding. + nodes: &[ + Read, + Parameter(0), + Binary(Mul, 1, 0), + Parameter(1), + Binary(Add, 2, 3), + Binary(Mul, 4, 0), + Parameter(2), + Binary(Add, 5, 6), + Binary(Mul, 7, 0), + Parameter(3), + Binary(Add, 8, 9), + ], + result: 10, + next_carries: &[], + }, +]; + +pub fn recipes() -> impl Iterator { + CATALOG + .iter() + .enumerate() + .map(|(index, recipe)| (RecipeId(index as u8), recipe)) +} + +/// Validate the shared semantics against the existing pointwise encoding. +pub fn validate() -> Result<(), String> { + crate::loop_recipe::validate_catalog(CATALOG, MAX_COEFFICIENTS, 0)?; + if AFFINE.recipe().name != "affine_map" + || CATALOG + .iter() + .any(|recipe| recipe.result != recipe.nodes.len() - 1) + { + return Err("invalid pointwise catalog identities or terminal result".into()); + } + Ok(()) +} diff --git a/src/recurrence.rs b/src/recurrence.rs new file mode 100644 index 00000000..c814d8f0 --- /dev/null +++ b/src/recurrence.rs @@ -0,0 +1,98 @@ +//! Ordered recurrence recipes consumed by checked-loop matching and native C +//! generation. The catalog initially describes the existing one-pole and direct +//! form I biquad kernels; operand encodings remain a separate backend concern. + +use crate::loop_recipe::{self, BinaryOp, Node, Recipe}; + +pub const MAX_COEFFICIENTS: usize = 5; +pub const MAX_CARRIES: usize = 4; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RecipeId(u8); + +impl RecipeId { + pub fn index(self) -> usize { + usize::from(self.0) + } + + pub fn recipe(self) -> &'static Recipe { + &CATALOG[self.index()] + } +} + +pub const ONE_POLE: RecipeId = RecipeId(0); +pub const BIQUAD: RecipeId = RecipeId(1); + +use BinaryOp::{Add, Mul, Sub}; +use Node::{Binary, Carry, Parameter, Read}; + +const CATALOG: &[Recipe] = &[ + Recipe { + name: "one_pole", + // x*feed + old*feedback; output and next state share the result. + nodes: &[ + Read, + Parameter(0), + Binary(Mul, 0, 1), + Carry(0), + Parameter(1), + Binary(Mul, 3, 4), + Binary(Add, 2, 5), + ], + result: 6, + next_carries: &[6], + }, + Recipe { + name: "biquad", + // b0*x + b1*x1 + b2*x2 - a1*y1 - a2*y2, separately rounded. + // Carry roles are x1, x2, y1, y2; history shifts use old snapshots. + nodes: &[ + Parameter(0), + Read, + Binary(Mul, 0, 1), + Parameter(1), + Carry(0), + Binary(Mul, 3, 4), + Binary(Add, 2, 5), + Parameter(2), + Carry(1), + Binary(Mul, 7, 8), + Binary(Add, 6, 9), + Parameter(3), + Carry(2), + Binary(Mul, 11, 12), + Binary(Sub, 10, 13), + Parameter(4), + Carry(3), + Binary(Mul, 15, 16), + Binary(Sub, 14, 17), + ], + result: 18, + next_carries: &[1, 4, 18, 12], + }, +]; + +pub fn recipes() -> impl Iterator { + CATALOG + .iter() + .enumerate() + .map(|(index, recipe)| (RecipeId(index as u8), recipe)) +} + +pub fn validate() -> Result<(), String> { + loop_recipe::validate_catalog(CATALOG, MAX_COEFFICIENTS, MAX_CARRIES)?; + // The retained native adapters require these identities and arities. They + // do not impose the arithmetic or the order of next-state updates. + for (id, name, coefficients, carries) in + [(ONE_POLE, "one_pole", 2, 1), (BIQUAD, "biquad", 5, 4)] + { + let recipe = id.recipe(); + if recipe.name != name + || recipe.coefficient_count() != coefficients + || recipe.carry_count() != carries + { + return Err(format!("{name}: incompatible native recurrence adapter")); + } + } + Ok(()) +} diff --git a/src/stack_codegen.rs b/src/stack_codegen.rs index 6c8c6e9d..f172fc37 100644 --- a/src/stack_codegen.rs +++ b/src/stack_codegen.rs @@ -11,6 +11,14 @@ use crate::expr::Expr; use crate::stack_ir::*; use crate::types::*; use std::collections::{HashMap, HashSet}; +use std::convert::TryFrom; + +/// A checked private input or exact literal awaiting native operand storage. +#[derive(Clone, Copy)] +enum LoopScalar { + Local(u16), + Constant(u64), +} /// Loop context for break/continue support. struct LoopContext { @@ -47,6 +55,9 @@ enum LocalKind { /// Code generator for the stack-based VM. pub struct StackCodegen { + /// Opt in to the shared precompiled native-loop consumer. + pub(crate) native_loops: bool, + /// The program being built. program: StackProgram, @@ -81,6 +92,7 @@ impl Default for StackCodegen { impl StackCodegen { pub fn new() -> Self { Self { + native_loops: false, program: StackProgram::new(), func_indices: HashMap::new(), compiled_functions: HashSet::new(), @@ -207,14 +219,35 @@ impl StackCodegen { let mut func = StackFunction::new(&*decl.name); func.param_count = decl.params.len() as u8; + let pending_function_count = self.pending_functions.len(); + let lambda_counter = self.lambda_counter; + let mut translator = FunctionTranslator::new( decl, decls, &mut self.pending_functions, &mut self.lambda_counter, &self.globals, + self.native_loops && instance.is_some(), ); - translator.translate(&mut func); + if !translator.translate(&mut func)? { + // Native setup is optional. Discard the complete unpublished draft + // and retry ordinary lowering if its extra slots exceed the encoding. + drop(translator); + self.pending_functions.truncate(pending_function_count); + self.lambda_counter = lambda_counter; + func = StackFunction::new(&*decl.name); + func.param_count = decl.params.len() as u8; + translator = FunctionTranslator::new( + decl, + decls, + &mut self.pending_functions, + &mut self.lambda_counter, + &self.globals, + false, + ); + translator.translate(&mut func)?; + } let idx = self.program.add_function(func); if let Some(instance) = instance { @@ -305,10 +338,17 @@ struct FunctionTranslator<'a> { variables: HashMap, /// Next available scalar local slot. - next_scalar: u16, + next_scalar: u32, /// Next available memory slot (in 8-byte units). - next_memory_slot: u16, + next_memory_slot: u32, + + /// Highest allocated memory start, including zero-sized objects. + max_memory_slot: Option, + + /// Analysis belongs to precisely this immutable checked body, after hoisting. + loops: HashMap, + native_loop_count: usize, /// Functions that are called and need to be compiled. pending_functions: &'a mut Vec, @@ -365,6 +405,7 @@ impl<'a> FunctionTranslator<'a> { pending_functions: &'a mut Vec, lambda_counter: &'a mut usize, globals: &'a HashMap, + native_loops: bool, ) -> Self { Self { decl, @@ -373,6 +414,13 @@ impl<'a> FunctionTranslator<'a> { next_scalar: 0, next_memory_slot: 0, + max_memory_slot: None, + loops: if native_loops { + crate::value_loops::analyze_function(decl) + } else { + HashMap::new() + }, + native_loop_count: 0, pending_functions, lambda_counter, calls_to_patch: Vec::new(), @@ -394,17 +442,20 @@ impl<'a> FunctionTranslator<'a> { /// Allocate a scalar local slot. fn alloc_scalar(&mut self) -> u16 { let slot = self.next_scalar; - self.next_scalar += 1; - slot + self.next_scalar = self.next_scalar.saturating_add(1); + // Draft operands are narrow, but a complete wide frame check runs before + // any function is published. Overflowing drafts are discarded. + slot as u16 } /// Allocate a memory-backed local slot. Returns the memory slot index. /// size is in bytes; we round up to 8-byte units. fn alloc_memory(&mut self, size: u32) -> u16 { let slot = self.next_memory_slot; - let slots_needed = ((size + 7) / 8) as u16; - self.next_memory_slot += slots_needed; - slot + self.max_memory_slot = Some(self.max_memory_slot.map_or(slot, |last| last.max(slot))); + let slots_needed = size / 8 + u32::from(size % 8 != 0); + self.next_memory_slot = self.next_memory_slot.saturating_add(slots_needed); + slot as u16 } /// Get the type of an expression. @@ -461,7 +512,7 @@ impl<'a> FunctionTranslator<'a> { } /// Translate the function body. - fn translate(&mut self, func: &mut StackFunction) { + fn translate(&mut self, func: &mut StackFunction) -> Result { // If return type is a pointer type, first parameter is output pointer. let has_sret = returns_via_pointer(self.decl.ret); if has_sret { @@ -479,7 +530,7 @@ impl<'a> FunctionTranslator<'a> { let ty = self.decl.arena.local(param.local).ty; if let Type::Reference(_) = &*ty { - while self.next_scalar <= param_slot { + while self.next_scalar <= u32::from(param_slot) { self.alloc_scalar(); } self.variables @@ -487,7 +538,7 @@ impl<'a> FunctionTranslator<'a> { } else if !self.is_ptr_type(&ty) { // Scalar parameter: already in local slot param_slot by calling convention. // Just make sure our allocator accounts for it. - while self.next_scalar <= param_slot { + while self.next_scalar <= u32::from(param_slot) { self.alloc_scalar(); } self.variables @@ -495,7 +546,7 @@ impl<'a> FunctionTranslator<'a> { } else { // Pointer-represented parameters are passed as addresses. // Keep the address value directly, matching the JIT/LLVM ABI. - while self.next_scalar <= param_slot { + while self.next_scalar <= u32::from(param_slot) { self.alloc_scalar(); } self.variables @@ -602,9 +653,25 @@ impl<'a> FunctionTranslator<'a> { func.emit(StackOp::ReturnVoid); } - func.local_count = self.next_scalar; - func.local_memory = self.next_memory_slot as u32 * 8; + let memory_starts_fit = self + .max_memory_slot + .is_none_or(|slot| slot.saturating_add(self.next_scalar) <= u32::from(u16::MAX)); + if self.next_scalar > u32::from(u16::MAX) + || !memory_starts_fit + || self.next_memory_slot.checked_mul(8).is_none() + { + if self.native_loop_count != 0 { + return Ok(false); + } + return Err(format!( + "stack function {} exceeds frame slot limits ({} scalar slots, {} memory slots)", + self.decl.name, self.next_scalar, self.next_memory_slot, + )); + } + func.local_count = self.next_scalar as u16; + func.local_memory = self.next_memory_slot * 8; func.has_return_value = !matches!(&*self.decl.ret, Type::Void) && !has_sret; + Ok(true) } /// Translate an expression. Pushes exactly one value onto the stack @@ -1002,7 +1069,7 @@ impl<'a> FunctionTranslator<'a> { start, end, body, .. } => { let var = self.decl.arena.binder(expr); - self.translate_for(var, *start, *end, *body, func); + self.translate_for(expr, var, *start, *end, *body, func); if !self.void_ctx { func.emit(StackOp::I64Const(0)); } @@ -2514,6 +2581,7 @@ impl<'a> FunctionTranslator<'a> { /// Translate a for loop. fn translate_for( &mut self, + expr: ExprID, var: LocalId, start_id: ExprID, end_id: ExprID, @@ -2532,6 +2600,10 @@ impl<'a> FunctionTranslator<'a> { let end_local = self.alloc_scalar(); func.emit(StackOp::LocalSet(end_local)); + // Bounds are evaluated once. The native path and scalar continuation + // share the same counter/end slots and publication boundary. + let kernel = self.try_emit_loop(expr, loop_var, end_local, func); + // The checked loop binding has its own local identity. let int_ty = mk_type(Type::Int32); @@ -2594,6 +2666,12 @@ impl<'a> FunctionTranslator<'a> { // Patch jumps. func.patch_jump(jump_to_end); + if let Some(index) = kernel { + let done = func.pos() as i32 - index as i32 - 1; + if let StackOp::NativeLoop(kernel) = &mut func.ops[index] { + kernel.done = done; + } + } let ctx = self.loop_stack.pop().unwrap(); for bp in ctx.break_patches { func.patch_jump(bp); @@ -2608,6 +2686,184 @@ impl<'a> FunctionTranslator<'a> { // Caller handles result push if needed. } + /// Select a precompiled implementation from a loop's checked value graph. + /// Unsupported value/effect shapes keep the existing loop lowering. + fn try_emit_loop( + &mut self, + expr: ExprID, + counter: u16, + end: u16, + func: &mut StackFunction, + ) -> Option { + use crate::value_loops::Scalar; + let plan = self.loops.get(&expr)?.clone(); + // Contracts supply typed coefficient/state roles. Resolve every + // scalar and span before emitting any native setup, then lower the + // common stream domain independently of its arithmetic body. + let (input, output, pending) = if let Some(map) = plan.pointwise() { + // Only the recipe's active prefix is materialized. Padding names + // slot zero but is never read by the generated implementation. + let mut coefficients = std::array::from_fn(|_| LoopScalar::Local(0)); + for (slot, value) in coefficients.iter_mut().zip(map.coefficients) { + *slot = self.loop_scalar(&plan, value)?; + } + ( + map.input, + map.output, + NativeStreamBody::Pointwise { + recipe: map.recipe, + coefficients, + }, + ) + } else if let Some(pole) = plan.one_pole() { + ( + pole.input, + pole.output, + NativeStreamBody::OnePole { + state: self.loop_scalar_local(plan.carries[pole.carry].binding)?, + feed: self.loop_scalar(&plan, pole.feed)?, + feedback: self.loop_scalar(&plan, pole.feedback)?, + }, + ) + } else if let Some(biquad) = plan.biquad() { + let [x1, x2, y1, y2] = biquad + .carries + .map(|carry| self.loop_scalar_local(plan.carries[carry].binding)); + let [b0, b1, b2, a1, a2] = biquad + .coefficients + .map(|value| self.loop_scalar(&plan, value)); + ( + biquad.input, + biquad.output, + NativeStreamBody::Biquad { + states: [x1?, x2?, y1?, y2?], + coefficients: [b0?, b1?, b2?, a1?, a2?], + }, + ) + } else { + return None; + }; + let input_expr = plan.streams[input].expression; + let output_expr = plan.streams[output].expression; + let input_length = self.loop_span_length(input_expr)?; + let output_length = self.loop_span_length(output_expr)?; + let body = pending.map_coefficients(|value| self.materialize_loop_scalar(value, func)); + let (input_slot, input_len) = self.materialize_loop_span(input_expr, input_length, func); + let (output_slot, output_len) = if input == output { + (input_slot, input_len) + } else { + self.materialize_loop_span(output_expr, output_length, func) + }; + let op = StackOp::NativeLoop(NativeLoopKernel { + scalar: match plan.ty { + Scalar::F32 => NativeScalar::F32, + Scalar::F64 => NativeScalar::F64, + }, + counter, + end, + done: 0, + spans: NativeStreamSlots { + input: input_slot, + output: output_slot, + input_len, + output_len, + }, + body, + }); + let index = func.pos(); + func.emit(op); + self.native_loop_count += 1; + Some(index) + } + + /// Only an owned numeric scalar slot can be a private native port. Captured + /// storage and reference/pointer slots keep ordinary lowering. + fn loop_scalar_local(&self, binding: LocalId) -> Option { + if self.captured_vars.contains(&binding) + || self.lambda_referenced.contains(&binding) + || !matches!( + &*self.decl.arena.local(binding).ty, + Type::Float32 | Type::Float64 + ) + { + return None; + } + match self.variables.get(&binding) { + Some(LocalKind::Scalar(slot)) => Some(*slot), + _ => None, + } + } + + /// Resolve invariant scalar ports while the outer lexical environment is live. + fn loop_scalar( + &self, + plan: &crate::value_loops::LoopRegion, + value: crate::value_loops::ValueId, + ) -> Option { + use crate::value_loops::{Operation, Scalar}; + match &plan.values.get(value.0)?.operation { + Operation::Input(binding) => Some(LoopScalar::Local(self.loop_scalar_local(*binding)?)), + Operation::Constant(text) => Some(LoopScalar::Constant(match plan.ty { + Scalar::F32 => u64::from(text.parse::().unwrap_or(0.0).to_bits()), + Scalar::F64 => text.parse::().unwrap_or(0.0).to_bits(), + })), + _ => None, + } + } + + fn materialize_loop_scalar(&mut self, value: LoopScalar, func: &mut StackFunction) -> u16 { + match value { + LoopScalar::Local(slot) => slot, + LoopScalar::Constant(bits) => { + let slot = self.alloc_scalar(); + func.emit(StackOp::FusedConstSet(bits as i64, slot)); + slot + } + } + } + + /// Solved array-to-slice coercions do not change the original storage ABI. + fn loop_span_length(&self, expression: ExprID) -> Option { + use crate::value_loops::SpanLength; + match &*self.representation_type(expression) { + Type::Array(_, ArraySize::Known(length)) => { + Some(SpanLength::Fixed(u32::try_from(*length).ok()?)) + } + Type::Slice(_) => Some(SpanLength::Slice), + _ => None, + } + } + + fn materialize_loop_span( + &mut self, + expression: ExprID, + length: crate::value_loops::SpanLength, + func: &mut StackFunction, + ) -> (u16, u16) { + use crate::value_loops::SpanLength; + self.translate_expr(expression, func); + let pointer = self.alloc_scalar(); + let length_slot = self.alloc_scalar(); + func.emit(StackOp::LocalSet(pointer)); + match length { + SpanLength::Fixed(length) => { + func.emit(StackOp::FusedConstSet(i64::from(length), length_slot)); + } + SpanLength::Slice => { + // First retain the header's length, then replace its address with + // the raw element pointer. Both are refreshed by ordinary lowering + // if an actual callback makes the kernel resume bytecode. + func.emit(StackOp::LocalGet(pointer)); + func.emit(StackOp::Load32Off(8)); + func.emit(StackOp::LocalSet(length_slot)); + func.emit(StackOp::LocalGet(pointer)); + func.emit(StackOp::Load64); + func.emit(StackOp::LocalSet(pointer)); + } + } + (pointer, length_slot) + } + /// Translate a field access. fn translate_field(&mut self, lhs_id: ExprID, name: Name, func: &mut StackFunction) { let lhs_ty = self.expr_type(lhs_id); diff --git a/src/stack_depth.rs b/src/stack_depth.rs index 1a345f5c..a9144eb7 100644 --- a/src/stack_depth.rs +++ b/src/stack_depth.rs @@ -30,6 +30,7 @@ pub fn compute_depths(func: &StackFunction) -> Vec { StackOp::FusedGetF32ConstFGtJumpIfZeroF(_, _, off) => Some(*off), StackOp::FusedF64ConstDGtJumpIfZeroD(_, off) => Some(*off), StackOp::FusedGetF64ConstDGtJumpIfZeroD(_, _, off) => Some(*off), + StackOp::NativeLoop(kernel) => Some(kernel.done), _ => None, }; if let Some(off) = off { @@ -205,6 +206,7 @@ pub fn stack_delta(op: &StackOp) -> i32 { | StackOp::FusedConstSet(_, _) | StackOp::FusedF32ConstSet(_, _) | StackOp::FusedGetAddImmSet(_, _, _) + | StackOp::NativeLoop(_) | StackOp::FusedGetGetILtJumpIfZero(_, _, _) | StackOp::FusedBoundsCheck1JumpIfZero(_, _) | StackOp::FusedBoundsCheck2JumpIfZero(_, _) diff --git a/src/stack_inline.rs b/src/stack_inline.rs index 1b32d006..f9281787 100644 --- a/src/stack_inline.rs +++ b/src/stack_inline.rs @@ -160,6 +160,7 @@ fn inline_calls_in(func: &mut StackFunction, bodies: &[Option>]) { StackOp::FusedGetF64ConstDGtJumpIfZeroD(_, _, off) => { Some((i as i64 + 1 + *off as i64) as usize) } + StackOp::NativeLoop(kernel) => Some((i as i64 + 1 + kernel.done as i64) as usize), _ => None, }; @@ -191,6 +192,9 @@ fn inline_calls_in(func: &mut StackFunction, bodies: &[Option>]) { StackOp::FusedGetF32ConstFGtJumpIfZeroF(_, _, o) => *o = new_off as i32, StackOp::FusedF64ConstDGtJumpIfZeroD(_, o) => *o = new_off as i32, StackOp::FusedGetF64ConstDGtJumpIfZeroD(_, _, o) => *o = new_off as i32, + StackOp::NativeLoop(kernel) => { + kernel.done = new_off as i32; + } _ => unreachable!(), } } diff --git a/src/stack_interp.c b/src/stack_interp.c index dfa09dbe..90f89fe3 100644 --- a/src/stack_interp.c +++ b/src/stack_interp.c @@ -206,18 +206,20 @@ static int64_t ipow(int64_t base, uint32_t exp) { // Decrement the cancel counter; on expiry invoke the callback and, if it // returns true, mark cancelled and break the tail-call chain. Use this // after applying any backward `pc = pc + 1 + off` (off < 0). -#define POLL_CANCEL() \ - do { \ - if (--ctx->cancel_counter <= 0) { \ - ctx->cancel_counter = CANCEL_CHECK_INTERVAL; \ - if (ctx->cancel_callback && ctx->cancel_callback(ctx->cancel_userdata)) { \ - ctx->cancelled = true; \ - ctx->trap_reason = STACK_TRAP_CANCELLED; \ - ctx->done = 1; \ - return; \ - } \ - } \ - } while(0) +static inline bool poll_cancel(Ctx* ctx) { + if (--ctx->cancel_counter <= 0) { + ctx->cancel_counter = CANCEL_CHECK_INTERVAL; + if (ctx->cancel_callback && ctx->cancel_callback(ctx->cancel_userdata)) { + ctx->cancelled = true; + ctx->trap_reason = STACK_TRAP_CANCELLED; + ctx->done = 1; + return true; + } + } + return false; +} + +#define POLL_CANCEL() do { if (poll_cancel(ctx)) return; } while (0) // Float TOS window push/pop. Spills/refills through `fsp`, a handler // argument pinned to a GPR by preserve_none — analogous to the integer @@ -1315,6 +1317,229 @@ HANDLER(op_fused_get_addimm_set) { NEXT(); } +// Whole-loop gateways retain the ordinary loop immediately after this op. +// Decode the compact and descriptor encodings into the same execution contract: +// two equally indexed spans, private coefficients/state, and a success edge. +typedef enum { +#define RECURRENCE_RECIPE(id, kind, name, coefficients, carried) NATIVE_##kind, +#include "recurrence_catalog.inc" +#undef RECURRENCE_RECIPE +#define POINTWISE_RECIPE(id, kind, name, coefficients) NATIVE_POINTWISE_##kind, +#include "pointwise_catalog.inc" +#undef POINTWISE_RECIPE +} NativeKernelKind; + +typedef struct { + int coefficients, carried; +} NativeKernelShape; + +static inline NativeKernelShape native_kernel_shape(NativeKernelKind kind) { + switch (kind) { +#define RECURRENCE_RECIPE(id, kind, name, coefficients, carried) \ + case NATIVE_##kind: return (NativeKernelShape) { coefficients, carried }; +#include "recurrence_catalog.inc" +#undef RECURRENCE_RECIPE +#define POINTWISE_RECIPE(id, kind, name, coefficients) \ + case NATIVE_POINTWISE_##kind: return (NativeKernelShape) { coefficients, 0 }; +#include "pointwise_catalog.inc" +#undef POINTWISE_RECIPE + } + __builtin_unreachable(); +} + +typedef struct { + uint16_t counter, end, left, right; + int64_t left_length, right_length, done; + uint16_t coefficients[5]; + uint16_t states[4]; +} NativeLoopOperands; + +static inline NativeLoopOperands native_stream_operands( + const Instruction* pc, const uint64_t* locals +) { + uint64_t slots = pc->imm[0], parameters = pc->imm[1]; + return (NativeLoopOperands) { + .counter = (uint16_t)slots, .end = (uint16_t)(slots >> 16), + .left = (uint16_t)(slots >> 32), .right = (uint16_t)(slots >> 48), + .left_length = (int64_t)locals[(uint16_t)(parameters >> 32)], + .right_length = (int64_t)locals[(uint16_t)(parameters >> 48)], + .coefficients = { (uint16_t)parameters, (uint16_t)(parameters >> 16) }, + .states = { (uint16_t)(pc->imm[2] >> 32) }, .done = (int32_t)pc->imm[2], + }; +} + +// Pointwise recipes use the stream encoding's spare high 32 bits of imm2 +// for coefficients two and three. Shape metadata keeps unused slots unread. +static inline NativeLoopOperands native_pointwise_operands( + const Instruction* pc, const uint64_t* locals +) { + NativeLoopOperands operands = native_stream_operands(pc, locals); + operands.coefficients[2] = (uint16_t)(pc->imm[2] >> 32); + operands.coefficients[3] = (uint16_t)(pc->imm[2] >> 48); + return operands; +} + +// Biquad's immutable uint16_t descriptor is owned by StackBackend: +// counter/end/input/output/input_len/output_len, b0/b1/b2/a1/a2, x1/x2/y1/y2. +static inline NativeLoopOperands native_biquad_operands( + const Instruction* pc, const uint64_t* locals +) { + const uint16_t* slots = (const uint16_t*)(uintptr_t)pc->imm[0]; + return (NativeLoopOperands) { + .counter = slots[0], .end = slots[1], .left = slots[2], .right = slots[3], + .left_length = (int64_t)locals[slots[4]], + .right_length = (int64_t)locals[slots[5]], + .coefficients = { slots[6], slots[7], slots[8], slots[9], slots[10] }, + .states = { slots[11], slots[12], slots[13], slots[14] }, + .done = (int32_t)pc->imm[2], + }; +} + +typedef struct { + int64_t index, end; + const uint8_t* left; + uint8_t* right; +} NativeLoop; + +typedef enum { NATIVE_CONTINUE, NATIVE_FALLBACK, NATIVE_CANCELLED } NativeLoopResult; + +// Empty/reversed ranges succeed without accessing either span. For writes, +// exact in-place and disjoint ranges are valid; partial overlap retains the +// original loop's ordered forward semantics by falling back before any work. +static inline bool native_loop_prepare( + NativeLoop* loop, const NativeLoopOperands* operands, const uint64_t* locals, + size_t width +) { + loop->index = (int64_t)locals[operands->counter]; + loop->end = (int64_t)locals[operands->end]; + if (loop->index >= loop->end) return true; + if (loop->index < 0 || loop->end > operands->left_length + || loop->end > operands->right_length + || (uint64_t)loop->end > SIZE_MAX / width) return false; + uintptr_t left = (uintptr_t)locals[operands->left]; + uintptr_t right = (uintptr_t)locals[operands->right]; + uintptr_t distance = left > right ? left - right : right - left; + uint64_t count_bytes = (uint64_t)(loop->end - loop->index) * width; + if (distance != 0 && distance < count_bytes) return false; + loop->left = (const uint8_t*)left; + loop->right = (uint8_t*)right; + return true; +} + +static inline int64_t native_loop_count(const NativeLoop* loop, const Ctx* ctx) { + int64_t count = loop->end - loop->index; + int32_t budget = ctx->cancel_counter > 0 ? ctx->cancel_counter : 1; + return count > budget ? budget : count; +} + +// Called after publishing every carried state and output in this chunk. Count +// the final iteration's backedge too. An actual callback ends native execution +// even when it does not cancel: the ordinary guard must reload anything that +// callback may have changed, including coefficients, pointers, bounds and state. +static inline NativeLoopResult native_loop_commit( + NativeLoop* loop, const NativeLoopOperands* operands, + Ctx* ctx, uint64_t* locals, int64_t count +) { + loop->index += count; + locals[operands->counter] = (uint64_t)loop->index; + ctx->cancel_counter -= (int32_t)count; + if (ctx->cancel_counter <= 0) { + bool called = ctx->cancel_callback != NULL; + ctx->cancel_counter = 1; + if (poll_cancel(ctx)) return NATIVE_CANCELLED; + if (called) return NATIVE_FALLBACK; + } + return NATIVE_CONTINUE; +} + +// This macro only instantiates typed arithmetic for f32 and f64. Kernel kind is +// a constant at each handler; forced inlining removes selection and unused state +// before native code generation. There is no per-sample operation dispatch. +// __builtin_memcpy preserves checked unaligned stores through vectorization; +// Darwin's fortified memcpy wrapper can otherwise survive until after that pass. +#include "pointwise_chunks.inc" +#include "recurrence_chunks.inc" + +#define DEFINE_NATIVE_KERNEL_TYPE(suffix, scalar, load) \ +typedef struct { scalar coefficients[5], carried[4]; } NativeState_##suffix; \ +static inline __attribute__((always_inline)) NativeState_##suffix native_state_##suffix( \ + NativeKernelKind kind, const NativeLoopOperands* operands, const uint64_t* locals \ +) { \ + NativeState_##suffix state = {0}; \ + NativeKernelShape shape = native_kernel_shape(kind); \ + for (int i = 0; i < shape.coefficients; ++i) state.coefficients[i] = load(locals + operands->coefficients[i]); \ + for (int i = 0; i < shape.carried; ++i) state.carried[i] = load(locals + operands->states[i]); \ + return state; \ +} \ +static inline __attribute__((always_inline)) void native_chunk_##suffix( \ + NativeKernelKind kind, const NativeLoop* loop, int64_t count, NativeState_##suffix* state \ +) { \ + const uint8_t* input = loop->left + (size_t)loop->index * sizeof(scalar); \ + uint8_t* output = loop->right + (size_t)loop->index * sizeof(scalar); \ + if (native_kernel_shape(kind).carried != 0) { \ + native_recurrence_##suffix(kind, input, output, count, state->coefficients, state->carried); \ + } else { \ + native_pointwise_##suffix(kind, input, output, count, state->coefficients); \ + } \ +} \ +static inline __attribute__((always_inline)) void native_publish_##suffix( \ + NativeKernelKind kind, const NativeLoopOperands* operands, uint64_t* locals, const NativeState_##suffix* state \ +) { \ + NativeKernelShape shape = native_kernel_shape(kind); \ + for (int i = 0; i < shape.carried; ++i) \ + __builtin_memcpy(locals + operands->states[i], &state->carried[i], sizeof(scalar)); \ +} + +DEFINE_NATIVE_KERNEL_TYPE(f32, float, load_f32_unaligned) +DEFINE_NATIVE_KERNEL_TYPE(f64, double, load_f64_unaligned) +#undef DEFINE_NATIVE_KERNEL_TYPE + +// One execution boundary for every precompiled kernel. Arithmetic and state +// publication are statically specialized above; all three operand windows pass +// through unchanged on success, guard rejection, cancellation and resumption. +#define NATIVE_LOOP_HANDLER(name, suffix, scalar, kind, decode) \ +HANDLER(name) { \ + NativeLoopOperands operands = decode(pc, locals); \ + NativeLoop loop; \ + if (!native_loop_prepare(&loop, &operands, locals, sizeof(scalar))) { NEXT(); } \ + if (loop.index < loop.end) { \ + NativeState_##suffix state = native_state_##suffix(kind, &operands, locals); \ + while (loop.index < loop.end) { \ + int64_t count = native_loop_count(&loop, ctx); \ + native_chunk_##suffix(kind, &loop, count, &state); \ + native_publish_##suffix(kind, &operands, locals, &state); \ + NativeLoopResult result = native_loop_commit(&loop, &operands, ctx, locals, count); \ + if (result == NATIVE_CANCELLED) return; \ + if (result == NATIVE_FALLBACK) { NEXT(); } \ + } \ + } \ + pc = pc + 1 + operands.done; \ + DISPATCH(); \ +} + +#define POINTWISE_RECIPE(id, kind, name, coefficients) \ +NATIVE_LOOP_HANDLER(op_##name##_f32, f32, float, NATIVE_POINTWISE_##kind, native_pointwise_operands) \ +NATIVE_LOOP_HANDLER(op_##name##_f64, f64, double, NATIVE_POINTWISE_##kind, native_pointwise_operands) +#include "pointwise_catalog.inc" +#undef POINTWISE_RECIPE +NATIVE_LOOP_HANDLER(op_one_pole_f32, f32, float, NATIVE_ONE_POLE, native_stream_operands) +NATIVE_LOOP_HANDLER(op_one_pole_f64, f64, double, NATIVE_ONE_POLE, native_stream_operands) +NATIVE_LOOP_HANDLER(op_biquad_f32, f32, float, NATIVE_BIQUAD, native_biquad_operands) +NATIVE_LOOP_HANDLER(op_biquad_f64, f64, double, NATIVE_BIQUAD, native_biquad_operands) +#undef NATIVE_LOOP_HANDLER + +// Backend construction resolves the recipe once. Executed instructions contain +// the specialized handler pointer; there is no recipe lookup in a sample loop. +void* stack_pointwise_handler(uint32_t recipe, bool double_precision) { + switch (recipe) { +#define POINTWISE_RECIPE(id, kind, name, coefficients) \ + case id: return double_precision ? (void*)op_##name##_f64 : (void*)op_##name##_f32; +#include "pointwise_catalog.inc" +#undef POINTWISE_RECIPE + default: return NULL; + } +} + // if !(locals[a] < locals[b]) jump -- no stack change HANDLER(op_fused_get_get_ilt_jiz) { if ((int64_t)locals[pc->imm[0]] >= (int64_t)locals[pc->imm[1]]) { @@ -2114,11 +2339,17 @@ HANDLER(op_fused_get_get_fmul_fsub_f) { f0 = f0 - a * b; NEXT(); } +// The source chain starts with its first product, not +0 plus that product: +// the latter would erase a negative zero. Codegen never sets mask bit 0; +// if present, it denotes unary negation of the initial rounded product. #define FMUL_SUM_HANDLER(name, TERMS) \ HANDLER(name) { \ uint8_t sub_mask = (uint8_t)pc->imm[2]; \ - float acc = 0.0f; \ - for (int i = 0; i < (TERMS); i++) { \ + float a0 = *(float*)((uint8_t*)locals + (size_t)imm_u8(pc, 0) * 8); \ + float b0 = *(float*)((uint8_t*)locals + (size_t)imm_u8(pc, 1) * 8); \ + float acc = a0 * b0; \ + if (sub_mask & 1u) acc = -acc; \ + for (int i = 1; i < (TERMS); i++) { \ uint8_t a_idx = imm_u8(pc, i * 2); \ uint8_t b_idx = imm_u8(pc, i * 2 + 1); \ float a = *(float*)((uint8_t*)locals + (size_t)a_idx * 8); \ @@ -2252,11 +2483,15 @@ HANDLER(op_fused_get_get_dmul_dsub_d) { NEXT(); } +// Same first-product and optional unary-negation semantics as the F handler. #define DMUL_SUM_HANDLER(name, TERMS) \ HANDLER(name) { \ uint8_t sub_mask = (uint8_t)pc->imm[2]; \ - double acc = 0.0; \ - for (int i = 0; i < (TERMS); i++) { \ + double a0 = *(double*)((uint8_t*)locals + (size_t)imm_u8(pc, 0) * 8); \ + double b0 = *(double*)((uint8_t*)locals + (size_t)imm_u8(pc, 1) * 8); \ + double acc = a0 * b0; \ + if (sub_mask & 1u) acc = -acc; \ + for (int i = 1; i < (TERMS); i++) { \ uint8_t a_idx = imm_u8(pc, i * 2); \ uint8_t b_idx = imm_u8(pc, i * 2 + 1); \ double a = *(double*)((uint8_t*)locals + (size_t)a_idx * 8); \ diff --git a/src/stack_interp_bridge.rs b/src/stack_interp_bridge.rs index e94acd24..47e30b73 100644 --- a/src/stack_interp_bridge.rs +++ b/src/stack_interp_bridge.rs @@ -3,7 +3,7 @@ //! Converts StackOp instructions into the C Instruction format //! (handler pointer + 3 immediates) and calls stack_interp_run. -use crate::stack_ir::{StackOp, StackProgram}; +use crate::stack_ir::{NativeLoopKernel, NativeScalar, NativeStreamBody, StackOp, StackProgram}; // C types matching stack_interp.h #[repr(C)] @@ -185,6 +185,11 @@ extern "C" { fn op_fused_get_get_ilt(); fn op_fused_addr_load32off(); fn op_fused_get_addimm_set(); + fn stack_pointwise_handler(recipe: u32, double: bool) -> *const (); + fn op_one_pole_f32(); + fn op_one_pole_f64(); + fn op_biquad_f32(); + fn op_biquad_f64(); fn op_fused_get_get_ilt_jiz(); fn op_fused_bounds_check1_jiz(); fn op_fused_bounds_check2_jiz(); @@ -514,6 +519,24 @@ fn handler_for(op: &StackOp) -> *const () { StackOp::FusedGetGetILt(_, _) => op_fused_get_get_ilt as *const (), StackOp::FusedAddrLoad32Off(_, _) => op_fused_addr_load32off as *const (), StackOp::FusedGetAddImmSet(_, _, _) => op_fused_get_addimm_set as *const (), + StackOp::NativeLoop(kernel) => match (&kernel.body, kernel.scalar) { + (NativeStreamBody::Pointwise { recipe, .. }, scalar) => { + // Resolve once while encoding. Execution dispatches directly + // to the generated, statically specialized native handler. + let handler = unsafe { + stack_pointwise_handler(recipe.index() as u32, scalar == NativeScalar::F64) + }; + assert!( + !handler.is_null(), + "pointwise handler missing from generated catalog" + ); + handler + } + (NativeStreamBody::OnePole { .. }, NativeScalar::F32) => op_one_pole_f32 as *const (), + (NativeStreamBody::OnePole { .. }, NativeScalar::F64) => op_one_pole_f64 as *const (), + (NativeStreamBody::Biquad { .. }, NativeScalar::F32) => op_biquad_f32 as *const (), + (NativeStreamBody::Biquad { .. }, NativeScalar::F64) => op_biquad_f64 as *const (), + }, StackOp::FusedGetGetILtJumpIfZero(_, _, _) => op_fused_get_get_ilt_jiz as *const (), StackOp::FusedBoundsCheck1JumpIfZero(_, _) => op_fused_bounds_check1_jiz as *const (), StackOp::FusedBoundsCheck2JumpIfZero(_, _) => op_fused_bounds_check2_jiz as *const (), @@ -805,6 +828,7 @@ fn encode_imm(op: &StackOp, func_idx: u32) -> [u64; 3] { } StackOp::FusedTeeSliceStore32(n, s, idx) => [*n as u64, *s as u64, *idx as u64], StackOp::FusedGetAddImmSet(s, v, d) => [*s as u64, *v as i64 as u64, *d as u64], + StackOp::NativeLoop(kernel) => encode_native_loop(kernel), StackOp::FusedGetGetILtJumpIfZero(a, b, off) => [*a as u64, *b as u64, *off as i64 as u64], StackOp::FusedBoundsCheck1JumpIfZero(p, off) => { let mut out = pack_u8_imms(p); @@ -996,13 +1020,17 @@ pub fn run(program: &StackProgram) -> i64 { /// repeatedly without re-allocating. Globals are passed per-call so a /// single backend can drive multiple independent globals buffers. /// -/// Pin-in-place semantics: once built, the struct must not move — `ctx` -/// holds raw pointers into the backend's own vectors. The FFI layer boxes -/// it (via `Box`) which keeps the address stable. +/// The encoded instructions and `ctx` hold raw pointers into owned backing +/// allocations. Those allocations must not resize after construction; moving +/// the backend itself is safe because it does not move the allocations. pub struct StackBackend { // Owned program data. The inner Vec never resizes after // construction, so the raw pointers stored in `func_metas` stay valid. _c_instructions: Vec>, + // Large native kernels name scalar slots through immutable descriptors. + // Freeze the arena before encoding pointers; neither calls nor moving the + // backend can invalidate its allocation. Small gateways stay inline. + _native_kernel_operands: Box<[u16]>, func_metas: Vec, call_stack: Vec, operand_stack: Vec, @@ -1012,15 +1040,96 @@ pub struct StackBackend { ctx: Ctx, } +#[cfg(test)] +mod native_loop_tests; + +fn encode_native_loop(kernel: &NativeLoopKernel) -> [u64; 3] { + let spans = &kernel.spans; + let body = &kernel.body; + let domain = u64::from(kernel.counter) + | (u64::from(kernel.end) << 16) + | (u64::from(spans.input) << 32) + | (u64::from(spans.output) << 48); + let lengths = (u64::from(spans.input_len) << 32) | (u64::from(spans.output_len) << 48); + match body { + NativeStreamBody::Pointwise { coefficients, .. } => [ + domain, + u64::from(coefficients[0]) | (u64::from(coefficients[1]) << 16) | lengths, + u64::from(kernel.done as u32) + | (u64::from(coefficients[2]) << 32) + | (u64::from(coefficients[3]) << 48), + ], + NativeStreamBody::OnePole { + feed, + feedback, + state, + } => [ + domain, + u64::from(*feed) | (u64::from(*feedback) << 16) | lengths, + u64::from(kernel.done as u32) | (u64::from(*state) << 32), + ], + // imm0 is filled from the frozen operand arena during backend construction. + NativeStreamBody::Biquad { .. } => [0, 0, kernel.done as i64 as u64], + } +} + +// Slot order is the native Biquad descriptor ABI; C reads six domain slots, +// five coefficients (b0, b1, b2, a1, a2), then four states (x1, x2, y1, y2). +fn native_kernel_operands(op: &StackOp) -> Option<[u16; 15]> { + match op { + StackOp::NativeLoop(NativeLoopKernel { + counter, + end, + spans, + body: + NativeStreamBody::Biquad { + coefficients, + states, + }, + .. + }) => Some([ + *counter, + *end, + spans.input, + spans.output, + spans.input_len, + spans.output_len, + coefficients[0], + coefficients[1], + coefficients[2], + coefficients[3], + coefficients[4], + states[0], + states[1], + states[2], + states[3], + ]), + _ => None, + } +} + impl StackBackend { pub fn new(program: &StackProgram) -> Self { + let native_kernel_operands: Box<[u16]> = program + .functions + .iter() + .flat_map(|function| &function.ops) + .filter_map(native_kernel_operands) + .flatten() + .collect(); + let mut operand_offset = 0; let mut c_instructions: Vec> = Vec::with_capacity(program.functions.len()); for (fi, func) in program.functions.iter().enumerate() { let mut instrs: Vec = Vec::with_capacity(func.ops.len() + 1); for op in func.ops.iter() { + let mut imm = encode_imm(op, fi as u32); + if let Some(operands) = self::native_kernel_operands(op) { + imm[0] = native_kernel_operands[operand_offset..].as_ptr() as u64; + operand_offset += operands.len(); + } instrs.push(Instruction { handler: handler_for(op), - imm: encode_imm(op, fi as u32), + imm, }); } instrs.push(Instruction { @@ -1084,6 +1193,7 @@ impl StackBackend { Self { _c_instructions: c_instructions, + _native_kernel_operands: native_kernel_operands, func_metas, call_stack, operand_stack, @@ -1192,10 +1302,7 @@ mod tests { let mut globals: Vec = vec![0u8; program.globals_size.max(1)]; backend.call_entry(program.entry, globals.as_mut_ptr()); - assert!( - !backend.cancelled(), - "assertion trap is not a cancellation" - ); + assert!(!backend.cancelled(), "assertion trap is not a cancellation"); assert_eq!( backend.trap_reason(), crate::cancel::TRAP_ASSERTION_FAILED, diff --git a/src/stack_interp_bridge/native_loop_tests.rs b/src/stack_interp_bridge/native_loop_tests.rs new file mode 100644 index 00000000..53219e9a --- /dev/null +++ b/src/stack_interp_bridge/native_loop_tests.rs @@ -0,0 +1,714 @@ +use super::*; +use crate::stack_ir::{NativeStreamSlots, StackFunction}; + +const LENGTH: usize = 240; +const STORAGE: usize = LENGTH + 4; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Edge { + Mixed, + SeparateRounding, + NegativeZero, +} + +#[derive(Clone, Copy, Debug)] +struct Case { + double: bool, + recurrence: bool, + biquad: bool, + edge: Edge, + // None is disjoint, Some(0) is in-place; +/-1 overlap by one sample. + overlap: Option, + start: i64, + end: i64, + input_len: i64, + output_len: i64, + prefix: i64, + callback: bool, + cancel: bool, + mutate: bool, +} + +fn bits(double: bool, value: f64) -> u64 { + if double { + value.to_bits() + } else { + (value as f32).to_bits() as u64 + } +} + +fn program(case: Case, native: bool, input: *const u8, output: *mut u8) -> StackProgram { + let mut function = StackFunction::new(if case.biquad { + "biquad_kernel" + } else { + "stream_kernel" + }); + function.local_count = if case.biquad { 20 } else { 12 }; + function.has_return_value = true; + let epsilon = if case.double { + f64::EPSILON + } else { + f32::EPSILON as f64 + }; + let (coefficients, states) = if case.biquad { + match case.edge { + Edge::Mixed => ( + [0.125, 0.25, 0.125, -0.75, 0.25], + [-0.25, 0.125, 0.0625, -0.125], + ), + Edge::SeparateRounding => ( + [1.0 + epsilon, 1.0, 0.0, 0.0, 0.0], + [-1.0 - 2.0 * epsilon, 0.0, 0.0, 0.0], + ), + Edge::NegativeZero => ([1.0, 1.0, 1.0, -1.0, -1.0], [-0.0; 4]), + } + } else { + let (first, second) = match (case.edge, case.recurrence) { + (Edge::NegativeZero, true) => (1.0, 1.0), + (Edge::NegativeZero, false) => (1.0, -0.0), + (Edge::SeparateRounding, true) => (1.0 + epsilon, 1.0), + (Edge::Mixed, true) => (0.5, 0.25), + (_, false) => (1.0 + epsilon, -1.0 - 2.0 * epsilon), + }; + let initial_state = if case.edge == Edge::SeparateRounding { + -1.0 - 2.0 * epsilon + } else { + -0.0 + }; + ( + [first, second, 0.0, 0.0, 0.0], + [initial_state, 0.0, 0.0, 0.0], + ) + }; + let locals = [ + case.start as u64, + case.end as u64, + input as u64, + output as u64, + bits(case.double, coefficients[0]), + bits(case.double, coefficients[1]), + bits(case.double, states[0]), + case.input_len as u64, + case.output_len as u64, + 0, + 0, + case.prefix as u64, + bits(case.double, coefficients[2]), + bits(case.double, coefficients[3]), + bits(case.double, coefficients[4]), + bits(case.double, states[1]), + bits(case.double, states[2]), + bits(case.double, states[3]), + 0, + 0, + ]; + for (slot, &value) in locals[..usize::from(function.local_count)] + .iter() + .enumerate() + { + function.emit(StackOp::I64Const(value as i64)); + function.emit(StackOp::LocalSet(slot as u16)); + } + function.emit(StackOp::FusedGetGetILtJumpIfZero(10, 11, 2)); + function.emit(StackOp::FusedGetAddImmSet(10, 1, 10)); + function.emit(StackOp::Jump(-3)); + function.emit(StackOp::I64Const(17)); + function.emit(StackOp::F32ConstF(19.0)); + function.emit(StackOp::F64ConstD(23.0)); + let gateway = function.pos(); + if native { + function.emit(StackOp::NativeLoop(NativeLoopKernel { + scalar: if case.double { + NativeScalar::F64 + } else { + NativeScalar::F32 + }, + counter: 0, + end: 1, + done: 0, + spans: NativeStreamSlots { + input: 2, + output: 3, + input_len: 7, + output_len: 8, + }, + body: if case.biquad { + NativeStreamBody::Biquad { + coefficients: [4, 5, 12, 13, 14], + states: [6, 15, 16, 17], + } + } else if case.recurrence { + NativeStreamBody::OnePole { + feed: 4, + feedback: 5, + state: 6, + } + } else { + NativeStreamBody::Pointwise { + recipe: crate::pointwise::AFFINE, + coefficients: [4, 5, 0, 0], + } + }, + })); + } + let guard = function.pos(); + function.emit(StackOp::FusedGetGetILtJumpIfZero(0, 1, 0)); + if case.biquad { + emit_scalar_biquad(&mut function, case.double); + } else { + emit_scalar_stream(&mut function, case.double, case.recurrence); + } + function.emit(StackOp::FusedGetAddImmSet(0, 1, 0)); + function.emit(StackOp::Jump(guard as i32 - function.pos() as i32 - 1)); + let done = function.pos(); + function.ops[guard] = StackOp::FusedGetGetILtJumpIfZero(0, 1, done as i32 - guard as i32 - 1); + if native { + match &mut function.ops[gateway] { + StackOp::NativeLoop(kernel) => kernel.done = done as i32 - gateway as i32 - 1, + _ => unreachable!(), + } + } + for op in [ + StackOp::I64Const(17), + StackOp::IEq, + StackOp::JumpIfNotZero(2), + StackOp::I64Const(0), + StackOp::Assert, + StackOp::F32ConstF(19.0), + StackOp::FEqF, + StackOp::JumpIfNotZero(2), + StackOp::I64Const(0), + StackOp::Assert, + StackOp::F64ConstD(23.0), + StackOp::DEqD, + StackOp::JumpIfNotZero(2), + StackOp::I64Const(0), + StackOp::Assert, + StackOp::FusedConstSet(1, 9), + StackOp::LocalGet(if case.biquad { 16 } else { 6 }), + StackOp::Return, + ] { + function.emit(op); + } + let mut program = StackProgram::new(); + program.entry = program.add_function(function); + program +} + +// Keep these scalar witnesses independent of native recipe matching/generation. +fn emit_scalar_biquad(function: &mut StackFunction, double: bool) { + let get = |slot| { + if double { + StackOp::LocalGetD(slot) + } else { + StackOp::LocalGetF(slot) + } + }; + let set = |slot| { + if double { + StackOp::LocalSetD(slot) + } else { + StackOp::LocalSetF(slot) + } + }; + function.emit(StackOp::LocalGet(2)); + function.emit(StackOp::LocalGet(0)); + function.emit(StackOp::I64Const(if double { 8 } else { 4 })); + function.emit(StackOp::IMul); + function.emit(StackOp::IAdd); + function.emit(if double { + StackOp::LoadF64D + } else { + StackOp::LoadF32F + }); + function.emit(set(18)); + for (term, (coefficient, state)) in [(4, 18), (5, 6), (12, 15), (13, 16), (14, 17)] + .iter() + .copied() + .enumerate() + { + function.emit(get(coefficient)); + function.emit(get(state)); + function.emit(if double { + StackOp::DMulD + } else { + StackOp::FMulF + }); + if term != 0 { + function.emit(match (double, term >= 3) { + (true, true) => StackOp::DSubD, + (true, false) => StackOp::DAddD, + (false, true) => StackOp::FSubF, + (false, false) => StackOp::FAddF, + }); + } + } + function.emit(set(19)); + for (source, destination) in [(6, 15), (18, 6), (16, 17), (19, 16)] { + function.emit(get(source)); + function.emit(set(destination)); + } + function.emit(StackOp::LocalGet(3)); + function.emit(StackOp::LocalGet(0)); + function.emit(StackOp::I64Const(if double { 8 } else { 4 })); + function.emit(StackOp::IMul); + function.emit(StackOp::IAdd); + function.emit(get(19)); + function.emit(if double { + StackOp::StoreF64D + } else { + StackOp::StoreF32F + }); +} + +fn emit_scalar_stream(function: &mut StackFunction, double: bool, recurrence: bool) { + for slot in [3, 2] { + function.emit(StackOp::LocalGet(slot)); + function.emit(StackOp::LocalGet(0)); + function.emit(StackOp::I64Const(if double { 8 } else { 4 })); + function.emit(StackOp::IMul); + function.emit(StackOp::IAdd); + } + function.emit(if double { + StackOp::LoadF64D + } else { + StackOp::LoadF32F + }); + function.emit(if double { + StackOp::LocalGetD(4) + } else { + StackOp::LocalGetF(4) + }); + function.emit(if double { + StackOp::DMulD + } else { + StackOp::FMulF + }); + if recurrence { + function.emit(if double { + StackOp::LocalGetD(6) + } else { + StackOp::LocalGetF(6) + }); + } + function.emit(if double { + StackOp::LocalGetD(5) + } else { + StackOp::LocalGetF(5) + }); + if recurrence { + function.emit(if double { + StackOp::DMulD + } else { + StackOp::FMulF + }); + } + function.emit(if double { + StackOp::DAddD + } else { + StackOp::FAddF + }); + if recurrence { + function.emit(if double { + StackOp::LocalTeeD(6) + } else { + StackOp::LocalTeeF(6) + }); + } + function.emit(if double { + StackOp::StoreF64D + } else { + StackOp::StoreF32F + }); +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct Snapshot { + counter: i64, + end: i64, + state: u64, + additional_states: [u64; 3], + finished: u64, + primary: Vec, + secondary: Vec, +} + +struct Probe { + case: Case, + frame: *mut u64, + primary: *mut u8, + secondary: *mut u8, + size: usize, + snapshots: Vec, +} + +impl Probe { + unsafe fn snapshot(&self) -> Snapshot { + Snapshot { + counter: *self.frame as i64, + end: *self.frame.add(1) as i64, + state: *self.frame.add(6), + additional_states: if self.case.biquad { + [ + *self.frame.add(15), + *self.frame.add(16), + *self.frame.add(17), + ] + } else { + [0; 3] + }, + finished: *self.frame.add(9), + primary: std::slice::from_raw_parts(self.primary, self.size).to_vec(), + secondary: std::slice::from_raw_parts(self.secondary, self.size).to_vec(), + } + } +} + +unsafe extern "C" fn observe(data: *mut u8) -> bool { + let probe = &mut *(data as *mut Probe); + probe.snapshots.push(probe.snapshot()); + if probe.case.mutate { + let width = if probe.case.double { 8 } else { 4 }; + *probe.frame -= 1; + *probe.frame.add(1) = (LENGTH - 1) as u64; + *probe.frame.add(2) = probe.secondary.add(1 + 2 * width) as u64; + *probe.frame.add(3) = probe.primary.add(1 + 2 * width) as u64; + *probe.frame.add(4) = bits(probe.case.double, 0.75); + *probe.frame.add(5) = bits(probe.case.double, -0.25); + *probe.frame.add(6) = bits(probe.case.double, -0.5); + if probe.case.biquad { + for (slot, value) in [ + (12, 0.25), + (13, -0.5), + (14, 0.125), + (15, 0.125), + (16, -0.25), + (17, 0.5), + ] { + *probe.frame.add(slot) = bits(probe.case.double, value); + } + } + let changed = bits(probe.case.double, 0.125).to_le_bytes(); + std::ptr::copy_nonoverlapping(changed.as_ptr(), probe.secondary.add(1 + 3 * width), width); + } + probe.case.cancel +} + +#[derive(Debug, PartialEq, Eq)] +struct Outcome { + result: i64, + cancelled: bool, + trap: u32, + budget: i32, + observed: Vec, + finished: Snapshot, +} + +fn run(case: Case, native: bool) -> Outcome { + let width = if case.double { 8 } else { 4 }; + let size = 1 + STORAGE * width; + // Deliberately unaligned streams with padding for invalid-guard cases. + let mut primary = vec![0xa5; size]; + let mut secondary = vec![0x5a; size]; + let epsilon = if case.double { + f64::EPSILON + } else { + f32::EPSILON as f64 + }; + let values = [-0.0, 1.0 + epsilon, -1.0, 0.125, -0.25, 2.0]; + for index in 0..STORAGE { + primary[1 + index * width..1 + (index + 1) * width].copy_from_slice( + &bits(case.double, values[index % values.len()]).to_le_bytes()[..width], + ); + secondary[1 + index * width..1 + (index + 1) * width] + .copy_from_slice(&bits(case.double, 7.0).to_le_bytes()[..width]); + } + if case.edge != Edge::Mixed { + let value = if case.edge == Edge::NegativeZero { + -0.0 + } else { + 1.0 + epsilon + }; + primary[1 + 2 * width..1 + 3 * width] + .copy_from_slice(&bits(case.double, value).to_le_bytes()[..width]); + } + let (input, output) = if case.start >= case.end { + (std::ptr::null(), std::ptr::null_mut()) + } else { + let input = unsafe { primary.as_mut_ptr().add(1 + 2 * width) }; + let output = if let Some(offset) = case.overlap { + unsafe { input.offset(offset * width as isize) } + } else { + unsafe { secondary.as_mut_ptr().add(1 + 2 * width) } + }; + (input.cast_const(), output) + }; + let program = program(case, native, input, output); + let mut backend = StackBackend::new(&program); + let mut probe = Probe { + case, + frame: backend.frame_stack.as_mut_ptr(), + primary: primary.as_mut_ptr(), + secondary: secondary.as_mut_ptr(), + size, + snapshots: Vec::new(), + }; + if case.callback { + backend.set_cancel_callback(Some(observe), (&mut probe as *mut Probe).cast()); + } + let result = backend.call_entry(program.entry, [0u8].as_mut_ptr()); + let finished = unsafe { probe.snapshot() }; + Outcome { + result, + cancelled: backend.cancelled(), + trap: backend.trap_reason(), + budget: backend.ctx.cancel_counter, + observed: probe.snapshots, + finished, + } +} + +fn case(double: bool, recurrence: bool) -> Case { + Case { + double, + recurrence, + biquad: false, + edge: Edge::Mixed, + overlap: None, + start: 0, + end: LENGTH as i64, + input_len: LENGTH as i64, + output_len: LENGTH as i64, + prefix: 0, + callback: true, + cancel: false, + mutate: false, + } +} + +fn poll_cases(base: Case) -> Vec { + let mut cases = Vec::new(); + for overlap in [None, Some(0), Some(-1), Some(1)] { + // Full budget, expiry on final sample, penultimate sample, and first sample. + for prefix in [0, 784, 785, 1023] { + for (callback, cancel) in [(false, false), (true, false), (true, true)] { + cases.push(Case { + overlap, + prefix, + callback, + cancel, + ..base + }); + } + } + } + cases +} + +fn assert_poll_case(case: Case) { + let ordinary = run(case, false); + let native = run(case, true); + assert_eq!(native, ordinary, "{case:?}"); + assert_eq!( + native.cancelled, + case.callback && case.cancel && case.prefix >= 784 + ); + assert_eq!(native.finished.finished, u64::from(!native.cancelled)); + assert_eq!( + native.observed.len(), + usize::from(case.callback && case.prefix >= 784) + ); + if let Some(snapshot) = native.observed.first() { + assert_eq!(snapshot.counter, 1024 - case.prefix); + } +} + +fn guard_and_mutation_cases(base: Case) -> [Case; 11] { + [ + Case { end: 0, ..base }, + Case { + start: 4, + end: 0, + ..base + }, + Case { + start: -1, + end: 1, + ..base + }, + Case { + end: LENGTH as i64 + 1, + ..base + }, + Case { + input_len: 0, + ..base + }, + Case { + output_len: -1, + ..base + }, + Case { + start: 3, + end: 239, + ..base + }, + Case { + prefix: 785, + mutate: true, + ..base + }, + Case { + prefix: 1023, + mutate: true, + ..base + }, + Case { + prefix: 1023, + mutate: true, + overlap: Some(0), + ..base + }, + Case { + prefix: 1023, + mutate: true, + cancel: true, + ..base + }, + ] +} + +#[test] +fn biquad_kernels_preserve_four_states_aliasing_and_poll_boundaries() { + for double in [false, true] { + let base = Case { + biquad: true, + ..case(double, true) + }; + for case in poll_cases(base) { + assert_poll_case(case); + } + for case in guard_and_mutation_cases(base) { + assert_eq!(run(case, true), run(case, false), "{case:?}"); + } + for edge in [Edge::NegativeZero, Edge::SeparateRounding] { + let case = Case { + edge, + end: 1, + ..base + }; + let native = run(case, true); + assert_eq!(native, run(case, false), "{case:?}"); + // y1, the third carried state, holds the output after one step. + assert_eq!( + native.finished.additional_states[1], + bits( + double, + if edge == Edge::NegativeZero { + -0.0 + } else { + 0.0 + } + ) + ); + } + } +} + +#[test] +fn native_operand_arena_survives_source_ir_drop_and_backend_move() { + let input = [bits(false, 2.0), bits(true, 2.0)]; + let mut output = [0u64; 2]; + let backend = { + let mut program = StackProgram::new(); + for (index, double) in [false, true].iter().copied().enumerate() { + let case = Case { + biquad: true, + end: 1, + ..case(double, true) + }; + let mut source = self::program( + case, + true, + (&input[index] as *const u64).cast(), + (&mut output[index] as *mut u64).cast(), + ); + program.add_function(source.functions.remove(0)); + } + crate::native_loop_tests::forbid_fallback(&mut program); + StackBackend::new(&program) + }; + let mut moved = Box::new(backend); + for (function, double) in [false, true].iter().copied().enumerate() { + assert_eq!( + moved.call_entry(function as u32, [0u8].as_mut_ptr()), + bits(double, 0.28125) as i64, + ); + assert_eq!(moved.trap_reason(), crate::cancel::TRAP_NONE); + assert_eq!(output[function], bits(double, 0.28125)); + } +} + +#[test] +fn stream_kernels_preserve_negative_zero_and_separate_multiply_rounding() { + for double in [false, true] { + for recurrence in [false, true] { + for edge in [Edge::SeparateRounding, Edge::NegativeZero] { + for overlap in [None, Some(0)] { + let case = Case { + edge, + overlap, + end: 1, + ..case(double, recurrence) + }; + let native = run(case, true); + assert_eq!(native, run(case, false), "{case:?}"); + let expected = bits( + double, + if edge == Edge::NegativeZero { + -0.0 + } else { + 0.0 + }, + ); + let width = if double { 8 } else { 4 }; + let output = if overlap.is_some() { + &native.finished.primary + } else { + &native.finished.secondary + }; + assert_eq!( + &output[1 + 2 * width..1 + 3 * width], + &expected.to_le_bytes()[..width] + ); + if recurrence { + assert_eq!(native.finished.state, expected); + } + } + } + } + } +} + +#[test] +fn stream_kernels_match_240_sample_outputs_state_and_inherited_polls() { + for double in [false, true] { + for recurrence in [false, true] { + for case in poll_cases(case(double, recurrence)) { + assert_poll_case(case); + } + } + } +} + +#[test] +fn stream_kernel_guards_and_callback_mutations_keep_forward_loop_semantics() { + for double in [false, true] { + for recurrence in [false, true] { + let base = case(double, recurrence); + for case in guard_and_mutation_cases(base) { + assert_eq!(run(case, true), run(case, false), "{case:?}"); + } + } + } +} diff --git a/src/stack_ir.rs b/src/stack_ir.rs index 22dc0c7c..cd329d58 100644 --- a/src/stack_ir.rs +++ b/src/stack_ir.rs @@ -38,6 +38,155 @@ impl StackExternRet { } } +/// Scalar interpretation shared by all native loop bodies. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum NativeScalar { + F32, + F64, +} + +/// Guarded native execution of a retained bytecode loop. Invalid ranges or +/// unsupported overlap fall through; completion skips the body through `done`. +/// Counter and carried values are published before polling. Operand windows +/// survive both completion and fallback. +#[derive(Clone, Debug, PartialEq)] +pub struct NativeLoopKernel { + pub scalar: NativeScalar, + pub counter: u16, + pub end: u16, + pub done: i32, + pub spans: NativeStreamSlots, + pub body: NativeStreamBody, +} + +/// Pointer and signed length slots for one indexed input/output stream pair. +/// Exact alias is supported; partially overlapping accessed spans use bytecode. +#[derive(Clone, Debug, PartialEq)] +pub struct NativeStreamSlots { + pub input: u16, + pub output: u16, + pub input_len: u16, + pub output_len: u16, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum NativeStreamBody { + /// An ordered, precompiled graph from the shared pointwise catalog. Only + /// the recipe's coefficient prefix is live; unused entries are padding. + Pointwise { + recipe: crate::pointwise::RecipeId, + coefficients: [C; crate::pointwise::MAX_COEFFICIENTS], + }, + /// Ordered `input * feed + state * feedback`, followed by an indexed store. + OnePole { feed: C, feedback: C, state: u16 }, + /// Ordered direct-form-I biquad. Coefficients are b0, b1, b2, a1, a2; + /// carried states are x1, x2, y1, y2. + Biquad { + coefficients: [C; 5], + states: [u16; 4], + }, +} + +impl NativeStreamBody { + /// Resolve coefficient representations without changing carried state slots. + pub fn map_coefficients(self, mut map: impl FnMut(C) -> D) -> NativeStreamBody { + match self { + Self::Pointwise { + recipe, + coefficients, + } => NativeStreamBody::Pointwise { + recipe, + coefficients: coefficients.map(map), + }, + Self::OnePole { + feed, + feedback, + state, + } => NativeStreamBody::OnePole { + feed: map(feed), + feedback: map(feedback), + state, + }, + Self::Biquad { + coefficients, + states, + } => NativeStreamBody::Biquad { + coefficients: coefficients.map(map), + states, + }, + } + } +} + +#[cfg(test)] +mod native_coordinate_tests { + use super::*; + + fn gateway(done: i32) -> StackOp { + StackOp::NativeLoop(NativeLoopKernel { + scalar: NativeScalar::F32, + counter: 0, + end: 1, + done, + spans: NativeStreamSlots { + input: 2, + output: 3, + input_len: 4, + output_len: 4, + }, + body: NativeStreamBody::Pointwise { + recipe: crate::pointwise::AFFINE, + coefficients: [5, 6, 0, 0], + }, + }) + } + + #[test] + fn inlining_remaps_native_success_edge() { + let mut callee = StackFunction::new("constant_add"); + callee.ops = vec![ + StackOp::I64Const(1), + StackOp::I64Const(2), + StackOp::IAdd, + StackOp::Return, + ]; + let call = StackOp::Call { + func: 1, + args: 0, + preserve: 0, + }; + let mut caller = StackFunction::new("native_caller"); + caller.local_count = 4; + caller.ops = vec![call.clone(), gateway(1), call, StackOp::ReturnVoid]; + let mut program = StackProgram::new(); + program.add_function(caller); + program.add_function(callee); + + crate::stack_inline::inline_trivial(&mut program); + + let caller = &program.functions[0]; + assert_eq!(caller.ops[3], gateway(3)); + assert_eq!(caller.ops[7], StackOp::ReturnVoid); + assert_eq!(crate::stack_depth::stack_delta(&caller.ops[3]), 0); + } + + #[test] + fn compaction_remaps_native_success_edge() { + let mut function = StackFunction::new("native_compaction"); + function.ops = vec![ + StackOp::Nop, + gateway(2), + StackOp::Nop, + StackOp::Nop, + StackOp::ReturnVoid, + ]; + + crate::stack_optimize::optimize(&mut function); + + assert_eq!(function.ops, vec![gateway(0), StackOp::ReturnVoid]); + } +} + /// Stack IR instruction. /// /// All values on the operand stack are 64-bit (i64/f64/pointer). @@ -259,6 +408,8 @@ pub enum StackOp { FusedAddrLoad32Off(u16, i32), /// locals[dst] = locals[src] + imm. Pop 0, push 0. FusedGetAddImmSet(u16, i32, u16), + /// Native whole-loop gateway; preserves all operand windows. + NativeLoop(NativeLoopKernel), /// if !(locals[a] < locals[b]) jump. Pop 0, push 0. FusedGetGetILtJumpIfZero(u16, u16, i32), /// if any packed `(idx < len)` check fails, jump. Pop 0, push 0. @@ -869,6 +1020,40 @@ impl fmt::Display for StackOp { StackOp::FusedGetAddImmSet(s, v, d) => { write!(f, "fused.get_addimm_set {} {} {}", s, v, d) } + StackOp::NativeLoop(kernel) => { + let width = match kernel.scalar { + NativeScalar::F32 => "f32", + NativeScalar::F64 => "f64", + }; + let spans = &kernel.spans; + match &kernel.body { + NativeStreamBody::Pointwise { + recipe, + coefficients, + } => { + write!(f, "pointwise_{}_{} counter={} end={} input={} output={} coefficients={:?} input_len={} output_len={} done={}", + recipe.recipe().name, width, kernel.counter, kernel.end, spans.input, spans.output, + &coefficients[..recipe.recipe().coefficient_count()], spans.input_len, spans.output_len, kernel.done) + } + NativeStreamBody::OnePole { + feed, + feedback, + state, + } => { + write!(f, "one_pole_{} counter={} end={} input={} output={} feed={} feedback={} state={} input_len={} output_len={} done={}", + width, kernel.counter, kernel.end, spans.input, spans.output, + feed, feedback, state, spans.input_len, spans.output_len, kernel.done) + } + NativeStreamBody::Biquad { + coefficients, + states, + } => { + write!(f, "biquad_{} counter={} end={} input={} output={} coefficients={:?} states={:?} input_len={} output_len={} done={}", + width, kernel.counter, kernel.end, spans.input, spans.output, + coefficients, states, spans.input_len, spans.output_len, kernel.done) + } + } + } StackOp::FusedGetGetILtJumpIfZero(a, b, o) => { write!(f, "fused.get_get_ilt_jiz {} {} {}", a, b, o) } diff --git a/src/stack_optimize.rs b/src/stack_optimize.rs index 9bc168c2..6f3cffc8 100644 --- a/src/stack_optimize.rs +++ b/src/stack_optimize.rs @@ -41,6 +41,7 @@ fn compute_jump_targets(ops: &[StackOp]) -> Vec { StackOp::FusedGetF32ConstFGtJumpIfZeroF(_, _, off) => Some(*off), StackOp::FusedF64ConstDGtJumpIfZeroD(_, off) => Some(*off), StackOp::FusedGetF64ConstDGtJumpIfZeroD(_, _, off) => Some(*off), + StackOp::NativeLoop(kernel) => Some(kernel.done), _ => None, }; if let Some(off) = off { @@ -376,10 +377,17 @@ fn packed_fmul_sum_fused_chain( fn fuse(func: &mut StackFunction) { let ops = &mut func.ops; let len = ops.len(); - let is_target = compute_jump_targets(ops); + let mut is_target = compute_jump_targets(ops); + // A native gateway is also a fusion boundary. The opcode itself carries + // that fact; no separate instruction-ownership metadata is needed. + for (index, op) in ops.iter().enumerate() { + if matches!(op, StackOp::NativeLoop(_)) { + is_target[index] = true; + } + } let mut i = 0; - // Helper: check if any instruction in range (i+1..i+n) is a jump target. + // A fusion may start at a jump target, but must not cross any boundary. let spans_target = |start: usize, count: usize| -> bool { for j in 1..count { if start + j < len && is_target[start + j] { @@ -390,6 +398,10 @@ fn fuse(func: &mut StackFunction) { }; while i < len { + if matches!(ops[i], StackOp::NativeLoop(_)) { + i += 1; + continue; + } // === Dead code elimination === // i64.const + drop → nop (dead Var initialization result) @@ -1429,6 +1441,17 @@ fn strip_nops(func: &mut StackFunction) { let new_off = target_new as i32 - new_idx[old] as i32 - 1; StackOp::FusedGetF64ConstDGtJumpIfZeroD(*n, *v, new_off) } + StackOp::NativeLoop(kernel) => { + let mut relocated = kernel.clone(); + let target_old = (old as i64 + 1 + kernel.done as i64) as usize; + let target_new = if target_old == len { + new_len + } else { + new_idx[target_old] + }; + relocated.done = target_new as i32 - new_idx[old] as i32 - 1; + StackOp::NativeLoop(relocated) + } other => other.clone(), }; new_ops.push(adjusted); diff --git a/src/stack_vm.rs b/src/stack_vm.rs index 1c42be0d..adcf1ce5 100644 --- a/src/stack_vm.rs +++ b/src/stack_vm.rs @@ -967,6 +967,9 @@ impl StackVM { } StackOp::Nop => {} + // The gateway's ordinary loop follows immediately. The Rust + // interpreter executes that retained implementation directly. + StackOp::NativeLoop(_) => {} // Fused superinstructions — not executed by the Rust VM. _ => unimplemented!("fused op {:?} not supported in Rust stack VM", op), diff --git a/src/value_loops.rs b/src/value_loops.rs new file mode 100644 index 00000000..6c54d35a --- /dev/null +++ b/src/value_loops.rs @@ -0,0 +1,553 @@ +//! Checked, ordered loop values and effects for experimental native selection. +//! +//! Discovery follows binding identities through scalar declarations and updates. +//! It records indexed reads and writes in source order, private loop invariants, +//! and incoming/final scalar state. Bounded kernel contracts consume this shared +//! representation; unsupported contracts keep the original bytecode loop. + +use crate::pointwise::{self, RecipeId}; +use crate::*; +use std::collections::{HashMap, HashSet}; +use std::convert::{TryFrom, TryInto}; + +mod recipes; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Scalar { + F32, + F64, +} + +impl Scalar { + pub(crate) fn of(ty: TypeID) -> Option { + match &*ty { + Type::Float32 => Some(Self::F32), + Type::Float64 => Some(Self::F64), + _ => None, + } + } +} + +/// Immutable value identity within one loop analysis, not a local storage slot. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +pub struct ValueId(pub usize); + +#[derive(Clone, Debug)] +pub enum Operation { + Input(LocalId), + Constant(String), + Binary(Binop, ValueId, ValueId), + Negate(ValueId), + ReadElement { stream: usize }, +} + +#[derive(Clone, Debug)] +pub struct Value { + pub ty: Scalar, + pub operation: Operation, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SpanLength { + Fixed(u32), + Slice, +} + +#[derive(Clone, Debug)] +pub struct MemoryStream { + pub binding: Reference, + /// Original base expression for representation-aware address/span lowering. + pub expression: ExprID, +} + +#[derive(Clone, Debug)] +pub struct ScalarInput { + pub binding: LocalId, + pub value: ValueId, +} + +#[derive(Clone, Debug)] +pub struct Carry { + pub binding: LocalId, + pub incoming: ValueId, + pub next: ValueId, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum LoopStep { + Value(ValueId), + StoreElement { stream: usize, value: ValueId }, +} + +#[derive(Clone, Debug)] +pub struct LoopRegion { + pub induction: LocalId, + pub values: Vec, + pub streams: Vec, + pub inputs: Vec, + pub carries: Vec, + pub tape: Vec, + pub ty: Scalar, +} + +#[derive(Clone, Debug)] +pub struct Pointwise { + pub input: usize, + pub output: usize, + pub recipe: RecipeId, + pub coefficients: Vec, +} + +#[derive(Clone, Copy, Debug)] +pub struct OnePole { + pub input: usize, + pub output: usize, + pub carry: usize, + pub feed: ValueId, + pub feedback: ValueId, +} + +#[derive(Clone, Copy, Debug)] +pub struct Biquad { + pub input: usize, + pub output: usize, + /// Ordered coefficient roles: b0, b1, b2, a1, a2. + pub coefficients: [ValueId; 5], + /// Carry roles x1, x2, y1, y2, independent of source declaration order. + pub carries: [usize; 4], +} + +impl LoopRegion { + fn operation(&self, value: ValueId) -> Option<&Operation> { + self.values.get(value.0).map(|value| &value.operation) + } + + fn binary(&self, value: ValueId, expected: Binop) -> Option<(ValueId, ValueId)> { + match self.operation(value)? { + Operation::Binary(operation, lhs, rhs) if *operation == expected => Some((*lhs, *rhs)), + _ => None, + } + } + + fn read(&self, value: ValueId) -> Option { + match self.operation(value)? { + Operation::ReadElement { stream } => Some(*stream), + _ => None, + } + } + + /// A coefficient is already private scalar state or an exact literal. + /// Arbitrary arithmetic remains per-iteration work until a contract covers it. + fn coefficient(&self, value: ValueId) -> bool { + match self.operation(value) { + Some(Operation::Constant(_)) => true, + Some(Operation::Input(binding)) => self + .inputs + .iter() + .any(|input| input.binding == *binding && input.value == value), + _ => false, + } + } + + /// Input/literal materialization has no loop effect. Every remaining value + /// and every store must occur exactly once in this source-ordered schedule. + /// In particular, this never drops an unused read or duplicates a shared one. + fn covers(&self, expected: &[LoopStep]) -> bool { + self.values.iter().all(|value| value.ty == self.ty) + && self + .tape + .iter() + .copied() + .filter(|step| match step { + LoopStep::Value(value) => !matches!( + self.operation(*value), + Some(Operation::Input(_) | Operation::Constant(_)) + ), + LoopStep::StoreElement { .. } => true, + }) + .eq(expected.iter().copied()) + } + + /// Match the same ordered recipe that generates the native arithmetic. + /// Nodes denote values, so a reused read or calculation must retain its + /// exact identity. The complete effect tape also rejects reordered work, + /// unused reads/arithmetic, and additional writes. + pub fn pointwise(&self) -> Option { + pointwise::recipes().find_map(|(id, recipe)| { + let matched = self.match_recipe(recipe)?; + Some(Pointwise { + input: matched.input, + output: matched.output, + recipe: id, + coefficients: matched.coefficients, + }) + }) + } + + /// Exact ordered one-pole state update followed by its output write: + /// `state = input[i] * feed + state * feedback; output[i] = state`. + pub fn one_pole(&self) -> Option { + let matched = self.match_recipe(crate::recurrence::ONE_POLE.recipe())?; + Some(OnePole { + input: matched.input, + output: matched.output, + carry: matched.carries[0], + feed: matched.coefficients[0], + feedback: matched.coefficients[1], + }) + } + + /// Direct form I with explicit, separately rounded arithmetic: + /// `b0*x + b1*x1 + b2*x2 - a1*y1 - a2*y2`. + /// The four final carry values must implement the two history shifts. + pub fn biquad(&self) -> Option { + let matched = self.match_recipe(crate::recurrence::BIQUAD.recipe())?; + Some(Biquad { + input: matched.input, + output: matched.output, + coefficients: matched.coefficients.try_into().ok()?, + carries: matched.carries.try_into().ok()?, + }) + } +} +fn local(function: &CheckedFunction, expression: ExprID) -> Option { + match function.arena.reference(expression) { + Some(Reference::Local(binding)) => Some(*binding), + _ => None, + } +} + +fn binder(function: &CheckedFunction, expression: ExprID) -> Option { + match function.arena[expression] { + Expr::Let(..) | Expr::Var(..) | Expr::For { .. } => Some(function.arena.binder(expression)), + _ => None, + } +} + +#[derive(Default)] +struct BodyShape { + definitions: HashSet, + assignments: Vec, + ty: Option, +} + +impl BodyShape { + fn collect(&mut self, function: &CheckedFunction, expression: ExprID) -> Option<()> { + match &function.arena[expression] { + Expr::Block(statements) => { + for &statement in statements { + self.collect(function, statement)?; + } + } + Expr::Let(_, initializer, _) | Expr::Var(_, Some(initializer), _) => { + self.definitions.insert(binder(function, expression)?); + self.ty = self + .ty + .or_else(|| Scalar::of(function.arena.ty(*initializer))); + } + Expr::Binop(Binop::Assign, lhs, rhs) => { + match function.arena[*lhs] { + Expr::Id(_) => { + let binding = local(function, *lhs)?; + if !self.assignments.contains(&binding) { + self.assignments.push(binding); + } + } + Expr::ArrayIndex(..) => {} + _ => return None, + } + self.ty = self.ty.or_else(|| Scalar::of(function.arena.ty(*rhs))); + } + _ => return None, + } + Some(()) + } +} + +struct Builder<'a> { + function: &'a CheckedFunction, + blocked: &'a HashSet, + definitions: HashSet, + region: LoopRegion, + bindings: HashMap, +} + +impl Builder<'_> { + fn add(&mut self, operation: Operation) -> ValueId { + let value = ValueId(self.region.values.len()); + self.region.values.push(Value { + ty: self.region.ty, + operation, + }); + self.region.tape.push(LoopStep::Value(value)); + value + } + + fn private_scalar(&self, binding: LocalId) -> bool { + !self.blocked.contains(&binding) + && Scalar::of(self.function.arena.local(binding).ty) == Some(self.region.ty) + } + + fn stream(&mut self, base: ExprID, index: ExprID) -> Option { + let function = self.function; + if !matches!(function.arena[base], Expr::Id(_)) + || !matches!(function.arena[index], Expr::Id(_)) + || local(function, index) != Some(self.region.induction) + { + return None; + } + let binding = function.arena.reference(base)?.clone(); + match &binding { + Reference::Local(binding) if !self.definitions.contains(binding) => {} + Reference::Instance(_) => {} + _ => return None, + } + let element = match &*function.arena.ty(base) { + Type::Array(element, ArraySize::Known(length)) => { + u32::try_from(*length).ok()?; + *element + } + Type::Slice(element) => *element, + _ => return None, + }; + if Scalar::of(element) != Some(self.region.ty) { + return None; + } + if let Some(stream) = self + .region + .streams + .iter() + .position(|stream| stream.binding == binding) + { + return Some(stream); + } + let stream = self.region.streams.len(); + self.region.streams.push(MemoryStream { + binding, + expression: base, + }); + Some(stream) + } + + fn expression(&mut self, expression: ExprID) -> Option { + let function = self.function; + if Scalar::of(function.arena.ty(expression)) != Some(self.region.ty) { + return None; + } + let operation = match &function.arena[expression] { + Expr::Id(_) => { + let binding = local(function, expression)?; + if let Some(value) = self.bindings.get(&binding) { + return Some(*value); + } + if self.definitions.contains(&binding) || !self.private_scalar(binding) { + return None; + } + let value = self.add(Operation::Input(binding)); + self.region.inputs.push(ScalarInput { binding, value }); + self.bindings.insert(binding, value); + return Some(value); + } + Expr::Real(value, _) => Operation::Constant(value.clone()), + Expr::ArrayIndex(base, index) => Operation::ReadElement { + stream: self.stream(*base, *index)?, + }, + Expr::Binop( + operation @ (Binop::Plus | Binop::Minus | Binop::Mult | Binop::Div), + lhs, + rhs, + ) => { + let lhs = self.expression(*lhs)?; + let rhs = self.expression(*rhs)?; + Operation::Binary(*operation, lhs, rhs) + } + Expr::Unop(Unop::Neg, operand) => Operation::Negate(self.expression(*operand)?), + _ => return None, + }; + Some(self.add(operation)) + } + + fn statement(&mut self, expression: ExprID) -> Option<()> { + let function = self.function; + match &function.arena[expression] { + Expr::Block(statements) => { + // LocalIds remain unique even after an inner lexical block. + for &statement in statements { + self.statement(statement)?; + } + } + Expr::Let(_, initializer, _) | Expr::Var(_, Some(initializer), _) => { + let binding = binder(function, expression)?; + if !self.private_scalar(binding) { + return None; + } + let value = self.expression(*initializer)?; + self.bindings.insert(binding, value); + } + Expr::Binop(Binop::Assign, lhs, rhs) => { + // Ordinary assignment evaluates its RHS before the lvalue. + let value = self.expression(*rhs)?; + match function.arena[*lhs] { + Expr::Id(_) => { + let binding = local(function, *lhs)?; + if !self.private_scalar(binding) || !self.bindings.contains_key(&binding) { + return None; + } + self.bindings.insert(binding, value); + if let Some(carry) = self + .region + .carries + .iter_mut() + .find(|carry| carry.binding == binding) + { + carry.next = value; + } + } + Expr::ArrayIndex(base, index) => { + let stream = self.stream(base, index)?; + self.region + .tape + .push(LoopStep::StoreElement { stream, value }); + } + _ => return None, + } + } + _ => return None, + } + Some(()) + } +} + +/// Observe checked HIR after safety checking. Selection and runtime span guards +/// establish a kernel's narrower executable contract. +fn analyze( + function: &CheckedFunction, + loop_expr: ExprID, + blocked: &HashSet, +) -> Option { + let Expr::For { body, .. } = function.arena[loop_expr] else { + return None; + }; + let induction = binder(function, loop_expr)?; + if blocked.contains(&induction) { + return None; + } + let mut shape = BodyShape::default(); + shape.collect(function, body)?; + let mut builder = Builder { + function, + blocked, + definitions: shape.definitions, + region: LoopRegion { + induction, + values: vec![], + streams: vec![], + inputs: vec![], + carries: vec![], + tape: vec![], + ty: shape.ty?, + }, + bindings: HashMap::new(), + }; + for binding in shape.assignments { + if builder.definitions.contains(&binding) { + continue; + } + if binding == induction || !builder.private_scalar(binding) { + return None; + } + let incoming = builder.add(Operation::Input(binding)); + builder.region.carries.push(Carry { + binding, + incoming, + next: incoming, + }); + builder.bindings.insert(binding, incoming); + } + builder.statement(body)?; + if builder.region.carries.is_empty() + && !builder + .region + .tape + .iter() + .any(|step| matches!(step, LoopStep::StoreElement { .. })) + { + return None; + } + Some(builder.region) +} + +/// Analyze one concrete, post-hoisting body. IDs and reports are body-local: +/// recompute after mutation or lambda extraction instead of transporting facts. +/// Captures and reference/unknown-call exposure remain conservative; identity +/// establishes neither purity nor absence of aliasing. +pub(crate) fn analyze_function(function: &CheckedFunction) -> HashMap { + fn expose(function: &CheckedFunction, id: ExprID, blocked: &mut HashSet) { + if let Some(binding) = local(function, id) { + blocked.insert(binding); + } + if matches!(function.arena[id], Expr::Lambda { .. }) { + return; + } + for child in function.arena[id].subexprs() { + expose(function, child, blocked); + } + } + + fn scan(function: &CheckedFunction, id: ExprID, blocked: &mut HashSet) { + if matches!(function.arena[id], Expr::Lambda { .. }) { + return; + } + if let Expr::Call(callee, args) = &function.arena[id] { + let callee_ty = function.arena.ty(*callee); + let params = match &*callee_ty { + Type::Func(domain, _) => match &**domain { + Type::Tuple(params) => Some(params), + _ => None, + }, + _ => None, + }; + for (position, arg) in args.iter().enumerate() { + if params + .and_then(|p| p.get(position)) + .is_none_or(|ty| ty.is_ptr()) + { + expose(function, *arg, blocked); + } + } + } + for child in function.arena[id].subexprs() { + scan(function, child, blocked); + } + } + + fn visit( + function: &CheckedFunction, + id: ExprID, + blocked: &HashSet, + loops: &mut HashMap, + ) { + // Generated lambdas keep ordinary lowering in this consumer. + if matches!(function.arena[id], Expr::Lambda { .. }) { + return; + } + if let Some(region) = analyze(function, id, blocked) { + loops.insert(id, region); + } + for child in function.arena[id].subexprs() { + visit(function, child, blocked, loops); + } + } + + let mut loops = HashMap::new(); + if let Some(body) = function.body { + let mut blocked = function.captured_locals(); + blocked.extend(function.closure_vars.iter().copied()); + scan(function, body, &mut blocked); + visit(function, body, &blocked, &mut loops); + } + loops +} + +#[cfg(test)] +mod tests; diff --git a/src/value_loops/recipes.rs b/src/value_loops/recipes.rs new file mode 100644 index 00000000..7be2f651 --- /dev/null +++ b/src/value_loops/recipes.rs @@ -0,0 +1,161 @@ +//! Match ordered native recipes against the checked loop's values and effects. +//! +//! Carry roles describe incoming values and their next-iteration roots. They do +//! not depend on declaration order or the order of equivalent scalar updates. + +use super::ValueId; +use super::{LoopRegion, LoopStep}; +use crate::loop_recipe::{BinaryOp, Node, Recipe}; +use crate::Binop; + +pub(super) struct MatchedRecipe { + pub input: usize, + pub output: usize, + pub coefficients: Vec, + pub carries: Vec, +} + +impl LoopRegion { + pub(super) fn match_recipe(&self, recipe: &Recipe) -> Option { + if self.carries.len() != recipe.carry_count() { + return None; + } + // Distinct state slots must also have distinct incoming value identities. + if self.carries.iter().enumerate().any(|(index, carry)| { + self.carries[..index] + .iter() + .any(|other| other.incoming == carry.incoming) + }) { + return None; + } + let LoopStep::StoreElement { + stream: output, + value, + } = *self.tape.last()? + else { + return None; + }; + self.match_recipe_carry_roles(recipe, output, value, &mut Vec::new()) + } + + fn match_recipe_carry_roles( + &self, + recipe: &Recipe, + output: usize, + result: ValueId, + carries: &mut Vec, + ) -> Option { + if carries.len() < self.carries.len() { + // Catalogs bound this to four carries (at most 24 permutations). + // Trying role assignments keeps output and next-state roots equal: + // a carry need not participate in the output calculation at all. + for carry in 0..self.carries.len() { + if carries.contains(&carry) { + continue; + } + carries.push(carry); + let matched = self.match_recipe_carry_roles(recipe, output, result, carries); + carries.pop(); + if matched.is_some() { + return matched; + } + } + return None; + } + + let mut bindings = vec![None; recipe.nodes.len()]; + let mut coefficients = vec![None; recipe.coefficient_count()]; + self.match_recipe_node( + recipe, + recipe.result, + result, + carries, + &mut bindings, + &mut coefficients, + )?; + for (role, node) in recipe.next_carries.iter().enumerate() { + self.match_recipe_node( + recipe, + *node, + self.carries[carries[role]].next, + carries, + &mut bindings, + &mut coefficients, + )?; + } + + let mut input = None; + let mut schedule = Vec::new(); + for (node, binding) in recipe.nodes.iter().zip(bindings) { + let binding = binding?; + match node { + Node::Parameter(_) | Node::Carry(_) => continue, + Node::Read => { + if input.replace(self.read(binding)?).is_some() { + return None; + } + } + Node::Binary(..) => {} + } + schedule.push(LoopStep::Value(binding)); + } + schedule.push(LoopStep::StoreElement { + stream: output, + value: result, + }); + self.covers(&schedule).then_some(MatchedRecipe { + input: input?, + output, + coefficients: coefficients.into_iter().collect::>()?, + carries: carries.clone(), + }) + } + + fn match_recipe_node( + &self, + recipe: &Recipe, + node: usize, + value: ValueId, + carries: &[usize], + bindings: &mut [Option], + coefficients: &mut [Option], + ) -> Option<()> { + if let Some(previous) = *bindings.get(node)? { + return (previous == value).then_some(()); + } + match recipe.nodes.get(node)? { + Node::Read => { + self.read(value)?; + } + Node::Parameter(role) => { + if !self.coefficient(value) { + return None; + } + let coefficient = coefficients.get_mut(usize::from(*role))?; + if coefficient.is_some_and(|previous| previous != value) { + return None; + } + // Different parameter roles may use the same private value. + *coefficient = Some(value); + } + Node::Carry(role) => { + let carry = self.carries.get(*carries.get(usize::from(*role))?)?; + if carry.incoming != value { + return None; + } + } + Node::Binary(operation, lhs_node, rhs_node) => { + let operation = match operation { + BinaryOp::Add => Binop::Plus, + BinaryOp::Sub => Binop::Minus, + BinaryOp::Mul => Binop::Mult, + }; + let (lhs, rhs) = self.binary(value, operation)?; + self.match_recipe_node(recipe, *lhs_node, lhs, carries, bindings, coefficients)?; + self.match_recipe_node(recipe, *rhs_node, rhs, carries, bindings, coefficients)?; + } + } + bindings[node] = Some(value); + Some(()) + } +} diff --git a/src/value_loops/tests.rs b/src/value_loops/tests.rs new file mode 100644 index 00000000..fdfcb809 --- /dev/null +++ b/src/value_loops/tests.rs @@ -0,0 +1,275 @@ +use super::*; + +fn inspect(source: &str) -> Vec { + inspect_with_prelude("", source) +} + +fn inspect_with_prelude(prelude: &str, source: &str) -> Vec { + let mut compiler = Compiler::new(); + compiler.quiet = true; + assert!( + compiler.parse(prelude, ""), + "{:?}", + compiler.last_errors + ); + assert!( + compiler.parse(source, "value-loops.lyte"), + "{:?}", + compiler.last_errors + ); + assert!(compiler.check(), "{:?}", compiler.last_errors); + compiler.specialize().unwrap(); + compiler + .specialized_program() + .unwrap() + .functions() + .flat_map(|(_, function)| analyze_function(function).into_values()) + .collect() +} + +fn loops(body: &str) -> Vec { + inspect(&format!( + "kernel(a: &[f32; 4], b: &[f32; 4], gain: f32, bias: f32) -> f32 {{ + var state = 1.0 + var coefficient = gain + for i in 0 .. 4 {{ {body} }} + state + }} + main {{ + var a: [f32; 4] + var b: [f32; 4] + print(kernel(a, b, 0.5, 0.25) as i32) + }}" + )) +} + +#[test] +fn pointwise_contract_uses_invariants_and_immutable_local_versions() { + for body in [ + "b[i] = a[i] * gain + bias", + "let x = a[i]; var y = x * gain; y = y + bias; b[i] = y", + "a[i] = a[i] * 0.5 + 1.0", + ] { + let regions = loops(body); + assert_eq!(regions.len(), 1); + assert_eq!( + regions[0].pointwise().map(|kernel| kernel.recipe), + Some(pointwise::AFFINE), + "missing affine map for {}", + body + ); + } + for body in [ + "let unused = a[i] * bias; b[i] = a[i] * gain + bias", + "coefficient = coefficient + bias; b[i] = a[i] * coefficient + bias", + "b[i] = a[i] * gain + bias; a[i] = b[i] * gain + bias", + ] { + let regions = loops(body); + assert!( + regions[0].pointwise().is_none(), + "unexpected map for {}", + body + ); + } + let regions = inspect( + "map(a: &[f32; 4], b: &[f32; 4], gain: &f32) { + for i in 0 .. 4 { b[i] = a[i] * gain + 1.0 } + } + main { + var a: [f32; 4] + var b: [f32; 4] + var gain = 0.5 + map(a, b, gain) + }", + ); + assert!( + regions.is_empty(), + "reference scalars cannot become invariants" + ); +} + +fn pointwise_loop(body: &str, scalar: &str) -> LoopRegion { + let mut regions = inspect(&format!( + "kernel(input: &[{scalar}; 4], output: &[{scalar}; 4], a: {scalar}, b: {scalar}, c: {scalar}, d: {scalar}) {{ + for i in 0 .. 4 {{ {body} }} + }} + main {{ + var input: [{scalar}; 4] + var output: [{scalar}; 4] + kernel(input, output, 0.5{scalar}, 0.25{scalar}, 0.125{scalar}, 0.0625{scalar}) + }}" + )); + assert_eq!(regions.len(), 1, "discovery lost {}", body); + regions.pop().unwrap() +} + +#[test] +fn pointwise_recipes_match_ordered_reuse_and_coefficient_roles() { + for scalar in ["f32", "f64"] { + for (name, body, coefficient_count) in [ + ("offset_scale", "output[i] = (input[i] - b) * a", 2), + ( + "cubic", + "let x = input[i]; output[i] = x - a * ((x*x)*x)", + 1, + ), + ( + "cubic", + "var x = input[i]; let original = x; var y = x*x; + y = y*x; y = a*y; x = original-y; output[i] = x", + 1, + ), + ( + "horner", + "let x = input[i]; output[i] = ((a*x+b)*x+c)*x+d", + 4, + ), + ( + "horner", + "let x = input[i]; var y = a*x; y = y+b; y = y*x; + y = y+c; y = y*x; y = y+d; output[i] = y", + 4, + ), + ] { + let region = pointwise_loop(body, scalar); + let kernel = region.pointwise().expect(body); + assert_eq!(kernel.recipe.recipe().name, name, "{}", body); + assert_eq!(kernel.coefficients.len(), coefficient_count); + assert_ne!(kernel.input, kernel.output); + } + let region = pointwise_loop("let x = input[i]; input[i] = ((a*x+a)*x+a)*x+a", scalar); + let kernel = region.pointwise().expect("aliased coefficient roles"); + assert_eq!(kernel.recipe.recipe().name, "horner"); + assert_eq!(kernel.input, kernel.output); + assert_eq!(kernel.coefficients.len(), 4); + assert!(kernel + .coefficients + .iter() + .all(|value| *value == kernel.coefficients[0])); + } +} + +#[test] +fn pointwise_recipes_reject_changed_identity_order_and_effects() { + for body in [ + // Repeated indexed reads are not reuse of one sampled value. + "output[i] = input[i] - a * ((input[i]*input[i])*input[i])", + // The read is shared, but changing association or operand order + // changes the recipe's separately rounded arithmetic. + "let x = input[i]; output[i] = x - a * (x*(x*x))", + "let x = input[i]; output[i] = x - ((x*x)*x) * a", + "let x = input[i]; output[i] = ((x*a+b)*x+c)*x+d", + // An apparently harmless local update changes later value uses. + "var x = input[i]; x = x*x; output[i] = x - a * ((x*x)*x)", + "let x = input[i]; let unused = input[i]; output[i] = x - a*((x*x)*x)", + "let x = input[i]; let unused = x*b; output[i] = x - a*((x*x)*x)", + "let x = input[i]; output[i] = x - a*((x*x)*x); input[i] = x", + ] { + let region = pointwise_loop(body, "f32"); + assert!( + region.pointwise().is_none(), + "unexpected kernel for {}", + body + ); + } +} + +fn biquad_loop(body: &str, scalar: &str) -> LoopRegion { + let mut regions = inspect(&format!( + "kernel(input: &[{scalar}; 4], output: &[{scalar}; 4], b0: {scalar}, b1: {scalar}, b2: {scalar}, a1: {scalar}, a2: {scalar}) -> {scalar} {{ + var y2 = 0.25{scalar} + var x1 = 0.5{scalar} + var y1 = 0.75{scalar} + var x2 = 1.0{scalar} + for i in 0 .. 4 {{ {body} }} + x1 + x2 + y1 + y2 + }} + main {{ + var input: [{scalar}; 4] + var output: [{scalar}; 4] + print(kernel(input, output, 0.5{scalar}, 0.25{scalar}, 0.125{scalar}, 0.0625{scalar}, 0.03125{scalar}) as i32) + }}" + )); + assert_eq!(regions.len(), 1); + regions.pop().unwrap() +} + +#[test] +fn biquad_contract_follows_four_state_roles_and_staged_values() { + for scalar in ["f32", "f64"] { + for body in [ + "let x = input[i] + let y = b0*x + b1*x1 + b2*x2 - a1*y1 - a2*y2 + x2 = x1; x1 = x; y2 = y1; y1 = y + output[i] = y", + "var x = input[i] + var y = b0*x + y = y + b1*x1 + y = y + b2*x2 + y = y - a1*y1 + y = y - a2*y2 + let old_input = x1 + let old_output = y1 + y1 = y; x1 = x; x2 = old_input; y2 = old_output + output[i] = y", + ] { + let region = biquad_loop(body, scalar); + let kernel = region.biquad().expect("missing biquad"); + assert_eq!(region.carries.len(), 4); + assert_eq!(region.inputs.len(), 5); + let [x1, x2, y1, y2] = kernel.carries.map(|carry| ®ion.carries[carry]); + assert_eq!(x2.next, x1.incoming); + assert_eq!(y2.next, y1.incoming); + assert!(matches!( + region.operation(x1.next), + Some(Operation::ReadElement { stream }) if *stream == kernel.input + )); + assert!(matches!( + region.tape.last(), + Some(LoopStep::StoreElement { stream, value }) + if *stream == kernel.output && *value == y1.next + )); + } + } +} + +#[test] +fn biquad_contract_rejects_changed_arithmetic_state_and_effects() { + let body = "let x = input[i] + let y = b0*x + b1*x1 + b2*x2 - a1*y1 - a2*y2 + x2 = x1; x1 = x; y2 = y1; y1 = y + output[i] = y"; + for changed in [ + body.replace("x2 = x1; x1 = x", "x1 = x; x2 = x1"), + body.replace("a2*y2", "a2*y1"), + body.replace("b0*x", "x*b0"), + body.replace("b0*x + b1*x1 + b2*x2", "b0*x + (b1*x1 + b2*x2)"), + body.replace("let y =", "let unused = input[i]; let y ="), + body.replace("output[i] = y", "output[i] = y; input[i] = x"), + ] { + let region = biquad_loop(&changed, "f32"); + assert!( + region.biquad().is_none(), + "unexpected biquad for {}", + changed + ); + } +} + +#[test] +fn reference_call_exposure_remains_conservative_for_owned_scalar_locals() { + let regions = inspect( + "observe(value: &f32) { print(value as i32) } + main { + var input: [f32; 4] + var output: [f32; 4] + var gain = 0.5 + observe(gain) + for i in 0 .. 4 { output[i] = input[i]*gain+1.0 } + }", + ); + assert!( + regions.is_empty(), + "reference-exposed scalar cannot become an invariant" + ); +} diff --git a/tests/cases/arith/packed_sum_signed_zero.lyte b/tests/cases/arith/packed_sum_signed_zero.lyte new file mode 100644 index 00000000..47641013 --- /dev/null +++ b/tests/cases/arith/packed_sum_signed_zero.lyte @@ -0,0 +1,31 @@ +// A sum of negative-zero products starts with its first product. Adding an +// artificial initial +0 changes its sign. atan2 exposes the sign without +// division by zero; the inline forms select Stack's packed sum handlers. +// expected stdout: +// compilation successful +// -3 +// -3 +// -3 +// -3 + +inline32(a: f32, b: f32, c: f32, d: f32) -> f32 { a * b + c * d } +split32(a: f32, b: f32, c: f32, d: f32) -> f32 { + let p = a * b + let q = c * d + p + q +} +inline64(a: f64, b: f64, c: f64, d: f64) -> f64 { a * b + c * d } +split64(a: f64, b: f64, c: f64, d: f64) -> f64 { + let p = a * b + let q = c * d + p + q +} + +main { + var z = -0.0 + print(atan2(inline32(z, 1.0, z, 1.0), -1.0) as i32) + print(atan2(split32(z, 1.0, z, 1.0), -1.0) as i32) + var z64 = -0.0f64 + print(atan2(inline64(z64, 1.0f64, z64, 1.0f64), -1.0f64) as i32) + print(atan2(split64(z64, 1.0f64, z64, 1.0f64), -1.0f64) as i32) +} From 3537caf780ac1a427b15b1dbf46174d425ea2c52 Mon Sep 17 00:00:00 2001 From: Izaak Branderhorst Date: Wed, 9 Sep 2026 23:25:48 +0200 Subject: [PATCH 2/3] Extend the pointwise recipe catalog Add gain, offset, square and smootherstep using recipe definitions and existing test infrastructure. These demonstrate zero and one coefficient ports plus reused intermediate values without additional interpreter machinery. --- src/native_loop_tests.rs | 75 ++++++++++++++++++++++++++++++++++++++++ src/pointwise.rs | 39 +++++++++++++++++++++ src/value_loops/tests.rs | 1 + 3 files changed, 115 insertions(+) diff --git a/src/native_loop_tests.rs b/src/native_loop_tests.rs index 44a4ab34..3f2754ee 100644 --- a/src/native_loop_tests.rs +++ b/src/native_loop_tests.rs @@ -83,6 +83,81 @@ fn execute_native(mut program: StackProgram, entries: &[&str]) -> Vec<(i64, Vec< expected } +#[test] +fn pointwise_extensions_execute_natively_in_both_widths() { + let inputs = [-0.0_f64, 0.0, 0.1, 0.25, 0.5, 0.75, 0.9, 1.0]; + for scalar in ["f32", "f64"] { + let cases: [(String, fn(f32) -> f32, fn(f64) -> f64); 4] = [ + ("let y = x*gain".into(), |x| x * -1.25, |x| x * -1.25), + ("let y = x+bias".into(), |x| x + 0.3, |x| x + 0.3), + ("let y = x*x".into(), |x| x * x, |x| x * x), + ( + format!( + "let cube = (x*x)*x + let curve = x*(x*6.0{scalar}-15.0{scalar})+10.0{scalar} + let y = cube*curve" + ), + |x| ((x * x) * x) * (x * (x * 6.0 - 15.0) + 10.0), + |x| ((x * x) * x) * (x * (x * 6.0 - 15.0) + 10.0), + ), + ]; + let values = inputs + .iter() + .map(|x| format!("{x:?}{scalar}")) + .collect::>() + .join(", "); + for (body, reference_f32, reference_f64) in cases { + // Independent scalar arithmetic, including separate rounding at each step. + let expected: Vec = inputs + .iter() + .flat_map(|&x| { + if scalar == "f32" { + reference_f32(x as f32).to_le_bytes().to_vec() + } else { + reference_f64(x).to_le_bytes().to_vec() + } + }) + .collect(); + for destination in ["output", "input"] { + let compiler = checked( + &format!( + r#" + var input: [{scalar}; 8] + var output: [{scalar}; 8] + main() -> i32 {{ + input = [{values}] + let gain = -1.25{scalar} + let bias = 0.3{scalar} + for i in 0 .. 8 {{ + let x = input[i] + {body} + {destination}[i] = y + }} + 0 + }} + "# + ), + &["main"], + ); + let (ordinary, native) = variants(&compiler); + let results = execute(&ordinary, &["main"]); + let offset = compiler + .globals_info_with_offset(crate::cancel::CANCEL_FLAG_RESERVED as usize) + .into_iter() + .find(|entry| entry.0 == destination) + .unwrap() + .1; + assert_eq!(&results[0].1[offset..offset + expected.len()], &expected); + assert_eq!( + execute_native(native, &["main"]), + results, + "{scalar}, {destination}, {body}" + ); + } + } + } +} + #[test] fn affine_slices_publish_output_and_reload_headers_after_callback() { struct Redirect { diff --git a/src/pointwise.rs b/src/pointwise.rs index 8fee0851..46254bc5 100644 --- a/src/pointwise.rs +++ b/src/pointwise.rs @@ -89,6 +89,45 @@ const CATALOG: &[Recipe] = &[ result: 10, next_carries: &[], }, + Recipe { + name: "smootherstep", + // ((x*x)*x) * (x*(x*a-b)+c); a=6, b=15, c=10 gives + // quintic smootherstep for normalized x, without clamping. + nodes: &[ + Read, + Binary(Mul, 0, 0), + Binary(Mul, 1, 0), + Parameter(0), + Binary(Mul, 0, 3), + Parameter(1), + Binary(Sub, 4, 5), + Binary(Mul, 0, 6), + Parameter(2), + Binary(Add, 7, 8), + Binary(Mul, 2, 9), + ], + result: 10, + next_carries: &[], + }, + Recipe { + name: "gain", + nodes: &[Read, Parameter(0), Binary(Mul, 0, 1)], + result: 2, + next_carries: &[], + }, + Recipe { + name: "offset", + nodes: &[Read, Parameter(0), Binary(Add, 0, 1)], + result: 2, + next_carries: &[], + }, + Recipe { + name: "square", + // Both operands reuse the same sampled input value. + nodes: &[Read, Binary(Mul, 0, 0)], + result: 1, + next_carries: &[], + }, ]; pub fn recipes() -> impl Iterator { diff --git a/src/value_loops/tests.rs b/src/value_loops/tests.rs index fdfcb809..d7caa5e0 100644 --- a/src/value_loops/tests.rs +++ b/src/value_loops/tests.rs @@ -153,6 +153,7 @@ fn pointwise_recipes_match_ordered_reuse_and_coefficient_roles() { fn pointwise_recipes_reject_changed_identity_order_and_effects() { for body in [ // Repeated indexed reads are not reuse of one sampled value. + "output[i] = input[i]*input[i]", "output[i] = input[i] - a * ((input[i]*input[i])*input[i])", // The read is shared, but changing association or operand order // changes the recipe's separately rounded arithmetic. From 03695a232b84e9080bc4936e6f80f24a99ef4212 Mon Sep 17 00:00:00 2001 From: Izaak Branderhorst Date: Thu, 10 Sep 2026 00:37:54 +0200 Subject: [PATCH 3/3] Benchmark native Stack recipes on complete audio blocks Compare ordinary optimized Stack and native loops across all ten recipes in f32 and f64. Measure complete 240-sample calls with host staging, persistent recurrence state, output consumption and a cancellation callback installed. Report paired timing medians and spread, with compilation and backend creation separately. Keep gain, biquad and smootherstep first in the output. Reuse the same host setup and output/state comparisons without additional dependencies. --- examples/native_loop_bench.rs | 326 ++++++++++++++++++++++++++++++++++ 1 file changed, 326 insertions(+) create mode 100644 examples/native_loop_bench.rs diff --git a/examples/native_loop_bench.rs b/examples/native_loop_bench.rs new file mode 100644 index 00000000..05820a8f --- /dev/null +++ b/examples/native_loop_bench.rs @@ -0,0 +1,326 @@ +//! Complete 240-sample processing calls, ordinary optimized Stack versus native loops. +//! Run: cargo run --release --example native_loop_bench -- [blocks-per-round] +//! Times include input/control copies, Lyte setup, host invocation/status checks, +//! and output copying/consumption. Recurrence history persists between calls. Inputs +//! are prepared outside timing; processing allocates nothing. A cancellation +//! callback is installed, but a fresh 1,024-backedge budget covers each call. +//! Five warmed, alternating paired rounds report median and min..max ns/block. +//! Frontend, Stack lowering, and backend construction are reported separately. +//! All ten recipes run in both precisions; the three headline cases appear first. +//! These are small buffered kernels, not full Audulus graphs or coefficient design. + +#[cfg(not(has_stack_interp))] +fn main() { + eprintln!("This example requires the C Stack interpreter (has_stack_interp)."); +} + +#[cfg(has_stack_interp)] +fn main() { + bench::run(); +} + +#[cfg(has_stack_interp)] +mod bench { + use lyte::stack_interp_bridge::StackBackend; + use lyte::stack_ir::{NativeStreamBody, StackOp}; + use lyte::{Compiler, Name, CANCEL_FLAG_RESERVED, TRAP_NONE}; + use std::{hint::black_box, ops::Range, time::Instant}; + + const SAMPLES: usize = 240; + const ROUNDS: usize = 5; + + struct Host { + backend: StackBackend, + globals: Vec, + entry: u32, + input: Range, + controls: Range, + output: Range, + history: Range, + copied_output: Vec, + polls: Box, + } + + unsafe extern "C" fn cancel(data: *mut u8) -> bool { + *data.cast::() += 1; + false + } + + impl Host { + fn build(compiler: &Compiler, native: bool, workload: &str) -> (Self, f64, f64) { + let start = Instant::now(); + let program = if native { + compiler.compile_stack_native_loops() + } else { + compiler.compile_stack() + } + .unwrap(); + let lowering_ms = start.elapsed().as_secs_f64() * 1e3; + let kernels: Vec<_> = program + .functions + .iter() + .flat_map(|f| &f.ops) + .filter_map(|op| { + if let StackOp::NativeLoop(kernel) = op { + Some(&kernel.body) + } else { + None + } + }) + .collect(); + assert_eq!(kernels.len(), usize::from(native)); + if native { + assert!( + match kernels[0] { + NativeStreamBody::Pointwise { recipe, .. } => + recipe.recipe().name == workload, + NativeStreamBody::Biquad { .. } => workload == "biquad", + NativeStreamBody::OnePole { .. } => workload == "one_pole", + }, + "unexpected native recipe for {}", + workload + ); + } + let start = Instant::now(); + let mut backend = StackBackend::new(&program); + let backend_ms = start.elapsed().as_secs_f64() * 1e3; + let mut polls = Box::new(0); + backend.set_cancel_callback(Some(cancel), (&mut *polls as *mut usize).cast()); + let info = compiler.globals_info_with_offset(CANCEL_FLAG_RESERVED as usize); + let range = |name| { + let (_, offset, size, _, _) = info.iter().find(|x| x.0 == name).unwrap(); + *offset..*offset + *size + }; + let output = range("output"); + ( + Self { + backend, + globals: vec![0; program.globals_size], + entry: program.entry_points[&Name::str("process")], + input: range("input"), + controls: range("controls"), + history: range("history"), + copied_output: vec![0; output.len()], + output, + polls, + }, + lowering_ms, + backend_ms, + ) + } + + fn process(&mut self, input: &[u8], controls: &[u8]) { + self.globals[self.input.clone()].copy_from_slice(input); + self.globals[self.controls.clone()].copy_from_slice(controls); + assert_eq!( + self.backend + .call_entry(self.entry, self.globals.as_mut_ptr()), + 0 + ); + assert!(!self.backend.cancelled()); + assert_eq!(self.backend.trap_reason(), TRAP_NONE); + self.copied_output + .copy_from_slice(&self.globals[self.output.clone()]); + black_box(&self.copied_output); + } + + fn reset(&mut self) { + self.globals.fill(0); + *self.polls = 0; + } + + fn measure(&mut self, bank: &[(Vec, Vec)], blocks: usize) -> f64 { + self.reset(); + for block in 0..2048 { + let (input, controls) = &bank[block % bank.len()]; + self.process(input, controls); + } + let start = Instant::now(); + for block in 0..blocks { + let (input, controls) = &bank[block % bank.len()]; + self.process(input, controls); + } + let ns = start.elapsed().as_secs_f64() * 1e9 / blocks as f64; + assert_eq!(*self.polls, 0); + ns + } + } + + fn source(scalar: &str, workload: &str) -> String { + let (count, setup, body, finish) = match workload { + "gain" => (1, "let gain = controls[0]", "let y = x*gain", ""), + "offset" => (1, "let bias = controls[0]", "let y = x+bias", ""), + // Lyte arrays are nonempty; square leaves its host control unused. + "square" => (1, "", "let y = x*x", ""), + "affine_map" => ( + 2, + "let gain = controls[0]; let bias = controls[1]", + "let y = x*gain+bias", + "", + ), + "offset_scale" => ( + 2, + "let bias = controls[0]; let gain = controls[1]", + "let y = (x-bias)*gain", + "", + ), + "cubic" => (1, "let k = controls[0]", "let y = x-k*((x*x)*x)", ""), + "horner" => ( + 4, + "let a = controls[0]; let b = controls[1] + let c = controls[2]; let d = controls[3]", + "let y = ((a*x+b)*x+c)*x+d", + "", + ), + "one_pole" => ( + 2, + "let feed = controls[0]; let feedback = controls[1]; var state = history[0]", + "state = x*feed+state*feedback; let y = state", + "history[0] = state", + ), + "smootherstep" => ( + 3, + "let a = controls[0]; let b = controls[1]; let c = controls[2]", + "let cube = (x*x)*x; let curve = x*(x*a-b)+c; let y = cube*curve", + "", + ), + "biquad" => ( + 5, + "let b0 = controls[0]; let b1 = controls[1]; let b2 = controls[2] + let a1 = controls[3]; let a2 = controls[4] + var x1 = history[0]; var x2 = history[1] + var y1 = history[2]; var y2 = history[3]", + "var y = b0*x; y = y+b1*x1; y = y+b2*x2; y = y-a1*y1; y = y-a2*y2 + x2 = x1; x1 = x; y2 = y1; y1 = y", + "history[0] = x1; history[1] = x2; history[2] = y1; history[3] = y2", + ), + _ => unreachable!(), + }; + format!( + "var input: [{scalar}; {SAMPLES}] + var output: [{scalar}; {SAMPLES}] + var controls: [{scalar}; {count}] + var history: [{scalar}; 4] + process() -> i32 {{ + {setup} + for i in 0 .. {SAMPLES} {{ + let x = input[i] + {body} + output[i] = y + }} + {finish} + 0 + }}" + ) + } + + fn bank(scalar: &str, workload: &str) -> Vec<(Vec, Vec)> { + let bytes = |values: &[f64]| -> Vec { + values + .iter() + .flat_map(|&value| { + if scalar == "f32" { + (value as f32).to_ne_bytes().to_vec() + } else { + value.to_ne_bytes().to_vec() + } + }) + .collect() + }; + (0..16) + .map(|block| { + let input: Vec<_> = (0..SAMPLES) + .map(|i| ((i * 17 + block * 11) % 241) as f64 / 240.0) + .collect(); + let controls = match workload { + "gain" => vec![0.5 + block as f64 / 64.0], + "offset" => vec![0.25 + block as f64 / 64.0], + "square" => vec![0.0], + "affine_map" => vec![0.5 + block as f64 / 64.0, 0.25], + "offset_scale" => vec![0.25, 0.5 + block as f64 / 64.0], + "cubic" => vec![0.25 + block as f64 / 128.0], + "horner" => vec![0.5 + block as f64 / 64.0, -0.25, 0.125, 0.0625], + "one_pole" => vec![0.25 + block as f64 / 1024.0, 0.75 - block as f64 / 1024.0], + "smootherstep" => vec![6.0, 15.0, 10.0], + "biquad" => vec![0.5 + block as f64 / 1024.0, 0.25, 0.125, 0.0625, 0.03125], + _ => unreachable!(), + }; + (bytes(&input), bytes(&controls)) + }) + .collect() + } + + pub fn run() { + let blocks = std::env::args() + .nth(1) + .map(|s| s.parse::().expect("blocks must be an integer")) + .unwrap_or(32768); + assert!(blocks > 0); + println!("240 samples/call; callback installed; {ROUNDS} paired rounds of {blocks} blocks"); + println!("ns/block: median [min..max]; setup times are single observations in ms"); + for scalar in ["f32", "f64"] { + for workload in [ + "gain", + "biquad", + "smootherstep", + "one_pole", + "affine_map", + "offset_scale", + "cubic", + "horner", + "offset", + "square", + ] { + let source = source(scalar, workload); + let start = Instant::now(); + let mut compiler = Compiler::new(); + compiler.quiet = true; + compiler.set_entry_points(&["process"]); + assert!( + compiler.parse(&source, "native_loop_bench.lyte"), + "{:?}", + compiler.last_errors + ); + assert!(compiler.check(), "{:?}", compiler.last_errors); + compiler.specialize().unwrap(); + let frontend_ms = start.elapsed().as_secs_f64() * 1e3; + let (mut ordinary, lower, backend) = Host::build(&compiler, false, workload); + let (mut native, native_lower, native_backend) = + Host::build(&compiler, true, workload); + let bank = bank(scalar, workload); + for block in 0..128 { + let (input, controls) = &bank[block % bank.len()]; + ordinary.process(input, controls); + native.process(input, controls); + assert_eq!( + ordinary.copied_output, native.copied_output, + "{scalar} {workload}, block {block}" + ); + assert_eq!( + ordinary.globals[ordinary.history.clone()], + native.globals[native.history.clone()] + ); + } + assert_eq!((*ordinary.polls, *native.polls), (0, 0)); + let mut times = [[0.0; ROUNDS]; 2]; + for round in 0..ROUNDS { + for variant in if round % 2 == 0 { [0, 1] } else { [1, 0] } { + let host = if variant == 0 { + &mut ordinary + } else { + &mut native + }; + times[variant][round] = host.measure(&bank, blocks); + } + } + for values in &mut times { + values.sort_by(f64::total_cmp); + } + let [a, b] = times; + println!("{scalar} {workload:12} Stack {:8.1} [{:.1}..{:.1}] native {:8.1} [{:.1}..{:.1}] {:5.2}x", + a[2], a[0], a[4], b[2], b[0], b[4], a[2]/b[2]); + println!(" setup: frontend {frontend_ms:.2}; Stack lower {lower:.2} + backend {backend:.2}; native lower {native_lower:.2} + backend {native_backend:.2}"); + } + } + } +}