diff --git a/build.rs b/build.rs index 286d984..3744182 100644 --- a/build.rs +++ b/build.rs @@ -1,7 +1,29 @@ use std::io::Write; use std::process::Command; +#[allow(dead_code)] +#[path = "src/loop_recipe.rs"] +mod loop_recipe; +#[allow(dead_code)] +#[path = "src/pointwise.rs"] +mod pointwise; +#[path = "build/pointwise.rs"] +mod pointwise_build; +#[allow(dead_code)] +#[path = "src/recurrence.rs"] +mod recurrence; +#[path = "build/recurrence.rs"] +mod recurrence_build; + fn main() { + let out_dir = std::env::var("OUT_DIR").unwrap(); + pointwise_build::generate(std::path::Path::new(&out_dir)); + recurrence_build::generate(std::path::Path::new(&out_dir)); + println!("cargo:rerun-if-changed=src/loop_recipe.rs"); + println!("cargo:rerun-if-changed=src/pointwise.rs"); + println!("cargo:rerun-if-changed=build/pointwise.rs"); + println!("cargo:rerun-if-changed=src/recurrence.rs"); + println!("cargo:rerun-if-changed=build/recurrence.rs"); // Compile the C stack interpreter with preserve_none + musttail. // Clang-only feature (skip on GCC). The handler signature keeps // the TOS window (t0..t3), the int stack pointer, the float @@ -17,6 +39,7 @@ fn main() { cc::Build::new() .file("src/stack_interp.c") .include("src") + .include(&out_dir) .opt_level(3) .flag("-std=c11") .flag("-Wno-unused-parameter") @@ -63,7 +86,6 @@ fn main() { println!("cargo:rerun-if-changed=src/stack_interp.c"); println!("cargo:rerun-if-changed=src/stack_interp.h"); // Determine sizeof(jmp_buf) on this platform by compiling and running a C snippet. - let out_dir = std::env::var("OUT_DIR").unwrap(); let c_path = format!("{}/jmpbuf_size.c", out_dir); let bin_path = format!("{}/jmpbuf_size", out_dir); diff --git a/build/pointwise.rs b/build/pointwise.rs new file mode 100644 index 0000000..26b3c0c --- /dev/null +++ b/build/pointwise.rs @@ -0,0 +1,112 @@ +//! Build-time expansion of the shared pointwise recipes into native C bodies. +//! +//! Every recipe produces typed SSA statements in its declared order. Handler +//! selection happens while the backend is constructed; constant kernel kinds +//! remove the generated switch before the interpreter is compiled. + +use crate::pointwise::{self, BinaryOp, Node, Recipe}; +use std::fmt::Write; +use std::path::Path; + +pub fn generate(out_dir: &Path) { + pointwise::validate().expect("invalid native pointwise recipe catalog"); + // The compact native encoding has exactly four coefficient slots. + assert_eq!(pointwise::MAX_COEFFICIENTS, 4); + + let mut catalog = String::from("// Generated from src/pointwise.rs; do not edit.\n"); + for (id, recipe) in pointwise::recipes() { + writeln!( + catalog, + "POINTWISE_RECIPE({}, {}, {}, {})", + id.index(), + recipe.name.to_ascii_uppercase(), + recipe.name, + recipe.coefficient_count(), + ) + .unwrap(); + } + std::fs::write(out_dir.join("pointwise_catalog.inc"), catalog) + .expect("failed to write native pointwise catalog"); + + let mut chunks = String::from("// Generated from src/pointwise.rs; do not edit.\n"); + for (suffix, scalar, load) in [ + ("f32", "float", "load_f32_unaligned"), + ("f64", "double", "load_f64_unaligned"), + ] { + writeln!( + chunks, + "static inline __attribute__((always_inline)) void native_pointwise_{suffix}(\n\ + NativeKernelKind kind, const uint8_t* input, uint8_t* output,\n\ + int64_t count, const {scalar}* coefficients\n\ + ) {{\n switch (kind) {{" + ) + .unwrap(); + for (_, recipe) in pointwise::recipes() { + writeln!( + chunks, + " case NATIVE_POINTWISE_{}: {{", + recipe.name.to_ascii_uppercase() + ) + .unwrap(); + for (index, node) in recipe.nodes.iter().enumerate() { + if let Node::Parameter(parameter) = node { + writeln!( + chunks, + " {scalar} v{index} = coefficients[{parameter}];" + ) + .unwrap(); + } + } + // The one-pointer spelling lets LLVM prove exact in-place access + // safe for SIMD without making a restrict promise about host spans. + chunks.push_str(" if (input == output) {\n"); + emit_loop(&mut chunks, recipe, scalar, load, "output"); + chunks.push_str(" } else {\n"); + emit_loop(&mut chunks, recipe, scalar, load, "input"); + chunks.push_str(" }\n return;\n }\n"); + } + chunks.push_str(" default: __builtin_unreachable();\n }\n}\n"); + } + std::fs::write(out_dir.join("pointwise_chunks.inc"), chunks) + .expect("failed to write native pointwise arithmetic"); +} + +fn emit_loop(code: &mut String, recipe: &Recipe, scalar: &str, load: &str, input: &str) { + writeln!( + code, + " for (int64_t item = 0; item < count; ++item) {{" + ) + .unwrap(); + writeln!( + code, + " size_t offset = (size_t)item * sizeof({scalar});" + ) + .unwrap(); + for (index, node) in recipe.nodes.iter().enumerate() { + let expression = match node { + Node::Read => format!("{load}({input} + offset)"), + Node::Parameter(_) => continue, + Node::Carry(_) => unreachable!("validated pointwise recipes have no carries"), + Node::Binary(operator, left, right) => { + let operator = match operator { + BinaryOp::Add => '+', + BinaryOp::Sub => '-', + BinaryOp::Mul => '*', + }; + format!("v{left} {operator} v{right}") + } + }; + writeln!( + code, + " {scalar} v{index} = {expression};" + ) + .unwrap(); + } + writeln!( + code, + " __builtin_memcpy(output + offset, &v{}, sizeof(v{}));", + recipe.result, recipe.result, + ) + .unwrap(); + code.push_str(" }\n"); +} diff --git a/build/recurrence.rs b/build/recurrence.rs new file mode 100644 index 0000000..2314e9a --- /dev/null +++ b/build/recurrence.rs @@ -0,0 +1,113 @@ +//! Expand the checked recurrence recipes into statically specialized C loops. +//! +//! Carry nodes name iteration-entry values. The next-carry map is published only +//! after every node has been evaluated, so history shifts do not depend on the +//! order in which mutable C carry variables are assigned. + +use crate::loop_recipe::{BinaryOp, Node}; +use crate::recurrence; +use std::fmt::Write; +use std::path::Path; + +pub fn generate(out_dir: &Path) { + recurrence::validate().expect("invalid native recurrence recipe catalog"); + // Preserve the existing native boundary and biquad descriptor capacities. + assert_eq!(recurrence::MAX_COEFFICIENTS, 5); + assert_eq!(recurrence::MAX_CARRIES, 4); + + let mut catalog = String::from("// Generated from src/recurrence.rs; do not edit.\n"); + for (id, recipe) in recurrence::recipes() { + writeln!( + catalog, + "RECURRENCE_RECIPE({}, {}, {}, {}, {})", + id.index(), + recipe.name.to_ascii_uppercase(), + recipe.name, + recipe.coefficient_count(), + recipe.carry_count(), + ) + .unwrap(); + } + std::fs::write(out_dir.join("recurrence_catalog.inc"), catalog) + .expect("failed to write native recurrence catalog"); + + let mut chunks = String::from("// Generated from src/recurrence.rs; do not edit.\n"); + for (suffix, scalar, load) in [ + ("f32", "float", "load_f32_unaligned"), + ("f64", "double", "load_f64_unaligned"), + ] { + writeln!( + chunks, + "static inline __attribute__((always_inline)) void native_recurrence_{suffix}(\n\ + NativeKernelKind kind, const uint8_t* input, uint8_t* output,\n\ + int64_t count, const {scalar}* coefficients, {scalar}* carried\n\ + ) {{\n switch (kind) {{" + ) + .unwrap(); + for (_, recipe) in recurrence::recipes() { + writeln!( + chunks, + " case NATIVE_{}: {{", + recipe.name.to_ascii_uppercase() + ) + .unwrap(); + for (index, node) in recipe.nodes.iter().enumerate() { + if let Node::Parameter(parameter) = node { + writeln!( + chunks, + " {scalar} v{index} = coefficients[{parameter}];" + ) + .unwrap(); + } + } + for carry in 0..recipe.carry_count() { + writeln!(chunks, " {scalar} c{carry} = carried[{carry}];").unwrap(); + } + writeln!( + chunks, + " for (int64_t item = 0; item < count; ++item) {{" + ) + .unwrap(); + writeln!( + chunks, + " size_t offset = (size_t)item * sizeof({scalar});" + ) + .unwrap(); + for (index, node) in recipe.nodes.iter().enumerate() { + let expression = match node { + Node::Read => format!("{load}(input + offset)"), + Node::Parameter(_) => continue, + Node::Carry(carry) => format!("c{carry}"), + Node::Binary(operator, left, right) => { + let operator = match operator { + BinaryOp::Add => '+', + BinaryOp::Sub => '-', + BinaryOp::Mul => '*', + }; + format!("v{left} {operator} v{right}") + } + }; + writeln!(chunks, " {scalar} v{index} = {expression};").unwrap(); + } + // Every root is an SSA value from the completed iteration. Updating + // c0 cannot change a later root which refers to the old c0 snapshot. + for (carry, next) in recipe.next_carries.iter().enumerate() { + writeln!(chunks, " c{carry} = v{next};").unwrap(); + } + writeln!( + chunks, + " __builtin_memcpy(output + offset, &v{}, sizeof(v{}));", + recipe.result, recipe.result, + ) + .unwrap(); + chunks.push_str(" }\n"); + for carry in 0..recipe.carry_count() { + writeln!(chunks, " carried[{carry}] = c{carry};").unwrap(); + } + chunks.push_str(" return;\n }\n"); + } + chunks.push_str(" default: __builtin_unreachable();\n }\n}\n"); + } + std::fs::write(out_dir.join("recurrence_chunks.inc"), chunks) + .expect("failed to write native recurrence arithmetic"); +} diff --git a/examples/native_loop_bench.rs b/examples/native_loop_bench.rs new file mode 100644 index 0000000..05820a8 --- /dev/null +++ b/examples/native_loop_bench.rs @@ -0,0 +1,326 @@ +//! Complete 240-sample processing calls, ordinary optimized Stack versus native loops. +//! Run: cargo run --release --example native_loop_bench -- [blocks-per-round] +//! Times include input/control copies, Lyte setup, host invocation/status checks, +//! and output copying/consumption. Recurrence history persists between calls. Inputs +//! are prepared outside timing; processing allocates nothing. A cancellation +//! callback is installed, but a fresh 1,024-backedge budget covers each call. +//! Five warmed, alternating paired rounds report median and min..max ns/block. +//! Frontend, Stack lowering, and backend construction are reported separately. +//! All ten recipes run in both precisions; the three headline cases appear first. +//! These are small buffered kernels, not full Audulus graphs or coefficient design. + +#[cfg(not(has_stack_interp))] +fn main() { + eprintln!("This example requires the C Stack interpreter (has_stack_interp)."); +} + +#[cfg(has_stack_interp)] +fn main() { + bench::run(); +} + +#[cfg(has_stack_interp)] +mod bench { + use lyte::stack_interp_bridge::StackBackend; + use lyte::stack_ir::{NativeStreamBody, StackOp}; + use lyte::{Compiler, Name, CANCEL_FLAG_RESERVED, TRAP_NONE}; + use std::{hint::black_box, ops::Range, time::Instant}; + + const SAMPLES: usize = 240; + const ROUNDS: usize = 5; + + struct Host { + backend: StackBackend, + globals: Vec, + entry: u32, + input: Range, + controls: Range, + output: Range, + history: Range, + copied_output: Vec, + polls: Box, + } + + unsafe extern "C" fn cancel(data: *mut u8) -> bool { + *data.cast::() += 1; + false + } + + impl Host { + fn build(compiler: &Compiler, native: bool, workload: &str) -> (Self, f64, f64) { + let start = Instant::now(); + let program = if native { + compiler.compile_stack_native_loops() + } else { + compiler.compile_stack() + } + .unwrap(); + let lowering_ms = start.elapsed().as_secs_f64() * 1e3; + let kernels: Vec<_> = program + .functions + .iter() + .flat_map(|f| &f.ops) + .filter_map(|op| { + if let StackOp::NativeLoop(kernel) = op { + Some(&kernel.body) + } else { + None + } + }) + .collect(); + assert_eq!(kernels.len(), usize::from(native)); + if native { + assert!( + match kernels[0] { + NativeStreamBody::Pointwise { recipe, .. } => + recipe.recipe().name == workload, + NativeStreamBody::Biquad { .. } => workload == "biquad", + NativeStreamBody::OnePole { .. } => workload == "one_pole", + }, + "unexpected native recipe for {}", + workload + ); + } + let start = Instant::now(); + let mut backend = StackBackend::new(&program); + let backend_ms = start.elapsed().as_secs_f64() * 1e3; + let mut polls = Box::new(0); + backend.set_cancel_callback(Some(cancel), (&mut *polls as *mut usize).cast()); + let info = compiler.globals_info_with_offset(CANCEL_FLAG_RESERVED as usize); + let range = |name| { + let (_, offset, size, _, _) = info.iter().find(|x| x.0 == name).unwrap(); + *offset..*offset + *size + }; + let output = range("output"); + ( + Self { + backend, + globals: vec![0; program.globals_size], + entry: program.entry_points[&Name::str("process")], + input: range("input"), + controls: range("controls"), + history: range("history"), + copied_output: vec![0; output.len()], + output, + polls, + }, + lowering_ms, + backend_ms, + ) + } + + fn process(&mut self, input: &[u8], controls: &[u8]) { + self.globals[self.input.clone()].copy_from_slice(input); + self.globals[self.controls.clone()].copy_from_slice(controls); + assert_eq!( + self.backend + .call_entry(self.entry, self.globals.as_mut_ptr()), + 0 + ); + assert!(!self.backend.cancelled()); + assert_eq!(self.backend.trap_reason(), TRAP_NONE); + self.copied_output + .copy_from_slice(&self.globals[self.output.clone()]); + black_box(&self.copied_output); + } + + fn reset(&mut self) { + self.globals.fill(0); + *self.polls = 0; + } + + fn measure(&mut self, bank: &[(Vec, Vec)], blocks: usize) -> f64 { + self.reset(); + for block in 0..2048 { + let (input, controls) = &bank[block % bank.len()]; + self.process(input, controls); + } + let start = Instant::now(); + for block in 0..blocks { + let (input, controls) = &bank[block % bank.len()]; + self.process(input, controls); + } + let ns = start.elapsed().as_secs_f64() * 1e9 / blocks as f64; + assert_eq!(*self.polls, 0); + ns + } + } + + fn source(scalar: &str, workload: &str) -> String { + let (count, setup, body, finish) = match workload { + "gain" => (1, "let gain = controls[0]", "let y = x*gain", ""), + "offset" => (1, "let bias = controls[0]", "let y = x+bias", ""), + // Lyte arrays are nonempty; square leaves its host control unused. + "square" => (1, "", "let y = x*x", ""), + "affine_map" => ( + 2, + "let gain = controls[0]; let bias = controls[1]", + "let y = x*gain+bias", + "", + ), + "offset_scale" => ( + 2, + "let bias = controls[0]; let gain = controls[1]", + "let y = (x-bias)*gain", + "", + ), + "cubic" => (1, "let k = controls[0]", "let y = x-k*((x*x)*x)", ""), + "horner" => ( + 4, + "let a = controls[0]; let b = controls[1] + let c = controls[2]; let d = controls[3]", + "let y = ((a*x+b)*x+c)*x+d", + "", + ), + "one_pole" => ( + 2, + "let feed = controls[0]; let feedback = controls[1]; var state = history[0]", + "state = x*feed+state*feedback; let y = state", + "history[0] = state", + ), + "smootherstep" => ( + 3, + "let a = controls[0]; let b = controls[1]; let c = controls[2]", + "let cube = (x*x)*x; let curve = x*(x*a-b)+c; let y = cube*curve", + "", + ), + "biquad" => ( + 5, + "let b0 = controls[0]; let b1 = controls[1]; let b2 = controls[2] + let a1 = controls[3]; let a2 = controls[4] + var x1 = history[0]; var x2 = history[1] + var y1 = history[2]; var y2 = history[3]", + "var y = b0*x; y = y+b1*x1; y = y+b2*x2; y = y-a1*y1; y = y-a2*y2 + x2 = x1; x1 = x; y2 = y1; y1 = y", + "history[0] = x1; history[1] = x2; history[2] = y1; history[3] = y2", + ), + _ => unreachable!(), + }; + format!( + "var input: [{scalar}; {SAMPLES}] + var output: [{scalar}; {SAMPLES}] + var controls: [{scalar}; {count}] + var history: [{scalar}; 4] + process() -> i32 {{ + {setup} + for i in 0 .. {SAMPLES} {{ + let x = input[i] + {body} + output[i] = y + }} + {finish} + 0 + }}" + ) + } + + fn bank(scalar: &str, workload: &str) -> Vec<(Vec, Vec)> { + let bytes = |values: &[f64]| -> Vec { + values + .iter() + .flat_map(|&value| { + if scalar == "f32" { + (value as f32).to_ne_bytes().to_vec() + } else { + value.to_ne_bytes().to_vec() + } + }) + .collect() + }; + (0..16) + .map(|block| { + let input: Vec<_> = (0..SAMPLES) + .map(|i| ((i * 17 + block * 11) % 241) as f64 / 240.0) + .collect(); + let controls = match workload { + "gain" => vec![0.5 + block as f64 / 64.0], + "offset" => vec![0.25 + block as f64 / 64.0], + "square" => vec![0.0], + "affine_map" => vec![0.5 + block as f64 / 64.0, 0.25], + "offset_scale" => vec![0.25, 0.5 + block as f64 / 64.0], + "cubic" => vec![0.25 + block as f64 / 128.0], + "horner" => vec![0.5 + block as f64 / 64.0, -0.25, 0.125, 0.0625], + "one_pole" => vec![0.25 + block as f64 / 1024.0, 0.75 - block as f64 / 1024.0], + "smootherstep" => vec![6.0, 15.0, 10.0], + "biquad" => vec![0.5 + block as f64 / 1024.0, 0.25, 0.125, 0.0625, 0.03125], + _ => unreachable!(), + }; + (bytes(&input), bytes(&controls)) + }) + .collect() + } + + pub fn run() { + let blocks = std::env::args() + .nth(1) + .map(|s| s.parse::().expect("blocks must be an integer")) + .unwrap_or(32768); + assert!(blocks > 0); + println!("240 samples/call; callback installed; {ROUNDS} paired rounds of {blocks} blocks"); + println!("ns/block: median [min..max]; setup times are single observations in ms"); + for scalar in ["f32", "f64"] { + for workload in [ + "gain", + "biquad", + "smootherstep", + "one_pole", + "affine_map", + "offset_scale", + "cubic", + "horner", + "offset", + "square", + ] { + let source = source(scalar, workload); + let start = Instant::now(); + let mut compiler = Compiler::new(); + compiler.quiet = true; + compiler.set_entry_points(&["process"]); + assert!( + compiler.parse(&source, "native_loop_bench.lyte"), + "{:?}", + compiler.last_errors + ); + assert!(compiler.check(), "{:?}", compiler.last_errors); + compiler.specialize().unwrap(); + let frontend_ms = start.elapsed().as_secs_f64() * 1e3; + let (mut ordinary, lower, backend) = Host::build(&compiler, false, workload); + let (mut native, native_lower, native_backend) = + Host::build(&compiler, true, workload); + let bank = bank(scalar, workload); + for block in 0..128 { + let (input, controls) = &bank[block % bank.len()]; + ordinary.process(input, controls); + native.process(input, controls); + assert_eq!( + ordinary.copied_output, native.copied_output, + "{scalar} {workload}, block {block}" + ); + assert_eq!( + ordinary.globals[ordinary.history.clone()], + native.globals[native.history.clone()] + ); + } + assert_eq!((*ordinary.polls, *native.polls), (0, 0)); + let mut times = [[0.0; ROUNDS]; 2]; + for round in 0..ROUNDS { + for variant in if round % 2 == 0 { [0, 1] } else { [1, 0] } { + let host = if variant == 0 { + &mut ordinary + } else { + &mut native + }; + times[variant][round] = host.measure(&bank, blocks); + } + } + for values in &mut times { + values.sort_by(f64::total_cmp); + } + let [a, b] = times; + println!("{scalar} {workload:12} Stack {:8.1} [{:.1}..{:.1}] native {:8.1} [{:.1}..{:.1}] {:5.2}x", + a[2], a[0], a[4], b[2], b[0], b[4], a[2]/b[2]); + println!(" setup: frontend {frontend_ms:.2}; Stack lower {lower:.2} + backend {backend:.2}; native lower {native_lower:.2} + backend {native_backend:.2}"); + } + } + } +} diff --git a/src/compiler.rs b/src/compiler.rs index 5f35c5c..060732c 100644 --- a/src/compiler.rs +++ b/src/compiler.rs @@ -1097,7 +1097,21 @@ impl Compiler { /// Compile to stack-based IR (for Silverfir-nano-style interpreters). pub fn compile_stack(&self) -> Result { + self.compile_stack_with_native_loops(false) + } + + /// Experimental consumer of concrete checked loops. Unmatched loops and + /// native runtime guard failures retain ordinary Stack execution. + pub fn compile_stack_native_loops(&self) -> Result { + self.compile_stack_with_native_loops(true) + } + + fn compile_stack_with_native_loops( + &self, + native_loops: bool, + ) -> Result { let mut codegen = crate::stack_codegen::StackCodegen::new(); + codegen.native_loops = native_loops; let entry_points = self.effective_entry_points(); let mut program = codegen.compile_multi(self.specialized_program()?, &entry_points)?; // Inline trivial leaf functions (like cmp(a, b) -> a - b) so their diff --git a/src/lib.rs b/src/lib.rs index ebd91dd..9d55496 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -87,6 +87,14 @@ pub mod stack_ir; pub mod stack_codegen; +pub mod loop_recipe; +pub mod pointwise; +pub mod recurrence; +pub mod value_loops; + +#[cfg(all(test, has_stack_interp))] +mod native_loop_tests; + pub mod stack_optimize; pub mod stack_depth; diff --git a/src/loop_recipe.rs b/src/loop_recipe.rs new file mode 100644 index 0000000..370041c --- /dev/null +++ b/src/loop_recipe.rs @@ -0,0 +1,182 @@ +//! Compiler-independent ordered values shared by native loop matching and C +//! generation. Carry updates are simultaneous next-state roots, not assignments +//! to execute in role order. The output is the single terminal memory effect. + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum BinaryOp { + Add, + Sub, + Mul, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Node { + Read, + Parameter(u8), + Carry(u8), + Binary(BinaryOp, usize, usize), +} + +#[derive(Clone, Copy, Debug)] +pub struct Recipe { + pub name: &'static str, + pub nodes: &'static [Node], + /// Value written by the terminal output effect, after all ordered work. + pub result: usize, + /// Final value for each incoming carry role. All roots refer to the same + /// iteration's immutable values, including its incoming carry snapshots. + pub next_carries: &'static [usize], +} + +impl Recipe { + pub fn coefficient_count(&self) -> usize { + self.nodes + .iter() + .filter_map(|node| match node { + Node::Parameter(role) => Some(usize::from(*role) + 1), + _ => None, + }) + .max() + .unwrap_or(0) + } + + pub fn carry_count(&self) -> usize { + self.next_carries.len() + } +} + +/// Validate semantic recipes before either consumer uses them. Physical operand +/// encodings supply their own limits. No work node may be unused: native C could +/// remove it while the matcher promises coverage of the complete ordered tape. +pub fn validate_catalog( + catalog: &[Recipe], + max_coefficients: usize, + max_carries: usize, +) -> Result<(), String> { + use Node::*; + if catalog.len() > usize::from(u8::MAX) + 1 { + return Err("native recipe catalog exceeds identity range".into()); + } + let mut names = std::collections::HashSet::new(); + for recipe in catalog { + let valid_name = recipe + .name + .bytes() + .all(|c| c.is_ascii_lowercase() || c == b'_'); + if recipe.name.is_empty() || !valid_name || !names.insert(recipe.name) { + return Err(format!("invalid or duplicate recipe name: {}", recipe.name)); + } + if recipe.result >= recipe.nodes.len() + || recipe + .next_carries + .iter() + .any(|&next| next >= recipe.nodes.len()) + || recipe.carry_count() > max_carries + { + return Err(format!("{}: invalid output or carry roots", recipe.name)); + } + let mut reads = 0; + let mut parameters = vec![false; max_coefficients]; + let mut carries = vec![false; recipe.carry_count()]; + for (index, node) in recipe.nodes.iter().enumerate() { + let (roles, role, label) = match *node { + Read => { + reads += 1; + continue; + } + Parameter(role) => (&mut parameters, usize::from(role), "parameter"), + Carry(role) => (&mut carries, usize::from(role), "carry"), + Binary(_, left, right) => { + if left >= index || right >= index { + return Err(format!("{}: operand is not an earlier value", recipe.name)); + } + continue; + } + }; + if role >= roles.len() || roles[role] { + return Err(format!("{}: invalid {label} role", recipe.name)); + } + roles[role] = true; + } + if reads != 1 + || parameters[..recipe.coefficient_count()].contains(&false) + || carries.contains(&false) + { + return Err(format!( + "{}: expected one read and complete parameter/carry roles", + recipe.name + )); + } + let mut live = vec![false; recipe.nodes.len()]; + live[recipe.result] = true; + for &next in recipe.next_carries { + live[next] = true; + } + for index in (0..recipe.nodes.len()).rev() { + if live[index] { + if let Binary(_, left, right) = recipe.nodes[index] { + live[left] = true; + live[right] = true; + } + } + } + if live.contains(&false) { + return Err(format!( + "{}: unused node would violate work coverage", + recipe.name + )); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use Node::*; + + #[test] + fn output_and_next_state_are_independent_roots_with_complete_roles() { + let recipe = Recipe { + name: "state_roots", + nodes: &[ + Read, + Carry(0), + Carry(1), + Binary(BinaryOp::Add, 0, 1), + Binary(BinaryOp::Mul, 2, 3), + ], + result: 0, + next_carries: &[3, 4], + }; + assert!(validate_catalog(&[recipe], 0, 2).is_ok()); + for invalid in [ + Recipe { + next_carries: &[3, 5], + ..recipe + }, + Recipe { + next_carries: &[3], + ..recipe + }, + Recipe { + next_carries: &[3, 2], + ..recipe + }, + Recipe { + nodes: &[Read, Carry(0), Carry(0)], + result: 0, + next_carries: &[1, 2], + ..recipe + }, + Recipe { + nodes: &[Read, Carry(0), Binary(BinaryOp::Add, 0, 3)], + result: 0, + next_carries: &[2], + ..recipe + }, + ] { + assert!(validate_catalog(&[invalid], 0, 2).is_err()); + } + } +} diff --git a/src/native_loop_tests.rs b/src/native_loop_tests.rs new file mode 100644 index 0000000..3f2754e --- /dev/null +++ b/src/native_loop_tests.rs @@ -0,0 +1,589 @@ +//! End-to-end consumers of checked identities and the optional native-loop path. +//! Raw alias/cancellation/frame-boundary contracts are covered by bridge tests. + +use crate::stack_interp_bridge::StackBackend; +use crate::stack_ir::{NativeStreamBody, StackOp, StackProgram}; +use crate::*; + +fn checked(source: &str, entries: &[&str]) -> Compiler { + let mut compiler = Compiler::new(); + compiler.quiet = true; + compiler.set_entry_points(entries); + assert!( + compiler.parse(source, "native_loop_tests.lyte"), + "{:?}", + compiler.last_errors + ); + assert!(compiler.check(), "{:?}", compiler.last_errors); + compiler.specialize().unwrap(); + compiler +} + +fn native_bodies(program: &StackProgram) -> Vec<&NativeStreamBody> { + program + .functions + .iter() + .flat_map(|function| &function.ops) + .filter_map(|op| { + if let StackOp::NativeLoop(kernel) = op { + Some(&kernel.body) + } else { + None + } + }) + .collect() +} + +fn variants(compiler: &Compiler) -> (StackProgram, StackProgram) { + let ordinary = compiler.compile_stack().unwrap(); + assert!( + native_bodies(&ordinary).is_empty(), + "ordinary compilation must remain opt-out" + ); + let selected = compiler.compile_stack_native_loops().unwrap(); + (ordinary, selected) +} + +fn execute(program: &StackProgram, entries: &[&str]) -> Vec<(i64, Vec)> { + let mut backend = Box::new(StackBackend::new(program)); + let mut globals = vec![0u8; program.globals_size.max(1)]; + entries + .iter() + .map(|entry| { + let function = program.entry_points[&Name::str(entry)]; + let result = backend.call_entry(function, globals.as_mut_ptr()); + assert!(!backend.cancelled()); + assert_eq!(backend.trap_reason(), crate::cancel::TRAP_NONE); + (result, globals.clone()) + }) + .collect() +} + +// Replace the retained loop entry with a trap without changing jump coordinates. +// Valid, non-polling examples must execute native work to reach their results. +pub(crate) fn forbid_fallback(program: &mut StackProgram) { + let mut gateways = 0; + for function in &mut program.functions { + for index in 0..function.ops.len() { + if let StackOp::NativeLoop(kernel) = &function.ops[index] { + assert!(kernel.done >= 2); + function.ops[index + 1] = StackOp::I64Const(0); + function.ops[index + 2] = StackOp::Assert; + gateways += 1; + } + } + } + assert!(gateways > 0, "expected executable native work"); +} + +fn execute_native(mut program: StackProgram, entries: &[&str]) -> Vec<(i64, Vec)> { + let expected = execute(&program, entries); + forbid_fallback(&mut program); + assert_eq!(execute(&program, entries), expected); + expected +} + +#[test] +fn pointwise_extensions_execute_natively_in_both_widths() { + let inputs = [-0.0_f64, 0.0, 0.1, 0.25, 0.5, 0.75, 0.9, 1.0]; + for scalar in ["f32", "f64"] { + let cases: [(String, fn(f32) -> f32, fn(f64) -> f64); 4] = [ + ("let y = x*gain".into(), |x| x * -1.25, |x| x * -1.25), + ("let y = x+bias".into(), |x| x + 0.3, |x| x + 0.3), + ("let y = x*x".into(), |x| x * x, |x| x * x), + ( + format!( + "let cube = (x*x)*x + let curve = x*(x*6.0{scalar}-15.0{scalar})+10.0{scalar} + let y = cube*curve" + ), + |x| ((x * x) * x) * (x * (x * 6.0 - 15.0) + 10.0), + |x| ((x * x) * x) * (x * (x * 6.0 - 15.0) + 10.0), + ), + ]; + let values = inputs + .iter() + .map(|x| format!("{x:?}{scalar}")) + .collect::>() + .join(", "); + for (body, reference_f32, reference_f64) in cases { + // Independent scalar arithmetic, including separate rounding at each step. + let expected: Vec = inputs + .iter() + .flat_map(|&x| { + if scalar == "f32" { + reference_f32(x as f32).to_le_bytes().to_vec() + } else { + reference_f64(x).to_le_bytes().to_vec() + } + }) + .collect(); + for destination in ["output", "input"] { + let compiler = checked( + &format!( + r#" + var input: [{scalar}; 8] + var output: [{scalar}; 8] + main() -> i32 {{ + input = [{values}] + let gain = -1.25{scalar} + let bias = 0.3{scalar} + for i in 0 .. 8 {{ + let x = input[i] + {body} + {destination}[i] = y + }} + 0 + }} + "# + ), + &["main"], + ); + let (ordinary, native) = variants(&compiler); + let results = execute(&ordinary, &["main"]); + let offset = compiler + .globals_info_with_offset(crate::cancel::CANCEL_FLAG_RESERVED as usize) + .into_iter() + .find(|entry| entry.0 == destination) + .unwrap() + .1; + assert_eq!(&results[0].1[offset..offset + expected.len()], &expected); + assert_eq!( + execute_native(native, &["main"]), + results, + "{scalar}, {destination}, {body}" + ); + } + } + } +} + +#[test] +fn affine_slices_publish_output_and_reload_headers_after_callback() { + struct Redirect { + headers: [*mut u8; 2], + replacements: [*const u8; 2], + output: *const u8, + size: usize, + observed: Vec, + } + unsafe extern "C" fn redirect(data: *mut u8) -> bool { + let probe = &mut *data.cast::(); + probe.observed = std::slice::from_raw_parts(probe.output, probe.size).to_vec(); + for (header, replacement) in probe.headers.iter().copied().zip(probe.replacements) { + std::ptr::write_unaligned(header.cast::<*const u8>(), replacement); + } + false + } + + for scalar in ["f32", "f64"] { + let compiler = checked( + &format!( + "var input: [{scalar}] + var output: [{scalar}] + process() -> i32 {{ + var prefix = 0 + for j in 0 .. 1022 {{ prefix = prefix + 1 }} + if input.len >= 4 && output.len >= 4 {{ + for i in 0 .. 4 {{ output[i] = input[i]*0.5{scalar}+1.0{scalar} }} + }} + prefix + }}" + ), + &["process"], + ); + let info = compiler.globals_info_with_offset(crate::cancel::CANCEL_FLAG_RESERVED as usize); + let offsets = + ["input", "output"].map(|name| info.iter().find(|entry| entry.0 == name).unwrap().1); + let bytes = |values: &[f64]| -> Vec { + values + .iter() + .flat_map(|&value| { + if scalar == "f64" { + value.to_le_bytes().to_vec() + } else { + (value as f32).to_le_bytes().to_vec() + } + }) + .collect() + }; + for callback in [false, true] { + let (ordinary, mut native) = variants(&compiler); + if !callback { + forbid_fallback(&mut native); + } + for program in [ordinary, native] { + let input = bytes(&[2.0, 4.0, 6.0, 8.0]); + let replacement = bytes(&[10.0, 12.0, 14.0, 16.0]); + let mut output = bytes(&[0.0; 4]); + let mut redirected = output.clone(); + let mut globals = vec![0u8; program.globals_size]; + let mut probe = Redirect { + headers: offsets.map(|offset| unsafe { globals.as_mut_ptr().add(offset) }), + replacements: [replacement.as_ptr(), redirected.as_mut_ptr()], + output: output.as_ptr(), + size: output.len(), + observed: Vec::new(), + }; + for (offset, pointer) in offsets.iter().zip([input.as_ptr(), output.as_mut_ptr()]) { + globals[*offset..*offset + 8].copy_from_slice(&(pointer as u64).to_le_bytes()); + globals[*offset + 8..*offset + 12].copy_from_slice(&4i32.to_le_bytes()); + } + let mut backend = StackBackend::new(&program); + if callback { + backend + .set_cancel_callback(Some(redirect), (&mut probe as *mut Redirect).cast()); + } + assert_eq!( + backend.call_entry(program.entry, globals.as_mut_ptr()), + 1022 + ); + assert_eq!(backend.trap_reason(), crate::cancel::TRAP_NONE); + if callback { + assert_eq!(probe.observed, bytes(&[2.0, 3.0, 0.0, 0.0])); + assert_eq!(output, probe.observed); + assert_eq!(redirected, bytes(&[0.0, 0.0, 8.0, 9.0])); + } else { + assert_eq!(output, bytes(&[2.0, 3.0, 4.0, 5.0])); + } + } + } + } +} + +#[test] +fn captured_outer_and_uncaptured_shadow_select_affine_and_preserve_closure() { + let compiler = checked( + r#" + kernel(input: &[f32; 4], output: &[f32; 4]) -> f32 { + let gain = 0.25 + let read = || { gain } + { + let gain = 0.5 + for i in 0 .. 4 { output[i] = input[i]*gain+1.0 } + } + read() + } + main() -> i32 { + var input = [2.0, 4.0, 6.0, 8.0] + var output: [f32; 4] + let captured = kernel(input, output) + ((captured + output[0]+output[1]+output[2]+output[3])*100.0) as i32 + } + "#, + &["main"], + ); + let (ordinary, selected) = variants(&compiler); + assert!(matches!( + native_bodies(&selected).as_slice(), + [NativeStreamBody::Pointwise { .. }] + )); + let expected = execute(&ordinary, &["main"]); + assert_eq!(expected[0].0, 1425); + assert_eq!(execute_native(selected, &["main"]), expected); + + let mut renamed = compiler.specialized_program().unwrap().clone(); + for decl in &mut renamed.decls.decls { + let Decl::Func(function) = decl else { continue }; + for local in &mut function.arena.locals { + local.name = Name::str("same"); + } + for id in function.arena.ids() { + let mut expr = function.arena[id].clone(); + match &mut expr { + Expr::Id(name) + | Expr::TypeApp(name, _) + | Expr::Let(name, ..) + | Expr::Var(name, ..) + | Expr::For { var: name, .. } => *name = Name::str("same"), + Expr::Lambda { params, .. } => { + for param in params { + param.name = Name::str("same"); + } + } + _ => {} + } + function.arena.replace(id, expr, function.arena.ty(id)); + } + } + renamed.validate().unwrap(); + let mut codegen = crate::stack_codegen::StackCodegen::new(); + codegen.native_loops = true; + let mut program = codegen.compile(&renamed).unwrap(); + crate::stack_inline::inline_trivial(&mut program); + for function in &mut program.functions { + crate::stack_rebase_lm::rebase(function); + crate::stack_optimize::optimize(function); + crate::stack_rebase_lm::patch_call_preserve(function); + } + assert_eq!(execute_native(program, &["main"]), expected); +} + +#[test] +fn one_pole_snapshots_and_state_survive_repeated_entry_calls() { + for scalar in ["f32", "f64"] { + let compiler = checked( + &format!( + r#" + var input: [{scalar}; 4] + var output: [{scalar}; 4] + var history: {scalar} + init() {{ + input[0] = 2.0{scalar}; input[1] = 4.0{scalar} + input[2] = 6.0{scalar}; input[3] = 8.0{scalar} + history = 1.0{scalar} + }} + process() -> i32 {{ + var state = history + let gain = 0.5{scalar} + let feedback = 0.25{scalar} + for i in 0 .. 4 {{ + let before = state + var x = input[i] + var feed = x*gain + let decay = before*feedback + state = feed+decay + output[i] = state + }} + history = state + (state*1024.0{scalar}) as i32 + }} + "# + ), + &["init", "process"], + ); + let (ordinary, selected) = variants(&compiler); + assert!(matches!( + native_bodies(&selected).as_slice(), + [NativeStreamBody::OnePole { .. }] + )); + let entries = ["init", "process", "process", "process", "init", "process"]; + let expected = execute(&ordinary, &entries); + assert_eq!(expected[1].0, 5012); + assert_ne!(expected[1].0, expected[2].0); + assert_eq!(expected[1], expected[5]); + assert_eq!(execute_native(selected, &entries), expected, "{scalar}"); + } +} + +#[test] +fn biquad_staged_history_updates_preserve_every_carried_state() { + for scalar in ["f32", "f64"] { + let compiler = checked( + &format!( + r#" + var input: [{scalar}; 4] + var output: [{scalar}; 4] + var history_x1: {scalar} + var history_x2: {scalar} + var history_y1: {scalar} + var history_y2: {scalar} + init() {{ + input[0] = 2.0{scalar}; input[1] = 4.0{scalar} + input[2] = 6.0{scalar}; input[3] = 8.0{scalar} + history_x1 = 0.5{scalar}; history_x2 = 1.0{scalar} + history_y1 = 0.75{scalar}; history_y2 = 0.25{scalar} + }} + process() -> i32 {{ + var y2 = history_y2 + var x1 = history_x1 + var y1 = history_y1 + var x2 = history_x2 + let b0 = 0.5{scalar}; let b1 = 0.25{scalar}; let b2 = 0.125{scalar} + let a1 = 0.0625{scalar}; let a2 = 0.03125{scalar} + for i in 0 .. 4 {{ + let x = input[i] + var y = b0*x + y = y+b1*x1 + y = y+b2*x2 + y = y-a1*y1 + y = y-a2*y2 + let old_input = x1 + let old_output = y1 + y1 = y; x1 = x; x2 = old_input; y2 = old_output + output[i] = y + }} + history_x1 = x1; history_x2 = x2 + history_y1 = y1; history_y2 = y2 + ((x1+x2+y1+y2)*1024.0{scalar}) as i32 + }} + "# + ), + &["init", "process"], + ); + let (ordinary, selected) = variants(&compiler); + assert!(matches!( + native_bodies(&selected).as_slice(), + [NativeStreamBody::Biquad { .. }] + )); + let entries = ["init", "process", "process", "process", "init", "process"]; + let expected = execute(&ordinary, &entries); + assert_ne!(expected[1].0, expected[2].0); + assert_eq!(expected[1], expected[5]); + assert_eq!(execute_native(selected, &entries), expected, "{scalar}"); + } +} + +#[test] +fn same_typed_generic_globals_select_distinct_native_streams() { + let compiler = checked( + r#" + var input: [f32; 4] + var output: [f32; 4] + main() -> i32 { + input⟨i32⟩[0] = 2.0 + input⟨f32⟩[0] = 8.0 + for i in 0 .. 4 { output⟨i32⟩[i] = input⟨i32⟩[i]*0.5+1.0 } + for i in 0 .. 4 { output⟨f32⟩[i] = input⟨f32⟩[i]*0.5+1.0 } + (output⟨i32⟩[0]*100.0 + output⟨f32⟩[0]) as i32 + } + "#, + &["main"], + ); + let program = compiler.specialized_program().unwrap(); + let inputs: Vec<_> = program + .globals() + .filter(|(id, _)| { + let source = program.instances[id.index()].definition; + program.globals().any(|(other, _)| { + other != *id && program.instances[other.index()].definition == source + }) + }) + .collect(); + assert_eq!( + inputs.len(), + 4, + "two concrete instances of each same-typed global" + ); + let (ordinary, selected) = variants(&compiler); + assert_eq!(native_bodies(&selected).len(), 2); + assert!(native_bodies(&selected) + .iter() + .all(|body| matches!(body, NativeStreamBody::Pointwise { .. }))); + let expected = execute(&ordinary, &["main", "main"]); + assert_eq!(expected[0].0, 205); + assert_eq!(execute_native(selected, &["main", "main"]), expected); +} + +#[test] +fn unsupported_indexing_calls_and_captured_inputs_remain_ordinary() { + for (body, captured) in [ + ("output[i] = input[i+1]*gain+1.0", false), + ("output[i] = twice(input[i])", false), + ("output[i] = input[i]*gain+1.0", true), + ] { + let capture = if captured { + "let read = || { gain }" + } else { + "" + }; + let result = if captured { "read()" } else { "gain" }; + let compiler = checked( + &format!( + r#" + twice(x: f32) -> f32 {{ x+x }} + kernel(input: &[f32; 4], output: &[f32; 4]) -> f32 {{ + let gain = 0.5 + {capture} + for i in 0 .. 3 {{ {body} }} + {result} + }} + main() -> i32 {{ + var input = [2.0, 4.0, 6.0, 8.0] + var output: [f32; 4] + let value = kernel(input, output) + ((value+output[0]+output[1]+output[2])*100.0) as i32 + }} + "# + ), + &["main"], + ); + let (ordinary, selected) = variants(&compiler); + assert!( + native_bodies(&selected).is_empty(), + "unexpected native selection for {}", + body + ); + assert_eq!(execute(&selected, &["main"]), execute(&ordinary, &["main"])); + } + + let compiler = checked( + "main() -> i32 { + var input = [2.0, 4.0, 6.0, 8.0] + var output: [f32; 4] + let map = |gain: f32| { + for i in 0 .. 4 { output[i] = input[i]*gain+1.0 } + } + map(0.5) + (output[0]+output[1]+output[2]+output[3]) as i32 + }", + &["main"], + ); + let (ordinary, selected) = variants(&compiler); + assert!(native_bodies(&selected).is_empty()); + let expected = execute(&ordinary, &["main"]); + assert_eq!(expected[0].0, 14); + assert_eq!(execute(&selected, &["main"]), expected); +} + +#[test] +fn native_setup_frame_overflow_retries_the_exact_ordinary_function() { + fn source(padding: usize, padding_first: bool) -> String { + let allocation = format!( + "var padding: [f32; {padding}]; padding[{}] = gain;\n\ + var marker: [f32; 1]; marker[0] = padding[{}];", + padding - 1, + padding - 1, + ); + let map = "for i in 0 .. 4 { output[i] = input[i]*gain+bias }"; + let body = if padding_first { + format!("{allocation}\n{map}") + } else { + format!("{map}\n{allocation}") + }; + format!( + r#" + helper(x: f32) -> f32 {{ x }} + kernel(input: &[f32; 4], output: &[f32; 4], gain: f32, bias: f32) -> f32 {{ + {body} + let finish = |x| x + helper(finish(marker[0]+output[3])) + }} + main() -> i32 {{ + var input: [f32; 4] + var output: [f32; 4] + kernel(input, output, 2.0, 0.5) as i32 + }} + "# + ) + } + + for padding_first in [true, false] { + let small = checked(&source(16, padding_first), &["main"]); + let (ordinary, selected) = variants(&small); + assert_eq!(native_bodies(&selected).len(), 1); + let kernel = ordinary + .functions + .iter() + .find(|f| f.name == "kernel") + .unwrap(); + let frame_slots = usize::from(kernel.local_count) + kernel.local_memory as usize / 8; + // Add only array storage; the rest of this function's slot demand stays + // unchanged. Its ordinary frame fits, but native setup needs extra slots. + let padding = 16 + 2 * (usize::from(u16::MAX) - frame_slots); + let compiler = checked(&source(padding, padding_first), &["main"]); + let (ordinary, selected) = variants(&compiler); + assert!(native_bodies(&selected).is_empty()); + assert_eq!(ordinary.entry_points, selected.entry_points); + assert_eq!(ordinary.functions.len(), selected.functions.len()); + for (prior, retried) in ordinary.functions.iter().zip(&selected.functions) { + assert_eq!(prior.name, retried.name); + assert_eq!(prior.local_count, retried.local_count); + assert_eq!(prior.local_memory, retried.local_memory); + assert_eq!(prior.ops, retried.ops); + } + let expected = execute(&ordinary, &["main"]); + assert_eq!(expected[0].0, 2); + assert_eq!(execute(&selected, &["main"]), expected); + } +} diff --git a/src/pointwise.rs b/src/pointwise.rs new file mode 100644 index 0000000..46254bc --- /dev/null +++ b/src/pointwise.rs @@ -0,0 +1,151 @@ +//! Ordered pointwise recipes shared by checked-loop matching and C generation. +//! +//! This module is deliberately independent of compiler types: build.rs reads the +//! same graph that runtime selection matches. Nodes are SSA values in evaluation +//! order, not algebraic expressions to reassociate. The finite catalog bounds +//! precompiled code growth; it is not interpreted during sample processing. + +pub const MAX_COEFFICIENTS: usize = 4; + +pub use crate::loop_recipe::{BinaryOp, Node, Recipe}; + +/// A catalog identity; construction only occurs through the shared catalog. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RecipeId(u8); + +impl RecipeId { + pub fn index(self) -> usize { + usize::from(self.0) + } + + pub fn recipe(self) -> &'static Recipe { + &CATALOG[self.index()] + } +} + +/// The affine map recipe `input * gain + bias`. +pub const AFFINE: RecipeId = RecipeId(0); + +use BinaryOp::{Add, Mul, Sub}; +use Node::{Binary, Parameter, Read}; + +const CATALOG: &[Recipe] = &[ + Recipe { + name: "affine_map", + // x*g+b + nodes: &[ + Read, + Parameter(0), + Binary(Mul, 0, 1), + Parameter(1), + Binary(Add, 2, 3), + ], + result: 4, + next_carries: &[], + }, + Recipe { + name: "offset_scale", + // (x-b)*g: subtraction precedes multiplication. + nodes: &[ + Read, + Parameter(0), + Binary(Sub, 0, 1), + Parameter(1), + Binary(Mul, 2, 3), + ], + result: 4, + next_carries: &[], + }, + Recipe { + name: "cubic", + // x-k*((x*x)*x), with one read reused by every occurrence of x. + nodes: &[ + Read, + Parameter(0), + Binary(Mul, 0, 0), + Binary(Mul, 2, 0), + Binary(Mul, 1, 3), + Binary(Sub, 0, 4), + ], + result: 5, + next_carries: &[], + }, + Recipe { + name: "horner", + // ((a*x+b)*x+c)*x+d, preserving each multiply/add rounding. + nodes: &[ + Read, + Parameter(0), + Binary(Mul, 1, 0), + Parameter(1), + Binary(Add, 2, 3), + Binary(Mul, 4, 0), + Parameter(2), + Binary(Add, 5, 6), + Binary(Mul, 7, 0), + Parameter(3), + Binary(Add, 8, 9), + ], + result: 10, + next_carries: &[], + }, + Recipe { + name: "smootherstep", + // ((x*x)*x) * (x*(x*a-b)+c); a=6, b=15, c=10 gives + // quintic smootherstep for normalized x, without clamping. + nodes: &[ + Read, + Binary(Mul, 0, 0), + Binary(Mul, 1, 0), + Parameter(0), + Binary(Mul, 0, 3), + Parameter(1), + Binary(Sub, 4, 5), + Binary(Mul, 0, 6), + Parameter(2), + Binary(Add, 7, 8), + Binary(Mul, 2, 9), + ], + result: 10, + next_carries: &[], + }, + Recipe { + name: "gain", + nodes: &[Read, Parameter(0), Binary(Mul, 0, 1)], + result: 2, + next_carries: &[], + }, + Recipe { + name: "offset", + nodes: &[Read, Parameter(0), Binary(Add, 0, 1)], + result: 2, + next_carries: &[], + }, + Recipe { + name: "square", + // Both operands reuse the same sampled input value. + nodes: &[Read, Binary(Mul, 0, 0)], + result: 1, + next_carries: &[], + }, +]; + +pub fn recipes() -> impl Iterator { + CATALOG + .iter() + .enumerate() + .map(|(index, recipe)| (RecipeId(index as u8), recipe)) +} + +/// Validate the shared semantics against the existing pointwise encoding. +pub fn validate() -> Result<(), String> { + crate::loop_recipe::validate_catalog(CATALOG, MAX_COEFFICIENTS, 0)?; + if AFFINE.recipe().name != "affine_map" + || CATALOG + .iter() + .any(|recipe| recipe.result != recipe.nodes.len() - 1) + { + return Err("invalid pointwise catalog identities or terminal result".into()); + } + Ok(()) +} diff --git a/src/recurrence.rs b/src/recurrence.rs new file mode 100644 index 0000000..c814d8f --- /dev/null +++ b/src/recurrence.rs @@ -0,0 +1,98 @@ +//! Ordered recurrence recipes consumed by checked-loop matching and native C +//! generation. The catalog initially describes the existing one-pole and direct +//! form I biquad kernels; operand encodings remain a separate backend concern. + +use crate::loop_recipe::{self, BinaryOp, Node, Recipe}; + +pub const MAX_COEFFICIENTS: usize = 5; +pub const MAX_CARRIES: usize = 4; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RecipeId(u8); + +impl RecipeId { + pub fn index(self) -> usize { + usize::from(self.0) + } + + pub fn recipe(self) -> &'static Recipe { + &CATALOG[self.index()] + } +} + +pub const ONE_POLE: RecipeId = RecipeId(0); +pub const BIQUAD: RecipeId = RecipeId(1); + +use BinaryOp::{Add, Mul, Sub}; +use Node::{Binary, Carry, Parameter, Read}; + +const CATALOG: &[Recipe] = &[ + Recipe { + name: "one_pole", + // x*feed + old*feedback; output and next state share the result. + nodes: &[ + Read, + Parameter(0), + Binary(Mul, 0, 1), + Carry(0), + Parameter(1), + Binary(Mul, 3, 4), + Binary(Add, 2, 5), + ], + result: 6, + next_carries: &[6], + }, + Recipe { + name: "biquad", + // b0*x + b1*x1 + b2*x2 - a1*y1 - a2*y2, separately rounded. + // Carry roles are x1, x2, y1, y2; history shifts use old snapshots. + nodes: &[ + Parameter(0), + Read, + Binary(Mul, 0, 1), + Parameter(1), + Carry(0), + Binary(Mul, 3, 4), + Binary(Add, 2, 5), + Parameter(2), + Carry(1), + Binary(Mul, 7, 8), + Binary(Add, 6, 9), + Parameter(3), + Carry(2), + Binary(Mul, 11, 12), + Binary(Sub, 10, 13), + Parameter(4), + Carry(3), + Binary(Mul, 15, 16), + Binary(Sub, 14, 17), + ], + result: 18, + next_carries: &[1, 4, 18, 12], + }, +]; + +pub fn recipes() -> impl Iterator { + CATALOG + .iter() + .enumerate() + .map(|(index, recipe)| (RecipeId(index as u8), recipe)) +} + +pub fn validate() -> Result<(), String> { + loop_recipe::validate_catalog(CATALOG, MAX_COEFFICIENTS, MAX_CARRIES)?; + // The retained native adapters require these identities and arities. They + // do not impose the arithmetic or the order of next-state updates. + for (id, name, coefficients, carries) in + [(ONE_POLE, "one_pole", 2, 1), (BIQUAD, "biquad", 5, 4)] + { + let recipe = id.recipe(); + if recipe.name != name + || recipe.coefficient_count() != coefficients + || recipe.carry_count() != carries + { + return Err(format!("{name}: incompatible native recurrence adapter")); + } + } + Ok(()) +} diff --git a/src/stack_codegen.rs b/src/stack_codegen.rs index 6c8c6e9..f172fc3 100644 --- a/src/stack_codegen.rs +++ b/src/stack_codegen.rs @@ -11,6 +11,14 @@ use crate::expr::Expr; use crate::stack_ir::*; use crate::types::*; use std::collections::{HashMap, HashSet}; +use std::convert::TryFrom; + +/// A checked private input or exact literal awaiting native operand storage. +#[derive(Clone, Copy)] +enum LoopScalar { + Local(u16), + Constant(u64), +} /// Loop context for break/continue support. struct LoopContext { @@ -47,6 +55,9 @@ enum LocalKind { /// Code generator for the stack-based VM. pub struct StackCodegen { + /// Opt in to the shared precompiled native-loop consumer. + pub(crate) native_loops: bool, + /// The program being built. program: StackProgram, @@ -81,6 +92,7 @@ impl Default for StackCodegen { impl StackCodegen { pub fn new() -> Self { Self { + native_loops: false, program: StackProgram::new(), func_indices: HashMap::new(), compiled_functions: HashSet::new(), @@ -207,14 +219,35 @@ impl StackCodegen { let mut func = StackFunction::new(&*decl.name); func.param_count = decl.params.len() as u8; + let pending_function_count = self.pending_functions.len(); + let lambda_counter = self.lambda_counter; + let mut translator = FunctionTranslator::new( decl, decls, &mut self.pending_functions, &mut self.lambda_counter, &self.globals, + self.native_loops && instance.is_some(), ); - translator.translate(&mut func); + if !translator.translate(&mut func)? { + // Native setup is optional. Discard the complete unpublished draft + // and retry ordinary lowering if its extra slots exceed the encoding. + drop(translator); + self.pending_functions.truncate(pending_function_count); + self.lambda_counter = lambda_counter; + func = StackFunction::new(&*decl.name); + func.param_count = decl.params.len() as u8; + translator = FunctionTranslator::new( + decl, + decls, + &mut self.pending_functions, + &mut self.lambda_counter, + &self.globals, + false, + ); + translator.translate(&mut func)?; + } let idx = self.program.add_function(func); if let Some(instance) = instance { @@ -305,10 +338,17 @@ struct FunctionTranslator<'a> { variables: HashMap, /// Next available scalar local slot. - next_scalar: u16, + next_scalar: u32, /// Next available memory slot (in 8-byte units). - next_memory_slot: u16, + next_memory_slot: u32, + + /// Highest allocated memory start, including zero-sized objects. + max_memory_slot: Option, + + /// Analysis belongs to precisely this immutable checked body, after hoisting. + loops: HashMap, + native_loop_count: usize, /// Functions that are called and need to be compiled. pending_functions: &'a mut Vec, @@ -365,6 +405,7 @@ impl<'a> FunctionTranslator<'a> { pending_functions: &'a mut Vec, lambda_counter: &'a mut usize, globals: &'a HashMap, + native_loops: bool, ) -> Self { Self { decl, @@ -373,6 +414,13 @@ impl<'a> FunctionTranslator<'a> { next_scalar: 0, next_memory_slot: 0, + max_memory_slot: None, + loops: if native_loops { + crate::value_loops::analyze_function(decl) + } else { + HashMap::new() + }, + native_loop_count: 0, pending_functions, lambda_counter, calls_to_patch: Vec::new(), @@ -394,17 +442,20 @@ impl<'a> FunctionTranslator<'a> { /// Allocate a scalar local slot. fn alloc_scalar(&mut self) -> u16 { let slot = self.next_scalar; - self.next_scalar += 1; - slot + self.next_scalar = self.next_scalar.saturating_add(1); + // Draft operands are narrow, but a complete wide frame check runs before + // any function is published. Overflowing drafts are discarded. + slot as u16 } /// Allocate a memory-backed local slot. Returns the memory slot index. /// size is in bytes; we round up to 8-byte units. fn alloc_memory(&mut self, size: u32) -> u16 { let slot = self.next_memory_slot; - let slots_needed = ((size + 7) / 8) as u16; - self.next_memory_slot += slots_needed; - slot + self.max_memory_slot = Some(self.max_memory_slot.map_or(slot, |last| last.max(slot))); + let slots_needed = size / 8 + u32::from(size % 8 != 0); + self.next_memory_slot = self.next_memory_slot.saturating_add(slots_needed); + slot as u16 } /// Get the type of an expression. @@ -461,7 +512,7 @@ impl<'a> FunctionTranslator<'a> { } /// Translate the function body. - fn translate(&mut self, func: &mut StackFunction) { + fn translate(&mut self, func: &mut StackFunction) -> Result { // If return type is a pointer type, first parameter is output pointer. let has_sret = returns_via_pointer(self.decl.ret); if has_sret { @@ -479,7 +530,7 @@ impl<'a> FunctionTranslator<'a> { let ty = self.decl.arena.local(param.local).ty; if let Type::Reference(_) = &*ty { - while self.next_scalar <= param_slot { + while self.next_scalar <= u32::from(param_slot) { self.alloc_scalar(); } self.variables @@ -487,7 +538,7 @@ impl<'a> FunctionTranslator<'a> { } else if !self.is_ptr_type(&ty) { // Scalar parameter: already in local slot param_slot by calling convention. // Just make sure our allocator accounts for it. - while self.next_scalar <= param_slot { + while self.next_scalar <= u32::from(param_slot) { self.alloc_scalar(); } self.variables @@ -495,7 +546,7 @@ impl<'a> FunctionTranslator<'a> { } else { // Pointer-represented parameters are passed as addresses. // Keep the address value directly, matching the JIT/LLVM ABI. - while self.next_scalar <= param_slot { + while self.next_scalar <= u32::from(param_slot) { self.alloc_scalar(); } self.variables @@ -602,9 +653,25 @@ impl<'a> FunctionTranslator<'a> { func.emit(StackOp::ReturnVoid); } - func.local_count = self.next_scalar; - func.local_memory = self.next_memory_slot as u32 * 8; + let memory_starts_fit = self + .max_memory_slot + .is_none_or(|slot| slot.saturating_add(self.next_scalar) <= u32::from(u16::MAX)); + if self.next_scalar > u32::from(u16::MAX) + || !memory_starts_fit + || self.next_memory_slot.checked_mul(8).is_none() + { + if self.native_loop_count != 0 { + return Ok(false); + } + return Err(format!( + "stack function {} exceeds frame slot limits ({} scalar slots, {} memory slots)", + self.decl.name, self.next_scalar, self.next_memory_slot, + )); + } + func.local_count = self.next_scalar as u16; + func.local_memory = self.next_memory_slot * 8; func.has_return_value = !matches!(&*self.decl.ret, Type::Void) && !has_sret; + Ok(true) } /// Translate an expression. Pushes exactly one value onto the stack @@ -1002,7 +1069,7 @@ impl<'a> FunctionTranslator<'a> { start, end, body, .. } => { let var = self.decl.arena.binder(expr); - self.translate_for(var, *start, *end, *body, func); + self.translate_for(expr, var, *start, *end, *body, func); if !self.void_ctx { func.emit(StackOp::I64Const(0)); } @@ -2514,6 +2581,7 @@ impl<'a> FunctionTranslator<'a> { /// Translate a for loop. fn translate_for( &mut self, + expr: ExprID, var: LocalId, start_id: ExprID, end_id: ExprID, @@ -2532,6 +2600,10 @@ impl<'a> FunctionTranslator<'a> { let end_local = self.alloc_scalar(); func.emit(StackOp::LocalSet(end_local)); + // Bounds are evaluated once. The native path and scalar continuation + // share the same counter/end slots and publication boundary. + let kernel = self.try_emit_loop(expr, loop_var, end_local, func); + // The checked loop binding has its own local identity. let int_ty = mk_type(Type::Int32); @@ -2594,6 +2666,12 @@ impl<'a> FunctionTranslator<'a> { // Patch jumps. func.patch_jump(jump_to_end); + if let Some(index) = kernel { + let done = func.pos() as i32 - index as i32 - 1; + if let StackOp::NativeLoop(kernel) = &mut func.ops[index] { + kernel.done = done; + } + } let ctx = self.loop_stack.pop().unwrap(); for bp in ctx.break_patches { func.patch_jump(bp); @@ -2608,6 +2686,184 @@ impl<'a> FunctionTranslator<'a> { // Caller handles result push if needed. } + /// Select a precompiled implementation from a loop's checked value graph. + /// Unsupported value/effect shapes keep the existing loop lowering. + fn try_emit_loop( + &mut self, + expr: ExprID, + counter: u16, + end: u16, + func: &mut StackFunction, + ) -> Option { + use crate::value_loops::Scalar; + let plan = self.loops.get(&expr)?.clone(); + // Contracts supply typed coefficient/state roles. Resolve every + // scalar and span before emitting any native setup, then lower the + // common stream domain independently of its arithmetic body. + let (input, output, pending) = if let Some(map) = plan.pointwise() { + // Only the recipe's active prefix is materialized. Padding names + // slot zero but is never read by the generated implementation. + let mut coefficients = std::array::from_fn(|_| LoopScalar::Local(0)); + for (slot, value) in coefficients.iter_mut().zip(map.coefficients) { + *slot = self.loop_scalar(&plan, value)?; + } + ( + map.input, + map.output, + NativeStreamBody::Pointwise { + recipe: map.recipe, + coefficients, + }, + ) + } else if let Some(pole) = plan.one_pole() { + ( + pole.input, + pole.output, + NativeStreamBody::OnePole { + state: self.loop_scalar_local(plan.carries[pole.carry].binding)?, + feed: self.loop_scalar(&plan, pole.feed)?, + feedback: self.loop_scalar(&plan, pole.feedback)?, + }, + ) + } else if let Some(biquad) = plan.biquad() { + let [x1, x2, y1, y2] = biquad + .carries + .map(|carry| self.loop_scalar_local(plan.carries[carry].binding)); + let [b0, b1, b2, a1, a2] = biquad + .coefficients + .map(|value| self.loop_scalar(&plan, value)); + ( + biquad.input, + biquad.output, + NativeStreamBody::Biquad { + states: [x1?, x2?, y1?, y2?], + coefficients: [b0?, b1?, b2?, a1?, a2?], + }, + ) + } else { + return None; + }; + let input_expr = plan.streams[input].expression; + let output_expr = plan.streams[output].expression; + let input_length = self.loop_span_length(input_expr)?; + let output_length = self.loop_span_length(output_expr)?; + let body = pending.map_coefficients(|value| self.materialize_loop_scalar(value, func)); + let (input_slot, input_len) = self.materialize_loop_span(input_expr, input_length, func); + let (output_slot, output_len) = if input == output { + (input_slot, input_len) + } else { + self.materialize_loop_span(output_expr, output_length, func) + }; + let op = StackOp::NativeLoop(NativeLoopKernel { + scalar: match plan.ty { + Scalar::F32 => NativeScalar::F32, + Scalar::F64 => NativeScalar::F64, + }, + counter, + end, + done: 0, + spans: NativeStreamSlots { + input: input_slot, + output: output_slot, + input_len, + output_len, + }, + body, + }); + let index = func.pos(); + func.emit(op); + self.native_loop_count += 1; + Some(index) + } + + /// Only an owned numeric scalar slot can be a private native port. Captured + /// storage and reference/pointer slots keep ordinary lowering. + fn loop_scalar_local(&self, binding: LocalId) -> Option { + if self.captured_vars.contains(&binding) + || self.lambda_referenced.contains(&binding) + || !matches!( + &*self.decl.arena.local(binding).ty, + Type::Float32 | Type::Float64 + ) + { + return None; + } + match self.variables.get(&binding) { + Some(LocalKind::Scalar(slot)) => Some(*slot), + _ => None, + } + } + + /// Resolve invariant scalar ports while the outer lexical environment is live. + fn loop_scalar( + &self, + plan: &crate::value_loops::LoopRegion, + value: crate::value_loops::ValueId, + ) -> Option { + use crate::value_loops::{Operation, Scalar}; + match &plan.values.get(value.0)?.operation { + Operation::Input(binding) => Some(LoopScalar::Local(self.loop_scalar_local(*binding)?)), + Operation::Constant(text) => Some(LoopScalar::Constant(match plan.ty { + Scalar::F32 => u64::from(text.parse::().unwrap_or(0.0).to_bits()), + Scalar::F64 => text.parse::().unwrap_or(0.0).to_bits(), + })), + _ => None, + } + } + + fn materialize_loop_scalar(&mut self, value: LoopScalar, func: &mut StackFunction) -> u16 { + match value { + LoopScalar::Local(slot) => slot, + LoopScalar::Constant(bits) => { + let slot = self.alloc_scalar(); + func.emit(StackOp::FusedConstSet(bits as i64, slot)); + slot + } + } + } + + /// Solved array-to-slice coercions do not change the original storage ABI. + fn loop_span_length(&self, expression: ExprID) -> Option { + use crate::value_loops::SpanLength; + match &*self.representation_type(expression) { + Type::Array(_, ArraySize::Known(length)) => { + Some(SpanLength::Fixed(u32::try_from(*length).ok()?)) + } + Type::Slice(_) => Some(SpanLength::Slice), + _ => None, + } + } + + fn materialize_loop_span( + &mut self, + expression: ExprID, + length: crate::value_loops::SpanLength, + func: &mut StackFunction, + ) -> (u16, u16) { + use crate::value_loops::SpanLength; + self.translate_expr(expression, func); + let pointer = self.alloc_scalar(); + let length_slot = self.alloc_scalar(); + func.emit(StackOp::LocalSet(pointer)); + match length { + SpanLength::Fixed(length) => { + func.emit(StackOp::FusedConstSet(i64::from(length), length_slot)); + } + SpanLength::Slice => { + // First retain the header's length, then replace its address with + // the raw element pointer. Both are refreshed by ordinary lowering + // if an actual callback makes the kernel resume bytecode. + func.emit(StackOp::LocalGet(pointer)); + func.emit(StackOp::Load32Off(8)); + func.emit(StackOp::LocalSet(length_slot)); + func.emit(StackOp::LocalGet(pointer)); + func.emit(StackOp::Load64); + func.emit(StackOp::LocalSet(pointer)); + } + } + (pointer, length_slot) + } + /// Translate a field access. fn translate_field(&mut self, lhs_id: ExprID, name: Name, func: &mut StackFunction) { let lhs_ty = self.expr_type(lhs_id); diff --git a/src/stack_depth.rs b/src/stack_depth.rs index 1a345f5..a9144eb 100644 --- a/src/stack_depth.rs +++ b/src/stack_depth.rs @@ -30,6 +30,7 @@ pub fn compute_depths(func: &StackFunction) -> Vec { StackOp::FusedGetF32ConstFGtJumpIfZeroF(_, _, off) => Some(*off), StackOp::FusedF64ConstDGtJumpIfZeroD(_, off) => Some(*off), StackOp::FusedGetF64ConstDGtJumpIfZeroD(_, _, off) => Some(*off), + StackOp::NativeLoop(kernel) => Some(kernel.done), _ => None, }; if let Some(off) = off { @@ -205,6 +206,7 @@ pub fn stack_delta(op: &StackOp) -> i32 { | StackOp::FusedConstSet(_, _) | StackOp::FusedF32ConstSet(_, _) | StackOp::FusedGetAddImmSet(_, _, _) + | StackOp::NativeLoop(_) | StackOp::FusedGetGetILtJumpIfZero(_, _, _) | StackOp::FusedBoundsCheck1JumpIfZero(_, _) | StackOp::FusedBoundsCheck2JumpIfZero(_, _) diff --git a/src/stack_inline.rs b/src/stack_inline.rs index 1b32d00..f928178 100644 --- a/src/stack_inline.rs +++ b/src/stack_inline.rs @@ -160,6 +160,7 @@ fn inline_calls_in(func: &mut StackFunction, bodies: &[Option>]) { StackOp::FusedGetF64ConstDGtJumpIfZeroD(_, _, off) => { Some((i as i64 + 1 + *off as i64) as usize) } + StackOp::NativeLoop(kernel) => Some((i as i64 + 1 + kernel.done as i64) as usize), _ => None, }; @@ -191,6 +192,9 @@ fn inline_calls_in(func: &mut StackFunction, bodies: &[Option>]) { StackOp::FusedGetF32ConstFGtJumpIfZeroF(_, _, o) => *o = new_off as i32, StackOp::FusedF64ConstDGtJumpIfZeroD(_, o) => *o = new_off as i32, StackOp::FusedGetF64ConstDGtJumpIfZeroD(_, _, o) => *o = new_off as i32, + StackOp::NativeLoop(kernel) => { + kernel.done = new_off as i32; + } _ => unreachable!(), } } diff --git a/src/stack_interp.c b/src/stack_interp.c index dfa09db..90f89fe 100644 --- a/src/stack_interp.c +++ b/src/stack_interp.c @@ -206,18 +206,20 @@ static int64_t ipow(int64_t base, uint32_t exp) { // Decrement the cancel counter; on expiry invoke the callback and, if it // returns true, mark cancelled and break the tail-call chain. Use this // after applying any backward `pc = pc + 1 + off` (off < 0). -#define POLL_CANCEL() \ - do { \ - if (--ctx->cancel_counter <= 0) { \ - ctx->cancel_counter = CANCEL_CHECK_INTERVAL; \ - if (ctx->cancel_callback && ctx->cancel_callback(ctx->cancel_userdata)) { \ - ctx->cancelled = true; \ - ctx->trap_reason = STACK_TRAP_CANCELLED; \ - ctx->done = 1; \ - return; \ - } \ - } \ - } while(0) +static inline bool poll_cancel(Ctx* ctx) { + if (--ctx->cancel_counter <= 0) { + ctx->cancel_counter = CANCEL_CHECK_INTERVAL; + if (ctx->cancel_callback && ctx->cancel_callback(ctx->cancel_userdata)) { + ctx->cancelled = true; + ctx->trap_reason = STACK_TRAP_CANCELLED; + ctx->done = 1; + return true; + } + } + return false; +} + +#define POLL_CANCEL() do { if (poll_cancel(ctx)) return; } while (0) // Float TOS window push/pop. Spills/refills through `fsp`, a handler // argument pinned to a GPR by preserve_none — analogous to the integer @@ -1315,6 +1317,229 @@ HANDLER(op_fused_get_addimm_set) { NEXT(); } +// Whole-loop gateways retain the ordinary loop immediately after this op. +// Decode the compact and descriptor encodings into the same execution contract: +// two equally indexed spans, private coefficients/state, and a success edge. +typedef enum { +#define RECURRENCE_RECIPE(id, kind, name, coefficients, carried) NATIVE_##kind, +#include "recurrence_catalog.inc" +#undef RECURRENCE_RECIPE +#define POINTWISE_RECIPE(id, kind, name, coefficients) NATIVE_POINTWISE_##kind, +#include "pointwise_catalog.inc" +#undef POINTWISE_RECIPE +} NativeKernelKind; + +typedef struct { + int coefficients, carried; +} NativeKernelShape; + +static inline NativeKernelShape native_kernel_shape(NativeKernelKind kind) { + switch (kind) { +#define RECURRENCE_RECIPE(id, kind, name, coefficients, carried) \ + case NATIVE_##kind: return (NativeKernelShape) { coefficients, carried }; +#include "recurrence_catalog.inc" +#undef RECURRENCE_RECIPE +#define POINTWISE_RECIPE(id, kind, name, coefficients) \ + case NATIVE_POINTWISE_##kind: return (NativeKernelShape) { coefficients, 0 }; +#include "pointwise_catalog.inc" +#undef POINTWISE_RECIPE + } + __builtin_unreachable(); +} + +typedef struct { + uint16_t counter, end, left, right; + int64_t left_length, right_length, done; + uint16_t coefficients[5]; + uint16_t states[4]; +} NativeLoopOperands; + +static inline NativeLoopOperands native_stream_operands( + const Instruction* pc, const uint64_t* locals +) { + uint64_t slots = pc->imm[0], parameters = pc->imm[1]; + return (NativeLoopOperands) { + .counter = (uint16_t)slots, .end = (uint16_t)(slots >> 16), + .left = (uint16_t)(slots >> 32), .right = (uint16_t)(slots >> 48), + .left_length = (int64_t)locals[(uint16_t)(parameters >> 32)], + .right_length = (int64_t)locals[(uint16_t)(parameters >> 48)], + .coefficients = { (uint16_t)parameters, (uint16_t)(parameters >> 16) }, + .states = { (uint16_t)(pc->imm[2] >> 32) }, .done = (int32_t)pc->imm[2], + }; +} + +// Pointwise recipes use the stream encoding's spare high 32 bits of imm2 +// for coefficients two and three. Shape metadata keeps unused slots unread. +static inline NativeLoopOperands native_pointwise_operands( + const Instruction* pc, const uint64_t* locals +) { + NativeLoopOperands operands = native_stream_operands(pc, locals); + operands.coefficients[2] = (uint16_t)(pc->imm[2] >> 32); + operands.coefficients[3] = (uint16_t)(pc->imm[2] >> 48); + return operands; +} + +// Biquad's immutable uint16_t descriptor is owned by StackBackend: +// counter/end/input/output/input_len/output_len, b0/b1/b2/a1/a2, x1/x2/y1/y2. +static inline NativeLoopOperands native_biquad_operands( + const Instruction* pc, const uint64_t* locals +) { + const uint16_t* slots = (const uint16_t*)(uintptr_t)pc->imm[0]; + return (NativeLoopOperands) { + .counter = slots[0], .end = slots[1], .left = slots[2], .right = slots[3], + .left_length = (int64_t)locals[slots[4]], + .right_length = (int64_t)locals[slots[5]], + .coefficients = { slots[6], slots[7], slots[8], slots[9], slots[10] }, + .states = { slots[11], slots[12], slots[13], slots[14] }, + .done = (int32_t)pc->imm[2], + }; +} + +typedef struct { + int64_t index, end; + const uint8_t* left; + uint8_t* right; +} NativeLoop; + +typedef enum { NATIVE_CONTINUE, NATIVE_FALLBACK, NATIVE_CANCELLED } NativeLoopResult; + +// Empty/reversed ranges succeed without accessing either span. For writes, +// exact in-place and disjoint ranges are valid; partial overlap retains the +// original loop's ordered forward semantics by falling back before any work. +static inline bool native_loop_prepare( + NativeLoop* loop, const NativeLoopOperands* operands, const uint64_t* locals, + size_t width +) { + loop->index = (int64_t)locals[operands->counter]; + loop->end = (int64_t)locals[operands->end]; + if (loop->index >= loop->end) return true; + if (loop->index < 0 || loop->end > operands->left_length + || loop->end > operands->right_length + || (uint64_t)loop->end > SIZE_MAX / width) return false; + uintptr_t left = (uintptr_t)locals[operands->left]; + uintptr_t right = (uintptr_t)locals[operands->right]; + uintptr_t distance = left > right ? left - right : right - left; + uint64_t count_bytes = (uint64_t)(loop->end - loop->index) * width; + if (distance != 0 && distance < count_bytes) return false; + loop->left = (const uint8_t*)left; + loop->right = (uint8_t*)right; + return true; +} + +static inline int64_t native_loop_count(const NativeLoop* loop, const Ctx* ctx) { + int64_t count = loop->end - loop->index; + int32_t budget = ctx->cancel_counter > 0 ? ctx->cancel_counter : 1; + return count > budget ? budget : count; +} + +// Called after publishing every carried state and output in this chunk. Count +// the final iteration's backedge too. An actual callback ends native execution +// even when it does not cancel: the ordinary guard must reload anything that +// callback may have changed, including coefficients, pointers, bounds and state. +static inline NativeLoopResult native_loop_commit( + NativeLoop* loop, const NativeLoopOperands* operands, + Ctx* ctx, uint64_t* locals, int64_t count +) { + loop->index += count; + locals[operands->counter] = (uint64_t)loop->index; + ctx->cancel_counter -= (int32_t)count; + if (ctx->cancel_counter <= 0) { + bool called = ctx->cancel_callback != NULL; + ctx->cancel_counter = 1; + if (poll_cancel(ctx)) return NATIVE_CANCELLED; + if (called) return NATIVE_FALLBACK; + } + return NATIVE_CONTINUE; +} + +// This macro only instantiates typed arithmetic for f32 and f64. Kernel kind is +// a constant at each handler; forced inlining removes selection and unused state +// before native code generation. There is no per-sample operation dispatch. +// __builtin_memcpy preserves checked unaligned stores through vectorization; +// Darwin's fortified memcpy wrapper can otherwise survive until after that pass. +#include "pointwise_chunks.inc" +#include "recurrence_chunks.inc" + +#define DEFINE_NATIVE_KERNEL_TYPE(suffix, scalar, load) \ +typedef struct { scalar coefficients[5], carried[4]; } NativeState_##suffix; \ +static inline __attribute__((always_inline)) NativeState_##suffix native_state_##suffix( \ + NativeKernelKind kind, const NativeLoopOperands* operands, const uint64_t* locals \ +) { \ + NativeState_##suffix state = {0}; \ + NativeKernelShape shape = native_kernel_shape(kind); \ + for (int i = 0; i < shape.coefficients; ++i) state.coefficients[i] = load(locals + operands->coefficients[i]); \ + for (int i = 0; i < shape.carried; ++i) state.carried[i] = load(locals + operands->states[i]); \ + return state; \ +} \ +static inline __attribute__((always_inline)) void native_chunk_##suffix( \ + NativeKernelKind kind, const NativeLoop* loop, int64_t count, NativeState_##suffix* state \ +) { \ + const uint8_t* input = loop->left + (size_t)loop->index * sizeof(scalar); \ + uint8_t* output = loop->right + (size_t)loop->index * sizeof(scalar); \ + if (native_kernel_shape(kind).carried != 0) { \ + native_recurrence_##suffix(kind, input, output, count, state->coefficients, state->carried); \ + } else { \ + native_pointwise_##suffix(kind, input, output, count, state->coefficients); \ + } \ +} \ +static inline __attribute__((always_inline)) void native_publish_##suffix( \ + NativeKernelKind kind, const NativeLoopOperands* operands, uint64_t* locals, const NativeState_##suffix* state \ +) { \ + NativeKernelShape shape = native_kernel_shape(kind); \ + for (int i = 0; i < shape.carried; ++i) \ + __builtin_memcpy(locals + operands->states[i], &state->carried[i], sizeof(scalar)); \ +} + +DEFINE_NATIVE_KERNEL_TYPE(f32, float, load_f32_unaligned) +DEFINE_NATIVE_KERNEL_TYPE(f64, double, load_f64_unaligned) +#undef DEFINE_NATIVE_KERNEL_TYPE + +// One execution boundary for every precompiled kernel. Arithmetic and state +// publication are statically specialized above; all three operand windows pass +// through unchanged on success, guard rejection, cancellation and resumption. +#define NATIVE_LOOP_HANDLER(name, suffix, scalar, kind, decode) \ +HANDLER(name) { \ + NativeLoopOperands operands = decode(pc, locals); \ + NativeLoop loop; \ + if (!native_loop_prepare(&loop, &operands, locals, sizeof(scalar))) { NEXT(); } \ + if (loop.index < loop.end) { \ + NativeState_##suffix state = native_state_##suffix(kind, &operands, locals); \ + while (loop.index < loop.end) { \ + int64_t count = native_loop_count(&loop, ctx); \ + native_chunk_##suffix(kind, &loop, count, &state); \ + native_publish_##suffix(kind, &operands, locals, &state); \ + NativeLoopResult result = native_loop_commit(&loop, &operands, ctx, locals, count); \ + if (result == NATIVE_CANCELLED) return; \ + if (result == NATIVE_FALLBACK) { NEXT(); } \ + } \ + } \ + pc = pc + 1 + operands.done; \ + DISPATCH(); \ +} + +#define POINTWISE_RECIPE(id, kind, name, coefficients) \ +NATIVE_LOOP_HANDLER(op_##name##_f32, f32, float, NATIVE_POINTWISE_##kind, native_pointwise_operands) \ +NATIVE_LOOP_HANDLER(op_##name##_f64, f64, double, NATIVE_POINTWISE_##kind, native_pointwise_operands) +#include "pointwise_catalog.inc" +#undef POINTWISE_RECIPE +NATIVE_LOOP_HANDLER(op_one_pole_f32, f32, float, NATIVE_ONE_POLE, native_stream_operands) +NATIVE_LOOP_HANDLER(op_one_pole_f64, f64, double, NATIVE_ONE_POLE, native_stream_operands) +NATIVE_LOOP_HANDLER(op_biquad_f32, f32, float, NATIVE_BIQUAD, native_biquad_operands) +NATIVE_LOOP_HANDLER(op_biquad_f64, f64, double, NATIVE_BIQUAD, native_biquad_operands) +#undef NATIVE_LOOP_HANDLER + +// Backend construction resolves the recipe once. Executed instructions contain +// the specialized handler pointer; there is no recipe lookup in a sample loop. +void* stack_pointwise_handler(uint32_t recipe, bool double_precision) { + switch (recipe) { +#define POINTWISE_RECIPE(id, kind, name, coefficients) \ + case id: return double_precision ? (void*)op_##name##_f64 : (void*)op_##name##_f32; +#include "pointwise_catalog.inc" +#undef POINTWISE_RECIPE + default: return NULL; + } +} + // if !(locals[a] < locals[b]) jump -- no stack change HANDLER(op_fused_get_get_ilt_jiz) { if ((int64_t)locals[pc->imm[0]] >= (int64_t)locals[pc->imm[1]]) { @@ -2114,11 +2339,17 @@ HANDLER(op_fused_get_get_fmul_fsub_f) { f0 = f0 - a * b; NEXT(); } +// The source chain starts with its first product, not +0 plus that product: +// the latter would erase a negative zero. Codegen never sets mask bit 0; +// if present, it denotes unary negation of the initial rounded product. #define FMUL_SUM_HANDLER(name, TERMS) \ HANDLER(name) { \ uint8_t sub_mask = (uint8_t)pc->imm[2]; \ - float acc = 0.0f; \ - for (int i = 0; i < (TERMS); i++) { \ + float a0 = *(float*)((uint8_t*)locals + (size_t)imm_u8(pc, 0) * 8); \ + float b0 = *(float*)((uint8_t*)locals + (size_t)imm_u8(pc, 1) * 8); \ + float acc = a0 * b0; \ + if (sub_mask & 1u) acc = -acc; \ + for (int i = 1; i < (TERMS); i++) { \ uint8_t a_idx = imm_u8(pc, i * 2); \ uint8_t b_idx = imm_u8(pc, i * 2 + 1); \ float a = *(float*)((uint8_t*)locals + (size_t)a_idx * 8); \ @@ -2252,11 +2483,15 @@ HANDLER(op_fused_get_get_dmul_dsub_d) { NEXT(); } +// Same first-product and optional unary-negation semantics as the F handler. #define DMUL_SUM_HANDLER(name, TERMS) \ HANDLER(name) { \ uint8_t sub_mask = (uint8_t)pc->imm[2]; \ - double acc = 0.0; \ - for (int i = 0; i < (TERMS); i++) { \ + double a0 = *(double*)((uint8_t*)locals + (size_t)imm_u8(pc, 0) * 8); \ + double b0 = *(double*)((uint8_t*)locals + (size_t)imm_u8(pc, 1) * 8); \ + double acc = a0 * b0; \ + if (sub_mask & 1u) acc = -acc; \ + for (int i = 1; i < (TERMS); i++) { \ uint8_t a_idx = imm_u8(pc, i * 2); \ uint8_t b_idx = imm_u8(pc, i * 2 + 1); \ double a = *(double*)((uint8_t*)locals + (size_t)a_idx * 8); \ diff --git a/src/stack_interp_bridge.rs b/src/stack_interp_bridge.rs index e94acd2..47e30b7 100644 --- a/src/stack_interp_bridge.rs +++ b/src/stack_interp_bridge.rs @@ -3,7 +3,7 @@ //! Converts StackOp instructions into the C Instruction format //! (handler pointer + 3 immediates) and calls stack_interp_run. -use crate::stack_ir::{StackOp, StackProgram}; +use crate::stack_ir::{NativeLoopKernel, NativeScalar, NativeStreamBody, StackOp, StackProgram}; // C types matching stack_interp.h #[repr(C)] @@ -185,6 +185,11 @@ extern "C" { fn op_fused_get_get_ilt(); fn op_fused_addr_load32off(); fn op_fused_get_addimm_set(); + fn stack_pointwise_handler(recipe: u32, double: bool) -> *const (); + fn op_one_pole_f32(); + fn op_one_pole_f64(); + fn op_biquad_f32(); + fn op_biquad_f64(); fn op_fused_get_get_ilt_jiz(); fn op_fused_bounds_check1_jiz(); fn op_fused_bounds_check2_jiz(); @@ -514,6 +519,24 @@ fn handler_for(op: &StackOp) -> *const () { StackOp::FusedGetGetILt(_, _) => op_fused_get_get_ilt as *const (), StackOp::FusedAddrLoad32Off(_, _) => op_fused_addr_load32off as *const (), StackOp::FusedGetAddImmSet(_, _, _) => op_fused_get_addimm_set as *const (), + StackOp::NativeLoop(kernel) => match (&kernel.body, kernel.scalar) { + (NativeStreamBody::Pointwise { recipe, .. }, scalar) => { + // Resolve once while encoding. Execution dispatches directly + // to the generated, statically specialized native handler. + let handler = unsafe { + stack_pointwise_handler(recipe.index() as u32, scalar == NativeScalar::F64) + }; + assert!( + !handler.is_null(), + "pointwise handler missing from generated catalog" + ); + handler + } + (NativeStreamBody::OnePole { .. }, NativeScalar::F32) => op_one_pole_f32 as *const (), + (NativeStreamBody::OnePole { .. }, NativeScalar::F64) => op_one_pole_f64 as *const (), + (NativeStreamBody::Biquad { .. }, NativeScalar::F32) => op_biquad_f32 as *const (), + (NativeStreamBody::Biquad { .. }, NativeScalar::F64) => op_biquad_f64 as *const (), + }, StackOp::FusedGetGetILtJumpIfZero(_, _, _) => op_fused_get_get_ilt_jiz as *const (), StackOp::FusedBoundsCheck1JumpIfZero(_, _) => op_fused_bounds_check1_jiz as *const (), StackOp::FusedBoundsCheck2JumpIfZero(_, _) => op_fused_bounds_check2_jiz as *const (), @@ -805,6 +828,7 @@ fn encode_imm(op: &StackOp, func_idx: u32) -> [u64; 3] { } StackOp::FusedTeeSliceStore32(n, s, idx) => [*n as u64, *s as u64, *idx as u64], StackOp::FusedGetAddImmSet(s, v, d) => [*s as u64, *v as i64 as u64, *d as u64], + StackOp::NativeLoop(kernel) => encode_native_loop(kernel), StackOp::FusedGetGetILtJumpIfZero(a, b, off) => [*a as u64, *b as u64, *off as i64 as u64], StackOp::FusedBoundsCheck1JumpIfZero(p, off) => { let mut out = pack_u8_imms(p); @@ -996,13 +1020,17 @@ pub fn run(program: &StackProgram) -> i64 { /// repeatedly without re-allocating. Globals are passed per-call so a /// single backend can drive multiple independent globals buffers. /// -/// Pin-in-place semantics: once built, the struct must not move — `ctx` -/// holds raw pointers into the backend's own vectors. The FFI layer boxes -/// it (via `Box`) which keeps the address stable. +/// The encoded instructions and `ctx` hold raw pointers into owned backing +/// allocations. Those allocations must not resize after construction; moving +/// the backend itself is safe because it does not move the allocations. pub struct StackBackend { // Owned program data. The inner Vec never resizes after // construction, so the raw pointers stored in `func_metas` stay valid. _c_instructions: Vec>, + // Large native kernels name scalar slots through immutable descriptors. + // Freeze the arena before encoding pointers; neither calls nor moving the + // backend can invalidate its allocation. Small gateways stay inline. + _native_kernel_operands: Box<[u16]>, func_metas: Vec, call_stack: Vec, operand_stack: Vec, @@ -1012,15 +1040,96 @@ pub struct StackBackend { ctx: Ctx, } +#[cfg(test)] +mod native_loop_tests; + +fn encode_native_loop(kernel: &NativeLoopKernel) -> [u64; 3] { + let spans = &kernel.spans; + let body = &kernel.body; + let domain = u64::from(kernel.counter) + | (u64::from(kernel.end) << 16) + | (u64::from(spans.input) << 32) + | (u64::from(spans.output) << 48); + let lengths = (u64::from(spans.input_len) << 32) | (u64::from(spans.output_len) << 48); + match body { + NativeStreamBody::Pointwise { coefficients, .. } => [ + domain, + u64::from(coefficients[0]) | (u64::from(coefficients[1]) << 16) | lengths, + u64::from(kernel.done as u32) + | (u64::from(coefficients[2]) << 32) + | (u64::from(coefficients[3]) << 48), + ], + NativeStreamBody::OnePole { + feed, + feedback, + state, + } => [ + domain, + u64::from(*feed) | (u64::from(*feedback) << 16) | lengths, + u64::from(kernel.done as u32) | (u64::from(*state) << 32), + ], + // imm0 is filled from the frozen operand arena during backend construction. + NativeStreamBody::Biquad { .. } => [0, 0, kernel.done as i64 as u64], + } +} + +// Slot order is the native Biquad descriptor ABI; C reads six domain slots, +// five coefficients (b0, b1, b2, a1, a2), then four states (x1, x2, y1, y2). +fn native_kernel_operands(op: &StackOp) -> Option<[u16; 15]> { + match op { + StackOp::NativeLoop(NativeLoopKernel { + counter, + end, + spans, + body: + NativeStreamBody::Biquad { + coefficients, + states, + }, + .. + }) => Some([ + *counter, + *end, + spans.input, + spans.output, + spans.input_len, + spans.output_len, + coefficients[0], + coefficients[1], + coefficients[2], + coefficients[3], + coefficients[4], + states[0], + states[1], + states[2], + states[3], + ]), + _ => None, + } +} + impl StackBackend { pub fn new(program: &StackProgram) -> Self { + let native_kernel_operands: Box<[u16]> = program + .functions + .iter() + .flat_map(|function| &function.ops) + .filter_map(native_kernel_operands) + .flatten() + .collect(); + let mut operand_offset = 0; let mut c_instructions: Vec> = Vec::with_capacity(program.functions.len()); for (fi, func) in program.functions.iter().enumerate() { let mut instrs: Vec = Vec::with_capacity(func.ops.len() + 1); for op in func.ops.iter() { + let mut imm = encode_imm(op, fi as u32); + if let Some(operands) = self::native_kernel_operands(op) { + imm[0] = native_kernel_operands[operand_offset..].as_ptr() as u64; + operand_offset += operands.len(); + } instrs.push(Instruction { handler: handler_for(op), - imm: encode_imm(op, fi as u32), + imm, }); } instrs.push(Instruction { @@ -1084,6 +1193,7 @@ impl StackBackend { Self { _c_instructions: c_instructions, + _native_kernel_operands: native_kernel_operands, func_metas, call_stack, operand_stack, @@ -1192,10 +1302,7 @@ mod tests { let mut globals: Vec = vec![0u8; program.globals_size.max(1)]; backend.call_entry(program.entry, globals.as_mut_ptr()); - assert!( - !backend.cancelled(), - "assertion trap is not a cancellation" - ); + assert!(!backend.cancelled(), "assertion trap is not a cancellation"); assert_eq!( backend.trap_reason(), crate::cancel::TRAP_ASSERTION_FAILED, diff --git a/src/stack_interp_bridge/native_loop_tests.rs b/src/stack_interp_bridge/native_loop_tests.rs new file mode 100644 index 0000000..53219e9 --- /dev/null +++ b/src/stack_interp_bridge/native_loop_tests.rs @@ -0,0 +1,714 @@ +use super::*; +use crate::stack_ir::{NativeStreamSlots, StackFunction}; + +const LENGTH: usize = 240; +const STORAGE: usize = LENGTH + 4; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Edge { + Mixed, + SeparateRounding, + NegativeZero, +} + +#[derive(Clone, Copy, Debug)] +struct Case { + double: bool, + recurrence: bool, + biquad: bool, + edge: Edge, + // None is disjoint, Some(0) is in-place; +/-1 overlap by one sample. + overlap: Option, + start: i64, + end: i64, + input_len: i64, + output_len: i64, + prefix: i64, + callback: bool, + cancel: bool, + mutate: bool, +} + +fn bits(double: bool, value: f64) -> u64 { + if double { + value.to_bits() + } else { + (value as f32).to_bits() as u64 + } +} + +fn program(case: Case, native: bool, input: *const u8, output: *mut u8) -> StackProgram { + let mut function = StackFunction::new(if case.biquad { + "biquad_kernel" + } else { + "stream_kernel" + }); + function.local_count = if case.biquad { 20 } else { 12 }; + function.has_return_value = true; + let epsilon = if case.double { + f64::EPSILON + } else { + f32::EPSILON as f64 + }; + let (coefficients, states) = if case.biquad { + match case.edge { + Edge::Mixed => ( + [0.125, 0.25, 0.125, -0.75, 0.25], + [-0.25, 0.125, 0.0625, -0.125], + ), + Edge::SeparateRounding => ( + [1.0 + epsilon, 1.0, 0.0, 0.0, 0.0], + [-1.0 - 2.0 * epsilon, 0.0, 0.0, 0.0], + ), + Edge::NegativeZero => ([1.0, 1.0, 1.0, -1.0, -1.0], [-0.0; 4]), + } + } else { + let (first, second) = match (case.edge, case.recurrence) { + (Edge::NegativeZero, true) => (1.0, 1.0), + (Edge::NegativeZero, false) => (1.0, -0.0), + (Edge::SeparateRounding, true) => (1.0 + epsilon, 1.0), + (Edge::Mixed, true) => (0.5, 0.25), + (_, false) => (1.0 + epsilon, -1.0 - 2.0 * epsilon), + }; + let initial_state = if case.edge == Edge::SeparateRounding { + -1.0 - 2.0 * epsilon + } else { + -0.0 + }; + ( + [first, second, 0.0, 0.0, 0.0], + [initial_state, 0.0, 0.0, 0.0], + ) + }; + let locals = [ + case.start as u64, + case.end as u64, + input as u64, + output as u64, + bits(case.double, coefficients[0]), + bits(case.double, coefficients[1]), + bits(case.double, states[0]), + case.input_len as u64, + case.output_len as u64, + 0, + 0, + case.prefix as u64, + bits(case.double, coefficients[2]), + bits(case.double, coefficients[3]), + bits(case.double, coefficients[4]), + bits(case.double, states[1]), + bits(case.double, states[2]), + bits(case.double, states[3]), + 0, + 0, + ]; + for (slot, &value) in locals[..usize::from(function.local_count)] + .iter() + .enumerate() + { + function.emit(StackOp::I64Const(value as i64)); + function.emit(StackOp::LocalSet(slot as u16)); + } + function.emit(StackOp::FusedGetGetILtJumpIfZero(10, 11, 2)); + function.emit(StackOp::FusedGetAddImmSet(10, 1, 10)); + function.emit(StackOp::Jump(-3)); + function.emit(StackOp::I64Const(17)); + function.emit(StackOp::F32ConstF(19.0)); + function.emit(StackOp::F64ConstD(23.0)); + let gateway = function.pos(); + if native { + function.emit(StackOp::NativeLoop(NativeLoopKernel { + scalar: if case.double { + NativeScalar::F64 + } else { + NativeScalar::F32 + }, + counter: 0, + end: 1, + done: 0, + spans: NativeStreamSlots { + input: 2, + output: 3, + input_len: 7, + output_len: 8, + }, + body: if case.biquad { + NativeStreamBody::Biquad { + coefficients: [4, 5, 12, 13, 14], + states: [6, 15, 16, 17], + } + } else if case.recurrence { + NativeStreamBody::OnePole { + feed: 4, + feedback: 5, + state: 6, + } + } else { + NativeStreamBody::Pointwise { + recipe: crate::pointwise::AFFINE, + coefficients: [4, 5, 0, 0], + } + }, + })); + } + let guard = function.pos(); + function.emit(StackOp::FusedGetGetILtJumpIfZero(0, 1, 0)); + if case.biquad { + emit_scalar_biquad(&mut function, case.double); + } else { + emit_scalar_stream(&mut function, case.double, case.recurrence); + } + function.emit(StackOp::FusedGetAddImmSet(0, 1, 0)); + function.emit(StackOp::Jump(guard as i32 - function.pos() as i32 - 1)); + let done = function.pos(); + function.ops[guard] = StackOp::FusedGetGetILtJumpIfZero(0, 1, done as i32 - guard as i32 - 1); + if native { + match &mut function.ops[gateway] { + StackOp::NativeLoop(kernel) => kernel.done = done as i32 - gateway as i32 - 1, + _ => unreachable!(), + } + } + for op in [ + StackOp::I64Const(17), + StackOp::IEq, + StackOp::JumpIfNotZero(2), + StackOp::I64Const(0), + StackOp::Assert, + StackOp::F32ConstF(19.0), + StackOp::FEqF, + StackOp::JumpIfNotZero(2), + StackOp::I64Const(0), + StackOp::Assert, + StackOp::F64ConstD(23.0), + StackOp::DEqD, + StackOp::JumpIfNotZero(2), + StackOp::I64Const(0), + StackOp::Assert, + StackOp::FusedConstSet(1, 9), + StackOp::LocalGet(if case.biquad { 16 } else { 6 }), + StackOp::Return, + ] { + function.emit(op); + } + let mut program = StackProgram::new(); + program.entry = program.add_function(function); + program +} + +// Keep these scalar witnesses independent of native recipe matching/generation. +fn emit_scalar_biquad(function: &mut StackFunction, double: bool) { + let get = |slot| { + if double { + StackOp::LocalGetD(slot) + } else { + StackOp::LocalGetF(slot) + } + }; + let set = |slot| { + if double { + StackOp::LocalSetD(slot) + } else { + StackOp::LocalSetF(slot) + } + }; + function.emit(StackOp::LocalGet(2)); + function.emit(StackOp::LocalGet(0)); + function.emit(StackOp::I64Const(if double { 8 } else { 4 })); + function.emit(StackOp::IMul); + function.emit(StackOp::IAdd); + function.emit(if double { + StackOp::LoadF64D + } else { + StackOp::LoadF32F + }); + function.emit(set(18)); + for (term, (coefficient, state)) in [(4, 18), (5, 6), (12, 15), (13, 16), (14, 17)] + .iter() + .copied() + .enumerate() + { + function.emit(get(coefficient)); + function.emit(get(state)); + function.emit(if double { + StackOp::DMulD + } else { + StackOp::FMulF + }); + if term != 0 { + function.emit(match (double, term >= 3) { + (true, true) => StackOp::DSubD, + (true, false) => StackOp::DAddD, + (false, true) => StackOp::FSubF, + (false, false) => StackOp::FAddF, + }); + } + } + function.emit(set(19)); + for (source, destination) in [(6, 15), (18, 6), (16, 17), (19, 16)] { + function.emit(get(source)); + function.emit(set(destination)); + } + function.emit(StackOp::LocalGet(3)); + function.emit(StackOp::LocalGet(0)); + function.emit(StackOp::I64Const(if double { 8 } else { 4 })); + function.emit(StackOp::IMul); + function.emit(StackOp::IAdd); + function.emit(get(19)); + function.emit(if double { + StackOp::StoreF64D + } else { + StackOp::StoreF32F + }); +} + +fn emit_scalar_stream(function: &mut StackFunction, double: bool, recurrence: bool) { + for slot in [3, 2] { + function.emit(StackOp::LocalGet(slot)); + function.emit(StackOp::LocalGet(0)); + function.emit(StackOp::I64Const(if double { 8 } else { 4 })); + function.emit(StackOp::IMul); + function.emit(StackOp::IAdd); + } + function.emit(if double { + StackOp::LoadF64D + } else { + StackOp::LoadF32F + }); + function.emit(if double { + StackOp::LocalGetD(4) + } else { + StackOp::LocalGetF(4) + }); + function.emit(if double { + StackOp::DMulD + } else { + StackOp::FMulF + }); + if recurrence { + function.emit(if double { + StackOp::LocalGetD(6) + } else { + StackOp::LocalGetF(6) + }); + } + function.emit(if double { + StackOp::LocalGetD(5) + } else { + StackOp::LocalGetF(5) + }); + if recurrence { + function.emit(if double { + StackOp::DMulD + } else { + StackOp::FMulF + }); + } + function.emit(if double { + StackOp::DAddD + } else { + StackOp::FAddF + }); + if recurrence { + function.emit(if double { + StackOp::LocalTeeD(6) + } else { + StackOp::LocalTeeF(6) + }); + } + function.emit(if double { + StackOp::StoreF64D + } else { + StackOp::StoreF32F + }); +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct Snapshot { + counter: i64, + end: i64, + state: u64, + additional_states: [u64; 3], + finished: u64, + primary: Vec, + secondary: Vec, +} + +struct Probe { + case: Case, + frame: *mut u64, + primary: *mut u8, + secondary: *mut u8, + size: usize, + snapshots: Vec, +} + +impl Probe { + unsafe fn snapshot(&self) -> Snapshot { + Snapshot { + counter: *self.frame as i64, + end: *self.frame.add(1) as i64, + state: *self.frame.add(6), + additional_states: if self.case.biquad { + [ + *self.frame.add(15), + *self.frame.add(16), + *self.frame.add(17), + ] + } else { + [0; 3] + }, + finished: *self.frame.add(9), + primary: std::slice::from_raw_parts(self.primary, self.size).to_vec(), + secondary: std::slice::from_raw_parts(self.secondary, self.size).to_vec(), + } + } +} + +unsafe extern "C" fn observe(data: *mut u8) -> bool { + let probe = &mut *(data as *mut Probe); + probe.snapshots.push(probe.snapshot()); + if probe.case.mutate { + let width = if probe.case.double { 8 } else { 4 }; + *probe.frame -= 1; + *probe.frame.add(1) = (LENGTH - 1) as u64; + *probe.frame.add(2) = probe.secondary.add(1 + 2 * width) as u64; + *probe.frame.add(3) = probe.primary.add(1 + 2 * width) as u64; + *probe.frame.add(4) = bits(probe.case.double, 0.75); + *probe.frame.add(5) = bits(probe.case.double, -0.25); + *probe.frame.add(6) = bits(probe.case.double, -0.5); + if probe.case.biquad { + for (slot, value) in [ + (12, 0.25), + (13, -0.5), + (14, 0.125), + (15, 0.125), + (16, -0.25), + (17, 0.5), + ] { + *probe.frame.add(slot) = bits(probe.case.double, value); + } + } + let changed = bits(probe.case.double, 0.125).to_le_bytes(); + std::ptr::copy_nonoverlapping(changed.as_ptr(), probe.secondary.add(1 + 3 * width), width); + } + probe.case.cancel +} + +#[derive(Debug, PartialEq, Eq)] +struct Outcome { + result: i64, + cancelled: bool, + trap: u32, + budget: i32, + observed: Vec, + finished: Snapshot, +} + +fn run(case: Case, native: bool) -> Outcome { + let width = if case.double { 8 } else { 4 }; + let size = 1 + STORAGE * width; + // Deliberately unaligned streams with padding for invalid-guard cases. + let mut primary = vec![0xa5; size]; + let mut secondary = vec![0x5a; size]; + let epsilon = if case.double { + f64::EPSILON + } else { + f32::EPSILON as f64 + }; + let values = [-0.0, 1.0 + epsilon, -1.0, 0.125, -0.25, 2.0]; + for index in 0..STORAGE { + primary[1 + index * width..1 + (index + 1) * width].copy_from_slice( + &bits(case.double, values[index % values.len()]).to_le_bytes()[..width], + ); + secondary[1 + index * width..1 + (index + 1) * width] + .copy_from_slice(&bits(case.double, 7.0).to_le_bytes()[..width]); + } + if case.edge != Edge::Mixed { + let value = if case.edge == Edge::NegativeZero { + -0.0 + } else { + 1.0 + epsilon + }; + primary[1 + 2 * width..1 + 3 * width] + .copy_from_slice(&bits(case.double, value).to_le_bytes()[..width]); + } + let (input, output) = if case.start >= case.end { + (std::ptr::null(), std::ptr::null_mut()) + } else { + let input = unsafe { primary.as_mut_ptr().add(1 + 2 * width) }; + let output = if let Some(offset) = case.overlap { + unsafe { input.offset(offset * width as isize) } + } else { + unsafe { secondary.as_mut_ptr().add(1 + 2 * width) } + }; + (input.cast_const(), output) + }; + let program = program(case, native, input, output); + let mut backend = StackBackend::new(&program); + let mut probe = Probe { + case, + frame: backend.frame_stack.as_mut_ptr(), + primary: primary.as_mut_ptr(), + secondary: secondary.as_mut_ptr(), + size, + snapshots: Vec::new(), + }; + if case.callback { + backend.set_cancel_callback(Some(observe), (&mut probe as *mut Probe).cast()); + } + let result = backend.call_entry(program.entry, [0u8].as_mut_ptr()); + let finished = unsafe { probe.snapshot() }; + Outcome { + result, + cancelled: backend.cancelled(), + trap: backend.trap_reason(), + budget: backend.ctx.cancel_counter, + observed: probe.snapshots, + finished, + } +} + +fn case(double: bool, recurrence: bool) -> Case { + Case { + double, + recurrence, + biquad: false, + edge: Edge::Mixed, + overlap: None, + start: 0, + end: LENGTH as i64, + input_len: LENGTH as i64, + output_len: LENGTH as i64, + prefix: 0, + callback: true, + cancel: false, + mutate: false, + } +} + +fn poll_cases(base: Case) -> Vec { + let mut cases = Vec::new(); + for overlap in [None, Some(0), Some(-1), Some(1)] { + // Full budget, expiry on final sample, penultimate sample, and first sample. + for prefix in [0, 784, 785, 1023] { + for (callback, cancel) in [(false, false), (true, false), (true, true)] { + cases.push(Case { + overlap, + prefix, + callback, + cancel, + ..base + }); + } + } + } + cases +} + +fn assert_poll_case(case: Case) { + let ordinary = run(case, false); + let native = run(case, true); + assert_eq!(native, ordinary, "{case:?}"); + assert_eq!( + native.cancelled, + case.callback && case.cancel && case.prefix >= 784 + ); + assert_eq!(native.finished.finished, u64::from(!native.cancelled)); + assert_eq!( + native.observed.len(), + usize::from(case.callback && case.prefix >= 784) + ); + if let Some(snapshot) = native.observed.first() { + assert_eq!(snapshot.counter, 1024 - case.prefix); + } +} + +fn guard_and_mutation_cases(base: Case) -> [Case; 11] { + [ + Case { end: 0, ..base }, + Case { + start: 4, + end: 0, + ..base + }, + Case { + start: -1, + end: 1, + ..base + }, + Case { + end: LENGTH as i64 + 1, + ..base + }, + Case { + input_len: 0, + ..base + }, + Case { + output_len: -1, + ..base + }, + Case { + start: 3, + end: 239, + ..base + }, + Case { + prefix: 785, + mutate: true, + ..base + }, + Case { + prefix: 1023, + mutate: true, + ..base + }, + Case { + prefix: 1023, + mutate: true, + overlap: Some(0), + ..base + }, + Case { + prefix: 1023, + mutate: true, + cancel: true, + ..base + }, + ] +} + +#[test] +fn biquad_kernels_preserve_four_states_aliasing_and_poll_boundaries() { + for double in [false, true] { + let base = Case { + biquad: true, + ..case(double, true) + }; + for case in poll_cases(base) { + assert_poll_case(case); + } + for case in guard_and_mutation_cases(base) { + assert_eq!(run(case, true), run(case, false), "{case:?}"); + } + for edge in [Edge::NegativeZero, Edge::SeparateRounding] { + let case = Case { + edge, + end: 1, + ..base + }; + let native = run(case, true); + assert_eq!(native, run(case, false), "{case:?}"); + // y1, the third carried state, holds the output after one step. + assert_eq!( + native.finished.additional_states[1], + bits( + double, + if edge == Edge::NegativeZero { + -0.0 + } else { + 0.0 + } + ) + ); + } + } +} + +#[test] +fn native_operand_arena_survives_source_ir_drop_and_backend_move() { + let input = [bits(false, 2.0), bits(true, 2.0)]; + let mut output = [0u64; 2]; + let backend = { + let mut program = StackProgram::new(); + for (index, double) in [false, true].iter().copied().enumerate() { + let case = Case { + biquad: true, + end: 1, + ..case(double, true) + }; + let mut source = self::program( + case, + true, + (&input[index] as *const u64).cast(), + (&mut output[index] as *mut u64).cast(), + ); + program.add_function(source.functions.remove(0)); + } + crate::native_loop_tests::forbid_fallback(&mut program); + StackBackend::new(&program) + }; + let mut moved = Box::new(backend); + for (function, double) in [false, true].iter().copied().enumerate() { + assert_eq!( + moved.call_entry(function as u32, [0u8].as_mut_ptr()), + bits(double, 0.28125) as i64, + ); + assert_eq!(moved.trap_reason(), crate::cancel::TRAP_NONE); + assert_eq!(output[function], bits(double, 0.28125)); + } +} + +#[test] +fn stream_kernels_preserve_negative_zero_and_separate_multiply_rounding() { + for double in [false, true] { + for recurrence in [false, true] { + for edge in [Edge::SeparateRounding, Edge::NegativeZero] { + for overlap in [None, Some(0)] { + let case = Case { + edge, + overlap, + end: 1, + ..case(double, recurrence) + }; + let native = run(case, true); + assert_eq!(native, run(case, false), "{case:?}"); + let expected = bits( + double, + if edge == Edge::NegativeZero { + -0.0 + } else { + 0.0 + }, + ); + let width = if double { 8 } else { 4 }; + let output = if overlap.is_some() { + &native.finished.primary + } else { + &native.finished.secondary + }; + assert_eq!( + &output[1 + 2 * width..1 + 3 * width], + &expected.to_le_bytes()[..width] + ); + if recurrence { + assert_eq!(native.finished.state, expected); + } + } + } + } + } +} + +#[test] +fn stream_kernels_match_240_sample_outputs_state_and_inherited_polls() { + for double in [false, true] { + for recurrence in [false, true] { + for case in poll_cases(case(double, recurrence)) { + assert_poll_case(case); + } + } + } +} + +#[test] +fn stream_kernel_guards_and_callback_mutations_keep_forward_loop_semantics() { + for double in [false, true] { + for recurrence in [false, true] { + let base = case(double, recurrence); + for case in guard_and_mutation_cases(base) { + assert_eq!(run(case, true), run(case, false), "{case:?}"); + } + } + } +} diff --git a/src/stack_ir.rs b/src/stack_ir.rs index 22dc0c7..cd329d5 100644 --- a/src/stack_ir.rs +++ b/src/stack_ir.rs @@ -38,6 +38,155 @@ impl StackExternRet { } } +/// Scalar interpretation shared by all native loop bodies. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum NativeScalar { + F32, + F64, +} + +/// Guarded native execution of a retained bytecode loop. Invalid ranges or +/// unsupported overlap fall through; completion skips the body through `done`. +/// Counter and carried values are published before polling. Operand windows +/// survive both completion and fallback. +#[derive(Clone, Debug, PartialEq)] +pub struct NativeLoopKernel { + pub scalar: NativeScalar, + pub counter: u16, + pub end: u16, + pub done: i32, + pub spans: NativeStreamSlots, + pub body: NativeStreamBody, +} + +/// Pointer and signed length slots for one indexed input/output stream pair. +/// Exact alias is supported; partially overlapping accessed spans use bytecode. +#[derive(Clone, Debug, PartialEq)] +pub struct NativeStreamSlots { + pub input: u16, + pub output: u16, + pub input_len: u16, + pub output_len: u16, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum NativeStreamBody { + /// An ordered, precompiled graph from the shared pointwise catalog. Only + /// the recipe's coefficient prefix is live; unused entries are padding. + Pointwise { + recipe: crate::pointwise::RecipeId, + coefficients: [C; crate::pointwise::MAX_COEFFICIENTS], + }, + /// Ordered `input * feed + state * feedback`, followed by an indexed store. + OnePole { feed: C, feedback: C, state: u16 }, + /// Ordered direct-form-I biquad. Coefficients are b0, b1, b2, a1, a2; + /// carried states are x1, x2, y1, y2. + Biquad { + coefficients: [C; 5], + states: [u16; 4], + }, +} + +impl NativeStreamBody { + /// Resolve coefficient representations without changing carried state slots. + pub fn map_coefficients(self, mut map: impl FnMut(C) -> D) -> NativeStreamBody { + match self { + Self::Pointwise { + recipe, + coefficients, + } => NativeStreamBody::Pointwise { + recipe, + coefficients: coefficients.map(map), + }, + Self::OnePole { + feed, + feedback, + state, + } => NativeStreamBody::OnePole { + feed: map(feed), + feedback: map(feedback), + state, + }, + Self::Biquad { + coefficients, + states, + } => NativeStreamBody::Biquad { + coefficients: coefficients.map(map), + states, + }, + } + } +} + +#[cfg(test)] +mod native_coordinate_tests { + use super::*; + + fn gateway(done: i32) -> StackOp { + StackOp::NativeLoop(NativeLoopKernel { + scalar: NativeScalar::F32, + counter: 0, + end: 1, + done, + spans: NativeStreamSlots { + input: 2, + output: 3, + input_len: 4, + output_len: 4, + }, + body: NativeStreamBody::Pointwise { + recipe: crate::pointwise::AFFINE, + coefficients: [5, 6, 0, 0], + }, + }) + } + + #[test] + fn inlining_remaps_native_success_edge() { + let mut callee = StackFunction::new("constant_add"); + callee.ops = vec![ + StackOp::I64Const(1), + StackOp::I64Const(2), + StackOp::IAdd, + StackOp::Return, + ]; + let call = StackOp::Call { + func: 1, + args: 0, + preserve: 0, + }; + let mut caller = StackFunction::new("native_caller"); + caller.local_count = 4; + caller.ops = vec![call.clone(), gateway(1), call, StackOp::ReturnVoid]; + let mut program = StackProgram::new(); + program.add_function(caller); + program.add_function(callee); + + crate::stack_inline::inline_trivial(&mut program); + + let caller = &program.functions[0]; + assert_eq!(caller.ops[3], gateway(3)); + assert_eq!(caller.ops[7], StackOp::ReturnVoid); + assert_eq!(crate::stack_depth::stack_delta(&caller.ops[3]), 0); + } + + #[test] + fn compaction_remaps_native_success_edge() { + let mut function = StackFunction::new("native_compaction"); + function.ops = vec![ + StackOp::Nop, + gateway(2), + StackOp::Nop, + StackOp::Nop, + StackOp::ReturnVoid, + ]; + + crate::stack_optimize::optimize(&mut function); + + assert_eq!(function.ops, vec![gateway(0), StackOp::ReturnVoid]); + } +} + /// Stack IR instruction. /// /// All values on the operand stack are 64-bit (i64/f64/pointer). @@ -259,6 +408,8 @@ pub enum StackOp { FusedAddrLoad32Off(u16, i32), /// locals[dst] = locals[src] + imm. Pop 0, push 0. FusedGetAddImmSet(u16, i32, u16), + /// Native whole-loop gateway; preserves all operand windows. + NativeLoop(NativeLoopKernel), /// if !(locals[a] < locals[b]) jump. Pop 0, push 0. FusedGetGetILtJumpIfZero(u16, u16, i32), /// if any packed `(idx < len)` check fails, jump. Pop 0, push 0. @@ -869,6 +1020,40 @@ impl fmt::Display for StackOp { StackOp::FusedGetAddImmSet(s, v, d) => { write!(f, "fused.get_addimm_set {} {} {}", s, v, d) } + StackOp::NativeLoop(kernel) => { + let width = match kernel.scalar { + NativeScalar::F32 => "f32", + NativeScalar::F64 => "f64", + }; + let spans = &kernel.spans; + match &kernel.body { + NativeStreamBody::Pointwise { + recipe, + coefficients, + } => { + write!(f, "pointwise_{}_{} counter={} end={} input={} output={} coefficients={:?} input_len={} output_len={} done={}", + recipe.recipe().name, width, kernel.counter, kernel.end, spans.input, spans.output, + &coefficients[..recipe.recipe().coefficient_count()], spans.input_len, spans.output_len, kernel.done) + } + NativeStreamBody::OnePole { + feed, + feedback, + state, + } => { + write!(f, "one_pole_{} counter={} end={} input={} output={} feed={} feedback={} state={} input_len={} output_len={} done={}", + width, kernel.counter, kernel.end, spans.input, spans.output, + feed, feedback, state, spans.input_len, spans.output_len, kernel.done) + } + NativeStreamBody::Biquad { + coefficients, + states, + } => { + write!(f, "biquad_{} counter={} end={} input={} output={} coefficients={:?} states={:?} input_len={} output_len={} done={}", + width, kernel.counter, kernel.end, spans.input, spans.output, + coefficients, states, spans.input_len, spans.output_len, kernel.done) + } + } + } StackOp::FusedGetGetILtJumpIfZero(a, b, o) => { write!(f, "fused.get_get_ilt_jiz {} {} {}", a, b, o) } diff --git a/src/stack_optimize.rs b/src/stack_optimize.rs index 9bc168c..6f3cffc 100644 --- a/src/stack_optimize.rs +++ b/src/stack_optimize.rs @@ -41,6 +41,7 @@ fn compute_jump_targets(ops: &[StackOp]) -> Vec { StackOp::FusedGetF32ConstFGtJumpIfZeroF(_, _, off) => Some(*off), StackOp::FusedF64ConstDGtJumpIfZeroD(_, off) => Some(*off), StackOp::FusedGetF64ConstDGtJumpIfZeroD(_, _, off) => Some(*off), + StackOp::NativeLoop(kernel) => Some(kernel.done), _ => None, }; if let Some(off) = off { @@ -376,10 +377,17 @@ fn packed_fmul_sum_fused_chain( fn fuse(func: &mut StackFunction) { let ops = &mut func.ops; let len = ops.len(); - let is_target = compute_jump_targets(ops); + let mut is_target = compute_jump_targets(ops); + // A native gateway is also a fusion boundary. The opcode itself carries + // that fact; no separate instruction-ownership metadata is needed. + for (index, op) in ops.iter().enumerate() { + if matches!(op, StackOp::NativeLoop(_)) { + is_target[index] = true; + } + } let mut i = 0; - // Helper: check if any instruction in range (i+1..i+n) is a jump target. + // A fusion may start at a jump target, but must not cross any boundary. let spans_target = |start: usize, count: usize| -> bool { for j in 1..count { if start + j < len && is_target[start + j] { @@ -390,6 +398,10 @@ fn fuse(func: &mut StackFunction) { }; while i < len { + if matches!(ops[i], StackOp::NativeLoop(_)) { + i += 1; + continue; + } // === Dead code elimination === // i64.const + drop → nop (dead Var initialization result) @@ -1429,6 +1441,17 @@ fn strip_nops(func: &mut StackFunction) { let new_off = target_new as i32 - new_idx[old] as i32 - 1; StackOp::FusedGetF64ConstDGtJumpIfZeroD(*n, *v, new_off) } + StackOp::NativeLoop(kernel) => { + let mut relocated = kernel.clone(); + let target_old = (old as i64 + 1 + kernel.done as i64) as usize; + let target_new = if target_old == len { + new_len + } else { + new_idx[target_old] + }; + relocated.done = target_new as i32 - new_idx[old] as i32 - 1; + StackOp::NativeLoop(relocated) + } other => other.clone(), }; new_ops.push(adjusted); diff --git a/src/stack_vm.rs b/src/stack_vm.rs index 1c42be0..adcf1ce 100644 --- a/src/stack_vm.rs +++ b/src/stack_vm.rs @@ -967,6 +967,9 @@ impl StackVM { } StackOp::Nop => {} + // The gateway's ordinary loop follows immediately. The Rust + // interpreter executes that retained implementation directly. + StackOp::NativeLoop(_) => {} // Fused superinstructions — not executed by the Rust VM. _ => unimplemented!("fused op {:?} not supported in Rust stack VM", op), diff --git a/src/value_loops.rs b/src/value_loops.rs new file mode 100644 index 0000000..6c54d35 --- /dev/null +++ b/src/value_loops.rs @@ -0,0 +1,553 @@ +//! Checked, ordered loop values and effects for experimental native selection. +//! +//! Discovery follows binding identities through scalar declarations and updates. +//! It records indexed reads and writes in source order, private loop invariants, +//! and incoming/final scalar state. Bounded kernel contracts consume this shared +//! representation; unsupported contracts keep the original bytecode loop. + +use crate::pointwise::{self, RecipeId}; +use crate::*; +use std::collections::{HashMap, HashSet}; +use std::convert::{TryFrom, TryInto}; + +mod recipes; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Scalar { + F32, + F64, +} + +impl Scalar { + pub(crate) fn of(ty: TypeID) -> Option { + match &*ty { + Type::Float32 => Some(Self::F32), + Type::Float64 => Some(Self::F64), + _ => None, + } + } +} + +/// Immutable value identity within one loop analysis, not a local storage slot. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +pub struct ValueId(pub usize); + +#[derive(Clone, Debug)] +pub enum Operation { + Input(LocalId), + Constant(String), + Binary(Binop, ValueId, ValueId), + Negate(ValueId), + ReadElement { stream: usize }, +} + +#[derive(Clone, Debug)] +pub struct Value { + pub ty: Scalar, + pub operation: Operation, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SpanLength { + Fixed(u32), + Slice, +} + +#[derive(Clone, Debug)] +pub struct MemoryStream { + pub binding: Reference, + /// Original base expression for representation-aware address/span lowering. + pub expression: ExprID, +} + +#[derive(Clone, Debug)] +pub struct ScalarInput { + pub binding: LocalId, + pub value: ValueId, +} + +#[derive(Clone, Debug)] +pub struct Carry { + pub binding: LocalId, + pub incoming: ValueId, + pub next: ValueId, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum LoopStep { + Value(ValueId), + StoreElement { stream: usize, value: ValueId }, +} + +#[derive(Clone, Debug)] +pub struct LoopRegion { + pub induction: LocalId, + pub values: Vec, + pub streams: Vec, + pub inputs: Vec, + pub carries: Vec, + pub tape: Vec, + pub ty: Scalar, +} + +#[derive(Clone, Debug)] +pub struct Pointwise { + pub input: usize, + pub output: usize, + pub recipe: RecipeId, + pub coefficients: Vec, +} + +#[derive(Clone, Copy, Debug)] +pub struct OnePole { + pub input: usize, + pub output: usize, + pub carry: usize, + pub feed: ValueId, + pub feedback: ValueId, +} + +#[derive(Clone, Copy, Debug)] +pub struct Biquad { + pub input: usize, + pub output: usize, + /// Ordered coefficient roles: b0, b1, b2, a1, a2. + pub coefficients: [ValueId; 5], + /// Carry roles x1, x2, y1, y2, independent of source declaration order. + pub carries: [usize; 4], +} + +impl LoopRegion { + fn operation(&self, value: ValueId) -> Option<&Operation> { + self.values.get(value.0).map(|value| &value.operation) + } + + fn binary(&self, value: ValueId, expected: Binop) -> Option<(ValueId, ValueId)> { + match self.operation(value)? { + Operation::Binary(operation, lhs, rhs) if *operation == expected => Some((*lhs, *rhs)), + _ => None, + } + } + + fn read(&self, value: ValueId) -> Option { + match self.operation(value)? { + Operation::ReadElement { stream } => Some(*stream), + _ => None, + } + } + + /// A coefficient is already private scalar state or an exact literal. + /// Arbitrary arithmetic remains per-iteration work until a contract covers it. + fn coefficient(&self, value: ValueId) -> bool { + match self.operation(value) { + Some(Operation::Constant(_)) => true, + Some(Operation::Input(binding)) => self + .inputs + .iter() + .any(|input| input.binding == *binding && input.value == value), + _ => false, + } + } + + /// Input/literal materialization has no loop effect. Every remaining value + /// and every store must occur exactly once in this source-ordered schedule. + /// In particular, this never drops an unused read or duplicates a shared one. + fn covers(&self, expected: &[LoopStep]) -> bool { + self.values.iter().all(|value| value.ty == self.ty) + && self + .tape + .iter() + .copied() + .filter(|step| match step { + LoopStep::Value(value) => !matches!( + self.operation(*value), + Some(Operation::Input(_) | Operation::Constant(_)) + ), + LoopStep::StoreElement { .. } => true, + }) + .eq(expected.iter().copied()) + } + + /// Match the same ordered recipe that generates the native arithmetic. + /// Nodes denote values, so a reused read or calculation must retain its + /// exact identity. The complete effect tape also rejects reordered work, + /// unused reads/arithmetic, and additional writes. + pub fn pointwise(&self) -> Option { + pointwise::recipes().find_map(|(id, recipe)| { + let matched = self.match_recipe(recipe)?; + Some(Pointwise { + input: matched.input, + output: matched.output, + recipe: id, + coefficients: matched.coefficients, + }) + }) + } + + /// Exact ordered one-pole state update followed by its output write: + /// `state = input[i] * feed + state * feedback; output[i] = state`. + pub fn one_pole(&self) -> Option { + let matched = self.match_recipe(crate::recurrence::ONE_POLE.recipe())?; + Some(OnePole { + input: matched.input, + output: matched.output, + carry: matched.carries[0], + feed: matched.coefficients[0], + feedback: matched.coefficients[1], + }) + } + + /// Direct form I with explicit, separately rounded arithmetic: + /// `b0*x + b1*x1 + b2*x2 - a1*y1 - a2*y2`. + /// The four final carry values must implement the two history shifts. + pub fn biquad(&self) -> Option { + let matched = self.match_recipe(crate::recurrence::BIQUAD.recipe())?; + Some(Biquad { + input: matched.input, + output: matched.output, + coefficients: matched.coefficients.try_into().ok()?, + carries: matched.carries.try_into().ok()?, + }) + } +} +fn local(function: &CheckedFunction, expression: ExprID) -> Option { + match function.arena.reference(expression) { + Some(Reference::Local(binding)) => Some(*binding), + _ => None, + } +} + +fn binder(function: &CheckedFunction, expression: ExprID) -> Option { + match function.arena[expression] { + Expr::Let(..) | Expr::Var(..) | Expr::For { .. } => Some(function.arena.binder(expression)), + _ => None, + } +} + +#[derive(Default)] +struct BodyShape { + definitions: HashSet, + assignments: Vec, + ty: Option, +} + +impl BodyShape { + fn collect(&mut self, function: &CheckedFunction, expression: ExprID) -> Option<()> { + match &function.arena[expression] { + Expr::Block(statements) => { + for &statement in statements { + self.collect(function, statement)?; + } + } + Expr::Let(_, initializer, _) | Expr::Var(_, Some(initializer), _) => { + self.definitions.insert(binder(function, expression)?); + self.ty = self + .ty + .or_else(|| Scalar::of(function.arena.ty(*initializer))); + } + Expr::Binop(Binop::Assign, lhs, rhs) => { + match function.arena[*lhs] { + Expr::Id(_) => { + let binding = local(function, *lhs)?; + if !self.assignments.contains(&binding) { + self.assignments.push(binding); + } + } + Expr::ArrayIndex(..) => {} + _ => return None, + } + self.ty = self.ty.or_else(|| Scalar::of(function.arena.ty(*rhs))); + } + _ => return None, + } + Some(()) + } +} + +struct Builder<'a> { + function: &'a CheckedFunction, + blocked: &'a HashSet, + definitions: HashSet, + region: LoopRegion, + bindings: HashMap, +} + +impl Builder<'_> { + fn add(&mut self, operation: Operation) -> ValueId { + let value = ValueId(self.region.values.len()); + self.region.values.push(Value { + ty: self.region.ty, + operation, + }); + self.region.tape.push(LoopStep::Value(value)); + value + } + + fn private_scalar(&self, binding: LocalId) -> bool { + !self.blocked.contains(&binding) + && Scalar::of(self.function.arena.local(binding).ty) == Some(self.region.ty) + } + + fn stream(&mut self, base: ExprID, index: ExprID) -> Option { + let function = self.function; + if !matches!(function.arena[base], Expr::Id(_)) + || !matches!(function.arena[index], Expr::Id(_)) + || local(function, index) != Some(self.region.induction) + { + return None; + } + let binding = function.arena.reference(base)?.clone(); + match &binding { + Reference::Local(binding) if !self.definitions.contains(binding) => {} + Reference::Instance(_) => {} + _ => return None, + } + let element = match &*function.arena.ty(base) { + Type::Array(element, ArraySize::Known(length)) => { + u32::try_from(*length).ok()?; + *element + } + Type::Slice(element) => *element, + _ => return None, + }; + if Scalar::of(element) != Some(self.region.ty) { + return None; + } + if let Some(stream) = self + .region + .streams + .iter() + .position(|stream| stream.binding == binding) + { + return Some(stream); + } + let stream = self.region.streams.len(); + self.region.streams.push(MemoryStream { + binding, + expression: base, + }); + Some(stream) + } + + fn expression(&mut self, expression: ExprID) -> Option { + let function = self.function; + if Scalar::of(function.arena.ty(expression)) != Some(self.region.ty) { + return None; + } + let operation = match &function.arena[expression] { + Expr::Id(_) => { + let binding = local(function, expression)?; + if let Some(value) = self.bindings.get(&binding) { + return Some(*value); + } + if self.definitions.contains(&binding) || !self.private_scalar(binding) { + return None; + } + let value = self.add(Operation::Input(binding)); + self.region.inputs.push(ScalarInput { binding, value }); + self.bindings.insert(binding, value); + return Some(value); + } + Expr::Real(value, _) => Operation::Constant(value.clone()), + Expr::ArrayIndex(base, index) => Operation::ReadElement { + stream: self.stream(*base, *index)?, + }, + Expr::Binop( + operation @ (Binop::Plus | Binop::Minus | Binop::Mult | Binop::Div), + lhs, + rhs, + ) => { + let lhs = self.expression(*lhs)?; + let rhs = self.expression(*rhs)?; + Operation::Binary(*operation, lhs, rhs) + } + Expr::Unop(Unop::Neg, operand) => Operation::Negate(self.expression(*operand)?), + _ => return None, + }; + Some(self.add(operation)) + } + + fn statement(&mut self, expression: ExprID) -> Option<()> { + let function = self.function; + match &function.arena[expression] { + Expr::Block(statements) => { + // LocalIds remain unique even after an inner lexical block. + for &statement in statements { + self.statement(statement)?; + } + } + Expr::Let(_, initializer, _) | Expr::Var(_, Some(initializer), _) => { + let binding = binder(function, expression)?; + if !self.private_scalar(binding) { + return None; + } + let value = self.expression(*initializer)?; + self.bindings.insert(binding, value); + } + Expr::Binop(Binop::Assign, lhs, rhs) => { + // Ordinary assignment evaluates its RHS before the lvalue. + let value = self.expression(*rhs)?; + match function.arena[*lhs] { + Expr::Id(_) => { + let binding = local(function, *lhs)?; + if !self.private_scalar(binding) || !self.bindings.contains_key(&binding) { + return None; + } + self.bindings.insert(binding, value); + if let Some(carry) = self + .region + .carries + .iter_mut() + .find(|carry| carry.binding == binding) + { + carry.next = value; + } + } + Expr::ArrayIndex(base, index) => { + let stream = self.stream(base, index)?; + self.region + .tape + .push(LoopStep::StoreElement { stream, value }); + } + _ => return None, + } + } + _ => return None, + } + Some(()) + } +} + +/// Observe checked HIR after safety checking. Selection and runtime span guards +/// establish a kernel's narrower executable contract. +fn analyze( + function: &CheckedFunction, + loop_expr: ExprID, + blocked: &HashSet, +) -> Option { + let Expr::For { body, .. } = function.arena[loop_expr] else { + return None; + }; + let induction = binder(function, loop_expr)?; + if blocked.contains(&induction) { + return None; + } + let mut shape = BodyShape::default(); + shape.collect(function, body)?; + let mut builder = Builder { + function, + blocked, + definitions: shape.definitions, + region: LoopRegion { + induction, + values: vec![], + streams: vec![], + inputs: vec![], + carries: vec![], + tape: vec![], + ty: shape.ty?, + }, + bindings: HashMap::new(), + }; + for binding in shape.assignments { + if builder.definitions.contains(&binding) { + continue; + } + if binding == induction || !builder.private_scalar(binding) { + return None; + } + let incoming = builder.add(Operation::Input(binding)); + builder.region.carries.push(Carry { + binding, + incoming, + next: incoming, + }); + builder.bindings.insert(binding, incoming); + } + builder.statement(body)?; + if builder.region.carries.is_empty() + && !builder + .region + .tape + .iter() + .any(|step| matches!(step, LoopStep::StoreElement { .. })) + { + return None; + } + Some(builder.region) +} + +/// Analyze one concrete, post-hoisting body. IDs and reports are body-local: +/// recompute after mutation or lambda extraction instead of transporting facts. +/// Captures and reference/unknown-call exposure remain conservative; identity +/// establishes neither purity nor absence of aliasing. +pub(crate) fn analyze_function(function: &CheckedFunction) -> HashMap { + fn expose(function: &CheckedFunction, id: ExprID, blocked: &mut HashSet) { + if let Some(binding) = local(function, id) { + blocked.insert(binding); + } + if matches!(function.arena[id], Expr::Lambda { .. }) { + return; + } + for child in function.arena[id].subexprs() { + expose(function, child, blocked); + } + } + + fn scan(function: &CheckedFunction, id: ExprID, blocked: &mut HashSet) { + if matches!(function.arena[id], Expr::Lambda { .. }) { + return; + } + if let Expr::Call(callee, args) = &function.arena[id] { + let callee_ty = function.arena.ty(*callee); + let params = match &*callee_ty { + Type::Func(domain, _) => match &**domain { + Type::Tuple(params) => Some(params), + _ => None, + }, + _ => None, + }; + for (position, arg) in args.iter().enumerate() { + if params + .and_then(|p| p.get(position)) + .is_none_or(|ty| ty.is_ptr()) + { + expose(function, *arg, blocked); + } + } + } + for child in function.arena[id].subexprs() { + scan(function, child, blocked); + } + } + + fn visit( + function: &CheckedFunction, + id: ExprID, + blocked: &HashSet, + loops: &mut HashMap, + ) { + // Generated lambdas keep ordinary lowering in this consumer. + if matches!(function.arena[id], Expr::Lambda { .. }) { + return; + } + if let Some(region) = analyze(function, id, blocked) { + loops.insert(id, region); + } + for child in function.arena[id].subexprs() { + visit(function, child, blocked, loops); + } + } + + let mut loops = HashMap::new(); + if let Some(body) = function.body { + let mut blocked = function.captured_locals(); + blocked.extend(function.closure_vars.iter().copied()); + scan(function, body, &mut blocked); + visit(function, body, &blocked, &mut loops); + } + loops +} + +#[cfg(test)] +mod tests; diff --git a/src/value_loops/recipes.rs b/src/value_loops/recipes.rs new file mode 100644 index 0000000..7be2f65 --- /dev/null +++ b/src/value_loops/recipes.rs @@ -0,0 +1,161 @@ +//! Match ordered native recipes against the checked loop's values and effects. +//! +//! Carry roles describe incoming values and their next-iteration roots. They do +//! not depend on declaration order or the order of equivalent scalar updates. + +use super::ValueId; +use super::{LoopRegion, LoopStep}; +use crate::loop_recipe::{BinaryOp, Node, Recipe}; +use crate::Binop; + +pub(super) struct MatchedRecipe { + pub input: usize, + pub output: usize, + pub coefficients: Vec, + pub carries: Vec, +} + +impl LoopRegion { + pub(super) fn match_recipe(&self, recipe: &Recipe) -> Option { + if self.carries.len() != recipe.carry_count() { + return None; + } + // Distinct state slots must also have distinct incoming value identities. + if self.carries.iter().enumerate().any(|(index, carry)| { + self.carries[..index] + .iter() + .any(|other| other.incoming == carry.incoming) + }) { + return None; + } + let LoopStep::StoreElement { + stream: output, + value, + } = *self.tape.last()? + else { + return None; + }; + self.match_recipe_carry_roles(recipe, output, value, &mut Vec::new()) + } + + fn match_recipe_carry_roles( + &self, + recipe: &Recipe, + output: usize, + result: ValueId, + carries: &mut Vec, + ) -> Option { + if carries.len() < self.carries.len() { + // Catalogs bound this to four carries (at most 24 permutations). + // Trying role assignments keeps output and next-state roots equal: + // a carry need not participate in the output calculation at all. + for carry in 0..self.carries.len() { + if carries.contains(&carry) { + continue; + } + carries.push(carry); + let matched = self.match_recipe_carry_roles(recipe, output, result, carries); + carries.pop(); + if matched.is_some() { + return matched; + } + } + return None; + } + + let mut bindings = vec![None; recipe.nodes.len()]; + let mut coefficients = vec![None; recipe.coefficient_count()]; + self.match_recipe_node( + recipe, + recipe.result, + result, + carries, + &mut bindings, + &mut coefficients, + )?; + for (role, node) in recipe.next_carries.iter().enumerate() { + self.match_recipe_node( + recipe, + *node, + self.carries[carries[role]].next, + carries, + &mut bindings, + &mut coefficients, + )?; + } + + let mut input = None; + let mut schedule = Vec::new(); + for (node, binding) in recipe.nodes.iter().zip(bindings) { + let binding = binding?; + match node { + Node::Parameter(_) | Node::Carry(_) => continue, + Node::Read => { + if input.replace(self.read(binding)?).is_some() { + return None; + } + } + Node::Binary(..) => {} + } + schedule.push(LoopStep::Value(binding)); + } + schedule.push(LoopStep::StoreElement { + stream: output, + value: result, + }); + self.covers(&schedule).then_some(MatchedRecipe { + input: input?, + output, + coefficients: coefficients.into_iter().collect::>()?, + carries: carries.clone(), + }) + } + + fn match_recipe_node( + &self, + recipe: &Recipe, + node: usize, + value: ValueId, + carries: &[usize], + bindings: &mut [Option], + coefficients: &mut [Option], + ) -> Option<()> { + if let Some(previous) = *bindings.get(node)? { + return (previous == value).then_some(()); + } + match recipe.nodes.get(node)? { + Node::Read => { + self.read(value)?; + } + Node::Parameter(role) => { + if !self.coefficient(value) { + return None; + } + let coefficient = coefficients.get_mut(usize::from(*role))?; + if coefficient.is_some_and(|previous| previous != value) { + return None; + } + // Different parameter roles may use the same private value. + *coefficient = Some(value); + } + Node::Carry(role) => { + let carry = self.carries.get(*carries.get(usize::from(*role))?)?; + if carry.incoming != value { + return None; + } + } + Node::Binary(operation, lhs_node, rhs_node) => { + let operation = match operation { + BinaryOp::Add => Binop::Plus, + BinaryOp::Sub => Binop::Minus, + BinaryOp::Mul => Binop::Mult, + }; + let (lhs, rhs) = self.binary(value, operation)?; + self.match_recipe_node(recipe, *lhs_node, lhs, carries, bindings, coefficients)?; + self.match_recipe_node(recipe, *rhs_node, rhs, carries, bindings, coefficients)?; + } + } + bindings[node] = Some(value); + Some(()) + } +} diff --git a/src/value_loops/tests.rs b/src/value_loops/tests.rs new file mode 100644 index 0000000..d7caa5e --- /dev/null +++ b/src/value_loops/tests.rs @@ -0,0 +1,276 @@ +use super::*; + +fn inspect(source: &str) -> Vec { + inspect_with_prelude("", source) +} + +fn inspect_with_prelude(prelude: &str, source: &str) -> Vec { + let mut compiler = Compiler::new(); + compiler.quiet = true; + assert!( + compiler.parse(prelude, ""), + "{:?}", + compiler.last_errors + ); + assert!( + compiler.parse(source, "value-loops.lyte"), + "{:?}", + compiler.last_errors + ); + assert!(compiler.check(), "{:?}", compiler.last_errors); + compiler.specialize().unwrap(); + compiler + .specialized_program() + .unwrap() + .functions() + .flat_map(|(_, function)| analyze_function(function).into_values()) + .collect() +} + +fn loops(body: &str) -> Vec { + inspect(&format!( + "kernel(a: &[f32; 4], b: &[f32; 4], gain: f32, bias: f32) -> f32 {{ + var state = 1.0 + var coefficient = gain + for i in 0 .. 4 {{ {body} }} + state + }} + main {{ + var a: [f32; 4] + var b: [f32; 4] + print(kernel(a, b, 0.5, 0.25) as i32) + }}" + )) +} + +#[test] +fn pointwise_contract_uses_invariants_and_immutable_local_versions() { + for body in [ + "b[i] = a[i] * gain + bias", + "let x = a[i]; var y = x * gain; y = y + bias; b[i] = y", + "a[i] = a[i] * 0.5 + 1.0", + ] { + let regions = loops(body); + assert_eq!(regions.len(), 1); + assert_eq!( + regions[0].pointwise().map(|kernel| kernel.recipe), + Some(pointwise::AFFINE), + "missing affine map for {}", + body + ); + } + for body in [ + "let unused = a[i] * bias; b[i] = a[i] * gain + bias", + "coefficient = coefficient + bias; b[i] = a[i] * coefficient + bias", + "b[i] = a[i] * gain + bias; a[i] = b[i] * gain + bias", + ] { + let regions = loops(body); + assert!( + regions[0].pointwise().is_none(), + "unexpected map for {}", + body + ); + } + let regions = inspect( + "map(a: &[f32; 4], b: &[f32; 4], gain: &f32) { + for i in 0 .. 4 { b[i] = a[i] * gain + 1.0 } + } + main { + var a: [f32; 4] + var b: [f32; 4] + var gain = 0.5 + map(a, b, gain) + }", + ); + assert!( + regions.is_empty(), + "reference scalars cannot become invariants" + ); +} + +fn pointwise_loop(body: &str, scalar: &str) -> LoopRegion { + let mut regions = inspect(&format!( + "kernel(input: &[{scalar}; 4], output: &[{scalar}; 4], a: {scalar}, b: {scalar}, c: {scalar}, d: {scalar}) {{ + for i in 0 .. 4 {{ {body} }} + }} + main {{ + var input: [{scalar}; 4] + var output: [{scalar}; 4] + kernel(input, output, 0.5{scalar}, 0.25{scalar}, 0.125{scalar}, 0.0625{scalar}) + }}" + )); + assert_eq!(regions.len(), 1, "discovery lost {}", body); + regions.pop().unwrap() +} + +#[test] +fn pointwise_recipes_match_ordered_reuse_and_coefficient_roles() { + for scalar in ["f32", "f64"] { + for (name, body, coefficient_count) in [ + ("offset_scale", "output[i] = (input[i] - b) * a", 2), + ( + "cubic", + "let x = input[i]; output[i] = x - a * ((x*x)*x)", + 1, + ), + ( + "cubic", + "var x = input[i]; let original = x; var y = x*x; + y = y*x; y = a*y; x = original-y; output[i] = x", + 1, + ), + ( + "horner", + "let x = input[i]; output[i] = ((a*x+b)*x+c)*x+d", + 4, + ), + ( + "horner", + "let x = input[i]; var y = a*x; y = y+b; y = y*x; + y = y+c; y = y*x; y = y+d; output[i] = y", + 4, + ), + ] { + let region = pointwise_loop(body, scalar); + let kernel = region.pointwise().expect(body); + assert_eq!(kernel.recipe.recipe().name, name, "{}", body); + assert_eq!(kernel.coefficients.len(), coefficient_count); + assert_ne!(kernel.input, kernel.output); + } + let region = pointwise_loop("let x = input[i]; input[i] = ((a*x+a)*x+a)*x+a", scalar); + let kernel = region.pointwise().expect("aliased coefficient roles"); + assert_eq!(kernel.recipe.recipe().name, "horner"); + assert_eq!(kernel.input, kernel.output); + assert_eq!(kernel.coefficients.len(), 4); + assert!(kernel + .coefficients + .iter() + .all(|value| *value == kernel.coefficients[0])); + } +} + +#[test] +fn pointwise_recipes_reject_changed_identity_order_and_effects() { + for body in [ + // Repeated indexed reads are not reuse of one sampled value. + "output[i] = input[i]*input[i]", + "output[i] = input[i] - a * ((input[i]*input[i])*input[i])", + // The read is shared, but changing association or operand order + // changes the recipe's separately rounded arithmetic. + "let x = input[i]; output[i] = x - a * (x*(x*x))", + "let x = input[i]; output[i] = x - ((x*x)*x) * a", + "let x = input[i]; output[i] = ((x*a+b)*x+c)*x+d", + // An apparently harmless local update changes later value uses. + "var x = input[i]; x = x*x; output[i] = x - a * ((x*x)*x)", + "let x = input[i]; let unused = input[i]; output[i] = x - a*((x*x)*x)", + "let x = input[i]; let unused = x*b; output[i] = x - a*((x*x)*x)", + "let x = input[i]; output[i] = x - a*((x*x)*x); input[i] = x", + ] { + let region = pointwise_loop(body, "f32"); + assert!( + region.pointwise().is_none(), + "unexpected kernel for {}", + body + ); + } +} + +fn biquad_loop(body: &str, scalar: &str) -> LoopRegion { + let mut regions = inspect(&format!( + "kernel(input: &[{scalar}; 4], output: &[{scalar}; 4], b0: {scalar}, b1: {scalar}, b2: {scalar}, a1: {scalar}, a2: {scalar}) -> {scalar} {{ + var y2 = 0.25{scalar} + var x1 = 0.5{scalar} + var y1 = 0.75{scalar} + var x2 = 1.0{scalar} + for i in 0 .. 4 {{ {body} }} + x1 + x2 + y1 + y2 + }} + main {{ + var input: [{scalar}; 4] + var output: [{scalar}; 4] + print(kernel(input, output, 0.5{scalar}, 0.25{scalar}, 0.125{scalar}, 0.0625{scalar}, 0.03125{scalar}) as i32) + }}" + )); + assert_eq!(regions.len(), 1); + regions.pop().unwrap() +} + +#[test] +fn biquad_contract_follows_four_state_roles_and_staged_values() { + for scalar in ["f32", "f64"] { + for body in [ + "let x = input[i] + let y = b0*x + b1*x1 + b2*x2 - a1*y1 - a2*y2 + x2 = x1; x1 = x; y2 = y1; y1 = y + output[i] = y", + "var x = input[i] + var y = b0*x + y = y + b1*x1 + y = y + b2*x2 + y = y - a1*y1 + y = y - a2*y2 + let old_input = x1 + let old_output = y1 + y1 = y; x1 = x; x2 = old_input; y2 = old_output + output[i] = y", + ] { + let region = biquad_loop(body, scalar); + let kernel = region.biquad().expect("missing biquad"); + assert_eq!(region.carries.len(), 4); + assert_eq!(region.inputs.len(), 5); + let [x1, x2, y1, y2] = kernel.carries.map(|carry| ®ion.carries[carry]); + assert_eq!(x2.next, x1.incoming); + assert_eq!(y2.next, y1.incoming); + assert!(matches!( + region.operation(x1.next), + Some(Operation::ReadElement { stream }) if *stream == kernel.input + )); + assert!(matches!( + region.tape.last(), + Some(LoopStep::StoreElement { stream, value }) + if *stream == kernel.output && *value == y1.next + )); + } + } +} + +#[test] +fn biquad_contract_rejects_changed_arithmetic_state_and_effects() { + let body = "let x = input[i] + let y = b0*x + b1*x1 + b2*x2 - a1*y1 - a2*y2 + x2 = x1; x1 = x; y2 = y1; y1 = y + output[i] = y"; + for changed in [ + body.replace("x2 = x1; x1 = x", "x1 = x; x2 = x1"), + body.replace("a2*y2", "a2*y1"), + body.replace("b0*x", "x*b0"), + body.replace("b0*x + b1*x1 + b2*x2", "b0*x + (b1*x1 + b2*x2)"), + body.replace("let y =", "let unused = input[i]; let y ="), + body.replace("output[i] = y", "output[i] = y; input[i] = x"), + ] { + let region = biquad_loop(&changed, "f32"); + assert!( + region.biquad().is_none(), + "unexpected biquad for {}", + changed + ); + } +} + +#[test] +fn reference_call_exposure_remains_conservative_for_owned_scalar_locals() { + let regions = inspect( + "observe(value: &f32) { print(value as i32) } + main { + var input: [f32; 4] + var output: [f32; 4] + var gain = 0.5 + observe(gain) + for i in 0 .. 4 { output[i] = input[i]*gain+1.0 } + }", + ); + assert!( + regions.is_empty(), + "reference-exposed scalar cannot become an invariant" + ); +} diff --git a/tests/cases/arith/packed_sum_signed_zero.lyte b/tests/cases/arith/packed_sum_signed_zero.lyte new file mode 100644 index 0000000..4764101 --- /dev/null +++ b/tests/cases/arith/packed_sum_signed_zero.lyte @@ -0,0 +1,31 @@ +// A sum of negative-zero products starts with its first product. Adding an +// artificial initial +0 changes its sign. atan2 exposes the sign without +// division by zero; the inline forms select Stack's packed sum handlers. +// expected stdout: +// compilation successful +// -3 +// -3 +// -3 +// -3 + +inline32(a: f32, b: f32, c: f32, d: f32) -> f32 { a * b + c * d } +split32(a: f32, b: f32, c: f32, d: f32) -> f32 { + let p = a * b + let q = c * d + p + q +} +inline64(a: f64, b: f64, c: f64, d: f64) -> f64 { a * b + c * d } +split64(a: f64, b: f64, c: f64, d: f64) -> f64 { + let p = a * b + let q = c * d + p + q +} + +main { + var z = -0.0 + print(atan2(inline32(z, 1.0, z, 1.0), -1.0) as i32) + print(atan2(split32(z, 1.0, z, 1.0), -1.0) as i32) + var z64 = -0.0f64 + print(atan2(inline64(z64, 1.0f64, z64, 1.0f64), -1.0f64) as i32) + print(atan2(split64(z64, 1.0f64, z64, 1.0f64), -1.0f64) as i32) +}