From b159a949128b53cd19e173a0b36b419707190079 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 07:14:27 +0900 Subject: [PATCH 1/3] Bench: Add the TypedPython benchmark harness (CPython / Node / Java / Rust) Five programs (nbody, spectral-norm, fannkuch-redux, binary-trees, and a dict-heavy word-frequency count), each in gate-clean typed Python, Node, Java and Rust, all printing identical results. run.py verify checks agreement; run.py measure refuses to run above load 4.0 and writes results only with --write. The @compiled column reports n/a until the backend exists. No results are committed: the machine was at load 40+ throughout. Refs #24 --- benchmarks/typedpython/.gitignore | 4 + benchmarks/typedpython/java/BinaryTrees.java | 33 ++++ benchmarks/typedpython/java/Fannkuch.java | 39 +++++ benchmarks/typedpython/java/Nbody.java | 74 ++++++++ benchmarks/typedpython/java/SpectralNorm.java | 46 +++++ benchmarks/typedpython/java/Wordfreq.java | 51 ++++++ benchmarks/typedpython/js/binary_trees.mjs | 25 +++ benchmarks/typedpython/js/fannkuch.mjs | 34 ++++ benchmarks/typedpython/js/nbody.mjs | 68 ++++++++ benchmarks/typedpython/js/spectral_norm.mjs | 35 ++++ benchmarks/typedpython/js/wordfreq.mjs | 38 +++++ benchmarks/typedpython/py/binary_trees.py | 44 +++++ benchmarks/typedpython/py/fannkuch.py | 56 ++++++ benchmarks/typedpython/py/nbody.py | 106 ++++++++++++ benchmarks/typedpython/py/spectral_norm.py | 47 +++++ benchmarks/typedpython/py/wordfreq.py | 51 ++++++ benchmarks/typedpython/run.py | 161 ++++++++++++++++++ benchmarks/typedpython/rust/Cargo.lock | 7 + benchmarks/typedpython/rust/Cargo.toml | 29 ++++ .../typedpython/rust/src/bin/binary_trees.rs | 39 +++++ .../typedpython/rust/src/bin/fannkuch.rs | 55 ++++++ benchmarks/typedpython/rust/src/bin/nbody.rs | 78 +++++++++ .../typedpython/rust/src/bin/spectral_norm.rs | 45 +++++ .../typedpython/rust/src/bin/wordfreq.rs | 54 ++++++ 24 files changed, 1219 insertions(+) create mode 100644 benchmarks/typedpython/.gitignore create mode 100644 benchmarks/typedpython/java/BinaryTrees.java create mode 100644 benchmarks/typedpython/java/Fannkuch.java create mode 100644 benchmarks/typedpython/java/Nbody.java create mode 100644 benchmarks/typedpython/java/SpectralNorm.java create mode 100644 benchmarks/typedpython/java/Wordfreq.java create mode 100644 benchmarks/typedpython/js/binary_trees.mjs create mode 100644 benchmarks/typedpython/js/fannkuch.mjs create mode 100644 benchmarks/typedpython/js/nbody.mjs create mode 100644 benchmarks/typedpython/js/spectral_norm.mjs create mode 100644 benchmarks/typedpython/js/wordfreq.mjs create mode 100644 benchmarks/typedpython/py/binary_trees.py create mode 100644 benchmarks/typedpython/py/fannkuch.py create mode 100644 benchmarks/typedpython/py/nbody.py create mode 100644 benchmarks/typedpython/py/spectral_norm.py create mode 100644 benchmarks/typedpython/py/wordfreq.py create mode 100644 benchmarks/typedpython/run.py create mode 100644 benchmarks/typedpython/rust/Cargo.lock create mode 100644 benchmarks/typedpython/rust/Cargo.toml create mode 100644 benchmarks/typedpython/rust/src/bin/binary_trees.rs create mode 100644 benchmarks/typedpython/rust/src/bin/fannkuch.rs create mode 100644 benchmarks/typedpython/rust/src/bin/nbody.rs create mode 100644 benchmarks/typedpython/rust/src/bin/spectral_norm.rs create mode 100644 benchmarks/typedpython/rust/src/bin/wordfreq.rs diff --git a/benchmarks/typedpython/.gitignore b/benchmarks/typedpython/.gitignore new file mode 100644 index 000000000..cd831ffe7 --- /dev/null +++ b/benchmarks/typedpython/.gitignore @@ -0,0 +1,4 @@ +target/ +java-out/ +__pycache__/ +*.class diff --git a/benchmarks/typedpython/java/BinaryTrees.java b/benchmarks/typedpython/java/BinaryTrees.java new file mode 100644 index 000000000..5a4dedc1c --- /dev/null +++ b/benchmarks/typedpython/java/BinaryTrees.java @@ -0,0 +1,33 @@ +public class BinaryTrees { + static final class Node { + final Node left, right; + Node(Node left, Node right) { this.left = left; this.right = right; } + } + + static Node make(int depth) { + if (depth > 0) return new Node(make(depth - 1), make(depth - 1)); + return new Node(null, null); + } + + static int check(Node node) { + Node left = node.left, right = node.right; + if (left == null || right == null) return 1; + return 1 + check(left) + check(right); + } + + public static void main(String[] args) { + int n = Integer.parseInt(args[0]); + int minDepth = 4; + int maxDepth = Math.max(minDepth + 2, n); + int stretch = check(make(maxDepth + 1)); + Node longLived = make(maxDepth); + long total = 0; + for (int depth = minDepth; depth <= maxDepth; depth += 2) { + int iterations = 1 << (maxDepth - depth + minDepth); + long chk = 0; + for (int i = 0; i < iterations; i++) chk += check(make(depth)); + total += chk; + } + System.out.println(stretch + " " + total + " " + check(longLived)); + } +} diff --git a/benchmarks/typedpython/java/Fannkuch.java b/benchmarks/typedpython/java/Fannkuch.java new file mode 100644 index 000000000..2920f348e --- /dev/null +++ b/benchmarks/typedpython/java/Fannkuch.java @@ -0,0 +1,39 @@ +public class Fannkuch { + public static void main(String[] args) { + int n = Integer.parseInt(args[0]); + int[] perm1 = new int[n]; + for (int i = 0; i < n; i++) perm1[i] = i; + int[] count = new int[n]; + int maxFlips = 0, checksum = 0, permCount = 0, r = n; + while (true) { + while (r != 1) { count[r - 1] = r; r--; } + int[] perm = perm1.clone(); + int flips = 0; + int k = perm[0]; + while (k != 0) { + int lo = 0, hi = k; + while (lo < hi) { + int t = perm[lo]; perm[lo] = perm[hi]; perm[hi] = t; + lo++; hi--; + } + flips++; + k = perm[0]; + } + if (flips > maxFlips) maxFlips = flips; + if (permCount % 2 == 0) checksum += flips; else checksum -= flips; + boolean done = false; + while (true) { + if (r == n) { done = true; break; } + int p0 = perm1[0]; + for (int i = 0; i < r; i++) perm1[i] = perm1[i + 1]; + perm1[r] = p0; + count[r]--; + if (count[r] > 0) break; + r++; + } + if (done) break; + permCount++; + } + System.out.println(checksum + " " + maxFlips); + } +} diff --git a/benchmarks/typedpython/java/Nbody.java b/benchmarks/typedpython/java/Nbody.java new file mode 100644 index 000000000..97be4c0c3 --- /dev/null +++ b/benchmarks/typedpython/java/Nbody.java @@ -0,0 +1,74 @@ +import java.util.Locale; + +public class Nbody { + static final double PI = 3.141592653589793; + static final double SOLAR_MASS = 4 * PI * PI; + static final double DAYS = 365.24; + static final double[] x = {0.0, 4.84143144246472090e+00, 8.34336671824457987e+00, 1.28943695621391310e+01, 1.53796971148509165e+01}; + static final double[] y = {0.0, -1.16032004402742839e+00, 4.12479856412430479e+00, -1.51111514016986312e+01, -2.59193146099879641e+01}; + static final double[] z = {0.0, -1.03622044471123109e-01, -4.03523417114321381e-01, -2.23307578892655734e-01, 1.79258772950371181e-01}; + static final double[] vx = {0.0, 1.66007664274403694e-03 * DAYS, -2.76742510726862411e-03 * DAYS, 2.96460137564761618e-03 * DAYS, 2.68067772490389322e-03 * DAYS}; + static final double[] vy = {0.0, 7.69901118419740425e-03 * DAYS, 4.99852801234917238e-03 * DAYS, 2.37847173959480950e-03 * DAYS, 1.62824170038242295e-03 * DAYS}; + static final double[] vz = {0.0, -6.90460016972063023e-05 * DAYS, 2.30417297573763929e-05 * DAYS, -2.96589568540237556e-05 * DAYS, -9.51592254519715870e-05 * DAYS}; + static final double[] mass = {SOLAR_MASS, 9.54791938424326609e-04 * SOLAR_MASS, 2.85885980666130812e-04 * SOLAR_MASS, 4.36624404335156298e-05 * SOLAR_MASS, 5.15138902046611451e-05 * SOLAR_MASS}; + + static void offsetMomentum() { + double px = 0.0, py = 0.0, pz = 0.0; + for (int i = 0; i < 5; i++) { + px += vx[i] * mass[i]; + py += vy[i] * mass[i]; + pz += vz[i] * mass[i]; + } + vx[0] = -px / SOLAR_MASS; + vy[0] = -py / SOLAR_MASS; + vz[0] = -pz / SOLAR_MASS; + } + + static void advance(double dt, int steps) { + for (int s = 0; s < steps; s++) { + for (int i = 0; i < 5; i++) { + for (int j = i + 1; j < 5; j++) { + double dx = x[i] - x[j]; + double dy = y[i] - y[j]; + double dz = z[i] - z[j]; + double d2 = dx * dx + dy * dy + dz * dz; + double mag = dt / (d2 * Math.sqrt(d2)); + vx[i] -= dx * mass[j] * mag; + vy[i] -= dy * mass[j] * mag; + vz[i] -= dz * mass[j] * mag; + vx[j] += dx * mass[i] * mag; + vy[j] += dy * mass[i] * mag; + vz[j] += dz * mass[i] * mag; + } + } + for (int i = 0; i < 5; i++) { + x[i] += dt * vx[i]; + y[i] += dt * vy[i]; + z[i] += dt * vz[i]; + } + } + } + + static double energy() { + double e = 0.0; + for (int i = 0; i < 5; i++) { + e += 0.5 * mass[i] * (vx[i] * vx[i] + vy[i] * vy[i] + vz[i] * vz[i]); + for (int j = i + 1; j < 5; j++) { + double dx = x[i] - x[j]; + double dy = y[i] - y[j]; + double dz = z[i] - z[j]; + e -= mass[i] * mass[j] / Math.sqrt(dx * dx + dy * dy + dz * dz); + } + } + return e; + } + + public static void main(String[] args) { + int n = Integer.parseInt(args[0]); + offsetMomentum(); + double before = energy(); + advance(0.01, n); + double after = energy(); + System.out.println(String.format(Locale.ROOT, "%.9f %.9f", before, after)); + } +} diff --git a/benchmarks/typedpython/java/SpectralNorm.java b/benchmarks/typedpython/java/SpectralNorm.java new file mode 100644 index 000000000..1227c7819 --- /dev/null +++ b/benchmarks/typedpython/java/SpectralNorm.java @@ -0,0 +1,46 @@ +import java.util.Locale; + +public class SpectralNorm { + static double evalA(int i, int j) { + return 1.0 / ((i + j) * (i + j + 1) / 2 + i + 1); + } + + static void mulAv(int n, double[] v, double[] out) { + for (int i = 0; i < n; i++) { + double s = 0.0; + for (int j = 0; j < n; j++) s += evalA(i, j) * v[j]; + out[i] = s; + } + } + + static void mulAtv(int n, double[] v, double[] out) { + for (int i = 0; i < n; i++) { + double s = 0.0; + for (int j = 0; j < n; j++) s += evalA(j, i) * v[j]; + out[i] = s; + } + } + + static void mulAtAv(int n, double[] v, double[] out, double[] tmp) { + mulAv(n, v, tmp); + mulAtv(n, tmp, out); + } + + public static void main(String[] args) { + int n = Integer.parseInt(args[0]); + double[] u = new double[n]; + double[] v = new double[n]; + double[] tmp = new double[n]; + java.util.Arrays.fill(u, 1.0); + for (int k = 0; k < 10; k++) { + mulAtAv(n, u, v, tmp); + mulAtAv(n, v, u, tmp); + } + double vbv = 0.0, vv = 0.0; + for (int i = 0; i < n; i++) { + vbv += u[i] * v[i]; + vv += v[i] * v[i]; + } + System.out.println(String.format(Locale.ROOT, "%.9f", Math.sqrt(vbv / vv))); + } +} diff --git a/benchmarks/typedpython/java/Wordfreq.java b/benchmarks/typedpython/java/Wordfreq.java new file mode 100644 index 000000000..259c7ec07 --- /dev/null +++ b/benchmarks/typedpython/java/Wordfreq.java @@ -0,0 +1,51 @@ +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Wordfreq { + static final long VOCAB = 50000; + static final long MOD = 2147483647L; + + static long nextState(long s) { return (s * 48271) % MOD; } + + static String wordOf(long id) { + long x = id + 17576; + StringBuilder sb = new StringBuilder(); + while (x > 0) { + sb.append((char) (97 + x % 26)); + x /= 26; + } + return sb.reverse().toString(); + } + + public static void main(String[] args) { + int n = Integer.parseInt(args[0]); + Map counts = new HashMap<>(); + long s = 12345; + for (int i = 0; i < n; i++) { + s = nextState(s); + long a = s % VOCAB; + s = nextState(s); + long b = s % VOCAB; + counts.merge(wordOf(a * b / VOCAB), 1, Integer::sum); + } + long weighted = 0; + for (int c : counts.values()) weighted += (long) c * c; + List taken = new ArrayList<>(); + List parts = new ArrayList<>(); + for (int k = 0; k < 5; k++) { + String bestWord = ""; + int bestCount = -1; + for (Map.Entry e : counts.entrySet()) { + String w = e.getKey(); + int c = e.getValue(); + if (taken.contains(w)) continue; + if (c > bestCount || (c == bestCount && w.compareTo(bestWord) < 0)) { bestWord = w; bestCount = c; } + } + taken.add(bestWord); + parts.add(bestWord + ":" + bestCount); + } + System.out.println(counts.size() + " " + weighted + " " + String.join(",", parts)); + } +} diff --git a/benchmarks/typedpython/js/binary_trees.mjs b/benchmarks/typedpython/js/binary_trees.mjs new file mode 100644 index 000000000..e0d6b45a9 --- /dev/null +++ b/benchmarks/typedpython/js/binary_trees.mjs @@ -0,0 +1,25 @@ +class Node { + constructor(left, right) { this.left = left; this.right = right; } +} +function make(depth) { + if (depth > 0) return new Node(make(depth - 1), make(depth - 1)); + return new Node(null, null); +} +function check(node) { + const left = node.left, right = node.right; + if (left === null || right === null) return 1; + return 1 + check(left) + check(right); +} +const n = parseInt(process.argv[2], 10); +const minDepth = 4; +const maxDepth = Math.max(minDepth + 2, n); +const stretch = check(make(maxDepth + 1)); +const longLived = make(maxDepth); +let total = 0; +for (let depth = minDepth; depth <= maxDepth; depth += 2) { + const iterations = 1 << (maxDepth - depth + minDepth); + let chk = 0; + for (let i = 0; i < iterations; i++) chk += check(make(depth)); + total += chk; +} +console.log(`${stretch} ${total} ${check(longLived)}`); diff --git a/benchmarks/typedpython/js/fannkuch.mjs b/benchmarks/typedpython/js/fannkuch.mjs new file mode 100644 index 000000000..d9f0d4bf6 --- /dev/null +++ b/benchmarks/typedpython/js/fannkuch.mjs @@ -0,0 +1,34 @@ +function fannkuch(n) { + const perm1 = Array.from({ length: n }, (_, i) => i); + const count = new Array(n).fill(0); + let maxFlips = 0, checksum = 0, permCount = 0, r = n; + for (;;) { + while (r !== 1) { count[r - 1] = r; r--; } + const perm = perm1.slice(); + let flips = 0; + let k = perm[0]; + while (k !== 0) { + let lo = 0, hi = k; + while (lo < hi) { + const t = perm[lo]; perm[lo] = perm[hi]; perm[hi] = t; + lo++; hi--; + } + flips++; + k = perm[0]; + } + if (flips > maxFlips) maxFlips = flips; + if (permCount % 2 === 0) checksum += flips; else checksum -= flips; + for (;;) { + if (r === n) return [checksum, maxFlips]; + const p0 = perm1[0]; + for (let i = 0; i < r; i++) perm1[i] = perm1[i + 1]; + perm1[r] = p0; + count[r]--; + if (count[r] > 0) break; + r++; + } + permCount++; + } +} +const [checksum, maxFlips] = fannkuch(parseInt(process.argv[2], 10)); +console.log(`${checksum} ${maxFlips}`); diff --git a/benchmarks/typedpython/js/nbody.mjs b/benchmarks/typedpython/js/nbody.mjs new file mode 100644 index 000000000..41f8dac8d --- /dev/null +++ b/benchmarks/typedpython/js/nbody.mjs @@ -0,0 +1,68 @@ +const PI = 3.141592653589793; +const SOLAR_MASS = 4 * PI * PI; +const DAYS = 365.24; +const x = [0.0, 4.84143144246472090e+00, 8.34336671824457987e+00, 1.28943695621391310e+01, 1.53796971148509165e+01]; +const y = [0.0, -1.16032004402742839e+00, 4.12479856412430479e+00, -1.51111514016986312e+01, -2.59193146099879641e+01]; +const z = [0.0, -1.03622044471123109e-01, -4.03523417114321381e-01, -2.23307578892655734e-01, 1.79258772950371181e-01]; +const vx = [0.0, 1.66007664274403694e-03 * DAYS, -2.76742510726862411e-03 * DAYS, 2.96460137564761618e-03 * DAYS, 2.68067772490389322e-03 * DAYS]; +const vy = [0.0, 7.69901118419740425e-03 * DAYS, 4.99852801234917238e-03 * DAYS, 2.37847173959480950e-03 * DAYS, 1.62824170038242295e-03 * DAYS]; +const vz = [0.0, -6.90460016972063023e-05 * DAYS, 2.30417297573763929e-05 * DAYS, -2.96589568540237556e-05 * DAYS, -9.51592254519715870e-05 * DAYS]; +const mass = [SOLAR_MASS, 9.54791938424326609e-04 * SOLAR_MASS, 2.85885980666130812e-04 * SOLAR_MASS, 4.36624404335156298e-05 * SOLAR_MASS, 5.15138902046611451e-05 * SOLAR_MASS]; + +function offsetMomentum() { + let px = 0.0, py = 0.0, pz = 0.0; + for (let i = 0; i < 5; i++) { + px += vx[i] * mass[i]; + py += vy[i] * mass[i]; + pz += vz[i] * mass[i]; + } + vx[0] = -px / SOLAR_MASS; + vy[0] = -py / SOLAR_MASS; + vz[0] = -pz / SOLAR_MASS; +} + +function advance(dt, steps) { + for (let s = 0; s < steps; s++) { + for (let i = 0; i < 5; i++) { + for (let j = i + 1; j < 5; j++) { + const dx = x[i] - x[j]; + const dy = y[i] - y[j]; + const dz = z[i] - z[j]; + const d2 = dx * dx + dy * dy + dz * dz; + const mag = dt / (d2 * Math.sqrt(d2)); + vx[i] -= dx * mass[j] * mag; + vy[i] -= dy * mass[j] * mag; + vz[i] -= dz * mass[j] * mag; + vx[j] += dx * mass[i] * mag; + vy[j] += dy * mass[i] * mag; + vz[j] += dz * mass[i] * mag; + } + } + for (let i = 0; i < 5; i++) { + x[i] += dt * vx[i]; + y[i] += dt * vy[i]; + z[i] += dt * vz[i]; + } + } +} + +function energy() { + let e = 0.0; + for (let i = 0; i < 5; i++) { + e += 0.5 * mass[i] * (vx[i] * vx[i] + vy[i] * vy[i] + vz[i] * vz[i]); + for (let j = i + 1; j < 5; j++) { + const dx = x[i] - x[j]; + const dy = y[i] - y[j]; + const dz = z[i] - z[j]; + e -= mass[i] * mass[j] / Math.sqrt(dx * dx + dy * dy + dz * dz); + } + } + return e; +} + +const n = parseInt(process.argv[2], 10); +offsetMomentum(); +const before = energy(); +advance(0.01, n); +const after = energy(); +console.log(`${before.toFixed(9)} ${after.toFixed(9)}`); diff --git a/benchmarks/typedpython/js/spectral_norm.mjs b/benchmarks/typedpython/js/spectral_norm.mjs new file mode 100644 index 000000000..705dd13c7 --- /dev/null +++ b/benchmarks/typedpython/js/spectral_norm.mjs @@ -0,0 +1,35 @@ +function evalA(i, j) { + return 1.0 / (Math.floor((i + j) * (i + j + 1) / 2) + i + 1); +} +function mulAv(n, v, out) { + for (let i = 0; i < n; i++) { + let s = 0.0; + for (let j = 0; j < n; j++) s += evalA(i, j) * v[j]; + out[i] = s; + } +} +function mulAtv(n, v, out) { + for (let i = 0; i < n; i++) { + let s = 0.0; + for (let j = 0; j < n; j++) s += evalA(j, i) * v[j]; + out[i] = s; + } +} +function mulAtAv(n, v, out, tmp) { + mulAv(n, v, tmp); + mulAtv(n, tmp, out); +} +const n = parseInt(process.argv[2], 10); +const u = new Array(n).fill(1.0); +const v = new Array(n).fill(0.0); +const tmp = new Array(n).fill(0.0); +for (let k = 0; k < 10; k++) { + mulAtAv(n, u, v, tmp); + mulAtAv(n, v, u, tmp); +} +let vbv = 0.0, vv = 0.0; +for (let i = 0; i < n; i++) { + vbv += u[i] * v[i]; + vv += v[i] * v[i]; +} +console.log(Math.sqrt(vbv / vv).toFixed(9)); diff --git a/benchmarks/typedpython/js/wordfreq.mjs b/benchmarks/typedpython/js/wordfreq.mjs new file mode 100644 index 000000000..539d01b93 --- /dev/null +++ b/benchmarks/typedpython/js/wordfreq.mjs @@ -0,0 +1,38 @@ +const VOCAB = 50000; +const MOD = 2147483647; +function nextState(s) { return (s * 48271) % MOD; } +function wordOf(id) { + let x = id + 17576; + let w = ""; + while (x > 0) { + w = String.fromCharCode(97 + (x % 26)) + w; + x = Math.floor(x / 26); + } + return w; +} +const n = parseInt(process.argv[2], 10); +const counts = new Map(); +let s = 12345; +for (let i = 0; i < n; i++) { + s = nextState(s); + const a = s % VOCAB; + s = nextState(s); + const b = s % VOCAB; + const word = wordOf(Math.floor((a * b) / VOCAB)); + counts.set(word, (counts.get(word) ?? 0) + 1); +} +let weighted = 0; +for (const [, c] of counts) weighted += c * c; +const taken = []; +const parts = []; +for (let k = 0; k < 5; k++) { + let bestWord = ""; + let bestCount = -1; + for (const [w, c] of counts) { + if (taken.includes(w)) continue; + if (c > bestCount || (c === bestCount && w < bestWord)) { bestWord = w; bestCount = c; } + } + taken.push(bestWord); + parts.push(`${bestWord}:${bestCount}`); +} +console.log(`${counts.size} ${weighted} ${parts.join(",")}`); diff --git a/benchmarks/typedpython/py/binary_trees.py b/benchmarks/typedpython/py/binary_trees.py new file mode 100644 index 000000000..9ab44c379 --- /dev/null +++ b/benchmarks/typedpython/py/binary_trees.py @@ -0,0 +1,44 @@ +import sys + + +class Node: + left: "Node | None" + right: "Node | None" + + def __init__(self, left: "Node | None", right: "Node | None") -> None: + self.left = left + self.right = right + + +def make(depth: int) -> Node: + if depth > 0: + return Node(make(depth - 1), make(depth - 1)) + return Node(None, None) + + +def check(node: Node) -> int: + left: Node | None = node.left + right: Node | None = node.right + if left is None or right is None: + return 1 + return 1 + check(left) + check(right) + + +def main() -> None: + n: int = int(sys.argv[1]) + min_depth: int = 4 + max_depth: int = max(min_depth + 2, n) + stretch: int = check(make(max_depth + 1)) + long_lived: Node = make(max_depth) + total: int = 0 + for depth in range(min_depth, max_depth + 1, 2): + iterations: int = 1 << (max_depth - depth + min_depth) + chk: int = 0 + for _ in range(iterations): + chk += check(make(depth)) + total += chk + print(f"{stretch} {total} {check(long_lived)}") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/typedpython/py/fannkuch.py b/benchmarks/typedpython/py/fannkuch.py new file mode 100644 index 000000000..cdd9f5b4c --- /dev/null +++ b/benchmarks/typedpython/py/fannkuch.py @@ -0,0 +1,56 @@ +import sys + + +def fannkuch(n: int) -> tuple[int, int]: + perm1: list[int] = list(range(n)) + count: list[int] = [0] * n + max_flips: int = 0 + checksum: int = 0 + perm_count: int = 0 + r: int = n + while True: + while r != 1: + count[r - 1] = r + r -= 1 + perm: list[int] = perm1[:] + flips: int = 0 + k: int = perm[0] + while k != 0: + lo: int = 0 + hi: int = k + while lo < hi: + t: int = perm[lo] + perm[lo] = perm[hi] + perm[hi] = t + lo += 1 + hi -= 1 + flips += 1 + k = perm[0] + if flips > max_flips: + max_flips = flips + if perm_count % 2 == 0: + checksum += flips + else: + checksum -= flips + while True: + if r == n: + return checksum, max_flips + p0: int = perm1[0] + for i in range(r): + perm1[i] = perm1[i + 1] + perm1[r] = p0 + count[r] -= 1 + if count[r] > 0: + break + r += 1 + perm_count += 1 + + +def main() -> None: + n: int = int(sys.argv[1]) + checksum, max_flips = fannkuch(n) + print(f"{checksum} {max_flips}") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/typedpython/py/nbody.py b/benchmarks/typedpython/py/nbody.py new file mode 100644 index 000000000..c67838312 --- /dev/null +++ b/benchmarks/typedpython/py/nbody.py @@ -0,0 +1,106 @@ +import math +import sys + +PI: float = 3.141592653589793 +SOLAR_MASS: float = 4 * PI * PI +DAYS_PER_YEAR: float = 365.24 + + +def make_x() -> list[float]: + return [0.0, 4.84143144246472090e+00, 8.34336671824457987e+00, 1.28943695621391310e+01, 1.53796971148509165e+01] + + +def make_y() -> list[float]: + return [0.0, -1.16032004402742839e+00, 4.12479856412430479e+00, -1.51111514016986312e+01, -2.59193146099879641e+01] + + +def make_z() -> list[float]: + return [0.0, -1.03622044471123109e-01, -4.03523417114321381e-01, -2.23307578892655734e-01, 1.79258772950371181e-01] + + +def make_vx() -> list[float]: + return [0.0, 1.66007664274403694e-03 * DAYS_PER_YEAR, -2.76742510726862411e-03 * DAYS_PER_YEAR, + 2.96460137564761618e-03 * DAYS_PER_YEAR, 2.68067772490389322e-03 * DAYS_PER_YEAR] + + +def make_vy() -> list[float]: + return [0.0, 7.69901118419740425e-03 * DAYS_PER_YEAR, 4.99852801234917238e-03 * DAYS_PER_YEAR, + 2.37847173959480950e-03 * DAYS_PER_YEAR, 1.62824170038242295e-03 * DAYS_PER_YEAR] + + +def make_vz() -> list[float]: + return [0.0, -6.90460016972063023e-05 * DAYS_PER_YEAR, 2.30417297573763929e-05 * DAYS_PER_YEAR, + -2.96589568540237556e-05 * DAYS_PER_YEAR, -9.51592254519715870e-05 * DAYS_PER_YEAR] + + +def make_mass() -> list[float]: + return [SOLAR_MASS, 9.54791938424326609e-04 * SOLAR_MASS, 2.85885980666130812e-04 * SOLAR_MASS, + 4.36624404335156298e-05 * SOLAR_MASS, 5.15138902046611451e-05 * SOLAR_MASS] + + +def offset_momentum(vx: list[float], vy: list[float], vz: list[float], mass: list[float]) -> None: + px: float = 0.0 + py: float = 0.0 + pz: float = 0.0 + for i in range(5): + px += vx[i] * mass[i] + py += vy[i] * mass[i] + pz += vz[i] * mass[i] + vx[0] = -px / SOLAR_MASS + vy[0] = -py / SOLAR_MASS + vz[0] = -pz / SOLAR_MASS + + +def advance(dt: float, steps: int, x: list[float], y: list[float], z: list[float], + vx: list[float], vy: list[float], vz: list[float], mass: list[float]) -> None: + for _ in range(steps): + for i in range(5): + for j in range(i + 1, 5): + dx: float = x[i] - x[j] + dy: float = y[i] - y[j] + dz: float = z[i] - z[j] + d2: float = dx * dx + dy * dy + dz * dz + mag: float = dt / (d2 * math.sqrt(d2)) + vx[i] -= dx * mass[j] * mag + vy[i] -= dy * mass[j] * mag + vz[i] -= dz * mass[j] * mag + vx[j] += dx * mass[i] * mag + vy[j] += dy * mass[i] * mag + vz[j] += dz * mass[i] * mag + for i in range(5): + x[i] += dt * vx[i] + y[i] += dt * vy[i] + z[i] += dt * vz[i] + + +def energy(x: list[float], y: list[float], z: list[float], + vx: list[float], vy: list[float], vz: list[float], mass: list[float]) -> float: + e: float = 0.0 + for i in range(5): + e += 0.5 * mass[i] * (vx[i] * vx[i] + vy[i] * vy[i] + vz[i] * vz[i]) + for j in range(i + 1, 5): + dx: float = x[i] - x[j] + dy: float = y[i] - y[j] + dz: float = z[i] - z[j] + e -= mass[i] * mass[j] / math.sqrt(dx * dx + dy * dy + dz * dz) + return e + + +def main() -> None: + n: int = int(sys.argv[1]) + x = make_x() + y = make_y() + z = make_z() + vx = make_vx() + vy = make_vy() + vz = make_vz() + mass = make_mass() + offset_momentum(vx, vy, vz, mass) + before: float = energy(x, y, z, vx, vy, vz, mass) + advance(0.01, n, x, y, z, vx, vy, vz, mass) + after: float = energy(x, y, z, vx, vy, vz, mass) + print(f"{before:.9f} {after:.9f}") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/typedpython/py/spectral_norm.py b/benchmarks/typedpython/py/spectral_norm.py new file mode 100644 index 000000000..e4d29091a --- /dev/null +++ b/benchmarks/typedpython/py/spectral_norm.py @@ -0,0 +1,47 @@ +import math +import sys + + +def eval_a(i: int, j: int) -> float: + return 1.0 / ((i + j) * (i + j + 1) // 2 + i + 1) + + +def mul_av(n: int, v: list[float], out: list[float]) -> None: + for i in range(n): + s: float = 0.0 + for j in range(n): + s += eval_a(i, j) * v[j] + out[i] = s + + +def mul_atv(n: int, v: list[float], out: list[float]) -> None: + for i in range(n): + s: float = 0.0 + for j in range(n): + s += eval_a(j, i) * v[j] + out[i] = s + + +def mul_atav(n: int, v: list[float], out: list[float], tmp: list[float]) -> None: + mul_av(n, v, tmp) + mul_atv(n, tmp, out) + + +def main() -> None: + n: int = int(sys.argv[1]) + u: list[float] = [1.0] * n + v: list[float] = [0.0] * n + tmp: list[float] = [0.0] * n + for _ in range(10): + mul_atav(n, u, v, tmp) + mul_atav(n, v, u, tmp) + vbv: float = 0.0 + vv: float = 0.0 + for i in range(n): + vbv += u[i] * v[i] + vv += v[i] * v[i] + print(f"{math.sqrt(vbv / vv):.9f}") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/typedpython/py/wordfreq.py b/benchmarks/typedpython/py/wordfreq.py new file mode 100644 index 000000000..8e3542481 --- /dev/null +++ b/benchmarks/typedpython/py/wordfreq.py @@ -0,0 +1,51 @@ +import sys + +VOCAB: int = 50000 +MOD: int = 2147483647 + + +def next_state(s: int) -> int: + return (s * 48271) % MOD + + +def word_of(word_id: int) -> str: + x: int = word_id + 17576 + w: str = "" + while x > 0: + w = chr(97 + x % 26) + w + x //= 26 + return w + + +def main() -> None: + n: int = int(sys.argv[1]) + counts: dict[str, int] = {} + s: int = 12345 + for _ in range(n): + s = next_state(s) + a: int = s % VOCAB + s = next_state(s) + b: int = s % VOCAB + word: str = word_of(a * b // VOCAB) + counts[word] = counts.get(word, 0) + 1 + weighted: int = 0 + for w, c in counts.items(): + weighted += c * c + taken: list[str] = [] + parts: list[str] = [] + for _ in range(5): + best_word: str = "" + best_count: int = -1 + for w, c in counts.items(): + if w in taken: + continue + if c > best_count or (c == best_count and w < best_word): + best_word = w + best_count = c + taken.append(best_word) + parts.append(f"{best_word}:{best_count}") + print(f"{len(counts)} {weighted} {','.join(parts)}") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/typedpython/run.py b/benchmarks/typedpython/run.py new file mode 100644 index 000000000..80b3ed3e2 --- /dev/null +++ b/benchmarks/typedpython/run.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""TypedPython benchmark runner (issue #24). Stdlib only. + + python3 run.py verify + python3 run.py measure [--repeat N] [--python PY] [--force] [--write] + +`verify` runs every implementation at a small size and requires identical output. +`measure` reports the median wall time over N runs at benchmark size. The Java column runs +pre-compiled classes (javac -d java-out), so javac time is excluded; JVM startup is included. +The "TypedPython @compiled" column is a placeholder: that backend does not exist yet. +""" +import argparse +import datetime +import json +import os +import socket +import statistics +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parent +TARGET = ROOT / "target" +JAVA_OUT = ROOT / "java-out" +CARGO = os.environ.get("CARGO", "/Users/ibrew/.cargo/bin/cargo") +NODE = os.environ.get("NODE", "/Users/ibrew/.local/bin/node") +LOAD_LIMIT = 4.0 + +# name -> (py file, js file, java class, rust bin, verify size, benchmark size) +BENCHMARKS = { + "nbody": ("nbody.py", "nbody.mjs", "Nbody", "nbody", "5000", "1000000"), + "spectral-norm": ("spectral_norm.py", "spectral_norm.mjs", "SpectralNorm", "spectral_norm", "100", "800"), + "fannkuch-redux": ("fannkuch.py", "fannkuch.mjs", "Fannkuch", "fannkuch", "7", "10"), + "binary-trees": ("binary_trees.py", "binary_trees.mjs", "BinaryTrees", "binary_trees", "10", "16"), + "wordfreq": ("wordfreq.py", "wordfreq.mjs", "Wordfreq", "wordfreq", "100000", "3000000"), +} +COLUMNS = ["CPython", "Node", "Java", "Rust", "TypedPython @compiled"] +NOT_BUILT = "n/a (not built)" + + +def run_checked(cmd: list[str], cwd: Path | None = None, env: dict[str, str] | None = None) -> None: + result = subprocess.run(cmd, cwd=cwd, env=env, capture_output=True, text=True) + if result.returncode != 0: + sys.stderr.write(f"command failed ({result.returncode}): {' '.join(cmd)}\n{result.stdout}{result.stderr}") + sys.exit(1) + + +def build() -> None: + env = dict(os.environ, CARGO_TARGET_DIR=str(TARGET)) + run_checked([CARGO, "build", "--release", "--quiet"], cwd=ROOT / "rust", env=env) + JAVA_OUT.mkdir(exist_ok=True) + sources = [str(p) for p in sorted((ROOT / "java").glob("*.java"))] + run_checked(["javac", "-d", str(JAVA_OUT), *sources]) + + +def commands(name: str, size: str, python: str, java_source: bool) -> dict[str, list[str]]: + py, js, cls, rs, _, _ = BENCHMARKS[name] + java = ["java", str(ROOT / "java" / f"{cls}.java"), size] if java_source \ + else ["java", "-cp", str(JAVA_OUT), cls, size] + return { + "CPython": [python, str(ROOT / "py" / py), size], + "Node": [NODE, str(ROOT / "js" / js), size], + "Java": java, + "Rust": [str(TARGET / "release" / rs), size], + } + + +def run_once(cmd: list[str]) -> tuple[float, str]: + start = time.perf_counter() + result = subprocess.run(cmd, capture_output=True, text=True) + elapsed = time.perf_counter() - start + if result.returncode != 0: + sys.stderr.write(f"failed ({result.returncode}): {' '.join(cmd)}\n{result.stderr}") + sys.exit(1) + return elapsed, result.stdout.strip() + + +def verify(args: argparse.Namespace) -> int: + build() + bad = 0 + for name, spec in BENCHMARKS.items(): + outputs = {col: run_once(cmd)[1] for col, cmd in commands(name, spec[4], args.python, True).items()} + distinct = set(outputs.values()) + status = "OK " if len(distinct) == 1 else "FAIL" + print(f"{status} {name} (size {spec[4]}): {next(iter(outputs.values())) if len(distinct) == 1 else ''}") + if len(distinct) != 1: + bad += 1 + for col, out in outputs.items(): + print(f" {col}: {out}") + print("all outputs agree" if bad == 0 else f"{bad} benchmark(s) disagree") + return 1 if bad else 0 + + +def measure(args: argparse.Namespace) -> int: + load = os.getloadavg() + print(f"load average: {load[0]:.2f} {load[1]:.2f} {load[2]:.2f}") + print(subprocess.run(["uptime"], capture_output=True, text=True).stdout.strip()) + if load[0] > LOAD_LIMIT and not args.force: + print(f"refusing to measure: 1-minute load {load[0]:.2f} > {LOAD_LIMIT} (use --force to override)") + return 2 + build() + results: dict[str, dict[str, object]] = {} + for name, spec in BENCHMARKS.items(): + size = args.size.get(name, spec[5]) + row: dict[str, object] = {"size": size} + for col, cmd in commands(name, size, args.python, False).items(): + runs = [run_once(cmd) for _ in range(args.repeat)] + outs = {out for _, out in runs} + row[col] = {"median_s": statistics.median(t for t, _ in runs), + "runs_s": [t for t, _ in runs], "output": sorted(outs)} + if len(outs) != 1: + print(f"warning: {name}/{col} produced differing outputs across runs") + row["TypedPython @compiled"] = NOT_BUILT + outs_all = {tuple(row[c]["output"]) for c in COLUMNS[:4]} # type: ignore[index] + row["outputs_agree"] = len(outs_all) == 1 + results[name] = row + print(f"measured {name}", file=sys.stderr) + + print() + print("| benchmark | size | " + " | ".join(COLUMNS) + " |") + print("|---|---|" + "---|" * len(COLUMNS)) + for name, row in results.items(): + cells = [f"{row[c]['median_s']:.3f} s" for c in COLUMNS[:4]] # type: ignore[index] + flag = "" if row["outputs_agree"] else " (OUTPUTS DIFFER)" + print(f"| {name}{flag} | {row['size']} | " + " | ".join(cells + [NOT_BUILT]) + " |") + print(f"\nmedian of {args.repeat} runs, wall time; Java excludes javac, includes JVM startup.") + + if args.write: + out_dir = ROOT / "results" + out_dir.mkdir(exist_ok=True) + path = out_dir / f"{datetime.date.today().isoformat()}-{socket.gethostname()}.json" + payload = {"date": datetime.datetime.now().isoformat(timespec="seconds"), + "host": socket.gethostname(), "load_average": load, "repeat": args.repeat, + "python": subprocess.run([args.python, "--version"], capture_output=True, text=True).stdout.strip(), + "results": results} + path.write_text(json.dumps(payload, indent=2) + "\n") + print(f"wrote {path}") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="cmd", required=True) + v = sub.add_parser("verify") + v.add_argument("--python", default=sys.executable) + m = sub.add_parser("measure") + m.add_argument("--python", default=sys.executable) + m.add_argument("--repeat", type=int, default=5) + m.add_argument("--force", action="store_true") + m.add_argument("--write", action="store_true") + m.add_argument("--size", action="append", default=[], metavar="NAME=N", + help="override a benchmark size (for runner smoke tests only)") + args = parser.parse_args() + if args.cmd == "measure": + args.size = dict(s.split("=", 1) for s in args.size) + return verify(args) if args.cmd == "verify" else measure(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarks/typedpython/rust/Cargo.lock b/benchmarks/typedpython/rust/Cargo.lock new file mode 100644 index 000000000..f0c0de11c --- /dev/null +++ b/benchmarks/typedpython/rust/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "typedpython-bench" +version = "0.1.0" diff --git a/benchmarks/typedpython/rust/Cargo.toml b/benchmarks/typedpython/rust/Cargo.toml new file mode 100644 index 000000000..59dba371d --- /dev/null +++ b/benchmarks/typedpython/rust/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "typedpython-bench" +version = "0.1.0" +edition = "2021" + +[[bin]] +name = "nbody" +path = "src/bin/nbody.rs" + +[[bin]] +name = "spectral_norm" +path = "src/bin/spectral_norm.rs" + +[[bin]] +name = "fannkuch" +path = "src/bin/fannkuch.rs" + +[[bin]] +name = "binary_trees" +path = "src/bin/binary_trees.rs" + +[[bin]] +name = "wordfreq" +path = "src/bin/wordfreq.rs" + +[profile.release] +opt-level = 3 +lto = true +codegen-units = 1 diff --git a/benchmarks/typedpython/rust/src/bin/binary_trees.rs b/benchmarks/typedpython/rust/src/bin/binary_trees.rs new file mode 100644 index 000000000..cb8579d3e --- /dev/null +++ b/benchmarks/typedpython/rust/src/bin/binary_trees.rs @@ -0,0 +1,39 @@ +struct Node { + left: Option>, + right: Option>, +} + +fn make(depth: u32) -> Box { + if depth > 0 { + Box::new(Node { left: Some(make(depth - 1)), right: Some(make(depth - 1)) }) + } else { + Box::new(Node { left: None, right: None }) + } +} + +fn check(node: &Node) -> i64 { + match (&node.left, &node.right) { + (Some(l), Some(r)) => 1 + check(l) + check(r), + _ => 1, + } +} + +fn main() { + let n: u32 = std::env::args().nth(1).unwrap().parse().unwrap(); + let min_depth = 4u32; + let max_depth = std::cmp::max(min_depth + 2, n); + let stretch = check(&make(max_depth + 1)); + let long_lived = make(max_depth); + let mut total = 0i64; + let mut depth = min_depth; + while depth <= max_depth { + let iterations = 1u64 << (max_depth - depth + min_depth); + let mut chk = 0i64; + for _ in 0..iterations { + chk += check(&make(depth)); + } + total += chk; + depth += 2; + } + println!("{} {} {}", stretch, total, check(&long_lived)); +} diff --git a/benchmarks/typedpython/rust/src/bin/fannkuch.rs b/benchmarks/typedpython/rust/src/bin/fannkuch.rs new file mode 100644 index 000000000..e62828379 --- /dev/null +++ b/benchmarks/typedpython/rust/src/bin/fannkuch.rs @@ -0,0 +1,55 @@ +fn fannkuch(n: usize) -> (i64, i64) { + let mut perm1: Vec = (0..n).collect(); + let mut count = vec![0usize; n]; + let (mut max_flips, mut checksum, mut perm_count) = (0i64, 0i64, 0i64); + let mut r = n; + loop { + while r != 1 { + count[r - 1] = r; + r -= 1; + } + let mut perm = perm1.clone(); + let mut flips = 0i64; + let mut k = perm[0]; + while k != 0 { + let (mut lo, mut hi) = (0usize, k); + while lo < hi { + perm.swap(lo, hi); + lo += 1; + hi -= 1; + } + flips += 1; + k = perm[0]; + } + if flips > max_flips { + max_flips = flips; + } + if perm_count % 2 == 0 { + checksum += flips; + } else { + checksum -= flips; + } + loop { + if r == n { + return (checksum, max_flips); + } + let p0 = perm1[0]; + for i in 0..r { + perm1[i] = perm1[i + 1]; + } + perm1[r] = p0; + count[r] -= 1; + if count[r] > 0 { + break; + } + r += 1; + } + perm_count += 1; + } +} + +fn main() { + let n: usize = std::env::args().nth(1).unwrap().parse().unwrap(); + let (checksum, max_flips) = fannkuch(n); + println!("{} {}", checksum, max_flips); +} diff --git a/benchmarks/typedpython/rust/src/bin/nbody.rs b/benchmarks/typedpython/rust/src/bin/nbody.rs new file mode 100644 index 000000000..0ccc8d733 --- /dev/null +++ b/benchmarks/typedpython/rust/src/bin/nbody.rs @@ -0,0 +1,78 @@ +const PI: f64 = 3.141592653589793; +const SOLAR_MASS: f64 = 4.0 * PI * PI; +const DAYS: f64 = 365.24; + +struct Sys { + x: [f64; 5], y: [f64; 5], z: [f64; 5], + vx: [f64; 5], vy: [f64; 5], vz: [f64; 5], + mass: [f64; 5], +} + +fn offset_momentum(s: &mut Sys) { + let (mut px, mut py, mut pz) = (0.0f64, 0.0f64, 0.0f64); + for i in 0..5 { + px += s.vx[i] * s.mass[i]; + py += s.vy[i] * s.mass[i]; + pz += s.vz[i] * s.mass[i]; + } + s.vx[0] = -px / SOLAR_MASS; + s.vy[0] = -py / SOLAR_MASS; + s.vz[0] = -pz / SOLAR_MASS; +} + +fn advance(dt: f64, steps: i64, s: &mut Sys) { + for _ in 0..steps { + for i in 0..5 { + for j in (i + 1)..5 { + let dx = s.x[i] - s.x[j]; + let dy = s.y[i] - s.y[j]; + let dz = s.z[i] - s.z[j]; + let d2 = dx * dx + dy * dy + dz * dz; + let mag = dt / (d2 * d2.sqrt()); + s.vx[i] -= dx * s.mass[j] * mag; + s.vy[i] -= dy * s.mass[j] * mag; + s.vz[i] -= dz * s.mass[j] * mag; + s.vx[j] += dx * s.mass[i] * mag; + s.vy[j] += dy * s.mass[i] * mag; + s.vz[j] += dz * s.mass[i] * mag; + } + } + for i in 0..5 { + s.x[i] += dt * s.vx[i]; + s.y[i] += dt * s.vy[i]; + s.z[i] += dt * s.vz[i]; + } + } +} + +fn energy(s: &Sys) -> f64 { + let mut e = 0.0f64; + for i in 0..5 { + e += 0.5 * s.mass[i] * (s.vx[i] * s.vx[i] + s.vy[i] * s.vy[i] + s.vz[i] * s.vz[i]); + for j in (i + 1)..5 { + let dx = s.x[i] - s.x[j]; + let dy = s.y[i] - s.y[j]; + let dz = s.z[i] - s.z[j]; + e -= s.mass[i] * s.mass[j] / (dx * dx + dy * dy + dz * dz).sqrt(); + } + } + e +} + +fn main() { + let n: i64 = std::env::args().nth(1).unwrap().parse().unwrap(); + let mut s = Sys { + x: [0.0, 4.84143144246472090e+00, 8.34336671824457987e+00, 1.28943695621391310e+01, 1.53796971148509165e+01], + y: [0.0, -1.16032004402742839e+00, 4.12479856412430479e+00, -1.51111514016986312e+01, -2.59193146099879641e+01], + z: [0.0, -1.03622044471123109e-01, -4.03523417114321381e-01, -2.23307578892655734e-01, 1.79258772950371181e-01], + vx: [0.0, 1.66007664274403694e-03 * DAYS, -2.76742510726862411e-03 * DAYS, 2.96460137564761618e-03 * DAYS, 2.68067772490389322e-03 * DAYS], + vy: [0.0, 7.69901118419740425e-03 * DAYS, 4.99852801234917238e-03 * DAYS, 2.37847173959480950e-03 * DAYS, 1.62824170038242295e-03 * DAYS], + vz: [0.0, -6.90460016972063023e-05 * DAYS, 2.30417297573763929e-05 * DAYS, -2.96589568540237556e-05 * DAYS, -9.51592254519715870e-05 * DAYS], + mass: [SOLAR_MASS, 9.54791938424326609e-04 * SOLAR_MASS, 2.85885980666130812e-04 * SOLAR_MASS, 4.36624404335156298e-05 * SOLAR_MASS, 5.15138902046611451e-05 * SOLAR_MASS], + }; + offset_momentum(&mut s); + let before = energy(&s); + advance(0.01, n, &mut s); + let after = energy(&s); + println!("{:.9} {:.9}", before, after); +} diff --git a/benchmarks/typedpython/rust/src/bin/spectral_norm.rs b/benchmarks/typedpython/rust/src/bin/spectral_norm.rs new file mode 100644 index 000000000..5fe23a77b --- /dev/null +++ b/benchmarks/typedpython/rust/src/bin/spectral_norm.rs @@ -0,0 +1,45 @@ +fn eval_a(i: usize, j: usize) -> f64 { + 1.0 / (((i + j) * (i + j + 1) / 2 + i + 1) as f64) +} + +fn mul_av(n: usize, v: &[f64], out: &mut [f64]) { + for i in 0..n { + let mut s = 0.0f64; + for j in 0..n { + s += eval_a(i, j) * v[j]; + } + out[i] = s; + } +} + +fn mul_atv(n: usize, v: &[f64], out: &mut [f64]) { + for i in 0..n { + let mut s = 0.0f64; + for j in 0..n { + s += eval_a(j, i) * v[j]; + } + out[i] = s; + } +} + +fn mul_atav(n: usize, v: &[f64], out: &mut [f64], tmp: &mut [f64]) { + mul_av(n, v, tmp); + mul_atv(n, tmp, out); +} + +fn main() { + let n: usize = std::env::args().nth(1).unwrap().parse().unwrap(); + let mut u = vec![1.0f64; n]; + let mut v = vec![0.0f64; n]; + let mut tmp = vec![0.0f64; n]; + for _ in 0..10 { + mul_atav(n, &u, &mut v, &mut tmp); + mul_atav(n, &v, &mut u, &mut tmp); + } + let (mut vbv, mut vv) = (0.0f64, 0.0f64); + for i in 0..n { + vbv += u[i] * v[i]; + vv += v[i] * v[i]; + } + println!("{:.9}", (vbv / vv).sqrt()); +} diff --git a/benchmarks/typedpython/rust/src/bin/wordfreq.rs b/benchmarks/typedpython/rust/src/bin/wordfreq.rs new file mode 100644 index 000000000..b6f3e0dc3 --- /dev/null +++ b/benchmarks/typedpython/rust/src/bin/wordfreq.rs @@ -0,0 +1,54 @@ +use std::collections::HashMap; + +const VOCAB: u64 = 50000; +const MOD: u64 = 2147483647; + +fn next_state(s: u64) -> u64 { + (s * 48271) % MOD +} + +fn word_of(id: u64) -> String { + let mut x = id + 17576; + let mut bytes: Vec = Vec::new(); + while x > 0 { + bytes.push(97 + (x % 26) as u8); + x /= 26; + } + bytes.reverse(); + String::from_utf8(bytes).unwrap() +} + +fn main() { + let n: u64 = std::env::args().nth(1).unwrap().parse().unwrap(); + let mut counts: HashMap = HashMap::new(); + let mut s: u64 = 12345; + for _ in 0..n { + s = next_state(s); + let a = s % VOCAB; + s = next_state(s); + let b = s % VOCAB; + *counts.entry(word_of(a * b / VOCAB)).or_insert(0) += 1; + } + let mut weighted = 0i64; + for c in counts.values() { + weighted += c * c; + } + let mut taken: Vec = Vec::new(); + let mut parts: Vec = Vec::new(); + for _ in 0..5 { + let mut best_word = String::new(); + let mut best_count = -1i64; + for (w, &c) in &counts { + if taken.contains(w) { + continue; + } + if c > best_count || (c == best_count && *w < best_word) { + best_word = w.clone(); + best_count = c; + } + } + parts.push(format!("{}:{}", best_word, best_count)); + taken.push(best_word); + } + println!("{} {} {}", counts.len(), weighted, parts.join(",")); +} From 68d64a15b79fda658408f5fe631ca788dad65fae Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 12:06:22 +0900 Subject: [PATCH 2/3] Bench: Record the TypedPython baseline (idle machine, load 1.6) Median of 5, wall time, measured alone at load 1.60/1.55/1.47 (a first run at load 3.1 agreed within ~3%): | benchmark | CPython | Node | Java | Rust | |---|---|---|---|---| | nbody 1e6 | 4.143 s | 0.130 s | 0.187 s | 0.061 s | | spectral-norm 800 | 3.441 s | 0.076 s | 0.116 s | 0.027 s | | fannkuch-redux 10 | 6.196 s | 0.271 s | 0.269 s | 0.261 s | | binary-trees 16 | 3.938 s | 0.172 s | 0.163 s | 0.266 s | | wordfreq 3e6 | 2.509 s | 0.972 s | 0.343 s | 0.164 s | Java includes JVM startup, excludes javac. @compiled column: not built yet. Closes #24 --- .../2026-10-03-irackui-Macmini.local.json | 298 ++++++++++++++++++ 1 file changed, 298 insertions(+) create mode 100644 benchmarks/typedpython/results/2026-10-03-irackui-Macmini.local.json diff --git a/benchmarks/typedpython/results/2026-10-03-irackui-Macmini.local.json b/benchmarks/typedpython/results/2026-10-03-irackui-Macmini.local.json new file mode 100644 index 000000000..605b49ecb --- /dev/null +++ b/benchmarks/typedpython/results/2026-10-03-irackui-Macmini.local.json @@ -0,0 +1,298 @@ +{ + "date": "2026-10-03T12:06:11", + "host": "irackui-Macmini.local", + "load_average": [ + 1.60009765625, + 1.54541015625, + 1.46533203125 + ], + "repeat": 5, + "python": "Python 3.13.0", + "results": { + "nbody": { + "size": "1000000", + "CPython": { + "median_s": 4.143098374945112, + "runs_s": [ + 4.254897207953036, + 4.221641207928769, + 4.143098374945112, + 4.065442958031781, + 4.0204505829606205 + ], + "output": [ + "-0.169075164 -0.169086185" + ] + }, + "Node": { + "median_s": 0.13044004200492054, + "runs_s": [ + 0.24463354202453047, + 0.11540012492332608, + 0.11569995793979615, + 0.13044004200492054, + 0.13507441605906934 + ], + "output": [ + "-0.169075164 -0.169086185" + ] + }, + "Java": { + "median_s": 0.18665479100309312, + "runs_s": [ + 0.18403883394785225, + 0.18665479100309312, + 0.18962366704363376, + 0.19042083295062184, + 0.17468633397947997 + ], + "output": [ + "-0.169075164 -0.169086185" + ] + }, + "Rust": { + "median_s": 0.06139945797622204, + "runs_s": [ + 0.6079301250865683, + 0.06139945797622204, + 0.061590292025357485, + 0.06120649992953986, + 0.060862750047817826 + ], + "output": [ + "-0.169075164 -0.169086185" + ] + }, + "TypedPython @compiled": "n/a (not built)", + "outputs_agree": true + }, + "spectral-norm": { + "size": "800", + "CPython": { + "median_s": 3.440628374926746, + "runs_s": [ + 3.2999370829202235, + 3.4090167910326272, + 3.440628374926746, + 3.4689366659149528, + 3.444350333069451 + ], + "output": [ + "1.274224144" + ] + }, + "Node": { + "median_s": 0.07610966707579792, + "runs_s": [ + 0.08152287499979138, + 0.07610966707579792, + 0.07956900005228817, + 0.07009887497406453, + 0.07070591708179563 + ], + "output": [ + "1.274224144" + ] + }, + "Java": { + "median_s": 0.11629029200412333, + "runs_s": [ + 0.22726075001992285, + 0.11629029200412333, + 0.11338383296970278, + 0.11177445796784014, + 0.12072145799174905 + ], + "output": [ + "1.274224144" + ] + }, + "Rust": { + "median_s": 0.027435958036221564, + "runs_s": [ + 0.38848837499972433, + 0.027392458054237068, + 0.027403999934904277, + 0.027435958036221564, + 0.027970083989202976 + ], + "output": [ + "1.274224144" + ] + }, + "TypedPython @compiled": "n/a (not built)", + "outputs_agree": true + }, + "fannkuch-redux": { + "size": "10", + "CPython": { + "median_s": 6.196417958009988, + "runs_s": [ + 6.150234500062652, + 6.2536906249588355, + 6.23115054203663, + 6.0902174999937415, + 6.196417958009988 + ], + "output": [ + "73196 38" + ] + }, + "Node": { + "median_s": 0.27098512498196214, + "runs_s": [ + 0.3849453340517357, + 0.28581020794808865, + 0.27098512498196214, + 0.26284683402627707, + 0.26294095802586526 + ], + "output": [ + "73196 38" + ] + }, + "Java": { + "median_s": 0.2690964579815045, + "runs_s": [ + 0.3637276670197025, + 0.2642790839308873, + 0.28995737503282726, + 0.2690964579815045, + 0.25848933297675103 + ], + "output": [ + "73196 38" + ] + }, + "Rust": { + "median_s": 0.2610693749738857, + "runs_s": [ + 0.6989039160544053, + 0.2595427499618381, + 0.2610693749738857, + 0.2637084999587387, + 0.2530785839771852 + ], + "output": [ + "73196 38" + ] + }, + "TypedPython @compiled": "n/a (not built)", + "outputs_agree": true + }, + "binary-trees": { + "size": "16", + "CPython": { + "median_s": 3.9383531670318916, + "runs_s": [ + 3.9383531670318916, + 4.0666959170484915, + 3.971159166889265, + 3.920735542080365, + 3.825213333009742 + ], + "output": [ + "262143 14592688 131071" + ] + }, + "Node": { + "median_s": 0.17200924991630018, + "runs_s": [ + 0.17211587505880743, + 0.17382529203314334, + 0.17200924991630018, + 0.17031329206656665, + 0.17060462501831353 + ], + "output": [ + "262143 14592688 131071" + ] + }, + "Java": { + "median_s": 0.16285908303689212, + "runs_s": [ + 0.1594726659823209, + 0.16640029195696115, + 0.1694065419724211, + 0.16285908303689212, + 0.1576562919653952 + ], + "output": [ + "262143 14592688 131071" + ] + }, + "Rust": { + "median_s": 0.2658698750892654, + "runs_s": [ + 0.6191935000242665, + 0.2656571250408888, + 0.2635289159370586, + 0.2658698750892654, + 0.2694279169663787 + ], + "output": [ + "262143 14592688 131071" + ] + }, + "TypedPython @compiled": "n/a (not built)", + "outputs_agree": true + }, + "wordfreq": { + "size": "3000000", + "CPython": { + "median_s": 2.5091819160152227, + "runs_s": [ + 2.477464916999452, + 2.5331434999825433, + 2.43217316607479, + 2.5359187909634784, + 2.5091819160152227 + ], + "output": [ + "49165 362711646 baaa:808,baab:653,baad:613,baac:576,baaf:566" + ] + }, + "Node": { + "median_s": 0.9722600829554722, + "runs_s": [ + 0.9722600829554722, + 0.9506856669904664, + 1.0822530830046162, + 0.974125292035751, + 0.9677200420992449 + ], + "output": [ + "49165 362711646 baaa:808,baab:653,baad:613,baac:576,baaf:566" + ] + }, + "Java": { + "median_s": 0.3431811250047758, + "runs_s": [ + 0.3431811250047758, + 0.3633901249850169, + 0.33033525000791997, + 0.3405763750197366, + 0.3798126250039786 + ], + "output": [ + "49165 362711646 baaa:808,baab:653,baad:613,baac:576,baaf:566" + ] + }, + "Rust": { + "median_s": 0.16358300007414073, + "runs_s": [ + 0.5581229169620201, + 0.2117745839059353, + 0.16358300007414073, + 0.1619741659378633, + 0.1581742090638727 + ], + "output": [ + "49165 362711646 baaa:808,baab:653,baad:613,baac:576,baaf:566" + ] + }, + "TypedPython @compiled": "n/a (not built)", + "outputs_agree": true + } + } +} From a6f0d377b2cca4e336aa31068ec068cd6265b20c Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 13:43:05 +0900 Subject: [PATCH 3/3] Bench: Run the TypedPython @compiled column through the pipeline With --typedpython (and --typedpython-python for the compile step), each benchmark is copied with the module marker, compiled by typedpython.pipeline, and its main() runs from the built extension. verify includes the column in the agreement check; benchmarks with nothing compiled report n/a. verify: all 5 agree. Refs #24, #41 --- benchmarks/typedpython/.gitignore | 1 + benchmarks/typedpython/run.py | 82 +++++++++++++++++++++++++++---- 2 files changed, 74 insertions(+), 9 deletions(-) diff --git a/benchmarks/typedpython/.gitignore b/benchmarks/typedpython/.gitignore index cd831ffe7..4977b6307 100644 --- a/benchmarks/typedpython/.gitignore +++ b/benchmarks/typedpython/.gitignore @@ -2,3 +2,4 @@ target/ java-out/ __pycache__/ *.class +tp-build/ diff --git a/benchmarks/typedpython/run.py b/benchmarks/typedpython/run.py index 80b3ed3e2..7aeeac231 100644 --- a/benchmarks/typedpython/run.py +++ b/benchmarks/typedpython/run.py @@ -7,7 +7,11 @@ `verify` runs every implementation at a small size and requires identical output. `measure` reports the median wall time over N runs at benchmark size. The Java column runs pre-compiled classes (javac -d java-out), so javac time is excluded; JVM startup is included. -The "TypedPython @compiled" column is a placeholder: that backend does not exist yet. +The "TypedPython @compiled" column runs only with `--typedpython ` (the directory that holds +the `typedpython` package, python-multiplatform-ksp/src/main/python) and `--typedpython-python` (an +interpreter with the gate's dependencies, e.g. that module's .venv): each benchmark is copied with +the `# typedpython: compiled` marker, compiled through `typedpython.pipeline`, and its `main()` is +run from the built extension. Without the option the column reports n/a. """ import argparse import datetime @@ -22,6 +26,7 @@ ROOT = Path(__file__).resolve().parent TARGET = ROOT / "target" +TP_BUILD = ROOT / "tp-build" JAVA_OUT = ROOT / "java-out" CARGO = os.environ.get("CARGO", "/Users/ibrew/.cargo/bin/cargo") NODE = os.environ.get("NODE", "/Users/ibrew/.local/bin/node") @@ -54,16 +59,61 @@ def build() -> None: run_checked(["javac", "-d", str(JAVA_OUT), *sources]) -def commands(name: str, size: str, python: str, java_source: bool) -> dict[str, list[str]]: +COMPILE_SNIPPET = """ +import json, sys +from pathlib import Path +from typedpython import pipeline +r = pipeline.compile_module(Path(sys.argv[1]), Path(sys.argv[2])) +print(json.dumps({"extension": str(r.extension) if r.extension else None, + "compiled": [f.name for f in r.module.functions], "skipped": r.skipped})) +""" + +RUN_SNIPPET = """ +import importlib.machinery, importlib.util, sys +name, path = sys.argv[1], sys.argv[2] +loader = importlib.machinery.ExtensionFileLoader(name, path) +spec = importlib.util.spec_from_file_location(name, path, loader=loader) +mod = importlib.util.module_from_spec(spec) +loader.exec_module(mod) +sys.argv = [name] + sys.argv[3:] +mod.main() +""" + + +def build_compiled(tp_dir: Path, tp_python: str) -> dict[str, tuple[str, list[str]] | str]: + """Compile each benchmark with TypedPython; name -> (extension, compiled functions) or a reason.""" + built: dict[str, tuple[str, list[str]] | str] = {} + env = dict(os.environ, PYTHONPATH=str(tp_dir)) + for name, (py, *_rest) in BENCHMARKS.items(): + work = TP_BUILD / name + work.mkdir(parents=True, exist_ok=True) + source = work / py + source.write_text("# typedpython: compiled\n" + (ROOT / "py" / py).read_text()) + result = subprocess.run([tp_python, "-c", COMPILE_SNIPPET, str(source), str(work / "out")], + capture_output=True, text=True, env=env) + if result.returncode != 0: + built[name] = f"n/a (compile failed: {result.stderr.strip().splitlines()[-1:]})" + continue + info = json.loads(result.stdout.strip().splitlines()[-1]) + built[name] = (info["extension"], info["compiled"]) if info["extension"] else "n/a (nothing compiled)" + return built + + +def commands(name: str, size: str, python: str, java_source: bool, + compiled: dict[str, tuple[str, list[str]] | str] | None = None) -> dict[str, list[str]]: py, js, cls, rs, _, _ = BENCHMARKS[name] java = ["java", str(ROOT / "java" / f"{cls}.java"), size] if java_source \ else ["java", "-cp", str(JAVA_OUT), cls, size] - return { + cmds = { "CPython": [python, str(ROOT / "py" / py), size], "Node": [NODE, str(ROOT / "js" / js), size], "Java": java, "Rust": [str(TARGET / "release" / rs), size], } + entry = (compiled or {}).get(name) + if isinstance(entry, tuple): + cmds["TypedPython @compiled"] = [python, "-c", RUN_SNIPPET, Path(py).stem, entry[0], size] + return cmds def run_once(cmd: list[str]) -> tuple[float, str]: @@ -78,9 +128,11 @@ def run_once(cmd: list[str]) -> tuple[float, str]: def verify(args: argparse.Namespace) -> int: build() + compiled = build_compiled(Path(args.typedpython), args.typedpython_python) if args.typedpython else None bad = 0 for name, spec in BENCHMARKS.items(): - outputs = {col: run_once(cmd)[1] for col, cmd in commands(name, spec[4], args.python, True).items()} + outputs = {col: run_once(cmd)[1] + for col, cmd in commands(name, spec[4], args.python, True, compiled).items()} distinct = set(outputs.values()) status = "OK " if len(distinct) == 1 else "FAIL" print(f"{status} {name} (size {spec[4]}): {next(iter(outputs.values())) if len(distinct) == 1 else ''}") @@ -100,19 +152,25 @@ def measure(args: argparse.Namespace) -> int: print(f"refusing to measure: 1-minute load {load[0]:.2f} > {LOAD_LIMIT} (use --force to override)") return 2 build() + compiled = build_compiled(Path(args.typedpython), args.typedpython_python) if args.typedpython else None results: dict[str, dict[str, object]] = {} for name, spec in BENCHMARKS.items(): size = args.size.get(name, spec[5]) row: dict[str, object] = {"size": size} - for col, cmd in commands(name, size, args.python, False).items(): + entry = (compiled or {}).get(name, NOT_BUILT) + if isinstance(entry, tuple): + row["compiled_functions"] = entry[1] + for col, cmd in commands(name, size, args.python, False, compiled).items(): runs = [run_once(cmd) for _ in range(args.repeat)] outs = {out for _, out in runs} row[col] = {"median_s": statistics.median(t for t, _ in runs), "runs_s": [t for t, _ in runs], "output": sorted(outs)} if len(outs) != 1: print(f"warning: {name}/{col} produced differing outputs across runs") - row["TypedPython @compiled"] = NOT_BUILT - outs_all = {tuple(row[c]["output"]) for c in COLUMNS[:4]} # type: ignore[index] + if not isinstance(entry, tuple): + row["TypedPython @compiled"] = entry + measured = [c for c in COLUMNS if isinstance(row.get(c), dict)] + outs_all = {tuple(row[c]["output"]) for c in measured} # type: ignore[index] row["outputs_agree"] = len(outs_all) == 1 results[name] = row print(f"measured {name}", file=sys.stderr) @@ -121,9 +179,10 @@ def measure(args: argparse.Namespace) -> int: print("| benchmark | size | " + " | ".join(COLUMNS) + " |") print("|---|---|" + "---|" * len(COLUMNS)) for name, row in results.items(): - cells = [f"{row[c]['median_s']:.3f} s" for c in COLUMNS[:4]] # type: ignore[index] + cells = [f"{row[c]['median_s']:.3f} s" if isinstance(row.get(c), dict) else str(row.get(c, NOT_BUILT)) + for c in COLUMNS] # type: ignore[index] flag = "" if row["outputs_agree"] else " (OUTPUTS DIFFER)" - print(f"| {name}{flag} | {row['size']} | " + " | ".join(cells + [NOT_BUILT]) + " |") + print(f"| {name}{flag} | {row['size']} | " + " | ".join(cells) + " |") print(f"\nmedian of {args.repeat} runs, wall time; Java excludes javac, includes JVM startup.") if args.write: @@ -146,6 +205,11 @@ def main() -> int: v.add_argument("--python", default=sys.executable) m = sub.add_parser("measure") m.add_argument("--python", default=sys.executable) + for p in (v, m): + p.add_argument("--typedpython", metavar="DIR", + help="directory holding the typedpython package; enables the @compiled column") + p.add_argument("--typedpython-python", default=sys.executable, + help="interpreter for compiling (needs pyrefly 1.3.2)") m.add_argument("--repeat", type=int, default=5) m.add_argument("--force", action="store_true") m.add_argument("--write", action="store_true")