From ab5c5eaf2bf9bae5faecd7a0f7e04f479944fe2f Mon Sep 17 00:00:00 2001 From: Cyril Poder Date: Thu, 24 Sep 2026 03:28:31 +0200 Subject: [PATCH] detect: no cliff past ten thousand open correlations; .not() on a named pattern (varpulis #288, #289) Engine pinned at varpulis 1cb201b. The engine swept every partition's open runs on each event; it now visits only the partitions with a deadline due. bench/detect.sh PROGRAM=sequence against v0.3.4, same session: 30 000 open 2 080 -> 14 500 events/s, 10 000 open 11 900 -> 22 000, 1 000 open 19 600 -> 26 000; 100 000 open runs at 11 000/s in 450 MB. The sizing in bench/README.md and concepts/detects.md is re-measured. And .not(C) on a stream reading a named pattern cancels its runs (it did nothing). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01N3K1TGnWTvwYzt9rKuJXES --- CHANGELOG.md | 18 +++++++++ bench/README.md | 62 ++++++++++++++++++++----------- core/Cargo.lock | 20 +++++----- core/Cargo.toml | 2 +- docs/book/src/concepts/detects.md | 8 ++-- 5 files changed, 73 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b1c476a..dae56a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,24 @@ is `0`, minor versions may carry breaking changes — they are called out here. ## [Unreleased] +### Changed — no cliff past ten thousand open correlations (varpulis #289) +- **A correlation with many sequences open at once no longer falls off a + cliff.** The engine swept every partition's open runs on each event, to + expire them and confirm absences; since varpulis #289 it keeps their + deadlines in order and visits only the partitions with one due. + `bench/detect.sh` (`PROGRAM=sequence`), against v0.3.4 in the same + session: 30 000 open went from 2 080 to 14 500 events/s (and 145 to about + 100 MB), 10 000 from 11 900 to 22 000, 1 000 from 19 600 to 26 000; a + hundred thousand open now runs at 11 000 events/s in 450 MB. The sizing in + `bench/README.md` and `concepts/detects.md` is re-measured: what bounds a + unit is memory, about 4 KB an open sequence. + +### Fixed — a cancellation on a named pattern (varpulis #288) +- **`.not(C)` on a stream that reads a named pattern cancels its runs.** It + was accepted and silently did nothing, so `Unacked.not(Cancel)` still + raised the cancelled orders. It now cancels them as it does on an inline + sequence, per partition when the pattern has one. + ## [0.3.4] — 2026-09-24 ### Fixed — an absence is raised when its deadline passes (varpulis #287) diff --git a/bench/README.md b/bench/README.md index d2a6ee6..02e03a9 100644 --- a/bench/README.md +++ b/bench/README.md @@ -185,34 +185,51 @@ their second. `LAG` sets it directly. | Open at once | Rate | RSS | Unpartitioned | |---:|---:|---:|---:| -| 1 | 24 000 – 26 700/s | 10.9 MB | — | -| 100 | 22 400 – 26 800/s | 11.7 MB | — | -| 1 000 | 16 900 – 17 400/s | 14.2 MB | 4 400 – 4 500/s | -| 10 000 | 10 600 – 11 700/s | 38 – 44 MB | 3 500 – 3 600/s | -| 30 000 | 1 800 – 2 200/s | 130 – 136 MB | — | +| 1 | 28 600/s | 12.5 MB | — | +| 100 | 28 200 – 28 600/s | 13.3 MB | — | +| 1 000 | 25 900 – 26 200/s | 16.2 – 16.5 MB | 4 250 – 4 260/s | +| 10 000 | 21 600 – 22 600/s | 32 – 34 MB | — | +| 30 000 | 14 200 – 14 900/s | 88 – 109 MB | — | +| 100 000 (400 000 events) | 11 100 – 11 200/s | 448 MB | — | Three things fall out of that table. -**`.partition_by` is worth three to four times** as soon as anything is open: +**`.partition_by` is worth about six times** as soon as anything is open: without it the engine has to consider every open run for every event, and at -1 000 open that is 17 400/s against 4 500. Partition every correlation, on +1 000 open that is 26 000/s against 4 300. Partition every correlation, on the field the pair shares. -**An open correlation costs about 2.8 KB.** Ten thousand of them is 28 MB on -top of the unit's 11. - -**Past ten thousand open runs there is a cliff**: 30 000 open falls to -2 000/s and 130 MB — five times slower for three times the state. It is a -real limit of the current engine, not a measurement artefact, and it is the -number to stay under until it is fixed. +**What bounds a unit is memory, not a cliff.** An open correlation costs +about 2 KB at ten thousand and 4.5 KB at a hundred thousand; the rate falls +gently with the runs each partition carries (here 500 hosts, so 200 runs a +partition at a hundred thousand open). + +**The cliff this table used to show is gone.** Up to v0.3.4 the engine swept +every partition's open runs on each event, to expire them and to confirm +absences: 30 000 open fell to 2 000/s and 145 MB. Since varpulis #289 it +keeps their deadlines in order and visits only the partitions with one due. +Measured against the v0.3.4 runtime in the same session, same box: + +| Open at once | v0.3.4 | now | +|---:|---:|---:| +| 1 | 28 400 – 28 500/s | 28 600/s | +| 100 | 27 100 – 27 200/s | 28 200 – 28 600/s | +| 1 000 | 19 500 – 19 600/s | 25 900 – 26 200/s | +| 10 000 | 11 900/s | 21 600 – 22 600/s | +| 30 000 | 2 075 – 2 085/s (145 MB) | 14 200 – 14 900/s (88 – 109 MB) | +| 1 000, unpartitioned | 4 280 – 4 290/s | 4 250 – 4 260/s | Correlations spread across units the way flows do — 1 000 open in each: | Units | Aggregate | Per unit | RSS | |---|---:|---:|---:| -| 1 | 17 000/s | 17 000/s | 14.2 MB | -| 2 | 29 100/s | 14 500/s | 19.1 MB | -| 4 | 37 900/s | 9 500/s | 27.2 MB | +| 1 | 27 100 – 27 200/s | 27 100/s | 16.4 MB | +| 2 | 37 200 – 39 400/s | 18 600 – 19 700/s | 21.1 – 21.5 MB | +| 4 | 48 300 – 48 900/s | 12 100 – 12 200/s | 28.4 – 29.1 MB | + +Four correlation units go past the stateless ceiling above because they emit +one alert per pair, where the stateless rule emits one per event: publishing +is part of what saturates a process. ### Sizing, then @@ -224,13 +241,14 @@ open at once = first steps per second x how long a first step waits A rule whose first step fires 200 times a second and whose second step typically follows within ten seconds carries 2 000 open runs. From the table, -one unit handles that at around 15 000 events/s in 15 MB. +one unit handles that at around 25 000 events/s in 18 MB. - **A stateless rule:** budget 30 000 events/s and 10 MB per unit. -- **A correlation under 100 open:** 25 000 events/s, 11 MB. -- **At 1 000 open:** 17 000 events/s, 14 MB. **At 10 000:** 11 000 events/s, - 40 MB. Do not plan past that on one unit. -- **Always partition a correlation** — three to four times the throughput. +- **A correlation under 100 open:** 28 000 events/s, 13 MB. +- **At 1 000 open:** 26 000 events/s, 16 MB. **At 10 000:** 22 000 events/s, + 33 MB. **At 100 000:** 11 000 events/s, 450 MB: plan on memory, about 4 KB + an open run. +- **Always partition a correlation** — about six times the throughput. - **One instance saturates near 44 000 events/s** whatever the unit count. Beyond it, add instances: they share the durable consumers and the work (see *Clustering*). diff --git a/core/Cargo.lock b/core/Cargo.lock index d0fd33f..8cecf50 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -1846,7 +1846,7 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "varpulis-core" version = "0.11.0" -source = "git+https://github.com/varpulis/varpulis?rev=21f3ba49220615a926b1d96c16075fe11c67e3b9#21f3ba49220615a926b1d96c16075fe11c67e3b9" +source = "git+https://github.com/varpulis/varpulis?rev=1cb201b6b11b2808eaf03dfebf1027997dd52f5a#1cb201b6b11b2808eaf03dfebf1027997dd52f5a" dependencies = [ "chrono", "indexmap", @@ -1863,7 +1863,7 @@ dependencies = [ [[package]] name = "varpulis-dead-letter" version = "0.11.0" -source = "git+https://github.com/varpulis/varpulis?rev=21f3ba49220615a926b1d96c16075fe11c67e3b9#21f3ba49220615a926b1d96c16075fe11c67e3b9" +source = "git+https://github.com/varpulis/varpulis?rev=1cb201b6b11b2808eaf03dfebf1027997dd52f5a#1cb201b6b11b2808eaf03dfebf1027997dd52f5a" dependencies = [ "chrono", "serde", @@ -1875,7 +1875,7 @@ dependencies = [ [[package]] name = "varpulis-engine" version = "0.11.0" -source = "git+https://github.com/varpulis/varpulis?rev=21f3ba49220615a926b1d96c16075fe11c67e3b9#21f3ba49220615a926b1d96c16075fe11c67e3b9" +source = "git+https://github.com/varpulis/varpulis?rev=1cb201b6b11b2808eaf03dfebf1027997dd52f5a#1cb201b6b11b2808eaf03dfebf1027997dd52f5a" dependencies = [ "serde_json", "thiserror", @@ -1887,7 +1887,7 @@ dependencies = [ [[package]] name = "varpulis-hamlet" version = "0.11.0" -source = "git+https://github.com/varpulis/varpulis?rev=21f3ba49220615a926b1d96c16075fe11c67e3b9#21f3ba49220615a926b1d96c16075fe11c67e3b9" +source = "git+https://github.com/varpulis/varpulis?rev=1cb201b6b11b2808eaf03dfebf1027997dd52f5a#1cb201b6b11b2808eaf03dfebf1027997dd52f5a" dependencies = [ "rustc-hash", "smallvec", @@ -1898,7 +1898,7 @@ dependencies = [ [[package]] name = "varpulis-parser" version = "0.11.0" -source = "git+https://github.com/varpulis/varpulis?rev=21f3ba49220615a926b1d96c16075fe11c67e3b9#21f3ba49220615a926b1d96c16075fe11c67e3b9" +source = "git+https://github.com/varpulis/varpulis?rev=1cb201b6b11b2808eaf03dfebf1027997dd52f5a#1cb201b6b11b2808eaf03dfebf1027997dd52f5a" dependencies = [ "logos", "miette", @@ -1911,7 +1911,7 @@ dependencies = [ [[package]] name = "varpulis-pst" version = "0.11.0" -source = "git+https://github.com/varpulis/varpulis?rev=21f3ba49220615a926b1d96c16075fe11c67e3b9#21f3ba49220615a926b1d96c16075fe11c67e3b9" +source = "git+https://github.com/varpulis/varpulis?rev=1cb201b6b11b2808eaf03dfebf1027997dd52f5a#1cb201b6b11b2808eaf03dfebf1027997dd52f5a" dependencies = [ "hashlink", "rustc-hash", @@ -1921,7 +1921,7 @@ dependencies = [ [[package]] name = "varpulis-runtime" version = "0.11.0" -source = "git+https://github.com/varpulis/varpulis?rev=21f3ba49220615a926b1d96c16075fe11c67e3b9#21f3ba49220615a926b1d96c16075fe11c67e3b9" +source = "git+https://github.com/varpulis/varpulis?rev=1cb201b6b11b2808eaf03dfebf1027997dd52f5a#1cb201b6b11b2808eaf03dfebf1027997dd52f5a" dependencies = [ "chrono", "hashlink", @@ -1949,7 +1949,7 @@ dependencies = [ [[package]] name = "varpulis-sase" version = "0.11.0" -source = "git+https://github.com/varpulis/varpulis?rev=21f3ba49220615a926b1d96c16075fe11c67e3b9#21f3ba49220615a926b1d96c16075fe11c67e3b9" +source = "git+https://github.com/varpulis/varpulis?rev=1cb201b6b11b2808eaf03dfebf1027997dd52f5a#1cb201b6b11b2808eaf03dfebf1027997dd52f5a" dependencies = [ "chrono", "rustc-hash", @@ -1961,7 +1961,7 @@ dependencies = [ [[package]] name = "varpulis-simd" version = "0.11.0" -source = "git+https://github.com/varpulis/varpulis?rev=21f3ba49220615a926b1d96c16075fe11c67e3b9#21f3ba49220615a926b1d96c16075fe11c67e3b9" +source = "git+https://github.com/varpulis/varpulis?rev=1cb201b6b11b2808eaf03dfebf1027997dd52f5a#1cb201b6b11b2808eaf03dfebf1027997dd52f5a" dependencies = [ "varpulis-core", ] @@ -1969,7 +1969,7 @@ dependencies = [ [[package]] name = "varpulis-zdd" version = "0.11.0" -source = "git+https://github.com/varpulis/varpulis?rev=21f3ba49220615a926b1d96c16075fe11c67e3b9#21f3ba49220615a926b1d96c16075fe11c67e3b9" +source = "git+https://github.com/varpulis/varpulis?rev=1cb201b6b11b2808eaf03dfebf1027997dd52f5a#1cb201b6b11b2808eaf03dfebf1027997dd52f5a" dependencies = [ "rustc-hash", ] diff --git a/core/Cargo.toml b/core/Cargo.toml index c17a894..d418666 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -22,7 +22,7 @@ webpki-roots = "0.26" # The Varpulis CEP engine, embedded as a library (ADR-0031): compile a VPL # program, feed it events, publish its emits. No async runtime in its tree — # its own CI (scripts/check-engine-deps.py) fails if one ever appears. -varpulis-engine = { git = "https://github.com/varpulis/varpulis", rev = "21f3ba49220615a926b1d96c16075fe11c67e3b9" } +varpulis-engine = { git = "https://github.com/varpulis/varpulis", rev = "1cb201b6b11b2808eaf03dfebf1027997dd52f5a" } [[bin]] name = "vejas-runtime" diff --git a/docs/book/src/concepts/detects.md b/docs/book/src/concepts/detects.md index 3221011..1e077df 100644 --- a/docs/book/src/concepts/detects.md +++ b/docs/book/src/concepts/detects.md @@ -129,10 +129,10 @@ waiting for their second: open at once = first steps per second x how long a first step waits ``` -Under a hundred open, a unit keeps 25 000 events/s; at a thousand, 17 000; -at ten thousand, 11 000 and about 40 MB. Past that it falls away sharply, so -ten thousand open sequences is the number to stay under on one unit. -`.partition_by` on the field a pair shares is worth three to four times the +Under a hundred open, a unit keeps 28 000 events/s; at a thousand, 26 000; +at ten thousand, 22 000 in about 33 MB; at a hundred thousand, 11 000 in +450 MB. What bounds a unit is memory, about 4 KB an open sequence. +`.partition_by` on the field a pair shares is worth about six times the throughput as soon as anything is open — partition every correlation. One process saturates near 44 000 events/s whatever the unit count; beyond