From 067600c77ed5678cbd057120ecb29bb9e57d3187 Mon Sep 17 00:00:00 2001 From: Rachith Gattu Date: Mon, 14 Sep 2026 18:55:31 -0400 Subject: [PATCH 1/2] docs: fix rolling_trust_score description (it's not row-weighted) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The streaming docs say `rolling_trust_score` is "row-weighted trust over the recent window," but that's not actually what the code does — it's just a plain mean of the last `rolling_trust_window` batch scores. So a 1-row batch ends up counting exactly as much as a 999-row batch, which isn't what "row-weighted" implies. I checked git blame and the doc line and the implementation both came from the same original commit, so this isn't a regression — the docs have just been wrong since day one. The config docstring for `rolling_trust_window` already says "averaged," which matches the code, so this really is just a doc bug. To confirm, I ran the repro from the issue: - 1-row batch (all missing) → 0 trust - 999-row batch (clean) → 100 trust - rolling comes out to 50.0, not ~100 like row-weighting would give `cumulative_trust_score` on the other hand IS genuinely row-weighted, so I left that line alone and only fixed the `rolling_trust_score` row. Changes: - Corrected the docs table row to describe what the metric actually does - Added two tests pinning down the exact numbers from the issue's repro, so this doesn't quietly drift again - Changelog entry No changes to the actual trust-score logic — that'd be a real behavior change and probably deserves its own discussion if it's ever wanted. Fixes #349 Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 5 +++++ docs/streaming.md | 2 +- tests/test_streaming_state.py | 41 +++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cc50e0d..3f469028 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,6 +96,11 @@ adheres to [Semantic Versioning](https://semver.org/). - The missing-pyarrow error names the feature that needs it (for example Arrow output), and Parquet metadata reads no longer fail with `AttributeError` in a fresh process (#215). +- Docs: the streaming per-batch field table described `rolling_trust_score` as + row-weighted. It is an unweighted mean of the per-batch scores in the window, + so a 1-row batch counts as much as a 999-row one; only + `cumulative_trust_score` is row-weighted. Documentation fix only — the score + is unchanged (#349). ## [2.0.0] - 2026-07-20 diff --git a/docs/streaming.md b/docs/streaming.md index 8db5cc2e..56b74a9f 100644 --- a/docs/streaming.md +++ b/docs/streaming.md @@ -97,7 +97,7 @@ in `report.to_dict()`) carries: | `batch_id` | 1-based index of this batch | | `rows_in_batch` / `rows_seen_total` | rows in this batch / cumulative across the stream | | `batch_trust_score` | trust score of this cleaned batch (0-100) | -| `rolling_trust_score` | row-weighted trust over the recent window | +| `rolling_trust_score` | unweighted mean of the last `rolling_trust_window` batch scores | | `cumulative_trust_score` | row-weighted trust over the whole stream | | `schema_drift_detected` | whether drift was flagged for this batch | | `warmup_phase` | whether this batch was still in warmup | diff --git a/tests/test_streaming_state.py b/tests/test_streaming_state.py index 8279792b..faded52b 100644 --- a/tests/test_streaming_state.py +++ b/tests/test_streaming_state.py @@ -2,7 +2,9 @@ import numpy as np import pandas as pd +import pytest +from freshdata.streaming import StreamingCleaner from freshdata.streaming._state import ColumnState, StreamingState from freshdata.streaming._stats import BoundedCounter, ReservoirSampler, Welford @@ -90,6 +92,45 @@ def test_streaming_state_tracks_rows_and_trust(): assert abs(cumulative - (80 * 3 + 90 * 1) / 4) < 1e-9 # rows-weighted +# --- documented behaviour: the rolling score is NOT row-weighted ------------- +# docs/streaming.md called rolling_trust_score "row-weighted trust over the +# recent window", but it is an unweighted mean of the per-batch scores in the +# window — a 1-row batch counts exactly as much as a 999-row one. Only +# cumulative_trust_score is row-weighted. The docs were corrected to match; +# these tests pin the arithmetic so the two cannot drift apart again. + +def test_rolling_trust_is_unweighted_while_cumulative_is_row_weighted(): + state = StreamingState(rolling_trust_window=8) + state.record_trust(0.0, rows=1) # tiny, terrible batch + rolling, cumulative = state.record_trust(100.0, rows=999) # huge, clean batch + + assert rolling == 50.0 # (0 + 100) / 2 — rows ignored + assert abs(cumulative - (0 * 1 + 100 * 999) / 1000) < 1e-9 # 99.9, rows honoured + assert rolling != pytest.approx(cumulative) # the two must not be conflated + + +def test_rolling_trust_ignores_batch_size_end_to_end(): + """The issue's reproduction: a 1-row all-None batch, then 999 clean rows.""" + cleaner = StreamingCleaner(verbose=False) + + _, tiny_rep = cleaner.clean_batch(pd.DataFrame({"a": [None], "b": [None]})) + _, big_rep = cleaner.clean_batch(pd.DataFrame({ + "a": list(range(999)), "b": [float(i) for i in range(999)], + })) + + tiny, big = tiny_rep.streaming, big_rep.streaming + scores = [tiny["batch_trust_score"], big["batch_trust_score"]] + rows = [tiny["rows_in_batch"], big["rows_in_batch"]] + assert scores[0] < scores[1] and rows[0] < rows[1] # the batches really differ + + # Derived from the observed scores, so an unrelated change to how a batch is + # scored will not break this — only a change to the *weighting* will. + assert big["rolling_trust_score"] == pytest.approx(sum(scores) / 2) + assert big["rolling_trust_score"] != pytest.approx( + sum(s * r for s, r in zip(scores, rows)) / sum(rows) + ) + + def test_state_to_dict_is_json_friendly(): state = StreamingState() state.observe_batch(pd.DataFrame({"n": [1.0, 2.0], "c": ["x", "y"]}), From 6901e1cf8b50580cfa6e2e760912d4f6f0400ef4 Mon Sep 17 00:00:00 2001 From: Rachith Gattu Date: Mon, 14 Sep 2026 19:09:26 -0400 Subject: [PATCH 2/2] test: make the 1-row batch survive cleaning in the rolling-trust test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The end-to-end test claimed to reproduce the issue's 1-row-vs-999-row split, but it did not. `_build_rep_config` clears `drop_empty_columns` and leaves `drop_empty_rows` at its default, so the all-None row was dropped before the batch was scored — `rows_in_batch` came back as 0, and the test was really exercising a 0-vs-999 split where the small batch carries no weight under either rule. Pass `drop_empty_rows=False` so the all-None row survives, keeping the issue's literal reproduction rather than substituting a different input. The weak `rows[0] < rows[1]` assertion (which passed on 0 < 999) is now an exact `rows == [1, 999]`, pinning the thing that regressed, and the docstring records why the flag is needed. Verified by temporarily making the rolling score row-weighted: the test now fails 99.97 != 85.0, against a genuine 1-vs-999 split. Thanks to the automated review on the PR for catching this. Co-Authored-By: Claude Opus 5 --- tests/test_streaming_state.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/test_streaming_state.py b/tests/test_streaming_state.py index faded52b..602d0991 100644 --- a/tests/test_streaming_state.py +++ b/tests/test_streaming_state.py @@ -110,8 +110,15 @@ def test_rolling_trust_is_unweighted_while_cumulative_is_row_weighted(): def test_rolling_trust_ignores_batch_size_end_to_end(): - """The issue's reproduction: a 1-row all-None batch, then 999 clean rows.""" - cleaner = StreamingCleaner(verbose=False) + """The issue's reproduction: a 1-row all-None batch, then 999 clean rows. + + ``drop_empty_rows=False`` matters here: the streaming representation pass + clears ``drop_empty_columns`` but keeps the default row dropping, so the + all-None row would otherwise be removed before the batch is scored. That + leaves a 0-row batch, and a 0-vs-999 split cannot demonstrate anything + about weighting — the tiny batch would carry no weight under either rule. + """ + cleaner = StreamingCleaner(verbose=False, drop_empty_rows=False) _, tiny_rep = cleaner.clean_batch(pd.DataFrame({"a": [None], "b": [None]})) _, big_rep = cleaner.clean_batch(pd.DataFrame({ @@ -121,7 +128,8 @@ def test_rolling_trust_ignores_batch_size_end_to_end(): tiny, big = tiny_rep.streaming, big_rep.streaming scores = [tiny["batch_trust_score"], big["batch_trust_score"]] rows = [tiny["rows_in_batch"], big["rows_in_batch"]] - assert scores[0] < scores[1] and rows[0] < rows[1] # the batches really differ + assert rows == [1, 999] # the 1-vs-999 split the issue describes + assert scores[0] < scores[1] # and the batches really do score differently # Derived from the observed scores, so an unrelated change to how a batch is # scored will not break this — only a change to the *weighting* will.