From 42fc3068f790fa54014b2d468ed6eeb8845e203f Mon Sep 17 00:00:00 2001 From: AJ Rotolo Date: Thu, 24 Sep 2026 14:01:53 -0500 Subject: [PATCH 1/2] fix: keep string leaves out of JSON_EACH in list-member filters The list-member overload, Filter(type, x => x.Items, x => x.Member, value), walks the list with JSON_TREE and handed every node it yielded to JSON_EACH. JSON_TREE yields the scalar leaves as well as the element objects, and a string leaf -- any string member, or a DateTime, which both serializers write as text -- is not JSON, so JSON_EACH raised SQLite error 1 "malformed JSON". EXISTS stops at the first element that satisfies the predicate, so the error only surfaced for documents whose elements never match: the filter appeared to work until the store held a row it had to reject. The one existing test never set a string member, so it never reached a leaf. JSON_EACH now receives its input through a CASE that yields NULL for any node that is not an object, and JSON_EACH on NULL produces no rows. This is safe by SQL semantics rather than by planner placement: a JT.type conjunct in the WHERE clause is also coded in the outer loop today, but the CASE form does not depend on that. Arrays are excluded along with scalars, since a $.Member path cannot resolve on one, and JSON_TREE still visits nested lists of objects, so the depth searched is unchanged. Writing the NotEquals test surfaced a second defect on the same path: a DateTime value was rendered with ToString(), "1/1/2026 8:00:00 AM" under en-US, against a stored "2026-01-01T08:00:00Z", so no element ever compared equal and every document with a non-empty list matched. NotEquals now formats through DateTimeSerializationFormat as Equals already did. The top-level NotEquals branch has the same defect and is left for a separate change. The shape dates from the original inner-list commit and first shipped in v2.0.0. Tests: 318 passed, 4 pre-existing skips; 8 new. --- CHANGELOG.md | 19 +++ TychoDB.UnitTests/ListMemberFilterTests.cs | 143 +++++++++++++++++++++ TychoDB/FilterBuilder.cs | 7 +- 3 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 TychoDB.UnitTests/ListMemberFilterTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d0930a..a97a32e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,25 @@ transaction it is rolled back, without one the single failing statement is atomic on its own. (#32) +### Fixed + +- **List-member filters raised `malformed JSON` on documents with string-valued elements.** + The `Filter(FilterType, x => x.Items, x => x.Member, value)` overload walks the list with + `JSON_TREE` and handed every node it yielded to `JSON_EACH`. `JSON_TREE` yields the scalar + leaves as well as the element objects, and a string leaf — any `string` member, or a + `DateTime`, which both serializers write as text — is not JSON, so `JSON_EACH` failed with + SQLite error 1 `malformed JSON`. `EXISTS` stops at the first element that satisfies the + predicate, so the error surfaced only for documents whose elements never match: a filter + appeared to work until the store held a row it had to reject. Only object nodes are now + handed to `JSON_EACH`; scalar and array nodes contribute nothing, and nested lists of objects + are still searched. Present since the overload was introduced. +- **`NotEquals` on a `DateTime` list member compared against the culture-dependent + `ToString()` form.** `Filter(FilterType.NotEquals, x => x.Items, x => x.When, value)` rendered + the value as `DateTime.ToString()` (`1/1/2026 8:00:00 AM` under `en-US`) while the document + stores the serializer's format (`2026-01-01T08:00:00Z`), so no element ever compared equal and + every document with a non-empty list matched. The value is now formatted with the + serializer's `DateTimeSerializationFormat`, as `Equals` already was. + ## 5.0.0 — 2026-07-21 — Security & performance hardening This release closes a critical SQL-injection vector and a data-integrity bug, and diff --git a/TychoDB.UnitTests/ListMemberFilterTests.cs b/TychoDB.UnitTests/ListMemberFilterTests.cs new file mode 100644 index 0000000..66d2a44 --- /dev/null +++ b/TychoDB.UnitTests/ListMemberFilterTests.cs @@ -0,0 +1,143 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Linq.Expressions; +using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Shouldly; + +namespace TychoDB.UnitTests; + +/// +/// The list-member overload of Filter walks every node under the list, string leaves +/// included, and hands each one to JSON_EACH, which accepts only JSON. The walk stops +/// at the first element that satisfies the predicate, so only a document whose elements never +/// match makes it reach a leaf. Every element seeded here carries a serialized +/// and a plain string, and every filter has a document it must reject. +/// +[TestClass] +public class ListMemberFilterTests +{ + private static readonly DateTime FirstDay = new(2026, 1, 1, 8, 0, 0, DateTimeKind.Utc); + private static readonly DateTime SecondDay = new(2026, 1, 2, 8, 0, 0, DateTimeKind.Utc); + private static readonly DateTime ThirdDay = new(2026, 1, 3, 8, 0, 0, DateTimeKind.Utc); + + public static IEnumerable Serializers + { + get + { + yield return new object[] { new SystemTextJsonSerializer() }; + yield return new object[] { new NewtonsoftJsonSerializer() }; + } + } + + [TestMethod] + [DynamicData(nameof(Serializers))] + public async Task Equals_OnDateTimeMember_MatchesDocumentsWithSuchAnElement(IJsonSerializer jsonSerializer) + { + using var db = await ConnectAsync(jsonSerializer); + + var ids = await ReadIdsAsync(db, FilterType.Equals, x => x.When, FirstDay); + + ids.ShouldBe(new[] { "mixed", "uniform" }); + } + + [TestMethod] + [DynamicData(nameof(Serializers))] + public async Task NotEquals_OnDateTimeMember_MatchesDocumentsWithSuchAnElement(IJsonSerializer jsonSerializer) + { + using var db = await ConnectAsync(jsonSerializer); + + var ids = await ReadIdsAsync(db, FilterType.NotEquals, x => x.When, FirstDay); + + ids.ShouldBe(new[] { "mixed", "other" }); + } + + [TestMethod] + [DynamicData(nameof(Serializers))] + public async Task Equals_OnNumericMember_MatchesDocumentsWithSuchAnElement(IJsonSerializer jsonSerializer) + { + using var db = await ConnectAsync(jsonSerializer); + + var ids = await ReadIdsAsync(db, FilterType.Equals, x => x.Amount, 1.5d); + + ids.ShouldBe(new[] { "mixed", "uniform" }); + } + + [TestMethod] + [DynamicData(nameof(Serializers))] + public async Task NotEquals_OnNumericMember_MatchesDocumentsWithSuchAnElement(IJsonSerializer jsonSerializer) + { + using var db = await ConnectAsync(jsonSerializer); + + var ids = await ReadIdsAsync(db, FilterType.NotEquals, x => x.Amount, 1.5d); + + ids.ShouldBe(new[] { "mixed", "other" }); + } + + private static async Task ConnectAsync(IJsonSerializer jsonSerializer) + { + var db = new Tycho(Path.GetTempPath(), jsonSerializer, dbName: $"{Guid.NewGuid()}.db", rebuildCache: true, requireTypeRegistration: false); + await db.ConnectAsync(); + await db.WriteObjectsAsync(Seed(), x => x.Id); + + return db; + } + + private static async Task ReadIdsAsync(Tycho db, FilterType filterType, Expression> member, object value) + { + var results = + await db.ReadObjectsAsync( + filter: FilterBuilder.Create().Filter(filterType, x => x.Entries, member, value)); + + return results.Select(x => x.Id).OrderBy(x => x, StringComparer.Ordinal).ToArray(); + } + + private static Ledger[] Seed() => + new[] + { + new Ledger + { + Id = "mixed", + Entries = new List + { + new() { When = FirstDay, Amount = 1.5d, Note = "opening" }, + new() { When = SecondDay, Amount = 2d, Note = "deposit" }, + }, + }, + new Ledger + { + Id = "other", + Entries = new List + { + new() { When = ThirdDay, Amount = 3d, Note = "closing" }, + }, + }, + new Ledger + { + Id = "uniform", + Entries = new List + { + new() { When = FirstDay, Amount = 1.5d, Note = "opening" }, + new() { When = FirstDay, Amount = 1.5d, Note = "repeat" }, + }, + }, + }; + + public class Ledger + { + public string Id { get; set; } + + public List Entries { get; set; } + } + + public class Entry + { + public DateTime When { get; set; } + + public double Amount { get; set; } + + public string Note { get; set; } + } +} diff --git a/TychoDB/FilterBuilder.cs b/TychoDB/FilterBuilder.cs index 0387360..c4a6296 100644 --- a/TychoDB/FilterBuilder.cs +++ b/TychoDB/FilterBuilder.cs @@ -19,7 +19,7 @@ public class FilterBuilder private const string CastNumericPrefix = "CAST(JSON_EXTRACT(Data, '"; private const string CastNumericSuffix = "') as NUMERIC)"; private const string ExistsPrefix = "EXISTS(SELECT 1 FROM JSON_TREE(Data, '"; - private const string ExistsMiddle = "') AS JT, JSON_EACH(JT.Value, '"; + private const string ExistsMiddle = "') AS JT, JSON_EACH(CASE JT.type WHEN 'object' THEN JT.Value END, '"; private const string ExistsSuffix = "') AS VAL WHERE "; private const string ExistsEnd = ")"; private const string ValValue = "VAL.value"; @@ -705,6 +705,11 @@ private void BuildExistsFilter(StringBuilder commandBuilder, in Filter filter, I { commandBuilder.Append(ValValue).Append(IsNotNull).Append(ExistsEnd).AppendLine(); } + else if (filter.IsPropertyValuePathDateTime) + { + var dateTimeString = GetDateTimeString(filter.Value, jsonSerializer); + commandBuilder.Append(ValValue).Append(NotEquals).Append(parameters.Add(dateTimeString)).Append(ExistsEnd).AppendLine(); + } else { commandBuilder.Append(ValValue).Append(NotEquals); From fe328995d3f6b7275c363e211e6c407335afa91e Mon Sep 17 00:00:00 2001 From: Michael Stonis <120685+michaelstonis@users.noreply.github.com> Date: Mon, 5 Oct 2026 13:24:56 -0500 Subject: [PATCH 2/2] fix: keep lists of scalars filterable in list-member filters Passing only object nodes to JSON_EACH dropped the array node itself, so a list of scalars filtered on its own values (x => x.Numbers, x => x) returned no rows where v5.3.0 returned the matches. Array nodes are passed again; only scalar leaves are withheld, which also makes lists of strings filterable instead of raising malformed JSON. Adds tests for numeric and string lists and for a string member whose text is JSON, and moves the changelog entries from the released 5.3.0 section into a new 5.3.1 section. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 52 +++++++----- TychoDB.UnitTests/ListMemberFilterTests.cs | 96 ++++++++++++++++++++++ TychoDB/FilterBuilder.cs | 2 +- 3 files changed, 130 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a97a32e..17cb136 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,21 +1,6 @@ # Changelog -## 5.3.0 (unreleased) - -### Added - -- **`UpsertObjectAsync` reports whether the write inserted or updated.** Returns - `UpsertResult.Inserted` when no row existed for the key in that partition and - `UpsertResult.Updated` when one did and its data was replaced; stored contents are identical - to `WriteObjectAsync`. Implemented as `INSERT OR IGNORE` plus, only when nothing was inserted, - an in-place `UPDATE`, both under the connection gate and the transaction — so callers that keep - an incremental view of the store (a queue count, an added/removed signal) no longer need a - read-then-write pair guarded by a lock of their own. Registered-id and explicit key-selector - overloads, same strict-mode divergence guard. There is deliberately no failure value: any - failure (insert ignored for another constraint, update not affecting exactly one row, a - serializer or SQLite error) throws `TychoException` and leaves the row as it was — with a - transaction it is rolled back, without one the single failing statement is atomic on its own. - (#32) +## 5.3.1 (unreleased) ### Fixed @@ -26,15 +11,44 @@ `DateTime`, which both serializers write as text — is not JSON, so `JSON_EACH` failed with SQLite error 1 `malformed JSON`. `EXISTS` stops at the first element that satisfies the predicate, so the error surfaced only for documents whose elements never match: a filter - appeared to work until the store held a row it had to reject. Only object nodes are now - handed to `JSON_EACH`; scalar and array nodes contribute nothing, and nested lists of objects + appeared to work until the store held a row it had to reject. Only object and array nodes + are now handed to `JSON_EACH`; scalar leaves contribute nothing, and nested lists of objects are still searched. Present since the overload was introduced. + - **A string member whose text is JSON no longer matches as if it were an element.** A + string leaf that happened to parse — `Note = "{\"Amount\":99}"` — was searched like a list + element, so `Filter(FilterType.Equals, x => x.Items, x => x.Amount, 99)` returned a document + with no such element. Stored text can no longer decide which documents a list-member + filter matches, or which `DeleteObjectsAsync` removes. + - **Lists of strings can be filtered on their own values.** + `Filter(FilterType.Equals, x => x.Tags, x => x, "red")` raised the same `malformed JSON` + for any document it had to reject. It now returns the documents holding the value, as the + same call over a list of numbers or booleans already did. - **`NotEquals` on a `DateTime` list member compared against the culture-dependent `ToString()` form.** `Filter(FilterType.NotEquals, x => x.Items, x => x.When, value)` rendered the value as `DateTime.ToString()` (`1/1/2026 8:00:00 AM` under `en-US`) while the document stores the serializer's format (`2026-01-01T08:00:00Z`), so no element ever compared equal and every document with a non-empty list matched. The value is now formatted with the - serializer's `DateTimeSerializationFormat`, as `Equals` already was. + serializer's `DateTimeSerializationFormat`, as `Equals` already was. `NotEquals` on a list + member matches a document when *some* element differs from the value, not when no element + equals it; that meaning is unchanged, but it is now observable on lists whose elements hold + strings or dates. + +## 5.3.0 — 2026-09-02 + +### Added + +- **`UpsertObjectAsync` reports whether the write inserted or updated.** Returns + `UpsertResult.Inserted` when no row existed for the key in that partition and + `UpsertResult.Updated` when one did and its data was replaced; stored contents are identical + to `WriteObjectAsync`. Implemented as `INSERT OR IGNORE` plus, only when nothing was inserted, + an in-place `UPDATE`, both under the connection gate and the transaction — so callers that keep + an incremental view of the store (a queue count, an added/removed signal) no longer need a + read-then-write pair guarded by a lock of their own. Registered-id and explicit key-selector + overloads, same strict-mode divergence guard. There is deliberately no failure value: any + failure (insert ignored for another constraint, update not affecting exactly one row, a + serializer or SQLite error) throws `TychoException` and leaves the row as it was — with a + transaction it is rolled back, without one the single failing statement is atomic on its own. + (#32) ## 5.0.0 — 2026-07-21 — Security & performance hardening diff --git a/TychoDB.UnitTests/ListMemberFilterTests.cs b/TychoDB.UnitTests/ListMemberFilterTests.cs index 66d2a44..f9d7f1e 100644 --- a/TychoDB.UnitTests/ListMemberFilterTests.cs +++ b/TychoDB.UnitTests/ListMemberFilterTests.cs @@ -76,6 +76,69 @@ public async Task NotEquals_OnNumericMember_MatchesDocumentsWithSuchAnElement(IJ ids.ShouldBe(new[] { "mixed", "other" }); } + [TestMethod] + [DynamicData(nameof(Serializers))] + public async Task Equals_OnStringMemberHoldingJsonText_DoesNotSearchInsideTheString(IJsonSerializer jsonSerializer) + { + using var db = await ConnectAsync(jsonSerializer); + + // No element has Amount 99; only this Note, a string, spells one out. + var embedded = new Ledger + { + Id = "embedded", + Entries = new List { new() { When = ThirdDay, Amount = 3d, Note = "{\"Amount\":99}" } }, + }; + await db.WriteObjectAsync(embedded, x => x.Id); + + var ids = await ReadIdsAsync(db, FilterType.Equals, x => x.Amount, 99d); + + ids.ShouldBeEmpty(); + } + + [TestMethod] + [DynamicData(nameof(Serializers))] + public async Task Equals_OnNumericListItself_MatchesDocumentsHoldingTheValue(IJsonSerializer jsonSerializer) + { + using var db = await ConnectTalliesAsync(jsonSerializer); + + var ids = await ReadTallyIdsAsync(db, FilterBuilder.Create().Filter(FilterType.Equals, x => x.Numbers, x => x, 5)); + + ids.ShouldBe(new[] { "a" }); + } + + [TestMethod] + [DynamicData(nameof(Serializers))] + public async Task GreaterThan_OnNumericListItself_MatchesDocumentsHoldingSuchAValue(IJsonSerializer jsonSerializer) + { + using var db = await ConnectTalliesAsync(jsonSerializer); + + var ids = await ReadTallyIdsAsync(db, FilterBuilder.Create().Filter(FilterType.GreaterThan, x => x.Numbers, x => x, 2)); + + ids.ShouldBe(new[] { "a", "b" }); + } + + [TestMethod] + [DynamicData(nameof(Serializers))] + public async Task Equals_OnStringListItself_MatchesDocumentsHoldingTheValue(IJsonSerializer jsonSerializer) + { + using var db = await ConnectTalliesAsync(jsonSerializer); + + var ids = await ReadTallyIdsAsync(db, FilterBuilder.Create().Filter(FilterType.Equals, x => x.Tags, x => x, "red")); + + ids.ShouldBe(new[] { "a" }); + } + + [TestMethod] + [DynamicData(nameof(Serializers))] + public async Task Contains_OnStringListItself_MatchesDocumentsHoldingSuchAValue(IJsonSerializer jsonSerializer) + { + using var db = await ConnectTalliesAsync(jsonSerializer); + + var ids = await ReadTallyIdsAsync(db, FilterBuilder.Create().Filter(FilterType.Contains, x => x.Tags, x => x, "re")); + + ids.ShouldBe(new[] { "a", "b" }); + } + private static async Task ConnectAsync(IJsonSerializer jsonSerializer) { var db = new Tycho(Path.GetTempPath(), jsonSerializer, dbName: $"{Guid.NewGuid()}.db", rebuildCache: true, requireTypeRegistration: false); @@ -94,6 +157,30 @@ await db.ReadObjectsAsync( return results.Select(x => x.Id).OrderBy(x => x, StringComparer.Ordinal).ToArray(); } + private static async Task ConnectTalliesAsync(IJsonSerializer jsonSerializer) + { + var db = new Tycho(Path.GetTempPath(), jsonSerializer, dbName: $"{Guid.NewGuid()}.db", rebuildCache: true, requireTypeRegistration: false); + await db.ConnectAsync(); + await db.WriteObjectsAsync(SeedTallies(), x => x.Id); + + return db; + } + + private static async Task ReadTallyIdsAsync(Tycho db, FilterBuilder filter) + { + var results = await db.ReadObjectsAsync(filter: filter); + + return results.Select(x => x.Id).OrderBy(x => x, StringComparer.Ordinal).ToArray(); + } + + private static Tally[] SeedTallies() => + new[] + { + new Tally { Id = "a", Numbers = new List { 1, 5 }, Tags = new List { "red", "blue" } }, + new Tally { Id = "b", Numbers = new List { 2, 3 }, Tags = new List { "green" } }, + new Tally { Id = "c", Numbers = new List { 1 }, Tags = new List { "blue" } }, + }; + private static Ledger[] Seed() => new[] { @@ -132,6 +219,15 @@ public class Ledger public List Entries { get; set; } } + public class Tally + { + public string Id { get; set; } + + public List Numbers { get; set; } + + public List Tags { get; set; } + } + public class Entry { public DateTime When { get; set; } diff --git a/TychoDB/FilterBuilder.cs b/TychoDB/FilterBuilder.cs index c4a6296..8881184 100644 --- a/TychoDB/FilterBuilder.cs +++ b/TychoDB/FilterBuilder.cs @@ -19,7 +19,7 @@ public class FilterBuilder private const string CastNumericPrefix = "CAST(JSON_EXTRACT(Data, '"; private const string CastNumericSuffix = "') as NUMERIC)"; private const string ExistsPrefix = "EXISTS(SELECT 1 FROM JSON_TREE(Data, '"; - private const string ExistsMiddle = "') AS JT, JSON_EACH(CASE JT.type WHEN 'object' THEN JT.Value END, '"; + private const string ExistsMiddle = "') AS JT, JSON_EACH(CASE WHEN JT.type IN ('object', 'array') THEN JT.Value END, '"; private const string ExistsSuffix = "') AS VAL WHERE "; private const string ExistsEnd = ")"; private const string ValValue = "VAL.value";