diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d0930a..17cb136 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,39 @@ # Changelog -## 5.3.0 (unreleased) +## 5.3.1 (unreleased) + +### Fixed + +- **List-member filters raised `malformed JSON` on documents with string-valued elements.** + The `Filter(FilterType, x => x.Items, x => x.Member, value)` overload walks the list with + `JSON_TREE` and handed every node it yielded to `JSON_EACH`. `JSON_TREE` yields the scalar + leaves as well as the element objects, and a string leaf — any `string` member, or a + `DateTime`, which both serializers write as text — is not JSON, so `JSON_EACH` failed with + SQLite error 1 `malformed JSON`. `EXISTS` stops at the first element that satisfies the + predicate, so the error surfaced only for documents whose elements never match: a filter + appeared to work until the store held a row it had to reject. Only object and array nodes + are now handed to `JSON_EACH`; scalar leaves contribute nothing, and nested lists of objects + are still searched. Present since the overload was introduced. + - **A string member whose text is JSON no longer matches as if it were an element.** A + string leaf that happened to parse — `Note = "{\"Amount\":99}"` — was searched like a list + element, so `Filter(FilterType.Equals, x => x.Items, x => x.Amount, 99)` returned a document + with no such element. Stored text can no longer decide which documents a list-member + filter matches, or which `DeleteObjectsAsync` removes. + - **Lists of strings can be filtered on their own values.** + `Filter(FilterType.Equals, x => x.Tags, x => x, "red")` raised the same `malformed JSON` + for any document it had to reject. It now returns the documents holding the value, as the + same call over a list of numbers or booleans already did. +- **`NotEquals` on a `DateTime` list member compared against the culture-dependent + `ToString()` form.** `Filter(FilterType.NotEquals, x => x.Items, x => x.When, value)` rendered + the value as `DateTime.ToString()` (`1/1/2026 8:00:00 AM` under `en-US`) while the document + stores the serializer's format (`2026-01-01T08:00:00Z`), so no element ever compared equal and + every document with a non-empty list matched. The value is now formatted with the + serializer's `DateTimeSerializationFormat`, as `Equals` already was. `NotEquals` on a list + member matches a document when *some* element differs from the value, not when no element + equals it; that meaning is unchanged, but it is now observable on lists whose elements hold + strings or dates. + +## 5.3.0 — 2026-09-02 ### Added diff --git a/TychoDB.UnitTests/ListMemberFilterTests.cs b/TychoDB.UnitTests/ListMemberFilterTests.cs new file mode 100644 index 0000000..f9d7f1e --- /dev/null +++ b/TychoDB.UnitTests/ListMemberFilterTests.cs @@ -0,0 +1,239 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Linq.Expressions; +using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Shouldly; + +namespace TychoDB.UnitTests; + +/// +/// The list-member overload of Filter walks every node under the list, string leaves +/// included, and hands each one to JSON_EACH, which accepts only JSON. The walk stops +/// at the first element that satisfies the predicate, so only a document whose elements never +/// match makes it reach a leaf. Every element seeded here carries a serialized +/// and a plain string, and every filter has a document it must reject. +/// +[TestClass] +public class ListMemberFilterTests +{ + private static readonly DateTime FirstDay = new(2026, 1, 1, 8, 0, 0, DateTimeKind.Utc); + private static readonly DateTime SecondDay = new(2026, 1, 2, 8, 0, 0, DateTimeKind.Utc); + private static readonly DateTime ThirdDay = new(2026, 1, 3, 8, 0, 0, DateTimeKind.Utc); + + public static IEnumerable Serializers + { + get + { + yield return new object[] { new SystemTextJsonSerializer() }; + yield return new object[] { new NewtonsoftJsonSerializer() }; + } + } + + [TestMethod] + [DynamicData(nameof(Serializers))] + public async Task Equals_OnDateTimeMember_MatchesDocumentsWithSuchAnElement(IJsonSerializer jsonSerializer) + { + using var db = await ConnectAsync(jsonSerializer); + + var ids = await ReadIdsAsync(db, FilterType.Equals, x => x.When, FirstDay); + + ids.ShouldBe(new[] { "mixed", "uniform" }); + } + + [TestMethod] + [DynamicData(nameof(Serializers))] + public async Task NotEquals_OnDateTimeMember_MatchesDocumentsWithSuchAnElement(IJsonSerializer jsonSerializer) + { + using var db = await ConnectAsync(jsonSerializer); + + var ids = await ReadIdsAsync(db, FilterType.NotEquals, x => x.When, FirstDay); + + ids.ShouldBe(new[] { "mixed", "other" }); + } + + [TestMethod] + [DynamicData(nameof(Serializers))] + public async Task Equals_OnNumericMember_MatchesDocumentsWithSuchAnElement(IJsonSerializer jsonSerializer) + { + using var db = await ConnectAsync(jsonSerializer); + + var ids = await ReadIdsAsync(db, FilterType.Equals, x => x.Amount, 1.5d); + + ids.ShouldBe(new[] { "mixed", "uniform" }); + } + + [TestMethod] + [DynamicData(nameof(Serializers))] + public async Task NotEquals_OnNumericMember_MatchesDocumentsWithSuchAnElement(IJsonSerializer jsonSerializer) + { + using var db = await ConnectAsync(jsonSerializer); + + var ids = await ReadIdsAsync(db, FilterType.NotEquals, x => x.Amount, 1.5d); + + ids.ShouldBe(new[] { "mixed", "other" }); + } + + [TestMethod] + [DynamicData(nameof(Serializers))] + public async Task Equals_OnStringMemberHoldingJsonText_DoesNotSearchInsideTheString(IJsonSerializer jsonSerializer) + { + using var db = await ConnectAsync(jsonSerializer); + + // No element has Amount 99; only this Note, a string, spells one out. + var embedded = new Ledger + { + Id = "embedded", + Entries = new List { new() { When = ThirdDay, Amount = 3d, Note = "{\"Amount\":99}" } }, + }; + await db.WriteObjectAsync(embedded, x => x.Id); + + var ids = await ReadIdsAsync(db, FilterType.Equals, x => x.Amount, 99d); + + ids.ShouldBeEmpty(); + } + + [TestMethod] + [DynamicData(nameof(Serializers))] + public async Task Equals_OnNumericListItself_MatchesDocumentsHoldingTheValue(IJsonSerializer jsonSerializer) + { + using var db = await ConnectTalliesAsync(jsonSerializer); + + var ids = await ReadTallyIdsAsync(db, FilterBuilder.Create().Filter(FilterType.Equals, x => x.Numbers, x => x, 5)); + + ids.ShouldBe(new[] { "a" }); + } + + [TestMethod] + [DynamicData(nameof(Serializers))] + public async Task GreaterThan_OnNumericListItself_MatchesDocumentsHoldingSuchAValue(IJsonSerializer jsonSerializer) + { + using var db = await ConnectTalliesAsync(jsonSerializer); + + var ids = await ReadTallyIdsAsync(db, FilterBuilder.Create().Filter(FilterType.GreaterThan, x => x.Numbers, x => x, 2)); + + ids.ShouldBe(new[] { "a", "b" }); + } + + [TestMethod] + [DynamicData(nameof(Serializers))] + public async Task Equals_OnStringListItself_MatchesDocumentsHoldingTheValue(IJsonSerializer jsonSerializer) + { + using var db = await ConnectTalliesAsync(jsonSerializer); + + var ids = await ReadTallyIdsAsync(db, FilterBuilder.Create().Filter(FilterType.Equals, x => x.Tags, x => x, "red")); + + ids.ShouldBe(new[] { "a" }); + } + + [TestMethod] + [DynamicData(nameof(Serializers))] + public async Task Contains_OnStringListItself_MatchesDocumentsHoldingSuchAValue(IJsonSerializer jsonSerializer) + { + using var db = await ConnectTalliesAsync(jsonSerializer); + + var ids = await ReadTallyIdsAsync(db, FilterBuilder.Create().Filter(FilterType.Contains, x => x.Tags, x => x, "re")); + + ids.ShouldBe(new[] { "a", "b" }); + } + + private static async Task ConnectAsync(IJsonSerializer jsonSerializer) + { + var db = new Tycho(Path.GetTempPath(), jsonSerializer, dbName: $"{Guid.NewGuid()}.db", rebuildCache: true, requireTypeRegistration: false); + await db.ConnectAsync(); + await db.WriteObjectsAsync(Seed(), x => x.Id); + + return db; + } + + private static async Task ReadIdsAsync(Tycho db, FilterType filterType, Expression> member, object value) + { + var results = + await db.ReadObjectsAsync( + filter: FilterBuilder.Create().Filter(filterType, x => x.Entries, member, value)); + + return results.Select(x => x.Id).OrderBy(x => x, StringComparer.Ordinal).ToArray(); + } + + private static async Task ConnectTalliesAsync(IJsonSerializer jsonSerializer) + { + var db = new Tycho(Path.GetTempPath(), jsonSerializer, dbName: $"{Guid.NewGuid()}.db", rebuildCache: true, requireTypeRegistration: false); + await db.ConnectAsync(); + await db.WriteObjectsAsync(SeedTallies(), x => x.Id); + + return db; + } + + private static async Task ReadTallyIdsAsync(Tycho db, FilterBuilder filter) + { + var results = await db.ReadObjectsAsync(filter: filter); + + return results.Select(x => x.Id).OrderBy(x => x, StringComparer.Ordinal).ToArray(); + } + + private static Tally[] SeedTallies() => + new[] + { + new Tally { Id = "a", Numbers = new List { 1, 5 }, Tags = new List { "red", "blue" } }, + new Tally { Id = "b", Numbers = new List { 2, 3 }, Tags = new List { "green" } }, + new Tally { Id = "c", Numbers = new List { 1 }, Tags = new List { "blue" } }, + }; + + private static Ledger[] Seed() => + new[] + { + new Ledger + { + Id = "mixed", + Entries = new List + { + new() { When = FirstDay, Amount = 1.5d, Note = "opening" }, + new() { When = SecondDay, Amount = 2d, Note = "deposit" }, + }, + }, + new Ledger + { + Id = "other", + Entries = new List + { + new() { When = ThirdDay, Amount = 3d, Note = "closing" }, + }, + }, + new Ledger + { + Id = "uniform", + Entries = new List + { + new() { When = FirstDay, Amount = 1.5d, Note = "opening" }, + new() { When = FirstDay, Amount = 1.5d, Note = "repeat" }, + }, + }, + }; + + public class Ledger + { + public string Id { get; set; } + + public List Entries { get; set; } + } + + public class Tally + { + public string Id { get; set; } + + public List Numbers { get; set; } + + public List Tags { get; set; } + } + + public class Entry + { + public DateTime When { get; set; } + + public double Amount { get; set; } + + public string Note { get; set; } + } +} diff --git a/TychoDB/FilterBuilder.cs b/TychoDB/FilterBuilder.cs index 0387360..8881184 100644 --- a/TychoDB/FilterBuilder.cs +++ b/TychoDB/FilterBuilder.cs @@ -19,7 +19,7 @@ public class FilterBuilder private const string CastNumericPrefix = "CAST(JSON_EXTRACT(Data, '"; private const string CastNumericSuffix = "') as NUMERIC)"; private const string ExistsPrefix = "EXISTS(SELECT 1 FROM JSON_TREE(Data, '"; - private const string ExistsMiddle = "') AS JT, JSON_EACH(JT.Value, '"; + private const string ExistsMiddle = "') AS JT, JSON_EACH(CASE WHEN JT.type IN ('object', 'array') THEN JT.Value END, '"; private const string ExistsSuffix = "') AS VAL WHERE "; private const string ExistsEnd = ")"; private const string ValValue = "VAL.value"; @@ -705,6 +705,11 @@ private void BuildExistsFilter(StringBuilder commandBuilder, in Filter filter, I { commandBuilder.Append(ValValue).Append(IsNotNull).Append(ExistsEnd).AppendLine(); } + else if (filter.IsPropertyValuePathDateTime) + { + var dateTimeString = GetDateTimeString(filter.Value, jsonSerializer); + commandBuilder.Append(ValValue).Append(NotEquals).Append(parameters.Add(dateTimeString)).Append(ExistsEnd).AppendLine(); + } else { commandBuilder.Append(ValValue).Append(NotEquals);