From db8375732e87e9333b3eaa5a0ad0afcf3e5c77a8 Mon Sep 17 00:00:00 2001 From: Paul Irwin Date: Fri, 8 May 2026 16:09:38 -0600 Subject: [PATCH] Support collection fields and OData lambda operators (closes #6) Implements Collection(Edm.*) field types end-to-end: multi-valued indexing via Lucene's native multi-field model, JSON-array round-trip through a sidecar stored field, and OData any/all lambda support (Tags/any(t: t eq 'red'), Sizes/all(s: s ge 12), search.in inside lambdas, any() with no body). Co-Authored-By: Claude Opus 4.7 (1M context) --- .../EmulatorIntegrationTests.cs | 165 +++++ .../Product.cs | 15 + .../CollectionFieldTests.cs | 649 ++++++++++++++++++ .../Indexing/SearchFieldExtensions.cs | 118 +++- .../Searching/LuceneNetIndexSearcher.cs | 30 +- .../Searching/ODataQueryVisitor.cs | 153 ++++- 6 files changed, 1070 insertions(+), 60 deletions(-) create mode 100644 AzureSearchEmulator.UnitTests/CollectionFieldTests.cs diff --git a/AzureSearchEmulator.IntegrationTests/EmulatorIntegrationTests.cs b/AzureSearchEmulator.IntegrationTests/EmulatorIntegrationTests.cs index 60d1a7b..b6cfcb0 100644 --- a/AzureSearchEmulator.IntegrationTests/EmulatorIntegrationTests.cs +++ b/AzureSearchEmulator.IntegrationTests/EmulatorIntegrationTests.cs @@ -598,8 +598,173 @@ public async Task SearchDocuments_SearchIsMatchScoring_ShouldReturnResults() await indexClient.DeleteIndexAsync(indexName); } + // Collection field tests (issue #6) + + [Fact] + public async Task CollectionField_UploadAndGet_ReturnsArrayValues() + { + const string indexName = "test-collection-roundtrip"; + var indexClient = factory.CreateSearchIndexClient(); + var searchClient = factory.CreateSearchClient(indexName); + + await CreateTaggedProductIndexAsync(indexClient, indexName); + await UploadTaggedProductsAsync(searchClient); + + var doc = await searchClient.GetDocumentAsync("1"); + + Assert.NotNull(doc.Value); + Assert.Equal("1", doc.Value.Id); + Assert.Equal(new[] { "red", "cotton", "shirt" }, doc.Value.Tags); + Assert.Equal(new[] { 8, 10, 12 }, doc.Value.Sizes); + + await indexClient.DeleteIndexAsync(indexName); + } + + [Fact] + public async Task CollectionField_FilterAnyEqual_ReturnsMatchingDocuments() + { + const string indexName = "test-collection-any-eq"; + var indexClient = factory.CreateSearchIndexClient(); + var searchClient = factory.CreateSearchClient(indexName); + + await CreateTaggedProductIndexAsync(indexClient, indexName); + await UploadTaggedProductsAsync(searchClient); + + var options = new SearchOptions { Filter = "Tags/any(t: t eq 'cotton')", Size = 50 }; + var results = await searchClient.SearchAsync("*", options); + var ids = (await results.Value.GetResultsAsync().ToListAsync()) + .Select(r => r.Document.Id) + .OrderBy(id => id) + .ToList(); + + // Doc 1 (Red Shirt) and doc 4 (Green Socks) both have 'cotton' in their tags. + Assert.Equal(["1", "4"], ids); + } + + [Fact] + public async Task CollectionField_FilterAnySearchIn_ReturnsMatchingDocuments() + { + const string indexName = "test-collection-any-searchin"; + var indexClient = factory.CreateSearchIndexClient(); + var searchClient = factory.CreateSearchClient(indexName); + + await CreateTaggedProductIndexAsync(indexClient, indexName); + await UploadTaggedProductsAsync(searchClient); + + var options = new SearchOptions { Filter = "Tags/any(t: search.in(t, 'wool,denim'))", Size = 50 }; + var results = await searchClient.SearchAsync("*", options); + var ids = (await results.Value.GetResultsAsync().ToListAsync()) + .Select(r => r.Document.Id) + .OrderBy(id => id) + .ToList(); + + Assert.Equal(["2", "3"], ids); + } + + [Fact] + public async Task CollectionField_FilterAllNotEqual_ExcludesDocsContainingValue() + { + const string indexName = "test-collection-all-ne"; + var indexClient = factory.CreateSearchIndexClient(); + var searchClient = factory.CreateSearchClient(indexName); + + await CreateTaggedProductIndexAsync(indexClient, indexName); + await UploadTaggedProductsAsync(searchClient); + + // all(t: t ne 'cotton') — only docs whose tags do NOT contain 'cotton' should match. + // Doc 1 and doc 4 contain 'cotton', so only doc 2 (jeans) and doc 3 (hat) should match. + var options = new SearchOptions { Filter = "Tags/all(t: t ne 'cotton')", Size = 50 }; + var results = await searchClient.SearchAsync("*", options); + var ids = (await results.Value.GetResultsAsync().ToListAsync()) + .Select(r => r.Document.Id) + .OrderBy(id => id) + .ToList(); + + Assert.Equal(["2", "3"], ids); + } + + [Fact] + public async Task CollectionField_FilterAnyNumericRange_MatchesDocsWithAnyValueInRange() + { + const string indexName = "test-collection-any-numeric"; + var indexClient = factory.CreateSearchIndexClient(); + var searchClient = factory.CreateSearchClient(indexName); + + await CreateTaggedProductIndexAsync(indexClient, indexName); + await UploadTaggedProductsAsync(searchClient); + + // Doc 1 has Sizes [8,10,12] — 12 is >= 12. Doc 2 has Sizes [30,32,34] — all >= 12. + var options = new SearchOptions { Filter = "Sizes/any(s: s ge 12)", Size = 50 }; + var results = await searchClient.SearchAsync("*", options); + var ids = (await results.Value.GetResultsAsync().ToListAsync()) + .Select(r => r.Document.Id) + .OrderBy(id => id) + .ToList(); + + Assert.Equal(["1", "2"], ids); + } + + [Fact] + public async Task CollectionField_FullTextSearchOnSearchableCollection_MatchesAcrossElements() + { + const string indexName = "test-collection-fulltext"; + var indexClient = factory.CreateSearchIndexClient(); + var searchClient = factory.CreateSearchClient(indexName); + + await CreateTaggedProductIndexAsync(indexClient, indexName); + await UploadTaggedProductsAsync(searchClient); + + // Tags is searchable; a free-text search for 'denim' should match doc 2 even though + // 'denim' is just one entry in its tags array. + var options = new SearchOptions { SearchFields = { "Tags" }, Size = 50 }; + var results = await searchClient.SearchAsync("denim", options); + var items = await results.Value.GetResultsAsync().ToListAsync(); + + Assert.Single(items); + Assert.Equal("2", items[0].Document.Id); + } + // Helper Methods + private static async Task CreateTaggedProductIndexAsync(SearchIndexClient indexClient, string indexName) + { + try + { + await indexClient.DeleteIndexAsync(indexName); + } + catch (Azure.RequestFailedException ex) when (ex.Status == 404) + { + // expected + } + + var index = new SearchIndex(indexName) + { + Fields = + [ + new SearchField(nameof(TaggedProduct.Id), SearchFieldDataType.String) { IsKey = true, IsStored = true, IsFilterable = true }, + new SearchField(nameof(TaggedProduct.Name), SearchFieldDataType.String) { IsSearchable = true, IsStored = true }, + new SearchField(nameof(TaggedProduct.Tags), SearchFieldDataType.Collection(SearchFieldDataType.String)) { IsSearchable = true, IsFilterable = true, IsStored = true }, + new SearchField(nameof(TaggedProduct.Sizes), SearchFieldDataType.Collection(SearchFieldDataType.Int32)) { IsFilterable = true, IsStored = true } + ] + }; + + await indexClient.CreateIndexAsync(index); + return index; + } + + private static async Task UploadTaggedProductsAsync(SearchClient searchClient) + { + var documents = new List + { + new() { Id = "1", Name = "Red Shirt", Tags = ["red", "cotton", "shirt"], Sizes = [8, 10, 12] }, + new() { Id = "2", Name = "Blue Jeans", Tags = ["blue", "denim", "pants"], Sizes = [30, 32, 34] }, + new() { Id = "3", Name = "Wool Hat", Tags = ["wool", "warm"], Sizes = [] }, + new() { Id = "4", Name = "Green Socks", Tags = ["green", "cotton"], Sizes = [9, 10, 11] }, + }; + var batch = IndexDocumentsBatch.Upload(documents); + await searchClient.IndexDocumentsAsync(batch); + } + private static async Task CreateIndexAsync(SearchIndexClient indexClient, string indexName) { // Clean up any existing index diff --git a/AzureSearchEmulator.IntegrationTests/Product.cs b/AzureSearchEmulator.IntegrationTests/Product.cs index 06187f0..2ba5153 100644 --- a/AzureSearchEmulator.IntegrationTests/Product.cs +++ b/AzureSearchEmulator.IntegrationTests/Product.cs @@ -17,3 +17,18 @@ public class Product public bool InStock { get; init; } } + +/// +/// Product model with collection fields, used to exercise Collection(Edm.*) support +/// (issue #6) end-to-end through the Azure Search SDK. +/// +public class TaggedProduct +{ + public required string Id { get; init; } + + public required string Name { get; init; } + + public required string[] Tags { get; init; } + + public required int[] Sizes { get; init; } +} diff --git a/AzureSearchEmulator.UnitTests/CollectionFieldTests.cs b/AzureSearchEmulator.UnitTests/CollectionFieldTests.cs new file mode 100644 index 0000000..f2723c4 --- /dev/null +++ b/AzureSearchEmulator.UnitTests/CollectionFieldTests.cs @@ -0,0 +1,649 @@ +using System.Text.Json.Nodes; +using AzureSearchEmulator.Indexing; +using AzureSearchEmulator.Models; +using AzureSearchEmulator.SearchData; +using AzureSearchEmulator.Searching; +using Lucene.Net.Index; +using Lucene.Net.Search; +using Lucene.Net.Store; +using Microsoft.OData.UriParser; +using Xunit; + +namespace AzureSearchEmulator.UnitTests; + +/// +/// Tests for Collection(Edm.*) field support: indexing, retrieval, and OData lambda +/// (any/all) filtering. Mirrors the Azure Search collection field semantics — see +/// https://learn.microsoft.com/en-us/azure/search/search-query-odata-collection-operators. +/// +public class CollectionFieldTests : IDisposable +{ + private readonly LuceneTestHelper _helper; + private readonly LuceneNetIndexSearcher _searcher; + + public CollectionFieldTests() + { + var index = CreateIndex(); + var docs = CreateDocuments(index); + _helper = new LuceneTestHelper(index, docs); + _searcher = new LuceneNetIndexSearcher(new StubReaderFactory(_helper.Directory)); + } + + public void Dispose() => _helper.Dispose(); + + private static SearchIndex CreateIndex() => new() + { + Name = "products", + Fields = + [ + new SearchField { Name = "Id", Type = "Edm.String", Key = true, Searchable = false, Filterable = true }, + new SearchField { Name = "Name", Type = "Edm.String", Searchable = true }, + new SearchField { Name = "Tags", Type = "Collection(Edm.String)", Searchable = true, Filterable = true }, + new SearchField { Name = "Sizes", Type = "Collection(Edm.Int32)", Filterable = true }, + new SearchField { Name = "Prices", Type = "Collection(Edm.Double)", Filterable = true }, + new SearchField { Name = "Available", Type = "Collection(Edm.Boolean)", Filterable = true }, + ] + }; + + private static List CreateDocuments(SearchIndex index) + { + var rows = new List + { + new() + { + ["Id"] = "1", + ["Name"] = "Red shirt", + ["Tags"] = new JsonArray("red", "cotton", "shirt"), + ["Sizes"] = new JsonArray(8, 10, 12), + ["Prices"] = new JsonArray(19.99, 24.99), + ["Available"] = new JsonArray(true), + }, + new() + { + ["Id"] = "2", + ["Name"] = "Blue jeans", + ["Tags"] = new JsonArray("blue", "denim", "pants"), + ["Sizes"] = new JsonArray(30, 32, 34), + ["Prices"] = new JsonArray(49.99), + ["Available"] = new JsonArray(true, false), + }, + new() + { + ["Id"] = "3", + ["Name"] = "Red hat", + ["Tags"] = new JsonArray("red", "wool"), + ["Sizes"] = new JsonArray(), + ["Prices"] = new JsonArray(15.0, 18.0, 22.5), + ["Available"] = new JsonArray(false), + }, + new() + { + ["Id"] = "4", + ["Name"] = "Green socks", + ["Tags"] = new JsonArray("green", "cotton"), + ["Sizes"] = new JsonArray(9, 10, 11), + ["Prices"] = new JsonArray(5.99), + ["Available"] = new JsonArray(true), + }, + new() + { + ["Id"] = "5", + ["Name"] = "Untagged item", + ["Tags"] = new JsonArray(), + ["Sizes"] = new JsonArray(42), + ["Prices"] = new JsonArray(), + ["Available"] = new JsonArray(), + } + }; + + return rows.Select(row => + { + var doc = new Lucene.Net.Documents.Document(); + foreach (var field in index.Fields) + { + if (row[field.Name] is { } value) + { + foreach (var f in field.CreateFields(value)) + { + doc.Add(f); + } + } + } + return doc; + }).ToList(); + } + + private async Task> SearchIds(string filter) + { + var response = await _searcher.Search(_helper.Index, new SearchRequest + { + Search = "*", + Filter = filter, + Top = 50 + }); + return response.Results + .Select(r => r["Id"]!.GetValue()) + .OrderBy(id => id) + .ToList(); + } + + // ===== any(t: t eq value) ===== + + [Fact] + public async Task Any_StringEquality_MatchesDocsContainingValue() + { + var ids = await SearchIds("Tags/any(t: t eq 'red')"); + + Assert.Equal(["1", "3"], ids); + } + + [Fact] + public async Task Any_StringEquality_NoMatch_ReturnsEmpty() + { + var ids = await SearchIds("Tags/any(t: t eq 'purple')"); + + Assert.Empty(ids); + } + + [Fact] + public async Task Any_StringEquality_DifferentParameterName() + { + // The lambda parameter name is arbitrary — verify we honor whatever the user picks. + var ids = await SearchIds("Tags/any(tag: tag eq 'cotton')"); + + Assert.Equal(["1", "4"], ids); + } + + [Fact] + public async Task Any_IntEquality_MatchesDocsContainingValue() + { + var ids = await SearchIds("Sizes/any(s: s eq 10)"); + + Assert.Equal(["1", "4"], ids); + } + + [Fact] + public async Task Any_IntRange_MatchesDocsWithAnyValueInRange() + { + // Doc 1: 8,10,12 — has 8 in range + // Doc 4: 9,10,11 — has 9 in range + var ids = await SearchIds("Sizes/any(s: s lt 10)"); + + Assert.Equal(["1", "4"], ids); + } + + [Fact] + public async Task Any_DoubleRange_MatchesDocsWithAnyValueAboveThreshold() + { + // Doc 1: 19.99, 24.99 — has 24.99 > 20 + // Doc 2: 49.99 — > 20 + // Doc 3: 15, 18, 22.5 — has 22.5 > 20 + var ids = await SearchIds("Prices/any(p: p gt 20.0)"); + + Assert.Equal(["1", "2", "3"], ids); + } + + [Fact] + public async Task Any_BooleanEquality_MatchesDocsContainingValue() + { + // Doc 2 has both true and false in its Available collection. + var ids = await SearchIds("Available/any(a: a eq false)"); + + Assert.Equal(["2", "3"], ids); + } + + // ===== any() with no expression ===== + + [Fact] + public async Task Any_NoExpression_MatchesDocsWhereCollectionIsNonEmpty() + { + // Doc 5 has empty Tags; everyone else has at least one tag. + var ids = await SearchIds("Tags/any()"); + + Assert.Equal(["1", "2", "3", "4"], ids); + } + + [Fact] + public async Task Any_NoExpression_OnEmptyOnlyCollection_ReturnsEmpty() + { + // Doc 5 has empty Available; doc 1,3,4 have one entry; doc 2 has two. + var ids = await SearchIds("Available/any()"); + + Assert.Equal(["1", "2", "3", "4"], ids); + } + + // ===== all(t: t op value) ===== + + [Fact] + public async Task All_NotEqual_OnlyMatchesDocsWithoutValue() + { + // all(t: t ne 'red') matches docs whose Tags do NOT contain 'red'. + // Doc 5 has empty tags — vacuously satisfies "all", which is consistent with set semantics. + var ids = await SearchIds("Tags/all(t: t ne 'red')"); + + Assert.Contains("2", ids); + Assert.Contains("4", ids); + Assert.DoesNotContain("1", ids); + Assert.DoesNotContain("3", ids); + } + + [Fact] + public async Task All_GreaterThan_OnlyMatchesDocsWhereEveryValueExceedsThreshold() + { + // all(s: s gt 8) — every Size must be > 8 + // Doc 1: 8,10,12 → has 8 (fails) + // Doc 4: 9,10,11 → all > 8 (passes) + // Doc 2: 30,32,34 → all > 8 (passes) + // Doc 5: just 42 → all > 8 (passes) + // Doc 3: empty → vacuously true (passes) + var ids = await SearchIds("Sizes/all(s: s gt 8)"); + + Assert.Contains("2", ids); + Assert.Contains("4", ids); + Assert.Contains("5", ids); + Assert.DoesNotContain("1", ids); + } + + [Fact] + public async Task All_NotWithEqual_MatchesSameAsNotEqual() + { + // all(t: not (t eq 'red')) is equivalent to all(t: t ne 'red') + var ids = await SearchIds("Tags/all(t: not (t eq 'red'))"); + + Assert.DoesNotContain("1", ids); + Assert.DoesNotContain("3", ids); + Assert.Contains("2", ids); + Assert.Contains("4", ids); + } + + // ===== any with search.in ===== + + [Fact] + public async Task Any_SearchIn_MatchesAnyValueInList() + { + var ids = await SearchIds("Tags/any(t: search.in(t, 'red,blue'))"); + + Assert.Equal(["1", "2", "3"], ids); + } + + // ===== Combinations with top-level operators ===== + + [Fact] + public async Task Any_CombinedWithAnd_ReturnsIntersection() + { + // Tags contains 'cotton' AND Sizes contains 10 + var ids = await SearchIds("Tags/any(t: t eq 'cotton') and Sizes/any(s: s eq 10)"); + + Assert.Equal(["1", "4"], ids); + } + + [Fact] + public async Task Any_CombinedWithOr_ReturnsUnion() + { + var ids = await SearchIds("Tags/any(t: t eq 'wool') or Tags/any(t: t eq 'denim')"); + + Assert.Equal(["2", "3"], ids); + } + + [Fact] + public async Task NotAny_NegatesLambda() + { + var ids = await SearchIds("not Tags/any(t: t eq 'red')"); + + Assert.Contains("2", ids); + Assert.Contains("4", ids); + Assert.Contains("5", ids); // empty collection — never contains 'red' + Assert.DoesNotContain("1", ids); + Assert.DoesNotContain("3", ids); + } + + // ===== Document retrieval round-trips ===== + + [Fact] + public async Task GetDoc_ReturnsCollectionValuesAsArray() + { + var doc = await _searcher.GetDoc(_helper.Index, "1"); + + Assert.NotNull(doc); + var tags = doc!["Tags"] as JsonArray; + Assert.NotNull(tags); + Assert.Equal(3, tags!.Count); + Assert.Equal("red", tags[0]!.GetValue()); + Assert.Equal("cotton", tags[1]!.GetValue()); + Assert.Equal("shirt", tags[2]!.GetValue()); + + var sizes = doc["Sizes"] as JsonArray; + Assert.NotNull(sizes); + Assert.Equal([8, 10, 12], sizes!.Select(n => n!.GetValue()).ToArray()); + + var prices = doc["Prices"] as JsonArray; + Assert.NotNull(prices); + Assert.Equal(2, prices!.Count); + Assert.Equal(19.99, prices[0]!.GetValue()); + Assert.Equal(24.99, prices[1]!.GetValue()); + + var available = doc["Available"] as JsonArray; + Assert.NotNull(available); + Assert.Single(available!); + Assert.True(available[0]!.GetValue()); + } + + [Fact] + public async Task GetDoc_PreservesEmptyCollection() + { + var doc = await _searcher.GetDoc(_helper.Index, "5"); + + Assert.NotNull(doc); + var tags = doc!["Tags"] as JsonArray; + Assert.NotNull(tags); + Assert.Empty(tags!); + } + + [Fact] + public async Task Search_StarReturnsAllDocsWithCollectionValues() + { + var response = await _searcher.Search(_helper.Index, new SearchRequest { Search = "*", Top = 50 }); + + Assert.Equal(5, response.Results.Count); + var doc1 = response.Results.Single(r => r["Id"]!.GetValue() == "2"); + var tags = doc1["Tags"] as JsonArray; + Assert.NotNull(tags); + Assert.Equal(["blue", "denim", "pants"], tags!.Select(n => n!.GetValue()).ToArray()); + } + + // ===== Searchable collection (full-text) ===== + + [Fact] + public async Task Search_SearchableCollection_FullTextMatchesAcrossAllValues() + { + // Tags is Searchable=true. A free-text search for "denim" should hit doc 2, + // since 'denim' is one of its tag values. + var response = await _searcher.Search(_helper.Index, new SearchRequest + { + Search = "denim", + SearchFields = "Tags", + Top = 50 + }); + + Assert.Single(response.Results); + Assert.Equal("2", response.Results[0]["Id"]!.GetValue()); + } + + private class StubReaderFactory(Lucene.Net.Store.Directory directory) : ILuceneIndexReaderFactory + { + public IndexReader GetIndexReader(string indexName) => DirectoryReader.Open(directory); + public IndexReader RefreshReader(string indexName) => GetIndexReader(indexName); + public void ClearCachedReader(string indexName) { } + } +} + +/// +/// Lower-level tests targeting the ODataQueryVisitor's lambda translation directly, +/// without going through the full search pipeline. +/// +public class ODataQueryVisitor_LambdaTests : IDisposable +{ + private readonly LuceneTestHelper _helper; + private readonly IndexSearcher _searcher; + private readonly SearchIndex _index; + + public ODataQueryVisitor_LambdaTests() + { + _index = new SearchIndex + { + Name = "tagged", + Fields = + [ + new SearchField { Name = "Id", Type = "Edm.String", Key = true, Searchable = false, Filterable = true }, + new SearchField { Name = "Tags", Type = "Collection(Edm.String)", Filterable = true }, + new SearchField { Name = "Locked", Type = "Collection(Edm.String)", Filterable = false }, + ] + }; + + var docs = new List + { + new() { ["Id"] = "1", ["Tags"] = new JsonArray("a", "b") }, + new() { ["Id"] = "2", ["Tags"] = new JsonArray("b", "c") }, + new() { ["Id"] = "3", ["Tags"] = new JsonArray("c") }, + }; + + var luceneDocs = docs.Select(row => + { + var doc = new Lucene.Net.Documents.Document(); + foreach (var field in _index.Fields) + { + if (row[field.Name] is { } value) + { + foreach (var f in field.CreateFields(value)) + { + doc.Add(f); + } + } + } + return doc; + }).ToList(); + + _helper = new LuceneTestHelper(_index, luceneDocs); + _searcher = _helper.CreateSearcher(); + } + + public void Dispose() => _helper.Dispose(); + + private List Run(string filter) + { + var parser = new UriQueryExpressionParser(100); + var token = parser.ParseFilter(filter); + var query = token.Accept(new ODataQueryVisitor(_index)); + var docs = _searcher.Search(query, 100); + return docs.ScoreDocs + .Select(sd => _searcher.Doc(sd.Doc).Get("Id")) + .OrderBy(id => id) + .ToList(); + } + + [Fact] + public void All_NotEqual_String_MatchesDocsWhereNoValueEqualsTarget() + { + // Per Azure Search collection-operator rules, the supported lambda for string + // collections inside all() is "ne". Doc 1 (a,b) and doc 2 (b,c) both contain 'b'; + // only doc 3 (c) has no 'b'. + var ids = Run("Tags/all(t: t ne 'b')"); + Assert.Contains("3", ids); + Assert.DoesNotContain("1", ids); + Assert.DoesNotContain("2", ids); + } + + [Fact] + public void All_NotSearchIn_String_ExcludesDocsContainingListedValues() + { + // all(t: not search.in(t, 'a,b')) — doc must not contain 'a' or 'b'. + var ids = Run("Tags/all(t: not search.in(t, 'a,b'))"); + Assert.Contains("3", ids); + Assert.DoesNotContain("1", ids); + Assert.DoesNotContain("2", ids); + } + + [Fact] + public void Any_OnNonFilterableCollection_Throws() + { + var parser = new UriQueryExpressionParser(100); + var token = parser.ParseFilter("Locked/any(l: l eq 'x')"); + Assert.Throws(() => token.Accept(new ODataQueryVisitor(_index))); + } +} + +/// +/// End-to-end indexer tests: drive the real LuceneNetSearchIndexer (which routes through +/// Upload/Merge/MergeOrUpload actions) so collection JSON values are written to a real +/// in-memory Lucene directory and then queried through LuceneNetIndexSearcher. This +/// covers the sidecar collection-storage field through both the initial upload path +/// and the field-replacement logic in MergeDocument. +/// +public class LuceneNetSearchIndexer_CollectionTests : IDisposable +{ + private readonly RAMDirectory _directory = new(); + private readonly SearchIndex _index; + private readonly LuceneNetSearchIndexer _indexer; + private readonly LuceneNetIndexSearcher _searcher; + + public LuceneNetSearchIndexer_CollectionTests() + { + _index = new SearchIndex + { + Name = "tagged-products", + Fields = + [ + new SearchField { Name = "Id", Type = "Edm.String", Key = true, Searchable = false, Filterable = true }, + new SearchField { Name = "Name", Type = "Edm.String", Searchable = true }, + new SearchField { Name = "Tags", Type = "Collection(Edm.String)", Searchable = true, Filterable = true }, + new SearchField { Name = "Sizes", Type = "Collection(Edm.Int32)", Filterable = true }, + ] + }; + + var factory = new SharedDirectoryFactory(_directory); + _indexer = new LuceneNetSearchIndexer(factory, factory); + _searcher = new LuceneNetIndexSearcher(factory); + } + + public void Dispose() => _directory.Dispose(); + + private static JsonObject Doc(string id, string name, string[] tags, int[] sizes) => new() + { + ["Id"] = id, + ["Name"] = name, + ["Tags"] = new JsonArray(tags.Select(t => (JsonNode)t).ToArray()), + ["Sizes"] = new JsonArray(sizes.Select(s => (JsonNode)s).ToArray()), + }; + + [Fact] + public async Task Upload_ThenSearch_ReturnsCollectionValuesAndHonorsLambdaFilter() + { + var batch = new List + { + new UploadIndexDocumentAction(Doc("1", "Red Shirt", ["red", "cotton"], [8, 10, 12])), + new UploadIndexDocumentAction(Doc("2", "Blue Jeans", ["blue", "denim"], [30, 32])), + new UploadIndexDocumentAction(Doc("3", "Red Hat", ["red", "wool"], [])), + }; + + var result = _indexer.IndexDocuments(_index, batch); + Assert.All(result.Value, r => Assert.True(r.Status)); + + // Round-trip: GetDoc should return the collection as a JSON array. + var doc1 = await _searcher.GetDoc(_index, "1"); + Assert.NotNull(doc1); + var tags = doc1!["Tags"] as JsonArray; + Assert.NotNull(tags); + Assert.Equal(["red", "cotton"], tags!.Select(n => n!.GetValue()).ToArray()); + + var sizes = doc1["Sizes"] as JsonArray; + Assert.NotNull(sizes); + Assert.Equal([8, 10, 12], sizes!.Select(n => n!.GetValue()).ToArray()); + + // Lambda filter on string collection. + var redResults = await _searcher.Search(_index, new SearchRequest + { + Search = "*", + Filter = "Tags/any(t: t eq 'red')", + Top = 50 + }); + Assert.Equal(2, redResults.Results.Count); + Assert.Contains(redResults.Results, r => r["Id"]!.GetValue() == "1"); + Assert.Contains(redResults.Results, r => r["Id"]!.GetValue() == "3"); + + // Lambda filter on numeric collection. + var sizeResults = await _searcher.Search(_index, new SearchRequest + { + Search = "*", + Filter = "Sizes/any(s: s ge 30)", + Top = 50 + }); + Assert.Single(sizeResults.Results); + Assert.Equal("2", sizeResults.Results[0]["Id"]!.GetValue()); + + // any() with no body — should exclude doc 3 (empty Sizes collection). + var nonEmpty = await _searcher.Search(_index, new SearchRequest + { + Search = "*", + Filter = "Sizes/any()", + Top = 50 + }); + Assert.Equal(2, nonEmpty.Results.Count); + Assert.DoesNotContain(nonEmpty.Results, r => r["Id"]!.GetValue() == "3"); + } + + [Fact] + public async Task MergeOrUpload_ReplacesCollection_AndRemovesStaleSidecar() + { + // Initial upload. + _indexer.IndexDocuments(_index, [new UploadIndexDocumentAction(Doc("1", "Red Shirt", ["red", "cotton"], [8, 10]))]); + + // Now overwrite the same doc with a different collection. The merge logic must + // both clear the per-element fields and replace the sidecar JSON, otherwise the + // round-tripped collection would still contain stale values. + var update = new JsonObject + { + ["Id"] = "1", + ["Tags"] = new JsonArray("green", "linen", "shirt"), + ["Sizes"] = new JsonArray(14, 16), + }; + _indexer.IndexDocuments(_index, [new MergeOrUploadIndexDocumentAction(update)]); + + var doc = await _searcher.GetDoc(_index, "1"); + Assert.NotNull(doc); + Assert.Equal("Red Shirt", doc!["Name"]?.GetValue()); // unchanged scalar field is preserved by merge + var tags = doc["Tags"] as JsonArray; + Assert.NotNull(tags); + Assert.Equal(["green", "linen", "shirt"], tags!.Select(n => n!.GetValue()).ToArray()); + + // The lambda filter must reflect the new values, not the old ones. + var redHits = await _searcher.Search(_index, new SearchRequest + { + Search = "*", + Filter = "Tags/any(t: t eq 'red')", + Top = 50 + }); + Assert.Empty(redHits.Results); + + var greenHits = await _searcher.Search(_index, new SearchRequest + { + Search = "*", + Filter = "Tags/any(t: t eq 'green')", + Top = 50 + }); + Assert.Single(greenHits.Results); + } + + [Fact] + public async Task Upload_EmptyCollection_RoundTripsAsEmptyArray() + { + _indexer.IndexDocuments(_index, [new UploadIndexDocumentAction(Doc("1", "Bare Item", [], []))]); + + var doc = await _searcher.GetDoc(_index, "1"); + Assert.NotNull(doc); + var tags = doc!["Tags"] as JsonArray; + Assert.NotNull(tags); + Assert.Empty(tags!); + + // any() on the empty collection should NOT match. + var hits = await _searcher.Search(_index, new SearchRequest + { + Search = "*", + Filter = "Tags/any()", + Top = 50 + }); + Assert.Empty(hits.Results); + } + + /// + /// Single-RAMDirectory backed factory pair shared between indexer and searcher so + /// writes are visible to subsequent reads within the same test. + /// + private class SharedDirectoryFactory(Lucene.Net.Store.Directory directory) : ILuceneDirectoryFactory, ILuceneIndexReaderFactory + { + public Lucene.Net.Store.Directory GetDirectory(string indexName) => directory; + public void ClearCachedDirectory(string indexName) { } + public IndexReader GetIndexReader(string indexName) => DirectoryReader.Open(directory); + public IndexReader RefreshReader(string indexName) => GetIndexReader(indexName); + public void ClearCachedReader(string indexName) { } + } +} diff --git a/AzureSearchEmulator/Indexing/SearchFieldExtensions.cs b/AzureSearchEmulator/Indexing/SearchFieldExtensions.cs index ed12cb5..7072dee 100644 --- a/AzureSearchEmulator/Indexing/SearchFieldExtensions.cs +++ b/AzureSearchEmulator/Indexing/SearchFieldExtensions.cs @@ -1,4 +1,4 @@ -using System.Text.Json.Nodes; +using System.Text.Json.Nodes; using AzureSearchEmulator.Models; using Lucene.Net.Documents; using Lucene.Net.Index; @@ -8,33 +8,96 @@ namespace AzureSearchEmulator.Indexing; public static class SearchFieldExtensions { public static IEnumerable CreateFields(this SearchField field, JsonNode value) + { + if (field.Type.StartsWith("Collection(", StringComparison.Ordinal)) + { + return CreateCollectionFields(field, value); + } + + return CreateSingleValueFields(field, value); + } + + private static IEnumerable CreateSingleValueFields(SearchField field, JsonNode value) { var stored = field.Retrievable ? Field.Store.YES : Field.Store.NO; if (field.Type == "Edm.String") { var str = value.GetValue(); - var searchable = field.Searchable.GetValueOrDefault(true); - var filterable = field.Filterable; + return CreateStringFields(field, str, stored); + } + + return [CreateScalarField(field, field.Type, value, stored)]; + } + + private static IEnumerable CreateCollectionFields(SearchField field, JsonNode value) + { + if (value is not JsonArray array) + { + throw new InvalidOperationException( + $"Field '{field.Name}' is type {field.Type} but received a non-array JSON value."); + } + + var elementType = GetCollectionElementType(field.Type); - if (searchable) + // Collections store each element as a separate Lucene field with the same name. + // For retrievable collections we also persist the original JSON array under a + // sidecar field so we can faithfully round-trip the values (preserving order + // and type fidelity for numbers/booleans/dates) when reading the document back. + var stored = Field.Store.NO; + + var fields = new List(); + + foreach (var element in array) + { + if (element is null) { - yield return new TextField(field.Name, str, stored); - // Filter/sort/facet require a non-analyzed copy under the same field name - // so TermQuery-based filters match the raw literal (matches Azure semantics). - if (filterable || field.Sortable.GetValueOrDefault() || field.Facetable.GetValueOrDefault()) - { - yield return new StringField(field.Name, str, Field.Store.NO); - } + // Azure Search ignores null entries within a collection. + continue; + } + + if (elementType == "Edm.String") + { + fields.AddRange(CreateStringFields(field, element.GetValue(), stored)); } else { - yield return new StringField(field.Name, str, stored); + fields.Add(CreateScalarField(field, elementType, element, stored)); + } + } + + if (field.Retrievable) + { + fields.Add(new StoredField(GetCollectionStorageFieldName(field.Name), array.ToJsonString())); + } + + return fields; + } + + private static IEnumerable CreateStringFields(SearchField field, string str, Field.Store stored) + { + var searchable = field.Searchable.GetValueOrDefault(true); + var filterable = field.Filterable; + + if (searchable) + { + yield return new TextField(field.Name, str, stored); + // Filter/sort/facet require a non-analyzed copy under the same field name + // so TermQuery-based filters match the raw literal (matches Azure semantics). + if (filterable || field.Sortable.GetValueOrDefault() || field.Facetable.GetValueOrDefault()) + { + yield return new StringField(field.Name, str, Field.Store.NO); } - yield break; } + else + { + yield return new StringField(field.Name, str, stored); + } + } - yield return field.Type switch + private static IIndexableField CreateScalarField(SearchField field, string type, JsonNode value, Field.Store stored) + { + return type switch { "Edm.Int32" => new Int32Field(field.Name, value.GetValue(), stored), "Edm.Int64" => new Int64Field(field.Name, value.GetValue(), stored), @@ -43,15 +106,22 @@ public static IEnumerable CreateFields(this SearchField field, "Edm.DateTimeOffset" => new Int64Field(field.Name, value.GetValue().ToUnixTimeMilliseconds(), stored), "Edm.GeographyPoint" => throw new NotImplementedException(), "Edm.ComplexType" => throw new NotImplementedException(), - "Collection(Edm.String)" => throw new NotImplementedException(), - "Collection(Edm.Int32)" => throw new NotImplementedException(), - "Collection(Edm.Int64)" => throw new NotImplementedException(), - "Collection(Edm.Double)" => throw new NotImplementedException(), - "Collection(Edm.Boolean)" => throw new NotImplementedException(), - "Collection(Edm.DateTimeOffset)" => throw new NotImplementedException(), - "Collection(Edm.GeographyPoint)" => throw new NotImplementedException(), - "Collection(Edm.ComplexType)" => throw new NotImplementedException(), - _ => throw new InvalidOperationException($"Unsupported field type {field.Type}") + _ => throw new InvalidOperationException($"Unsupported field type {type}") }; } -} \ No newline at end of file + + public static string GetCollectionElementType(string fieldType) + { + if (!fieldType.StartsWith("Collection(", StringComparison.Ordinal) || !fieldType.EndsWith(")", StringComparison.Ordinal)) + { + throw new InvalidOperationException($"'{fieldType}' is not a collection type."); + } + + return fieldType.Substring("Collection(".Length, fieldType.Length - "Collection(".Length - 1); + } + + public static bool IsCollection(this SearchField field) + => field.Type.StartsWith("Collection(", StringComparison.Ordinal); + + public static string GetCollectionStorageFieldName(string fieldName) => "__azs_collection__" + fieldName; +} diff --git a/AzureSearchEmulator/Searching/LuceneNetIndexSearcher.cs b/AzureSearchEmulator/Searching/LuceneNetIndexSearcher.cs index 4625e19..4c4f3ff 100644 --- a/AzureSearchEmulator/Searching/LuceneNetIndexSearcher.cs +++ b/AzureSearchEmulator/Searching/LuceneNetIndexSearcher.cs @@ -1,5 +1,6 @@ using System.Text.Json; using System.Text.Json.Nodes; +using AzureSearchEmulator.Indexing; using AzureSearchEmulator.Models; using AzureSearchEmulator.SearchData; using Lucene.Net.Analysis; @@ -194,6 +195,7 @@ private static SortField GetSortField(SearchIndex index, string sort) private static SortFieldType GetSortFieldType(SearchField field) { + // Collection fields are not directly sortable in Azure Search; fall through to error. return field.Type switch { "Edm.String" => SortFieldType.STRING, @@ -204,15 +206,7 @@ private static SortFieldType GetSortFieldType(SearchField field) "Edm.DateTimeOffset" => SortFieldType.INT64, "Edm.GeographyPoint" => throw new NotImplementedException(), "Edm.ComplexType" => throw new NotImplementedException(), - "Collection(Edm.String)" => throw new NotImplementedException(), - "Collection(Edm.Int32)" => throw new NotImplementedException(), - "Collection(Edm.Int64)" => throw new NotImplementedException(), - "Collection(Edm.Double)" => throw new NotImplementedException(), - "Collection(Edm.Boolean)" => throw new NotImplementedException(), - "Collection(Edm.DateTimeOffset)" => throw new NotImplementedException(), - "Collection(Edm.GeographyPoint)" => throw new NotImplementedException(), - "Collection(Edm.ComplexType)" => throw new NotImplementedException(), - _ => throw new InvalidOperationException($"Unsupported field type {field.Type}") + _ => throw new InvalidOperationException($"Unsupported field type {field.Type} for sorting") }; } @@ -333,6 +327,16 @@ private static JsonObject ConvertSearchDoc(SearchIndex index, Lucene.Net.Documen foreach (var field in index.Fields.Where(i => i.Retrievable)) { + if (field.IsCollection()) + { + var storedJson = doc.Get(SearchFieldExtensions.GetCollectionStorageFieldName(field.Name)); + if (storedJson != null) + { + result[field.Name] = JsonNode.Parse(storedJson); + } + continue; + } + var docField = doc.GetField(field.Name); if (docField != null) @@ -347,14 +351,6 @@ private static JsonObject ConvertSearchDoc(SearchIndex index, Lucene.Net.Documen "Edm.DateTimeOffset" => docField.GetInt64Value() is long ms ? DateTimeOffset.FromUnixTimeMilliseconds(ms) : null, "Edm.GeographyPoint" => throw new NotImplementedException(), "Edm.ComplexType" => throw new NotImplementedException(), - "Collection(Edm.String)" => throw new NotImplementedException(), - "Collection(Edm.Int32)" => throw new NotImplementedException(), - "Collection(Edm.Int64)" => throw new NotImplementedException(), - "Collection(Edm.Double)" => throw new NotImplementedException(), - "Collection(Edm.Boolean)" => throw new NotImplementedException(), - "Collection(Edm.DateTimeOffset)" => throw new NotImplementedException(), - "Collection(Edm.GeographyPoint)" => throw new NotImplementedException(), - "Collection(Edm.ComplexType)" => throw new NotImplementedException(), _ => throw new InvalidOperationException($"Unsupported field type {field.Type}") }; } diff --git a/AzureSearchEmulator/Searching/ODataQueryVisitor.cs b/AzureSearchEmulator/Searching/ODataQueryVisitor.cs index 1d843db..0e6e098 100644 --- a/AzureSearchEmulator/Searching/ODataQueryVisitor.cs +++ b/AzureSearchEmulator/Searching/ODataQueryVisitor.cs @@ -14,14 +14,111 @@ namespace AzureSearchEmulator.Searching; public class ODataQueryVisitor(SearchIndex? index = null) : ISyntacticTreeVisitor { private readonly SearchIndex? _index = index; - public Query Visit(AllToken tokenIn) + + // When walking into a lambda (any/all), pushes the (path, parameter) context so that + // child RangeVariableTokens can be resolved back to the collection's field path. + private readonly Stack _lambdaContexts = new(); + + private record LambdaContext(string Path, string Parameter); + + public Query Visit(AllToken tokenIn) => VisitLambda(tokenIn, isAll: true); + + public Query Visit(AnyToken tokenIn) => VisitLambda(tokenIn, isAll: false); + + private Query VisitLambda(LambdaToken tokenIn, bool isAll) { - throw new NotImplementedException(); + var path = ResolveLambdaPath(tokenIn.Parent); + EnsureFilterable(path); + + // any() with no expression: matches docs where the collection is non-empty. + // Lucene equivalent: the field must have at least one indexed term. + if (string.IsNullOrEmpty(tokenIn.Parameter) || tokenIn.Expression is LiteralToken { Value: bool b } && b) + { + if (isAll) + { + // all() with no body is not meaningful in OData; treat as match-all. + return new MatchAllDocsQuery(); + } + + // Match docs that have the field present (any indexed value). + return new ConstantScoreQuery(new WildcardQuery(new Term(path, "*"))); + } + + _lambdaContexts.Push(new LambdaContext(path, tokenIn.Parameter)); + try + { + var inner = tokenIn.Expression.Accept(this); + + if (!isAll) + { + // any(t: P(t)): the per-value query already matches docs where at least one + // indexed value satisfies P, since multi-valued fields share a field name. + return inner; + } + + // all(t: P(t)) ≡ ¬any(t: ¬P(t)). Implemented as MatchAll MUST_NOT (¬P). + // We invert P at the leaf-comparison level so that range/term semantics still + // match the per-value indexing model: e.g. all(t: t ne 'x') becomes + // "no document has a value equal to 'x'". + var negated = NegateLambdaExpression(tokenIn.Expression); + + return new BooleanQuery + { + Clauses = + { + new BooleanClause(new MatchAllDocsQuery(), Occur.MUST), + new BooleanClause(negated, Occur.MUST_NOT) + } + }; + } + finally + { + _lambdaContexts.Pop(); + } } - public Query Visit(AnyToken tokenIn) + private Query NegateLambdaExpression(QueryToken expression) { - throw new NotImplementedException(); + // Logical NOT is rewritten by negating the leaf comparison so that the resulting + // Lucene query stays a clean "MUST_NOT P" against the multi-valued field, instead + // of nested boolean wrappers that produce empty result sets. + return expression switch + { + UnaryOperatorToken { OperatorKind: UnaryOperatorKind.Not } unary => unary.Operand.Accept(this), + BinaryOperatorToken bin => InvertBinary(bin), + _ => expression.Accept(this) + }; + } + + private Query InvertBinary(BinaryOperatorToken bin) + { + var inverted = bin.OperatorKind switch + { + BinaryOperatorKind.Equal => BinaryOperatorKind.NotEqual, + BinaryOperatorKind.NotEqual => BinaryOperatorKind.Equal, + BinaryOperatorKind.LessThan => BinaryOperatorKind.GreaterThanOrEqual, + BinaryOperatorKind.LessThanOrEqual => BinaryOperatorKind.GreaterThan, + BinaryOperatorKind.GreaterThan => BinaryOperatorKind.LessThanOrEqual, + BinaryOperatorKind.GreaterThanOrEqual => BinaryOperatorKind.LessThan, + _ => throw new NotImplementedException($"Cannot invert operator {bin.OperatorKind} inside all(...)") + }; + + var rebuilt = new BinaryOperatorToken(inverted, bin.Left, bin.Right); + return rebuilt.Accept(this); + } + + private string ResolveLambdaPath(QueryToken? parent) + { + // Build a slash-joined path from chained InnerPathTokens; for our flat schema this + // is normally just the collection field's name. + return parent switch + { + EndPathToken end => end.Identifier, + InnerPathToken inner => inner.NextToken == null + ? inner.Identifier + : ResolveLambdaPath(inner.NextToken) + "/" + inner.Identifier, + _ => throw new NotImplementedException("Lambda parent must be a path token") + }; } public Query Visit(BinaryOperatorToken tokenIn) @@ -42,11 +139,8 @@ public Query Visit(BinaryOperatorToken tokenIn) }; } - if (tokenIn is - { - Left: EndPathToken { Identifier: string path }, - Right: LiteralToken literalToken - }) + if (TryResolveComparisonPath(tokenIn.Left, out var path) + && tokenIn.Right is LiteralToken literalToken) { EnsureFilterable(path); return tokenIn.OperatorKind switch @@ -64,9 +158,9 @@ public Query Visit(BinaryOperatorToken tokenIn) // Handle "not field eq value" which OData parses as "(not field) eq value" if (tokenIn is { - Left: UnaryOperatorToken { OperatorKind: UnaryOperatorKind.Not, Operand: EndPathToken { Identifier: string negatedPath } }, + Left: UnaryOperatorToken { OperatorKind: UnaryOperatorKind.Not, Operand: { } negatedOperand }, Right: LiteralToken negatedLiteral - }) + } && TryResolveComparisonPath(negatedOperand, out var negatedPath)) { EnsureFilterable(negatedPath); var equalQuery = tokenIn.OperatorKind switch @@ -93,6 +187,30 @@ public Query Visit(BinaryOperatorToken tokenIn) throw new NotImplementedException(); } + private bool TryResolveComparisonPath(QueryToken token, out string path) + { + // A bare end-path is the simple case: "Field eq 'value'". + if (token is EndPathToken end) + { + path = end.Identifier; + return true; + } + + // Inside a lambda, the range variable resolves back to the collection's field path. + if (token is RangeVariableToken rv && _lambdaContexts.Count > 0) + { + var ctx = _lambdaContexts.Peek(); + if (string.Equals(rv.Name, ctx.Parameter, StringComparison.Ordinal)) + { + path = ctx.Path; + return true; + } + } + + path = string.Empty; + return false; + } + private void EnsureFilterable(string path) { if (_index is null) return; @@ -201,11 +319,8 @@ public Query Visit(CountSegmentToken tokenIn) public Query Visit(InToken tokenIn) { - if (tokenIn is - { - Left: EndPathToken { Identifier: string path }, - Right: LiteralToken { Value: string valueString } - }) + if (TryResolveComparisonPath(tokenIn.Left, out var path) + && tokenIn.Right is LiteralToken { Value: string valueString }) { valueString = valueString.TrimStart('(').TrimEnd(')'); @@ -257,7 +372,7 @@ public Query Visit(FunctionCallToken tokenIn) }; } - private static BooleanQuery VisitSearchIn(FunctionCallToken tokenIn) + private BooleanQuery VisitSearchIn(FunctionCallToken tokenIn) { var args = tokenIn.Arguments.ToList(); @@ -266,9 +381,9 @@ private static BooleanQuery VisitSearchIn(FunctionCallToken tokenIn) throw new ArgumentException("search.in requires two or three arguments"); } - if (args[0].ValueToken is not EndPathToken { Identifier: string path }) + if (!TryResolveComparisonPath(args[0].ValueToken, out var path)) { - throw new NotImplementedException("Passing anything other than an end path as the first parameter to search.in is not yet implemented"); + throw new NotImplementedException("Passing anything other than an end path or lambda variable as the first parameter to search.in is not yet implemented"); } if (args[1].ValueToken is not LiteralToken { Value: string inList })