From 85cca1872fc8dade28b93773fa3f8326b973a505 Mon Sep 17 00:00:00 2001 From: Paul Irwin Date: Thu, 23 Apr 2026 11:19:41 -0600 Subject: [PATCH] Fix $filter on searchable string fields (#27) Searchable Edm.String fields were indexed only as analyzed TextFields, so an exact TermQuery built from the filter literal never matched the lowercased, tokenized terms. Filterable/sortable/facetable searchable string fields now also emit a non-analyzed StringField under the same name, matching Azure's semantics where filters run on raw, non-analyzed content (https://learn.microsoft.com/azure/search/search-analyzers). Also: reject filters against fields with Filterable=false, and fix MergeDocument so dual-indexed fields aren't accidentally dropped during upsert. Co-Authored-By: Claude Opus 4.7 --- .../EmulatorIntegrationTests.cs | 86 +++++++++++- .../LuceneNetIndexSearcherTests.cs | 131 ++++++++++++++++++ .../Indexing/SearchFieldExtensions.cs | 29 +++- .../Indexing/UpsertIndexDocumentActionBase.cs | 17 ++- .../Searching/ODataQueryVisitor.cs | 13 ++ 5 files changed, 262 insertions(+), 14 deletions(-) diff --git a/AzureSearchEmulator.IntegrationTests/EmulatorIntegrationTests.cs b/AzureSearchEmulator.IntegrationTests/EmulatorIntegrationTests.cs index b054570..08d77b9 100644 --- a/AzureSearchEmulator.IntegrationTests/EmulatorIntegrationTests.cs +++ b/AzureSearchEmulator.IntegrationTests/EmulatorIntegrationTests.cs @@ -149,6 +149,90 @@ public async Task SearchDocuments_WithFilter_ShouldReturnResults() await indexClient.DeleteIndexAsync(indexName); } + [Fact] + public async Task SearchDocuments_WithFilter_OnSearchableStringField_ShouldReturnResults() + { + // Regression test for issue #27: filtering a Searchable+Filterable string field + // must work. Before the fix, the analyzer lowercased 'Electronics' at index time + // and the exact TermQuery at filter time found zero hits. + const string indexName = "test-filter-searchable-string"; + var indexClient = factory.CreateSearchIndexClient(); + var searchClient = factory.CreateSearchClient(indexName); + + await CreateIndexAsync(indexClient, indexName); + await UploadDocumentsAsync(searchClient); + + var options = new SearchOptions + { + Filter = "Category eq 'Electronics'", + Size = 50 + }; + var results = await searchClient.SearchAsync("*", options); + + Assert.NotNull(results); + var items = await results.Value.GetResultsAsync().ToListAsync(); + Assert.NotEmpty(items); + Assert.True(items.All(r => r.Document.Category == "Electronics"), + "All results should have Category = Electronics"); + + await indexClient.DeleteIndexAsync(indexName); + } + + [Fact] + public async Task SearchDocuments_SearchOnDualIndexedField_StillMatchesAnalyzedTokens() + { + // Dual-indexing (TextField + StringField under same name) must not break + // full-text search: a tokenized search for 'electronics' must still match. + const string indexName = "test-search-on-dual-indexed"; + var indexClient = factory.CreateSearchIndexClient(); + var searchClient = factory.CreateSearchClient(indexName); + + await CreateIndexAsync(indexClient, indexName); + await UploadDocumentsAsync(searchClient); + + var options = new SearchOptions { SearchFields = { "Category" }, Size = 50 }; + var results = await searchClient.SearchAsync("electronics", options); + var items = await results.Value.GetResultsAsync().ToListAsync(); + + Assert.NotEmpty(items); + Assert.True(items.All(r => r.Document.Category == "Electronics")); + + await indexClient.DeleteIndexAsync(indexName); + } + + [Fact] + public async Task SearchDocuments_FilterOnNonFilterableField_ShouldError() + { + // Azure rejects $filter against a non-filterable field; the emulator should too. + const string indexName = "test-filter-nonfilterable"; + var indexClient = factory.CreateSearchIndexClient(); + var searchClient = factory.CreateSearchClient(indexName); + + var index = new SearchIndex(indexName) + { + Fields = + [ + new SearchField(nameof(Product.Id), SearchFieldDataType.String) { IsKey = true, IsStored = true, IsSearchable = true, IsFilterable = true }, + new SearchField(nameof(Product.Name), SearchFieldDataType.String) { IsSearchable = true, IsFilterable = false, IsStored = true }, + new SearchField(nameof(Product.Description), SearchFieldDataType.String) { IsSearchable = true, IsStored = true }, + new SearchField(nameof(Product.Price), SearchFieldDataType.Double) { IsFilterable = true, IsSortable = true, IsStored = true }, + new SearchField(nameof(Product.Category), SearchFieldDataType.String) { IsSearchable = true, IsFilterable = true, IsStored = true }, + new SearchField(nameof(Product.InStock), SearchFieldDataType.Boolean) { IsFilterable = true, IsStored = true } + ] + }; + await indexClient.CreateIndexAsync(index); + await UploadDocumentsAsync(searchClient); + + var options = new SearchOptions { Filter = "Name eq 'Laptop Pro 15'", Size = 50 }; + await Assert.ThrowsAnyAsync(async () => + { + var results = await searchClient.SearchAsync("*", options); + await results.Value.GetResultsAsync().ToListAsync(); + }); + + await indexClient.DeleteIndexAsync(indexName); + } + [Fact] public async Task SearchDocuments_WithSorting_ShouldReturnSortedResults() { @@ -464,7 +548,7 @@ private static async Task CreateIndexAsync(SearchIndexClient indexC new SearchField(nameof(Product.Name), SearchFieldDataType.String) { IsSearchable = true, IsStored = true }, new SearchField(nameof(Product.Description), SearchFieldDataType.String) { IsSearchable = true, IsStored = true}, new SearchField(nameof(Product.Price), SearchFieldDataType.Double) { IsFilterable = true, IsSortable = true, IsStored = true }, - new SearchField(nameof(Product.Category), SearchFieldDataType.String) { IsFilterable = true, IsStored = true }, + new SearchField(nameof(Product.Category), SearchFieldDataType.String) { IsSearchable = true, IsFilterable = true, IsStored = true }, new SearchField(nameof(Product.InStock), SearchFieldDataType.Boolean) { IsFilterable = true, IsStored = true } ] }; diff --git a/AzureSearchEmulator.UnitTests/LuceneNetIndexSearcherTests.cs b/AzureSearchEmulator.UnitTests/LuceneNetIndexSearcherTests.cs index 240c670..d67f324 100644 --- a/AzureSearchEmulator.UnitTests/LuceneNetIndexSearcherTests.cs +++ b/AzureSearchEmulator.UnitTests/LuceneNetIndexSearcherTests.cs @@ -1,3 +1,5 @@ +using System.Text.Json.Nodes; +using AzureSearchEmulator.Indexing; using AzureSearchEmulator.Models; using AzureSearchEmulator.SearchData; using AzureSearchEmulator.Searching; @@ -928,3 +930,132 @@ private class StubIndexReaderFactory(Lucene.Net.Store.Directory directory) : ILu public void ClearCachedReader(string indexName) { } } } + +/// +/// Tests covering filter behavior for string fields indexed through the real +/// SearchFieldExtensions.CreateField path (which applies the per-field analyzer), +/// exercising combinations of Searchable and Filterable. See issue #27. +/// +public class LuceneNetIndexSearcher_SearchableFilterableTests +{ + private static (LuceneTestHelper helper, LuceneNetIndexSearcher searcher) Build(SearchIndex index, List docs) + { + var luceneDocs = docs.Select(d => + { + var doc = new Lucene.Net.Documents.Document(); + foreach (var field in index.Fields) + { + if (d[field.Name] is { } value) + { + foreach (var f in field.CreateFields(value)) + { + doc.Add(f); + } + } + } + return doc; + }).ToList(); + + var helper = new LuceneTestHelper(index, luceneDocs); + var searcher = new LuceneNetIndexSearcher(new StubReaderFactory(helper.Directory)); + return (helper, searcher); + } + + [Fact] + public async Task Filter_StringEquality_SearchableAndFilterable_ReturnsMatch() + { + var index = new SearchIndex + { + Name = "products", + Fields = + [ + new SearchField { Name = "Id", Type = "Edm.String", Key = true, Searchable = false, Filterable = true }, + new SearchField { Name = "Category", Type = "Edm.String", Searchable = true, Filterable = true }, + ] + }; + var docs = new List + { + new() { ["Id"] = "1", ["Category"] = "Electronics" }, + new() { ["Id"] = "2", ["Category"] = "Books" }, + }; + var (helper, searcher) = Build(index, docs); + using var _ = helper; + + var response = await searcher.Search(index, new SearchRequest + { + Search = "*", + Filter = "Category eq 'Electronics'", + Top = 50 + }); + + Assert.Single(response.Results); + Assert.Equal("1", response.Results[0]["Id"]?.GetValue()); + } + + [Fact] + public async Task Filter_StringEquality_Defaults_ReturnsMatch() + { + // Mirrors the bug reported in issue #27: fields where Searchable and Filterable + // are unset (both default to true for Edm.String) must still filter correctly. + var index = new SearchIndex + { + Name = "products", + Fields = + [ + new SearchField { Name = "Id", Type = "Edm.String", Key = true, Searchable = true, Filterable = true }, + new SearchField { Name = "Category", Type = "Edm.String" }, + ] + }; + var docs = new List + { + new() { ["Id"] = "1", ["Category"] = "Electronics" }, + new() { ["Id"] = "2", ["Category"] = "Books" }, + }; + var (helper, searcher) = Build(index, docs); + using var _ = helper; + + var response = await searcher.Search(index, new SearchRequest + { + Search = "*", + Filter = "Category eq 'Electronics'", + Top = 50 + }); + + Assert.Single(response.Results); + Assert.Equal("1", response.Results[0]["Id"]?.GetValue()); + } + + [Fact] + public async Task Filter_StringEquality_SearchableNotFilterable_ShouldError() + { + var index = new SearchIndex + { + Name = "posts", + Fields = + [ + new SearchField { Name = "Id", Type = "Edm.String", Key = true, Searchable = false, Filterable = true }, + new SearchField { Name = "Body", Type = "Edm.String", Searchable = true, Filterable = false }, + ] + }; + var docs = new List + { + new() { ["Id"] = "1", ["Body"] = "Hello world" }, + }; + var (helper, searcher) = Build(index, docs); + using var _ = helper; + + await Assert.ThrowsAnyAsync(() => searcher.Search(index, new SearchRequest + { + Search = "*", + Filter = "Body eq 'Hello world'", + Top = 50 + })); + } + + private class StubReaderFactory(Lucene.Net.Store.Directory directory) : ILuceneIndexReaderFactory + { + public IndexReader GetIndexReader(string indexName) => DirectoryReader.Open(directory); + public IndexReader RefreshReader(string indexName) => GetIndexReader(indexName); + public void ClearCachedReader(string indexName) { } + } +} diff --git a/AzureSearchEmulator/Indexing/SearchFieldExtensions.cs b/AzureSearchEmulator/Indexing/SearchFieldExtensions.cs index 70112ba..ed12cb5 100644 --- a/AzureSearchEmulator/Indexing/SearchFieldExtensions.cs +++ b/AzureSearchEmulator/Indexing/SearchFieldExtensions.cs @@ -7,14 +7,35 @@ namespace AzureSearchEmulator.Indexing; public static class SearchFieldExtensions { - public static IIndexableField CreateField(this SearchField field, JsonNode value) + public static IEnumerable CreateFields(this SearchField field, JsonNode value) { var stored = field.Retrievable ? Field.Store.YES : Field.Store.NO; - return field.Type switch + if (field.Type == "Edm.String") + { + var str = value.GetValue(); + var searchable = field.Searchable.GetValueOrDefault(true); + var filterable = field.Filterable; + + if (searchable) + { + yield return new TextField(field.Name, str, stored); + // Filter/sort/facet require a non-analyzed copy under the same field name + // so TermQuery-based filters match the raw literal (matches Azure semantics). + if (filterable || field.Sortable.GetValueOrDefault() || field.Facetable.GetValueOrDefault()) + { + yield return new StringField(field.Name, str, Field.Store.NO); + } + } + else + { + yield return new StringField(field.Name, str, stored); + } + yield break; + } + + yield return field.Type switch { - "Edm.String" when field.Searchable.GetValueOrDefault(true) => new TextField(field.Name, value.GetValue(), stored), - "Edm.String" when !field.Searchable.GetValueOrDefault(true) => new StringField(field.Name, value.GetValue(), stored), "Edm.Int32" => new Int32Field(field.Name, value.GetValue(), stored), "Edm.Int64" => new Int64Field(field.Name, value.GetValue(), stored), "Edm.Double" => new DoubleField(field.Name, value.GetValue(), stored), diff --git a/AzureSearchEmulator/Indexing/UpsertIndexDocumentActionBase.cs b/AzureSearchEmulator/Indexing/UpsertIndexDocumentActionBase.cs index c6219c9..a4e9368 100644 --- a/AzureSearchEmulator/Indexing/UpsertIndexDocumentActionBase.cs +++ b/AzureSearchEmulator/Indexing/UpsertIndexDocumentActionBase.cs @@ -33,7 +33,8 @@ protected IEnumerable GetDocFields(SearchIndex index) return from f in index.Fields join v in Item on f.Name equals v.Key where v.Value != null - select f.CreateField(v.Value!); // [!]: null checked by where clause + from indexField in f.CreateFields(v.Value!) // [!]: null checked by where clause + select indexField; } protected static void MergeDocument(IndexingContext context, Term keyTerm, IEnumerable docFields, bool uploadIfMissing) @@ -50,15 +51,13 @@ protected static void MergeDocument(IndexingContext context, Term keyTerm, IEnum var doc = docs.TotalHits == 0 ? new Document() : searcher.Doc(docs.ScoreDocs[0].Doc); - foreach (var docField in docFields) + var materialized = docFields.ToList(); + foreach (var name in materialized.Select(f => f.Name).Distinct()) + { + doc.RemoveFields(name); + } + foreach (var docField in materialized) { - var field = doc.GetField(docField.Name); - - if (field != null) - { - doc.RemoveField(docField.Name); - } - doc.Add(docField); } diff --git a/AzureSearchEmulator/Searching/ODataQueryVisitor.cs b/AzureSearchEmulator/Searching/ODataQueryVisitor.cs index 63c19e5..1d843db 100644 --- a/AzureSearchEmulator/Searching/ODataQueryVisitor.cs +++ b/AzureSearchEmulator/Searching/ODataQueryVisitor.cs @@ -48,6 +48,7 @@ public Query Visit(BinaryOperatorToken tokenIn) Right: LiteralToken literalToken }) { + EnsureFilterable(path); return tokenIn.OperatorKind switch { BinaryOperatorKind.Equal => HandleEqualComparison(path, literalToken), @@ -67,6 +68,7 @@ public Query Visit(BinaryOperatorToken tokenIn) Right: LiteralToken negatedLiteral }) { + EnsureFilterable(negatedPath); var equalQuery = tokenIn.OperatorKind switch { BinaryOperatorKind.Equal => HandleEqualComparison(negatedPath, negatedLiteral), @@ -91,6 +93,17 @@ public Query Visit(BinaryOperatorToken tokenIn) throw new NotImplementedException(); } + private void EnsureFilterable(string path) + { + if (_index is null) return; + var field = _index.Fields.FirstOrDefault(f => string.Equals(f.Name, path, StringComparison.OrdinalIgnoreCase)); + if (field is null) return; + if (!field.Filterable) + { + throw new InvalidOperationException($"Field '{field.Name}' is not filterable."); + } + } + private static Occur GetOccurFromOperator(BinaryOperatorKind operatorKind) { return operatorKind switch