diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/OutputConverter.cs b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/OutputConverter.cs index 27be4dcdea4..224d6525bba 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/OutputConverter.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/OutputConverter.cs @@ -15,6 +15,14 @@ using Microsoft.Agents.AI.Workflows; using Microsoft.Extensions.AI; using MeaiTextContent = Microsoft.Extensions.AI.TextContent; +using OpenAIContainerFileCitationMessageAnnotation = OpenAI.Responses.ContainerFileCitationMessageAnnotation; +using OpenAIFileCitationMessageAnnotation = OpenAI.Responses.FileCitationMessageAnnotation; +using OpenAIFilePathMessageAnnotation = OpenAI.Responses.FilePathMessageAnnotation; +using OpenAIMessageResponseItem = OpenAI.Responses.MessageResponseItem; +using OpenAIStreamingResponseOutputItemAddedUpdate = OpenAI.Responses.StreamingResponseOutputItemAddedUpdate; +using OpenAIStreamingResponseOutputItemDoneUpdate = OpenAI.Responses.StreamingResponseOutputItemDoneUpdate; +using OpenAIStreamingResponseOutputTextDeltaUpdate = OpenAI.Responses.StreamingResponseOutputTextDeltaUpdate; +using OpenAIStreamingResponseTextAnnotationAddedUpdate = OpenAI.Responses.StreamingResponseTextAnnotationAddedUpdate; namespace Microsoft.Agents.AI.Foundry.Hosting; @@ -100,13 +108,14 @@ public static async IAsyncEnumerable ConvertUpdatesToEvents continue; } + string? currentMessageId = ResolveMessageId(update); foreach (var content in update.Contents) { switch (content) { case MeaiTextContent textContent: { - if (!IsSameMessage(update.MessageId, previousMessageId) && currentMessageBuilder is not null) + if (!IsSameMessage(currentMessageId, previousMessageId) && currentMessageBuilder is not null) { foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText, accumulatedAnnotations)) { @@ -119,7 +128,10 @@ public static async IAsyncEnumerable ConvertUpdatesToEvents accumulatedAnnotations = null; } - previousMessageId = update.MessageId; + if (currentMessageId is { Length: > 0 }) + { + previousMessageId = currentMessageId; + } if (currentMessageBuilder is null) { @@ -138,14 +150,6 @@ public static async IAsyncEnumerable ConvertUpdatesToEvents yield return currentTextBuilder!.EmitDelta(textContent.Text); } - if (textContent.Annotations is { Count: > 0 }) - { - foreach (var sdkAnnotation in ConvertToSdkAnnotations(textContent.Annotations)) - { - (accumulatedAnnotations ??= []).Add(sdkAnnotation); - } - } - break; } @@ -336,6 +340,27 @@ public static async IAsyncEnumerable ConvertUpdatesToEvents default: break; } + + var isTextContent = content is MeaiTextContent; + var isAnnotationOnlyContentForCurrentMessage = + content.GetType() == typeof(AIContent) && + IsSameAnnotationMessage(currentMessageId, previousMessageId); + + // MEAI OpenAI sends streaming citations in a separate annotation-only AIContent after + // the text deltas. Accumulate them because AgentServer emits annotations only after + // output_text.done, and de-duplicate providers that report the same citation twice. + if ((isTextContent || isAnnotationOnlyContentForCurrentMessage) + && content.Annotations is { Count: > 0 } + && currentMessageBuilder is not null) + { + foreach (var sdkAnnotation in ConvertToSdkAnnotations(content.Annotations)) + { + if (accumulatedAnnotations?.Any(existing => AreEquivalentAnnotations(existing, sdkAnnotation)) is not true) + { + (accumulatedAnnotations ??= []).Add(sdkAnnotation); + } + } + } } } @@ -385,16 +410,93 @@ private static IEnumerable CloseCurrentMessage( private static bool IsSameMessage(string? currentId, string? previousId) => currentId is not { Length: > 0 } || previousId is not { Length: > 0 } || currentId == previousId; + private static bool IsSameAnnotationMessage(string? currentId, string? previousId) => + currentId is { Length: > 0 } + ? currentId == previousId + : previousId is not { Length: > 0 }; + + private static string? ResolveMessageId(AgentResponseUpdate update) + { + if (update.MessageId is { Length: > 0 }) + { + return update.MessageId; + } + + object? rawRepresentation = update.RawRepresentation is ChatResponseUpdate chatUpdate + ? chatUpdate.RawRepresentation + : update.RawRepresentation; + + return rawRepresentation switch + { + OpenAIStreamingResponseOutputTextDeltaUpdate textDelta => textDelta.ItemId, + OpenAIStreamingResponseTextAnnotationAddedUpdate annotationAdded => annotationAdded.ItemId, + OpenAIStreamingResponseOutputItemAddedUpdate { Item: OpenAIMessageResponseItem message } => message.Id, + OpenAIStreamingResponseOutputItemDoneUpdate { Item: OpenAIMessageResponseItem message } => message.Id, + _ => null, + }; + } + + private static bool AreEquivalentAnnotations(Annotation left, Annotation right) => + (left, right) switch + { + (UrlCitationBody leftUrl, UrlCitationBody rightUrl) => + leftUrl.Url == rightUrl.Url && + leftUrl.StartIndex == rightUrl.StartIndex && + leftUrl.EndIndex == rightUrl.EndIndex && + leftUrl.Title == rightUrl.Title, + (FileCitationBody leftFile, FileCitationBody rightFile) => + leftFile.FileId == rightFile.FileId && + leftFile.Index == rightFile.Index && + leftFile.Filename == rightFile.Filename, + (ContainerFileCitationBody leftContainerFile, ContainerFileCitationBody rightContainerFile) => + leftContainerFile.ContainerId == rightContainerFile.ContainerId && + leftContainerFile.FileId == rightContainerFile.FileId && + leftContainerFile.StartIndex == rightContainerFile.StartIndex && + leftContainerFile.EndIndex == rightContainerFile.EndIndex && + leftContainerFile.Filename == rightContainerFile.Filename, + (FilePath leftFilePath, FilePath rightFilePath) => + leftFilePath.FileId == rightFilePath.FileId && + leftFilePath.Index == rightFilePath.Index, + _ => false, + }; + /// /// Converts MEAI instances to Responses SDK objects. - /// Only with a URL and at least one - /// with explicit start/end indices is converted; all other shapes are skipped. + /// Only supported shapes are converted; all others are skipped. /// private static IEnumerable ConvertToSdkAnnotations(IList annotations) { foreach (var ann in annotations) { - if (ann is not CitationAnnotation citation || citation.Url is null) + if (ann is not CitationAnnotation citation) + { + continue; + } + + if (citation.RawRepresentation is OpenAIContainerFileCitationMessageAnnotation containerFileCitation) + { + yield return new ContainerFileCitationBody( + containerFileCitation.ContainerId, + containerFileCitation.FileId, + containerFileCitation.StartIndex, + containerFileCitation.EndIndex, + containerFileCitation.Filename); + continue; + } + + if (citation.RawRepresentation is OpenAIFileCitationMessageAnnotation fileCitation) + { + yield return new FileCitationBody(fileCitation.FileId, fileCitation.Index, fileCitation.Filename); + continue; + } + + if (citation.RawRepresentation is OpenAIFilePathMessageAnnotation filePath) + { + yield return new FilePath(filePath.FileId, filePath.Index); + continue; + } + + if (citation.Url is null) { continue; } diff --git a/dotnet/src/Shared/IntegrationTests/TestSettings.cs b/dotnet/src/Shared/IntegrationTests/TestSettings.cs index c2a7ab09730..ca830575fa4 100644 --- a/dotnet/src/Shared/IntegrationTests/TestSettings.cs +++ b/dotnet/src/Shared/IntegrationTests/TestSettings.cs @@ -22,6 +22,8 @@ internal static class TestSettings public const string AzureAIProjectEndpoint = "AZURE_AI_PROJECT_ENDPOINT"; // Azure AI Search (Foundry.Hosting integration tests, RAG scenario) + public const string AzureSearchConnectionId = "AZURE_SEARCH_CONNECTION_ID"; + public const string AzureSearchConnectionName = "AZURE_SEARCH_CONNECTION_NAME"; public const string AzureSearchEndpoint = "AZURE_SEARCH_ENDPOINT"; public const string AzureSearchIndexName = "AZURE_SEARCH_INDEX_NAME"; diff --git a/dotnet/tests/Foundry.Hosting.IntegrationTests.TestContainer/Program.cs b/dotnet/tests/Foundry.Hosting.IntegrationTests.TestContainer/Program.cs index 323495c29e4..47c4e83f884 100644 --- a/dotnet/tests/Foundry.Hosting.IntegrationTests.TestContainer/Program.cs +++ b/dotnet/tests/Foundry.Hosting.IntegrationTests.TestContainer/Program.cs @@ -3,6 +3,7 @@ using System.ComponentModel; using Azure; using Azure.AI.Projects; +using Azure.AI.Projects.Agents; using Azure.Identity; using Azure.Search.Documents; using Azure.Search.Documents.Models; @@ -43,6 +44,8 @@ "custom-storage" => CreateCustomStorageAgent(projectClient, deployment), "memory" => await CreateMemoryAgentAsync(projectClient, deployment).ConfigureAwait(false), "azure-search-rag" => CreateAzureSearchRagAgent(projectClient, deployment), + "azure-search-tool-annotations" => CreateAzureSearchToolAnnotationsAgent(projectClient, deployment), + "web-search-annotations" => CreateWebSearchAnnotationsAgent(projectClient, deployment), "session-files" => CreateSessionFilesAgent(projectClient, deployment), "agent-skills" => CreateAgentSkillsAgent(projectClient, deployment), "user-identity" => CreateUserIdentityAgent(projectClient, deployment), @@ -66,14 +69,12 @@ options.SteerableConversations = scenario == "steerable-long-running"; }); -// toolbox-oauth-consent scenario: pre-register a Foundry toolbox whose tool source is fronted by a -// per-user OAuth connection. IT_TOOLBOX_NAME names that toolbox (the fixture sets it). With the -// startup-deferral fix the container stays routable even though the toolbox cannot enumerate without -// a consented user, and the first user request surfaces an oauth_consent_request. -var consentToolboxName = Environment.GetEnvironmentVariable("IT_TOOLBOX_NAME"); -if (!string.IsNullOrEmpty(consentToolboxName)) +// Scenarios that consume a project toolbox set IT_TOOLBOX_NAME through their fixture. +// The hosting bridge resolves the toolbox through its MCP endpoint and adds its tools to every request. +var toolboxName = Environment.GetEnvironmentVariable("IT_TOOLBOX_NAME"); +if (!string.IsNullOrEmpty(toolboxName)) { - builder.Services.AddFoundryToolboxes(credential, consentToolboxName); + builder.Services.AddFoundryToolboxes(credential, toolboxName); } var app = builder.Build(); @@ -196,6 +197,59 @@ static AIAgent CreateAzureSearchRagAgent(AIProjectClient client, string deployme }); } +static AIAgent CreateWebSearchAnnotationsAgent(AIProjectClient client, string deployment) => + client.AsAIAgent(new ChatClientAgentOptions + { + Name = "web-search-annotations-agent", + Description = "Hosted web search annotation test agent.", + ChatOptions = new ChatOptions + { + ModelId = deployment, + Instructions = """ + Answer with current information from the web search results. + Include citations for the sources used in the answer. + """, + Tools = [new HostedWebSearchTool()], + ToolMode = ChatToolMode.RequireAny, + }, + }); + +static AIAgent CreateAzureSearchToolAnnotationsAgent(AIProjectClient client, string deployment) +{ + var connectionId = Environment.GetEnvironmentVariable("AZURE_SEARCH_CONNECTION_ID") + ?? throw new InvalidOperationException( + "AZURE_SEARCH_CONNECTION_ID is not set for IT_SCENARIO=azure-search-tool-annotations."); + var indexName = Environment.GetEnvironmentVariable("AZURE_SEARCH_INDEX_NAME") + ?? throw new InvalidOperationException( + "AZURE_SEARCH_INDEX_NAME is not set for IT_SCENARIO=azure-search-tool-annotations."); + var searchTool = FoundryAITool.CreateAzureAISearchTool(new AzureAISearchToolOptions( + [ + new AzureAISearchToolIndex + { + ProjectConnectionId = connectionId, + IndexName = indexName, + QueryType = AzureAISearchQueryType.Simple, + TopK = 3, + } + ])); + + return client.AsAIAgent(new ChatClientAgentOptions + { + Name = "azure-search-tool-annotations-agent", + Description = "Azure AI Search hosted tool annotation test agent.", + ChatOptions = new ChatOptions + { + ModelId = deployment, + Instructions = """ + Answer only from the Azure AI Search results. + Include citations for the sources used in the answer. + """, + Tools = [searchTool], + ToolMode = ChatToolMode.RequireAny, + }, + }); +} + static Func>> CreateAzureSearchAdapter(SearchClient client, int top = 3) => async (query, cancellationToken) => diff --git a/dotnet/tests/Foundry.Hosting.IntegrationTests/AzureSearchToolAnnotationsHostedAgentTests.cs b/dotnet/tests/Foundry.Hosting.IntegrationTests/AzureSearchToolAnnotationsHostedAgentTests.cs new file mode 100644 index 00000000000..5d85f8b1ba1 --- /dev/null +++ b/dotnet/tests/Foundry.Hosting.IntegrationTests/AzureSearchToolAnnotationsHostedAgentTests.cs @@ -0,0 +1,125 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Foundry.Hosting.IntegrationTests.Fixtures; +using OpenAI.Responses; + +#pragma warning disable OPENAI001 // Experimental Responses API surfaces + +namespace Foundry.Hosting.IntegrationTests; + +/// +/// Verifies that citations produced by the Foundry Azure AI Search hosted tool survive the nested +/// model call and are returned by the hosted Responses API. +/// +[Trait("Category", "FoundryHostedAgents")] +public sealed class AzureSearchToolAnnotationsHostedAgentTests( + AzureSearchToolAnnotationsHostedAgentFixture fixture) + : IClassFixture +{ + private static readonly TimeSpan s_timeout = TimeSpan.FromMinutes(3); + private readonly AzureSearchToolAnnotationsHostedAgentFixture _fixture = fixture; + + [Fact] + public async Task ResponsesApi_Streaming_EmitsUrlCitationAnnotationsAsync() + { + // Arrange + ResponsesClient responses = this._fixture.AgentOpenAIClient.GetProjectResponsesClient(); + CreateResponseOptions options = CreateRequest(); + using CancellationTokenSource timeout = new(s_timeout); + List annotations = []; + StreamingResponseCompletedUpdate? completed = null; + + // Act + await foreach (StreamingResponseUpdate update in responses + .CreateResponseStreamingAsync(options, timeout.Token) + .WithCancellation(timeout.Token)) + { + switch (update) + { + case StreamingResponseTextAnnotationAddedUpdate annotation: + using (JsonDocument document = JsonDocument.Parse(annotation.Annotation)) + { + annotations.Add(document.RootElement.Clone()); + } + break; + + case StreamingResponseCompletedUpdate completedUpdate: + completed = completedUpdate; + break; + + case StreamingResponseFailedUpdate failed: + throw new InvalidOperationException( + $"Hosted Azure AI Search response failed: {failed.Response.Error?.Message}"); + } + } + + // Assert + Assert.NotNull(completed); + Assert.Contains(annotations, IsValidUrlCitation); + Assert.Contains(GetFinalAnnotations(completed.Response), IsValidUrlCitation); + Assert.Contains("TR-CANARY-7821", completed.Response.GetOutputText(), StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task ResponsesApi_NonStreaming_ReturnsUrlCitationAnnotationsAsync() + { + // Arrange + ResponsesClient responses = this._fixture.AgentOpenAIClient.GetProjectResponsesClient(); + CreateResponseOptions options = CreateRequest(); + using CancellationTokenSource timeout = new(s_timeout); + + // Act + ResponseResult response = (await responses.CreateResponseAsync(options, timeout.Token)).Value; + + // Assert + Assert.True( + response.Status == ResponseStatus.Completed, + $"Hosted Azure AI Search response failed: {response.Error?.Message}"); + Assert.Contains("TR-CANARY-7821", response.GetOutputText(), StringComparison.OrdinalIgnoreCase); + Assert.Contains(GetFinalAnnotations(response), IsValidUrlCitation); + } + + private static CreateResponseOptions CreateRequest() + { + CreateResponseOptions options = new() + { + StoredOutputEnabled = false, + }; + options.InputItems.Add(ResponseItem.CreateUserMessageItem( + "What item code do I get with my return? Use Azure AI Search and cite the source.")); + return options; + } + + private static IEnumerable GetFinalAnnotations(ResponseResult response) => + response.OutputItems + .OfType() + .SelectMany(message => message.Content) + .SelectMany(part => part.OutputTextAnnotations) + .OfType(); + + private static bool IsValidUrlCitation(JsonElement annotation) => + annotation.TryGetProperty("type", out JsonElement type) && + type.GetString() == "url_citation" && + annotation.TryGetProperty("url", out JsonElement url) && + Uri.TryCreate(url.GetString(), UriKind.Absolute, out _) && + annotation.TryGetProperty("title", out JsonElement title) && + !string.IsNullOrWhiteSpace(title.GetString()) && + annotation.TryGetProperty("start_index", out JsonElement startIndex) && + startIndex.TryGetInt32(out int start) && + start >= 0 && + annotation.TryGetProperty("end_index", out JsonElement endIndex) && + endIndex.TryGetInt32(out int end) && + end >= start; + + private static bool IsValidUrlCitation(UriCitationMessageAnnotation annotation) => + annotation.Uri.IsAbsoluteUri && + !string.IsNullOrWhiteSpace(annotation.Title) && + annotation.StartIndex >= 0 && + annotation.EndIndex >= annotation.StartIndex; +} diff --git a/dotnet/tests/Foundry.Hosting.IntegrationTests/Fixtures/AzureSearchToolAnnotationsHostedAgentFixture.cs b/dotnet/tests/Foundry.Hosting.IntegrationTests/Fixtures/AzureSearchToolAnnotationsHostedAgentFixture.cs new file mode 100644 index 00000000000..d362b5473d6 --- /dev/null +++ b/dotnet/tests/Foundry.Hosting.IntegrationTests/Fixtures/AzureSearchToolAnnotationsHostedAgentFixture.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using AgentConformance.IntegrationTests.Support; +using Shared.IntegrationTests; + +namespace Foundry.Hosting.IntegrationTests.Fixtures; + +/// +/// Provisions a hosted agent that uses the Foundry Azure AI Search hosted tool and exposes its +/// citations through the hosted Responses API. +/// +public sealed class AzureSearchToolAnnotationsHostedAgentFixture : HostedAgentFixture +{ + private const string DefaultConnectionName = "azure-ai-search-contoso"; + + protected override string ScenarioName => "azure-search-tool-annotations"; + + protected override void ConfigureEnvironment(IDictionary environment) + { + var connectionName = + TestConfiguration.GetValue(TestSettings.AzureSearchConnectionName) ?? + DefaultConnectionName; + environment[TestSettings.AzureSearchConnectionId] = + this.ProjectClient.Connections.GetConnection(connectionName).Id; + environment[TestSettings.AzureSearchIndexName] = + TestConfiguration.GetRequiredValue(TestSettings.AzureSearchIndexName); + } +} diff --git a/dotnet/tests/Foundry.Hosting.IntegrationTests/Fixtures/WebSearchAnnotationsHostedAgentFixture.cs b/dotnet/tests/Foundry.Hosting.IntegrationTests/Fixtures/WebSearchAnnotationsHostedAgentFixture.cs new file mode 100644 index 00000000000..80ed518b035 --- /dev/null +++ b/dotnet/tests/Foundry.Hosting.IntegrationTests/Fixtures/WebSearchAnnotationsHostedAgentFixture.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Foundry.Hosting.IntegrationTests.Fixtures; + +/// +/// Provisions a hosted agent that uses +/// and exposes its output through the hosted Responses API. +/// +public sealed class WebSearchAnnotationsHostedAgentFixture : HostedAgentFixture +{ + protected override string ScenarioName => "web-search-annotations"; +} diff --git a/dotnet/tests/Foundry.Hosting.IntegrationTests/README.md b/dotnet/tests/Foundry.Hosting.IntegrationTests/README.md index c20ba717ff4..f29d7c59351 100644 --- a/dotnet/tests/Foundry.Hosting.IntegrationTests/README.md +++ b/dotnet/tests/Foundry.Hosting.IntegrationTests/README.md @@ -67,7 +67,8 @@ The container scenario injects `USER-ID:` via | `AZURE_AI_MODEL_DEPLOYMENT_NAME` | Foundry project | Model the agent uses. Defaults to `gpt-4o` inside the container. | | `IT_HOSTED_AGENT_IMAGE` | `scripts/it-build-image.ps1` | ACR image reference the agent points at. | | `AZURE_SEARCH_ENDPOINT` | Pre-provisioned Azure AI Search service | Endpoint for the `azure-search-rag` scenario. The index it points at must already exist with the schema and content described under **Azure AI Search index prerequisite** below. | -| `AZURE_SEARCH_INDEX_NAME` | Pre-provisioned Azure AI Search service | Name of the pre-seeded index for the `azure-search-rag` scenario. | +| `AZURE_SEARCH_INDEX_NAME` | Pre-provisioned Azure AI Search service | Name of the pre-seeded index used by both Search scenarios. | +| `AZURE_SEARCH_CONNECTION_NAME` | Foundry project connection | Optional connection name used by the `azure-search-tool-annotations` scenario. Defaults to `azure-ai-search-contoso`. | ## One-time bootstrap (per Foundry project) @@ -92,7 +93,13 @@ running the tests. The bootstrap script grants only `Azure AI User` on the Foundry project scope, which is what every hosted agent needs to receive inbound inference traffic. Scenarios that read from external data services need an additional grant on that service to the agent's managed -identity. Today only the `azure-search-rag` scenario falls into this category. +identity. Both Search scenarios need data-plane access, but they use different identities: + +- `azure-search-rag` calls `SearchClient` inside the container, so the hosted agent's managed + identity needs `Search Index Data Reader`. +- `azure-search-tool-annotations` sends a Foundry Azure AI Search hosted tool through a + `ProjectManagedIdentity` connection, so the Foundry account's managed identity needs the roles + listed below. For `it-azure-search-rag`, after the first bootstrap run, grant `Search Index Data Reader` on the Azure AI Search service to the agent's managed identity: @@ -115,6 +122,35 @@ az role assignment create ` Wait ~3 minutes after the grant for RBAC propagation before running the tests. +The `azure-search-tool-annotations` fixture resolves the pre-provisioned connection by name and +passes its full resource ID plus the index name to the container. The container builds the Azure AI +Search hosted tool descriptor sent with the model request. Creating this descriptor does not +provision a project resource. The connection must use `ProjectManagedIdentity`. Grant the Foundry +account's system-assigned managed identity the roles required by the hosted tool on the Search service: + +```powershell +az role assignment create ` + --assignee-object-id "" ` + --assignee-principal-type ServicePrincipal ` + --role "Search Index Data Contributor" ` + --scope "/subscriptions//resourceGroups//providers/Microsoft.Search/searchServices/" + +az role assignment create ` + --assignee-object-id "" ` + --assignee-principal-type ServicePrincipal ` + --role "Search Service Contributor" ` + --scope "/subscriptions//resourceGroups//providers/Microsoft.Search/searchServices/" +``` + +The integration environment must provision these roles and the connection before running +`AzureSearchToolAnnotationsHostedAgentTests`. + +Projects may also retain a dedicated `ai-search-toolbox` containing the same Azure AI Search tool, +connection, and index for toolbox integration tests. Keep that toolbox separate from the +`AzureSearchToolAnnotationsHostedAgentTests`: toolbox calls surface the search result as tool output, +while these tests specifically verify provider-generated `response.output_text.annotation.added` +events from the hosted tool descriptor. + If the search service has `authOptions = apiKeyOnly` (default for older deployments), Entra auth will return 403 regardless of role assignments. Flip it to `aadOrApiKey` first: @@ -124,8 +160,10 @@ az search service update -g -n --auth-options aadOrApiKey ### Azure AI Search index prerequisite (one time, out of band) -The `azure-search-rag` scenario assumes the index pointed at by `AZURE_SEARCH_INDEX_NAME` already -exists with the schema and Contoso Outdoors content the test asserts against. See +Both Search scenarios assume the index pointed at by `AZURE_SEARCH_INDEX_NAME` already exists +with the schema and Contoso Outdoors content the tests assert against. The index must include +retrievable source name and source URL fields so the hosted tool can return `url_citation` +annotations. See `dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-AzureSearchRag/README.md` for the schema and copy-pasteable provisioning snippet. Provisioning the index from your user identity needs `Search Index Data Contributor` on the search service scope. The search service @@ -244,6 +282,8 @@ human-only operation; CI only adds and deletes versions under existing agents. | `CustomStorageHostedAgentFixture` | `custom-storage` | `it-custom-storage` | Round trip with custom `IResponsesStorageProvider`; multi turn reads from the custom store (placeholder). | | `MemoryHostedAgentFixture` | `memory` | `it-memory` | `FoundryMemoryProvider` (scoped via `HostedSessionContext`) running inside the hosted agent recalls user preferences across multiple turns; the memory store name is randomised per fixture (`IT_MEMORY_STORE_ID`). | | `AzureSearchRagHostedAgentFixture` | `azure-search-rag` | `it-azure-search-rag` | RAG against a real Azure AI Search index seeded with Contoso Outdoors documents; verifies the model cites the retrieved sources. | +| `AzureSearchToolAnnotationsHostedAgentFixture` | `azure-search-tool-annotations` | `it-azure-search-tool-annotations` | Uses the Azure AI Search hosted tool with a pre-provisioned connection and verifies URL citation annotations through streaming and non-streaming Responses API calls. | +| `WebSearchAnnotationsHostedAgentFixture` | `web-search-annotations` | `it-web-search-annotations` | Calls `HostedWebSearchTool` and verifies URL citation annotations through streaming and non-streaming Responses API calls. | | `SessionFilesHostedAgentFixture` | `session-files` | `it-session-files` | End-to-end: upload via `AgentSessionFiles` (alpha) into a pinned `agent_session_id`, invoke the agent, assert it reads the file via the container's `ReadFile` tool. | | `AgentSkillsHostedAgentFixture` | `agent-skills` | `it-agent-skills` | Agent skills via `AgentSkillsProvider`: advertises two Contoso Outdoors skills (support-style, escalation-policy) in the system prompt, loads them on demand via `load_skill`, verifies canary tokens prove the skill was loaded. | | `ResilientWorkflowHostedAgentFixture` | `resilient-workflow` | `it-resilient-workflow` | Stored background workflow remains active without client traffic, completes after an intentional container process crash, and replays a complete 20-item countdown without a sequence cursor. | diff --git a/dotnet/tests/Foundry.Hosting.IntegrationTests/WebSearchAnnotationsHostedAgentTests.cs b/dotnet/tests/Foundry.Hosting.IntegrationTests/WebSearchAnnotationsHostedAgentTests.cs new file mode 100644 index 00000000000..a239a1e41d3 --- /dev/null +++ b/dotnet/tests/Foundry.Hosting.IntegrationTests/WebSearchAnnotationsHostedAgentTests.cs @@ -0,0 +1,123 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Foundry.Hosting.IntegrationTests.Fixtures; +using OpenAI.Responses; + +#pragma warning disable OPENAI001 // Experimental Responses API surfaces + +namespace Foundry.Hosting.IntegrationTests; + +/// +/// Verifies that citations produced by +/// survive the nested model call and are returned by the hosted Responses API. +/// +[Trait("Category", "FoundryHostedAgents")] +public sealed class WebSearchAnnotationsHostedAgentTests( + WebSearchAnnotationsHostedAgentFixture fixture) + : IClassFixture +{ + private static readonly TimeSpan s_timeout = TimeSpan.FromMinutes(3); + private readonly WebSearchAnnotationsHostedAgentFixture _fixture = fixture; + + [Fact] + public async Task ResponsesApi_Streaming_EmitsUrlCitationAnnotationsAsync() + { + // Arrange + ResponsesClient responses = this._fixture.AgentOpenAIClient.GetProjectResponsesClient(); + CreateResponseOptions options = CreateRequest(); + using CancellationTokenSource timeout = new(s_timeout); + List annotations = []; + StreamingResponseCompletedUpdate? completed = null; + + // Act + await foreach (StreamingResponseUpdate update in responses + .CreateResponseStreamingAsync(options, timeout.Token) + .WithCancellation(timeout.Token)) + { + switch (update) + { + case StreamingResponseTextAnnotationAddedUpdate annotation: + using (JsonDocument document = JsonDocument.Parse(annotation.Annotation)) + { + annotations.Add(document.RootElement.Clone()); + } + break; + + case StreamingResponseCompletedUpdate completedUpdate: + completed = completedUpdate; + break; + + case StreamingResponseFailedUpdate failed: + throw new InvalidOperationException( + $"Hosted web search response failed: {failed.Response.Error?.Message}"); + } + } + + // Assert + Assert.NotNull(completed); + Assert.Contains(annotations, IsValidUrlCitation); + Assert.Contains(GetFinalAnnotations(completed.Response), IsValidUrlCitation); + } + + [Fact] + public async Task ResponsesApi_NonStreaming_ReturnsUrlCitationAnnotationsAsync() + { + // Arrange + ResponsesClient responses = this._fixture.AgentOpenAIClient.GetProjectResponsesClient(); + CreateResponseOptions options = CreateRequest(); + using CancellationTokenSource timeout = new(s_timeout); + + // Act + ResponseResult response = (await responses.CreateResponseAsync(options, timeout.Token)).Value; + + // Assert + Assert.Equal(ResponseStatus.Completed, response.Status); + Assert.False(string.IsNullOrWhiteSpace(response.GetOutputText())); + Assert.Contains(GetFinalAnnotations(response), IsValidUrlCitation); + } + + private static CreateResponseOptions CreateRequest() + { + CreateResponseOptions options = new() + { + StoredOutputEnabled = false, + }; + options.InputItems.Add(ResponseItem.CreateUserMessageItem( + "Search the web for the official Microsoft .NET support policy. " + + "Report the current support end date for .NET 10 and cite the official source.")); + return options; + } + + private static IEnumerable GetFinalAnnotations(ResponseResult response) => + response.OutputItems + .OfType() + .SelectMany(message => message.Content) + .SelectMany(part => part.OutputTextAnnotations) + .OfType(); + + private static bool IsValidUrlCitation(JsonElement annotation) => + annotation.TryGetProperty("type", out JsonElement type) && + type.GetString() == "url_citation" && + annotation.TryGetProperty("url", out JsonElement url) && + Uri.TryCreate(url.GetString(), UriKind.Absolute, out _) && + annotation.TryGetProperty("title", out JsonElement title) && + !string.IsNullOrWhiteSpace(title.GetString()) && + annotation.TryGetProperty("start_index", out JsonElement startIndex) && + startIndex.TryGetInt32(out int start) && + start >= 0 && + annotation.TryGetProperty("end_index", out JsonElement endIndex) && + endIndex.TryGetInt32(out int end) && + end >= start; + + private static bool IsValidUrlCitation(UriCitationMessageAnnotation annotation) => + annotation.Uri.IsAbsoluteUri && + !string.IsNullOrWhiteSpace(annotation.Title) && + annotation.StartIndex >= 0 && + annotation.EndIndex >= annotation.StartIndex; +} diff --git a/dotnet/tests/Foundry.Hosting.IntegrationTests/scripts/it-bootstrap-agents.ps1 b/dotnet/tests/Foundry.Hosting.IntegrationTests/scripts/it-bootstrap-agents.ps1 index 9aa0472bd57..384c4361ea3 100644 --- a/dotnet/tests/Foundry.Hosting.IntegrationTests/scripts/it-bootstrap-agents.ps1 +++ b/dotnet/tests/Foundry.Hosting.IntegrationTests/scripts/it-bootstrap-agents.ps1 @@ -50,6 +50,8 @@ $Scenarios = @( 'custom-storage', 'memory', 'azure-search-rag', + 'azure-search-tool-annotations', + 'web-search-annotations', 'session-files', 'agent-skills', 'user-identity', diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/AgentFrameworkResponseHandlerTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/AgentFrameworkResponseHandlerTests.cs index 2b7ae5b0061..509a24a6ab9 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/AgentFrameworkResponseHandlerTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/AgentFrameworkResponseHandlerTests.cs @@ -1366,6 +1366,50 @@ await DrainEventsAsync(handler.CreateAsync( Assert.Equal("set by the container", raw.EndUserId); } + [Fact] + public async Task CreateAsync_ChatClientAnnotationOnlyUpdate_EmitsCitationAsync() + { + // Arrange + var annotation = new CitationAnnotation + { + Url = new Uri("https://example.com/doc"), + Title = "Example Document", + AnnotatedRegions = [new TextSpanAnnotatedRegion { StartIndex = 0, EndIndex = 5 }] + }; + var client = new Mock(); + client.Setup(c => c.GetStreamingResponseAsync( + It.IsAny>(), + It.IsAny(), + It.IsAny())) + .Returns(() => ToAsyncEnumerableUpdatesAsync( + new ChatResponseUpdate(ChatRole.Assistant, "Hello") { MessageId = "resp_msg_1" }, + new ChatResponseUpdate( + ChatRole.Assistant, + [new AIContent { Annotations = [annotation] }]) + { + MessageId = "resp_msg_1" + })); + + var agent = new ChatClientAgent(client.Object); + var handler = BuildHandlerWith(agent, new FakeHostedSessionIsolationKeyProvider(), new InMemoryAgentSessionStore()); + + // Act + var events = new List(); + await foreach (var evt in handler.CreateAsync( + NewConversationRequest("conv-citation", "a question", store: true), + NewContextServing("resp_" + new string('c', 46), []), + CancellationToken.None)) + { + events.Add(evt); + } + + // Assert + var annotationEvent = Assert.Single(events.OfType()); + var citation = Assert.IsType(annotationEvent.Annotation); + Assert.Equal(new Uri("https://example.com/doc"), citation.Url); + Assert.Equal("Example Document", citation.Title); + } + private static CreateResponse NewConversationRequest(string conversationId, string text, bool store) { var request = new CreateResponse { Model = "test", Store = store }; diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests.csproj b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests.csproj index f9e81d5a3ed..41c9906b830 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests.csproj +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests.csproj @@ -13,6 +13,7 @@ + diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/OutputConverterTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/OutputConverterTests.cs index d385a763edd..b56e432459e 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/OutputConverterTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/OutputConverterTests.cs @@ -10,7 +10,15 @@ using Microsoft.Agents.AI.Workflows; using Microsoft.Extensions.AI; using Moq; +using ContainerFileCitationMessageAnnotation = OpenAI.Responses.ContainerFileCitationMessageAnnotation; +using FileCitationMessageAnnotation = OpenAI.Responses.FileCitationMessageAnnotation; +using FilePathMessageAnnotation = OpenAI.Responses.FilePathMessageAnnotation; using MeaiTextContent = Microsoft.Extensions.AI.TextContent; +using OpenAIResponseItem = OpenAI.Responses.ResponseItem; +using OpenAIStreamingResponseOutputItemDoneUpdate = OpenAI.Responses.StreamingResponseOutputItemDoneUpdate; +using OpenAIStreamingResponseOutputTextDeltaUpdate = OpenAI.Responses.StreamingResponseOutputTextDeltaUpdate; + +#pragma warning disable OPENAI001 // Experimental Responses API surfaces namespace Microsoft.Agents.AI.Foundry.Hosting.UnitTests; @@ -1404,6 +1412,455 @@ public async Task ConvertUpdatesToEventsAsync_TextWithUrlCitationAnnotation_Emit Assert.IsType(events[^1]); } + /// An annotation-only update following text for the same message emits a url_citation event. + [Fact] + public async Task ConvertUpdatesToEventsAsync_AnnotationOnlyContentAfterText_EmitsAnnotationEventAsync() + { + // Arrange + var (stream, _) = CreateTestStream(); + var annotation = new CitationAnnotation + { + Url = new Uri("https://example.com/doc"), + Title = "Example Document", + AnnotatedRegions = [new TextSpanAnnotatedRegion { StartIndex = 0, EndIndex = 5 }] + }; + var updates = new[] + { + new AgentResponseUpdate + { + MessageId = "msg_1", + Contents = [new MeaiTextContent("Hello")] + }, + new AgentResponseUpdate + { + MessageId = "msg_1", + Contents = [new AIContent { Annotations = [annotation] }] + }, + }; + + // Act + var events = new List(); + await foreach (var evt in OutputConverter.ConvertUpdatesToEventsAsync(ToAsync(updates), stream)) + { + events.Add(evt); + } + + // Assert + var annotationEvent = Assert.Single(events.OfType()); + var urlCitation = Assert.IsType(annotationEvent.Annotation); + Assert.Equal(new Uri("https://example.com/doc"), urlCitation.Url); + Assert.Equal("Example Document", urlCitation.Title); + Assert.Equal(0L, urlCitation.StartIndex); + Assert.Equal(5L, urlCitation.EndIndex); + + var contentPartDone = Assert.Single(events.OfType()); + var donePart = Assert.IsType(contentPartDone.Part); + Assert.IsType(Assert.Single(donePart.Annotations)); + + var outputItemDone = Assert.Single(events.OfType()); + var doneMessage = Assert.IsType(outputItemDone.Item); + var doneText = Assert.IsType(Assert.Single(doneMessage.Content)); + Assert.IsType(Assert.Single(doneText.Annotations)); + } + + /// The same citation attached to text and a later annotation-only update is emitted once. + [Fact] + public async Task ConvertUpdatesToEventsAsync_DuplicateCitationAcrossContentUpdates_EmitsOnceAsync() + { + // Arrange + var (stream, _) = CreateTestStream(); + var annotation = new CitationAnnotation + { + Url = new Uri("https://example.com/doc"), + Title = "Example Document", + AnnotatedRegions = [new TextSpanAnnotatedRegion { StartIndex = 0, EndIndex = 5 }] + }; + var updates = new[] + { + new AgentResponseUpdate + { + MessageId = "msg_1", + Contents = [new MeaiTextContent("Hello") { Annotations = [annotation] }] + }, + new AgentResponseUpdate + { + MessageId = "msg_1", + Contents = [new AIContent { Annotations = [annotation] }] + }, + }; + + // Act + var events = new List(); + await foreach (var evt in OutputConverter.ConvertUpdatesToEventsAsync(ToAsync(updates), stream)) + { + events.Add(evt); + } + + // Assert + Assert.Single(events.OfType()); + } + + /// An annotation-only update for a different message is not attached to the open message. + [Fact] + public async Task ConvertUpdatesToEventsAsync_AnnotationForDifferentMessage_IsSkippedAsync() + { + // Arrange + var (stream, _) = CreateTestStream(); + var annotation = new CitationAnnotation + { + Url = new Uri("https://example.com/doc"), + Title = "Example Document", + AnnotatedRegions = [new TextSpanAnnotatedRegion { StartIndex = 0, EndIndex = 5 }] + }; + var updates = new[] + { + new AgentResponseUpdate + { + MessageId = "msg_1", + Contents = [new MeaiTextContent("Hello")] + }, + new AgentResponseUpdate + { + MessageId = "msg_2", + Contents = [new AIContent { Annotations = [annotation] }] + }, + }; + + // Act + var events = new List(); + await foreach (var evt in OutputConverter.ConvertUpdatesToEventsAsync(ToAsync(updates), stream)) + { + events.Add(evt); + } + + // Assert + Assert.Empty(events.OfType()); + } + + /// An annotation-only update without a message ID is not attached to the open message. + [Fact] + public async Task ConvertUpdatesToEventsAsync_AnnotationWithoutMessageId_IsSkippedAsync() + { + // Arrange + var (stream, _) = CreateTestStream(); + var annotation = new CitationAnnotation + { + Url = new Uri("https://example.com/doc"), + Title = "Example Document", + AnnotatedRegions = [new TextSpanAnnotatedRegion { StartIndex = 0, EndIndex = 5 }] + }; + var updates = new[] + { + new AgentResponseUpdate + { + MessageId = "msg_1", + Contents = [new MeaiTextContent("Hello")] + }, + new AgentResponseUpdate + { + Contents = [new AIContent { Annotations = [annotation] }] + }, + }; + + // Act + var events = new List(); + await foreach (var evt in OutputConverter.ConvertUpdatesToEventsAsync(ToAsync(updates), stream)) + { + events.Add(evt); + } + + // Assert + Assert.Empty(events.OfType()); + } + + /// Generic updates without message IDs can attach annotations to the only open message. + [Fact] + public async Task ConvertUpdatesToEventsAsync_TextAndAnnotationWithoutMessageIds_EmitsAnnotationAsync() + { + // Arrange + var (stream, _) = CreateTestStream(); + var annotation = new CitationAnnotation + { + Url = new Uri("https://example.com/doc"), + Title = "Example Document", + AnnotatedRegions = [new TextSpanAnnotatedRegion { StartIndex = 0, EndIndex = 5 }] + }; + var updates = new[] + { + new AgentResponseUpdate + { + Contents = [new MeaiTextContent("Hello")] + }, + new AgentResponseUpdate + { + Contents = [new AIContent { Annotations = [annotation] }] + }, + }; + + // Act + var events = new List(); + await foreach (var evt in OutputConverter.ConvertUpdatesToEventsAsync(ToAsync(updates), stream)) + { + events.Add(evt); + } + + // Assert + Assert.Single(events.OfType()); + } + + /// OpenAI item IDs recover correlation when flattened message IDs are absent. + [Fact] + public async Task ConvertUpdatesToEventsAsync_OpenAIRawItemIds_EmitsAnnotationAsync() + { + // Arrange + var (stream, _) = CreateTestStream(); + var annotation = new CitationAnnotation + { + Url = new Uri("https://example.com/doc"), + Title = "Example Document", + AnnotatedRegions = [new TextSpanAnnotatedRegion { StartIndex = 0, EndIndex = 5 }] + }; + var completedMessage = OpenAIResponseItem.CreateAssistantMessageItem("Hello"); + completedMessage.Id = "msg_raw"; + var updates = new[] + { + new AgentResponseUpdate + { + Contents = [new MeaiTextContent("Hello")], + RawRepresentation = new ChatResponseUpdate + { + RawRepresentation = new OpenAIStreamingResponseOutputTextDeltaUpdate + { + ItemId = "msg_raw" + } + } + }, + new AgentResponseUpdate + { + Contents = [new AIContent { Annotations = [annotation] }], + RawRepresentation = new ChatResponseUpdate + { + RawRepresentation = new OpenAIStreamingResponseOutputItemDoneUpdate + { + Item = completedMessage + } + } + }, + }; + + // Act + var events = new List(); + await foreach (var evt in OutputConverter.ConvertUpdatesToEventsAsync(ToAsync(updates), stream)) + { + events.Add(evt); + } + + // Assert + Assert.Single(events.OfType()); + } + + /// An annotation on non-text content is not attached to the open text message. + [Fact] + public async Task ConvertUpdatesToEventsAsync_AnnotationOnDataContent_IsSkippedAsync() + { + // Arrange + var (stream, _) = CreateTestStream(); + var annotation = new CitationAnnotation + { + Url = new Uri("https://example.com/image"), + Title = "Image source", + AnnotatedRegions = [new TextSpanAnnotatedRegion { StartIndex = 0, EndIndex = 5 }] + }; + var updates = new[] + { + new AgentResponseUpdate + { + MessageId = "msg_1", + Contents = [new MeaiTextContent("Hello")] + }, + new AgentResponseUpdate + { + MessageId = "msg_1", + Contents = + [ + new DataContent("data:image/png;base64,aWNv", "image/png") + { + Annotations = [annotation] + } + ] + }, + }; + + // Act + var events = new List(); + await foreach (var evt in OutputConverter.ConvertUpdatesToEventsAsync(ToAsync(updates), stream)) + { + events.Add(evt); + } + + // Assert + Assert.Empty(events.OfType()); + } + + /// An annotation-only update without an open text message does not create a message. + [Fact] + public async Task ConvertUpdatesToEventsAsync_AnnotationWithoutOpenMessage_IsSkippedAsync() + { + // Arrange + var (stream, _) = CreateTestStream(); + var annotation = new CitationAnnotation + { + Url = new Uri("https://example.com/doc"), + Title = "Example Document", + AnnotatedRegions = [new TextSpanAnnotatedRegion { StartIndex = 0, EndIndex = 5 }] + }; + var update = new AgentResponseUpdate + { + MessageId = "msg_1", + Contents = [new AIContent { Annotations = [annotation] }] + }; + + // Act + var events = new List(); + await foreach (var evt in OutputConverter.ConvertUpdatesToEventsAsync(ToAsync([update]), stream)) + { + events.Add(evt); + } + + // Assert + Assert.Single(events); + Assert.IsType(events[0]); + } + + /// An annotation-only file citation is emitted with its file metadata. + [Fact] + public async Task ConvertUpdatesToEventsAsync_AnnotationOnlyFileCitation_EmitsAnnotationEventAsync() + { + // Arrange + var (stream, _) = CreateTestStream(); + var annotation = new CitationAnnotation + { + FileId = "file_123", + Title = "report.pdf", + RawRepresentation = new FileCitationMessageAnnotation("file_123", 2, "report.pdf") + }; + var updates = new[] + { + new AgentResponseUpdate + { + MessageId = "msg_1", + Contents = [new MeaiTextContent("See the report")] + }, + new AgentResponseUpdate + { + MessageId = "msg_1", + Contents = [new AIContent { Annotations = [annotation] }] + }, + }; + + // Act + var events = new List(); + await foreach (var evt in OutputConverter.ConvertUpdatesToEventsAsync(ToAsync(updates), stream)) + { + events.Add(evt); + } + + // Assert + var annotationEvent = Assert.Single(events.OfType()); + var fileCitation = Assert.IsType(annotationEvent.Annotation); + Assert.Equal("file_123", fileCitation.FileId); + Assert.Equal(2L, fileCitation.Index); + Assert.Equal("report.pdf", fileCitation.Filename); + } + + /// An annotation-only file path is emitted with its file metadata. + [Fact] + public async Task ConvertUpdatesToEventsAsync_AnnotationOnlyFilePath_EmitsAnnotationEventAsync() + { + // Arrange + var (stream, _) = CreateTestStream(); + var annotation = new CitationAnnotation + { + FileId = "file_123", + RawRepresentation = new FilePathMessageAnnotation("file_123", 3) + }; + var updates = new[] + { + new AgentResponseUpdate + { + MessageId = "msg_1", + Contents = [new MeaiTextContent("Download the file")] + }, + new AgentResponseUpdate + { + MessageId = "msg_1", + Contents = [new AIContent { Annotations = [annotation] }] + }, + }; + + // Act + var events = new List(); + await foreach (var evt in OutputConverter.ConvertUpdatesToEventsAsync(ToAsync(updates), stream)) + { + events.Add(evt); + } + + // Assert + var annotationEvent = Assert.Single(events.OfType()); + var filePath = Assert.IsType(annotationEvent.Annotation); + Assert.Equal("file_123", filePath.FileId); + Assert.Equal(3L, filePath.Index); + } + + /// An annotation-only container file citation is emitted with its container and span metadata. + [Fact] + public async Task ConvertUpdatesToEventsAsync_AnnotationOnlyContainerFileCitation_EmitsAnnotationEventAsync() + { + // Arrange + var (stream, _) = CreateTestStream(); + var annotation = new CitationAnnotation + { + FileId = "file_123", + Title = "chart.png", + AnnotatedRegions = [new TextSpanAnnotatedRegion { StartIndex = 4, EndIndex = 9 }], + RawRepresentation = new ContainerFileCitationMessageAnnotation( + "container_123", + "file_123", + 4, + 9, + "chart.png") + }; + var updates = new[] + { + new AgentResponseUpdate + { + MessageId = "msg_1", + Contents = [new MeaiTextContent("See chart")] + }, + new AgentResponseUpdate + { + MessageId = "msg_1", + Contents = [new AIContent { Annotations = [annotation] }] + }, + }; + + // Act + var events = new List(); + await foreach (var evt in OutputConverter.ConvertUpdatesToEventsAsync(ToAsync(updates), stream)) + { + events.Add(evt); + } + + // Assert + var annotationEvent = Assert.Single(events.OfType()); + var containerCitation = Assert.IsType(annotationEvent.Annotation); + Assert.Equal("container_123", containerCitation.ContainerId); + Assert.Equal("file_123", containerCitation.FileId); + Assert.Equal(4L, containerCitation.StartIndex); + Assert.Equal(9L, containerCitation.EndIndex); + Assert.Equal("chart.png", containerCitation.Filename); + } + /// The content_part.done and output_item.done payloads carry the url_citation metadata, guarding against the empty-annotations regression where only the annotation.added event fires. [Fact] public async Task ConvertUpdatesToEventsAsync_TextWithUrlCitationAnnotation_DoneEventsCarryAnnotationMetadataAsync() diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/ServiceCollectionExtensionsTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/ServiceCollectionExtensionsTests.cs index 855e7b861ff..027572f3cea 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/ServiceCollectionExtensionsTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/ServiceCollectionExtensionsTests.cs @@ -2,13 +2,18 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Net; using System.Net.Http; +using System.Net.ServerSentEvents; using System.Text; +using System.Text.Json; +using System.Threading; using System.Threading.Tasks; using Azure.AI.AgentServer.Responses; using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting.Server; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.TestHost; using Microsoft.Extensions.AI; @@ -19,6 +24,8 @@ using Moq; using OpenAI.Responses; +#pragma warning disable OPENAI001 // Experimental Responses API surfaces + namespace Microsoft.Agents.AI.Foundry.Hosting.UnitTests; public class ServiceCollectionExtensionsTests @@ -324,6 +331,215 @@ public async Task MapFoundryResponses_HostedCreateWithoutCallId_ReturnsUnsupport Assert.Contains("2.0.0", body, StringComparison.Ordinal); } + [Fact] + public async Task MapFoundryResponses_StreamTrue_EmitsAllAnnotationKindsAsync() + { + // Arrange and Act + var (statusCode, mediaType, body) = await InvokeResponsesEndpointAsync(stream: true); + + // Assert + Assert.Equal(HttpStatusCode.OK, statusCode); + Assert.Equal("text/event-stream", mediaType); + + var events = await ParseSseEventsAsync(body); + var annotationEvents = events + .Where(e => e.GetProperty("type").GetString() == "response.output_text.annotation.added") + .Select(e => e.GetProperty("annotation")) + .ToArray(); + AssertAnnotations(annotationEvents); + + var contentPartDone = Assert.Single(events, e => e.GetProperty("type").GetString() == "response.content_part.done"); + AssertAnnotations(contentPartDone.GetProperty("part")); + + var outputItemDone = Assert.Single(events, e => e.GetProperty("type").GetString() == "response.output_item.done"); + var outputText = Assert.Single(outputItemDone.GetProperty("item").GetProperty("content").EnumerateArray()); + AssertAnnotations(outputText); + + var completed = Assert.Single(events, e => e.GetProperty("type").GetString() == "response.completed"); + Assert.Equal("response.completed", events[^1].GetProperty("type").GetString()); + var completedOutputItem = Assert.Single(completed.GetProperty("response").GetProperty("output").EnumerateArray()); + var completedOutputText = Assert.Single(completedOutputItem.GetProperty("content").EnumerateArray()); + AssertAnnotations(completedOutputText); + } + + [Fact] + public async Task MapFoundryResponses_StreamFalse_ReturnsAllAnnotationKindsAsync() + { + // Arrange and Act + var (statusCode, mediaType, body) = await InvokeResponsesEndpointAsync(stream: false); + + // Assert + Assert.Equal(HttpStatusCode.OK, statusCode); + Assert.Equal("application/json", mediaType); + + using var document = JsonDocument.Parse(body); + var outputItem = Assert.Single(document.RootElement.GetProperty("output").EnumerateArray()); + var outputText = Assert.Single(outputItem.GetProperty("content").EnumerateArray()); + AssertAnnotations(outputText); + } + + private static async Task<(HttpStatusCode StatusCode, string? MediaType, string Body)> InvokeResponsesEndpointAsync(bool stream) + { + var builder = WebApplication.CreateBuilder(); + builder.WebHost.UseTestServer(); + + AIAgent agent = new ChatClientAgent(CreateAnnotationChatClient()); + builder.Services.AddFoundryResponses(agent, new InMemoryAgentSessionStore()); + builder.Services.AddSingleton(new FakeHostedSessionIsolationKeyProvider()); + builder.Services.AddLogging(); + + await using var app = builder.Build(); + app.MapFoundryResponses(); + await app.StartAsync(); + + var testServer = app.Services.GetRequiredService() as TestServer + ?? throw new InvalidOperationException("TestServer not found"); + using var client = testServer.CreateClient(); + using var request = new HttpRequestMessage(HttpMethod.Post, "/responses") + { + Content = new StringContent(CreateRequestJson(stream), Encoding.UTF8, "application/json"), + }; + using var response = await client.SendAsync(request); + var body = await response.Content.ReadAsStringAsync(); + return (response.StatusCode, response.Content.Headers.ContentType?.MediaType, body); + } + + private static IChatClient CreateAnnotationChatClient() + { + var annotations = new AIAnnotation[] + { + new CitationAnnotation + { + Url = new Uri("https://example.com/doc"), + Title = "Example Document", + AnnotatedRegions = [new TextSpanAnnotatedRegion { StartIndex = 0, EndIndex = 5 }] + }, + new CitationAnnotation + { + FileId = "file_1", + Title = "report.pdf", + RawRepresentation = new FileCitationMessageAnnotation("file_1", 1, "report.pdf") + }, + new CitationAnnotation + { + FileId = "file_2", + RawRepresentation = new FilePathMessageAnnotation("file_2", 2) + }, + new CitationAnnotation + { + FileId = "file_3", + Title = "chart.png", + AnnotatedRegions = [new TextSpanAnnotatedRegion { StartIndex = 6, EndIndex = 11 }], + RawRepresentation = new ContainerFileCitationMessageAnnotation( + "container_1", + "file_3", + 6, + 11, + "chart.png") + }, + }; + + var client = new Mock(); + client.Setup(c => c.GetStreamingResponseAsync( + It.IsAny>(), + It.IsAny(), + It.IsAny())) + .Returns(() => ToAsyncEnumerableUpdatesAsync( + new ChatResponseUpdate(ChatRole.Assistant, "Hello sources") { MessageId = "msg_response" }, + new ChatResponseUpdate( + ChatRole.Assistant, + [new AIContent { Annotations = annotations }]) + { + MessageId = "msg_response" + })); + return client.Object; + } + + private static async IAsyncEnumerable ToAsyncEnumerableUpdatesAsync( + params ChatResponseUpdate[] updates) + { + foreach (var update in updates) + { + yield return update; + } + + await Task.CompletedTask; + } + + private static string CreateRequestJson(bool stream) => $$""" + { + "model": "test", + "stream": {{(stream ? "true" : "false")}}, + "input": [ + { + "type": "message", + "id": "msg_request", + "status": "completed", + "role": "user", + "content": [{ "type": "input_text", "text": "Hello" }] + } + ] + } + """; + + private static async Task> ParseSseEventsAsync(string body) + { + var events = new List(); + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(body)); + await foreach (var item in SseParser.Create(stream).EnumerateAsync()) + { + if (item.Data == "[DONE]") + { + continue; + } + + using var document = JsonDocument.Parse(item.Data); + Assert.Equal(document.RootElement.GetProperty("type").GetString(), item.EventType); + events.Add(document.RootElement.Clone()); + } + + return events; + } + + private static void AssertAnnotations(JsonElement outputText) => + AssertAnnotations(outputText.GetProperty("annotations").EnumerateArray().ToArray()); + + private static void AssertAnnotations(IReadOnlyCollection annotations) + { + Assert.Collection( + annotations, + annotation => + { + Assert.Equal("url_citation", annotation.GetProperty("type").GetString()); + Assert.Equal("https://example.com/doc", annotation.GetProperty("url").GetString()); + Assert.Equal("Example Document", annotation.GetProperty("title").GetString()); + Assert.Equal(0, annotation.GetProperty("start_index").GetInt64()); + Assert.Equal(5, annotation.GetProperty("end_index").GetInt64()); + }, + annotation => + { + Assert.Equal("file_citation", annotation.GetProperty("type").GetString()); + Assert.Equal("file_1", annotation.GetProperty("file_id").GetString()); + Assert.Equal(1, annotation.GetProperty("index").GetInt64()); + Assert.Equal("report.pdf", annotation.GetProperty("filename").GetString()); + }, + annotation => + { + Assert.Equal("file_path", annotation.GetProperty("type").GetString()); + Assert.Equal("file_2", annotation.GetProperty("file_id").GetString()); + Assert.Equal(2, annotation.GetProperty("index").GetInt64()); + }, + annotation => + { + Assert.Equal("container_file_citation", annotation.GetProperty("type").GetString()); + Assert.Equal("container_1", annotation.GetProperty("container_id").GetString()); + Assert.Equal("file_3", annotation.GetProperty("file_id").GetString()); + Assert.Equal(6, annotation.GetProperty("start_index").GetInt64()); + Assert.Equal(11, annotation.GetProperty("end_index").GetInt64()); + Assert.Equal("chart.png", annotation.GetProperty("filename").GetString()); + }); + } + private static async Task BuildTestHostAsync( Action configure, Action? configureBuilder = null)