diff --git a/python/packages/openai/agent_framework_openai/_chat_client.py b/python/packages/openai/agent_framework_openai/_chat_client.py index c9f2e6de300..b18d5bb2130 100644 --- a/python/packages/openai/agent_framework_openai/_chat_client.py +++ b/python/packages/openai/agent_framework_openai/_chat_client.py @@ -71,6 +71,7 @@ ParsedResponse, ) from openai.types.responses.response import Response as OpenAIResponse +from openai.types.responses.response_create_params import PromptCacheOptions from openai.types.responses.response_stream_event import ( ResponseStreamEvent as OpenAIResponseStreamEvent, ) @@ -87,6 +88,7 @@ from ._exceptions import OpenAIContentFilterException from ._shared import ( AzureTokenProvider, + _attach_prompt_cache_breakpoint, # pyright: ignore[reportPrivateUsage] load_openai_service_settings, maybe_append_azure_endpoint_guidance, ) @@ -231,6 +233,12 @@ class OpenAIChatOptions(ChatOptions[ResponseFormatT], Generic[ResponseFormatT], prompt_cache_retention: Literal["24h"] """Retention policy for prompt cache. Set to '24h' for extended caching.""" + prompt_cache_options: PromptCacheOptions + """Request-wide prompt cache policy for GPT-5.6 and later models. + Set mode to 'explicit' to use only the breakpoints set on content parts via + ``Content.additional_properties["prompt_cache_breakpoint"]``. + See: https://developers.openai.com/api/docs/guides/prompt-caching#prompt-cache-breakpoints""" + reasoning: ReasoningOptions """Configuration for reasoning models (gpt-5, o-series). See: https://platform.openai.com/docs/guides/reasoning""" @@ -1654,10 +1662,13 @@ def _prepare_content_for_openai( "text": content.text, "annotations": _annotations_to_output_text(getattr(content, "annotations", None)), } - return { - "type": "input_text", - "text": content.text, - } + return _attach_prompt_cache_breakpoint( + { + "type": "input_text", + "text": content.text, + }, + content, + ) case "text_reasoning": ret: dict[str, Any] = {"type": "reasoning", "summary": []} if content.id: @@ -1685,7 +1696,7 @@ def _prepare_content_for_openai( file_id = content.additional_properties.get("file_id") if content.additional_properties else None if file_id is not None: result["file_id"] = file_id - return result + return _attach_prompt_cache_breakpoint(result, content) if content.has_top_level_media_type("audio"): if content.media_type and "wav" in content.media_type: format = "wav" @@ -1713,7 +1724,7 @@ def _prepare_content_for_openai( } if filename: file_obj["filename"] = filename - return file_obj + return _attach_prompt_cache_breakpoint(file_obj, content) return {} case "function_call": if not content.call_id: diff --git a/python/packages/openai/agent_framework_openai/_chat_completion_client.py b/python/packages/openai/agent_framework_openai/_chat_completion_client.py index 37c2e5b52dd..f852bad9221 100644 --- a/python/packages/openai/agent_framework_openai/_chat_completion_client.py +++ b/python/packages/openai/agent_framework_openai/_chat_completion_client.py @@ -54,12 +54,14 @@ from openai.types.chat.chat_completion_message_custom_tool_call import ( ChatCompletionMessageCustomToolCall, ) -from openai.types.chat.completion_create_params import WebSearchOptions +from openai.types.chat.completion_create_params import PromptCacheOptions, WebSearchOptions from pydantic import BaseModel from ._exceptions import OpenAIContentFilterException from ._shared import ( + PROMPT_CACHE_BREAKPOINT_KEY, AzureTokenProvider, + _attach_prompt_cache_breakpoint, # pyright: ignore[reportPrivateUsage] load_openai_service_settings, maybe_append_azure_endpoint_guidance, ) @@ -157,6 +159,12 @@ class OpenAIChatCompletionOptions(ChatOptions[ResponseModelT], Generic[ResponseM """Output verbosity for GPT-5 family models. Lower values yield shorter responses. See: https://developers.openai.com/cookbook/examples/gpt-5/gpt-5_new_params_and_tools#1-verbosity-parameter""" + prompt_cache_options: PromptCacheOptions + """Request-wide prompt cache policy for GPT-5.6 and later models. + Set mode to 'explicit' to use only the breakpoints set on content parts via + ``Content.additional_properties["prompt_cache_breakpoint"]``. + See: https://developers.openai.com/api/docs/guides/prompt-caching#prompt-cache-breakpoints""" + OpenAIChatCompletionOptionsT = TypeVar( "OpenAIChatCompletionOptionsT", @@ -873,12 +881,24 @@ def _prepare_messages_for_openai( def _prepare_message_for_openai(self, message: Message) -> list[dict[str, Any]]: """Prepare a chat message for OpenAI.""" - # System/developer messages must use plain string content because some - # OpenAI-compatible endpoints reject list content for non-user roles. + # System/developer messages default to plain string content because some + # OpenAI-compatible endpoints reject list content for non-user roles. The + # exception is a prompt cache breakpoint on a text part: it can only live on + # a typed part, so opting in switches that message to list content. if message.role in ("system", "developer"): - texts = [content.text for content in message.contents if content.type == "text" and content.text] - if texts: - sys_args: dict[str, Any] = {"role": message.role, "content": "\n".join(texts)} + text_contents = [content for content in message.contents if content.type == "text" and content.text] + if text_contents: + # A prompt cache breakpoint lives on a typed part; plain-string content + # cannot carry it, so keep the typed list form when one is present. + content_value: str | list[dict[str, Any]] + if any( + (content.additional_properties or {}).get(PROMPT_CACHE_BREAKPOINT_KEY) is not None + for content in text_contents + ): + content_value = [self._prepare_content_for_openai(content) for content in text_contents] + else: + content_value = "\n".join(content.text for content in text_contents if content.text) + sys_args: dict[str, Any] = {"role": message.role, "content": content_value} if message.author_name: sys_args["name"] = message.author_name return [sys_args] @@ -969,6 +989,10 @@ def _prepare_message_for_openai(self, message: Message) -> list[dict[str, Any]]: text_item = cast(Mapping[str, Any], item) if text_item.get("type") != "text": break + if PROMPT_CACHE_BREAKPOINT_KEY in text_item: + # A plain string cannot carry a prompt cache breakpoint; + # keep the typed part form for this message. + break text_items.append(text_item) else: msg["content"] = "\n".join( @@ -981,6 +1005,11 @@ def _prepare_message_for_openai(self, message: Message) -> list[dict[str, Any]]: def _prepare_content_for_openai(self, content: Content) -> dict[str, Any]: """Prepare content for OpenAI.""" match content.type: + case "text": + return _attach_prompt_cache_breakpoint( + {"type": "text", "text": content.text}, + content, + ) case "function_call": args = json.dumps(content.arguments) if isinstance(content.arguments, Mapping) else content.arguments return { @@ -998,10 +1027,13 @@ def _prepare_content_for_openai(self, content: Content) -> dict[str, Any]: detail = content.additional_properties.get("detail") if isinstance(detail, str): image_url_obj["detail"] = detail - return { - "type": "image_url", - "image_url": image_url_obj, - } + return _attach_prompt_cache_breakpoint( + { + "type": "image_url", + "image_url": image_url_obj, + }, + content, + ) case "data" | "uri" if content.has_top_level_media_type("audio"): if content.media_type and "wav" in content.media_type: audio_format = "wav" @@ -1017,13 +1049,16 @@ def _prepare_content_for_openai(self, content: Content) -> dict[str, Any]: # Extract just the base64 part after "data:audio/format;base64," audio_data = audio_data.split(",", 1)[-1] # type: ignore[union-attr] - return { - "type": "input_audio", - "input_audio": { - "data": audio_data, - "format": audio_format, + return _attach_prompt_cache_breakpoint( + { + "type": "input_audio", + "input_audio": { + "data": audio_data, + "format": audio_format, + }, }, - } + content, + ) case "data" | "uri" if content.has_top_level_media_type("application") and content.uri.startswith("data:"): # type: ignore[union-attr] # All application/* media types should be treated as files for OpenAI filename = getattr(content, "filename", None) or ( @@ -1034,10 +1069,13 @@ def _prepare_content_for_openai(self, content: Content) -> dict[str, Any]: file_obj = {"file_data": content.uri} if filename: file_obj["filename"] = filename - return { - "type": "file", - "file": file_obj, - } + return _attach_prompt_cache_breakpoint( + { + "type": "file", + "file": file_obj, + }, + content, + ) case _: # Default fallback for all other content types return content.to_dict(exclude_none=True) diff --git a/python/packages/openai/agent_framework_openai/_shared.py b/python/packages/openai/agent_framework_openai/_shared.py index 1cf0d487f67..9a3009f83e5 100644 --- a/python/packages/openai/agent_framework_openai/_shared.py +++ b/python/packages/openai/agent_framework_openai/_shared.py @@ -5,7 +5,7 @@ import sys from collections.abc import Awaitable, Callable, Mapping, Sequence from copy import copy -from typing import TYPE_CHECKING, Any, Literal, Union +from typing import TYPE_CHECKING, Any, Literal, Union, cast from agent_framework._settings import SecretString, load_settings from agent_framework._telemetry import APP_INFO, prepend_agent_framework_to_user_agent @@ -24,6 +24,7 @@ from typing_extensions import TypedDict # pragma: no cover if TYPE_CHECKING: + from agent_framework import Content from azure.core.credentials import TokenCredential from azure.core.credentials_async import AsyncTokenCredential @@ -49,6 +50,25 @@ AzureTokenProvider = Callable[[], str | Awaitable[str]] +PROMPT_CACHE_BREAKPOINT_KEY = "prompt_cache_breakpoint" + + +def _attach_prompt_cache_breakpoint( # pyright: ignore[reportUnusedFunction] + part: dict[str, Any], content: Content +) -> dict[str, Any]: + """Copy a prompt cache breakpoint from content metadata onto an outgoing part. + + GPT-5.6 and later models accept an explicit cache breakpoint on supported content + blocks; users opt in per part via + ``Content.additional_properties["prompt_cache_breakpoint"]``. + """ + props = content.additional_properties + breakpoint_value = props.get(PROMPT_CACHE_BREAKPOINT_KEY) if props else None + if isinstance(breakpoint_value, Mapping): + part[PROMPT_CACHE_BREAKPOINT_KEY] = dict(cast("Mapping[str, Any]", breakpoint_value)) + return part + + class OpenAISettings(TypedDict, total=False): """OpenAI environment settings. diff --git a/python/packages/openai/pyproject.toml b/python/packages/openai/pyproject.toml index be5ece61e0d..4467c054376 100644 --- a/python/packages/openai/pyproject.toml +++ b/python/packages/openai/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ ] dependencies = [ "agent-framework-core>=1.11.0,<2", - "openai>=2.25.0,<3", + "openai>=2.45.0,<3", ] [tool.uv] diff --git a/python/packages/openai/tests/openai/test_openai_chat_client.py b/python/packages/openai/tests/openai/test_openai_chat_client.py index 7849f4b67dc..a21dd55887f 100644 --- a/python/packages/openai/tests/openai/test_openai_chat_client.py +++ b/python/packages/openai/tests/openai/test_openai_chat_client.py @@ -6934,3 +6934,71 @@ def test_prepare_messages_strips_mcp_items_under_storage() -> None: # endregion + + +# region Prompt cache breakpoints and options + + +def _breakpoint_text_content() -> Content: + return Content.from_text( + "This is a stable prefix that should be cached.", + additional_properties={"prompt_cache_breakpoint": {"mode": "explicit"}}, + ) + + +def test_prepare_messages_for_openai_text_prompt_cache_breakpoint() -> None: + """A text part carries an explicit prompt cache breakpoint onto the request.""" + client = OpenAIChatClient(api_key="test-api-key", model="test-model") + items = client._prepare_messages_for_openai( + [Message(role="user", contents=[_breakpoint_text_content()])], + request_uses_service_side_storage=False, + ) + part = items[0]["content"][0] + assert part["type"] == "input_text" + assert part["prompt_cache_breakpoint"] == {"mode": "explicit"} + + +def test_prepare_content_for_openai_image_prompt_cache_breakpoint() -> None: + """An image part carries an explicit prompt cache breakpoint onto the request.""" + client = OpenAIChatClient(api_key="test-api-key", model="test-model") + image = Content.from_uri( + uri="https://example.com/x.png", + media_type="image/png", + additional_properties={"prompt_cache_breakpoint": {"mode": "explicit"}}, + ) + part = client._prepare_content_for_openai("user", image) + assert part["type"] == "input_image" + assert part["prompt_cache_breakpoint"] == {"mode": "explicit"} + + +def test_prepare_content_for_openai_file_prompt_cache_breakpoint() -> None: + """A file part carries an explicit prompt cache breakpoint onto the request.""" + client = OpenAIChatClient(api_key="test-api-key", model="test-model") + file_content = Content.from_uri( + uri="data:application/pdf;base64,AAAA", + media_type="application/pdf", + additional_properties={"prompt_cache_breakpoint": {"mode": "explicit"}}, + ) + part = client._prepare_content_for_openai("user", file_content) + assert part["type"] == "input_file" + assert part["prompt_cache_breakpoint"] == {"mode": "explicit"} + + +def test_prepare_content_for_openai_no_prompt_cache_breakpoint_by_default() -> None: + """Parts without the property keep their existing shape.""" + client = OpenAIChatClient(api_key="test-api-key", model="test-model") + part = client._prepare_content_for_openai("user", Content.from_text("hello")) + assert part == {"type": "input_text", "text": "hello"} + + +async def test_prepare_options_prompt_cache_options_passthrough() -> None: + """Request-level prompt_cache_options reaches the Responses API run options.""" + client = OpenAIChatClient(api_key="test-api-key", model="test-model") + run_options = await client._prepare_options( + [Message(role="user", contents=[Content.from_text("hi")])], + {"model": "test-model", "prompt_cache_options": {"mode": "explicit", "ttl": "30m"}}, + ) + assert run_options["prompt_cache_options"] == {"mode": "explicit", "ttl": "30m"} + + +# endregion diff --git a/python/packages/openai/tests/openai/test_openai_chat_completion_client.py b/python/packages/openai/tests/openai/test_openai_chat_completion_client.py index 81d9963688b..7445017c19a 100644 --- a/python/packages/openai/tests/openai/test_openai_chat_completion_client.py +++ b/python/packages/openai/tests/openai/test_openai_chat_completion_client.py @@ -2088,3 +2088,74 @@ def test_streaming_chunk_with_null_delta_no_tool_calls_parsed( # endregion + + +# region Prompt cache breakpoints and options + + +def test_prepare_message_text_prompt_cache_breakpoint_keeps_parts() -> None: + """A text part with a breakpoint stays in list form and carries the key.""" + client = OpenAIChatCompletionClient(api_key="test-api-key", model="test-model") + content = Content.from_text( + "stable prefix", + additional_properties={"prompt_cache_breakpoint": {"mode": "explicit"}}, + ) + msgs = client._prepare_message_for_openai(Message(role="user", contents=[content])) + assert isinstance(msgs[0]["content"], list) + assert msgs[0]["content"][0] == { + "type": "text", + "text": "stable prefix", + "prompt_cache_breakpoint": {"mode": "explicit"}, + } + + +def test_prepare_message_text_without_breakpoint_flattens_to_string() -> None: + """Text-only content without a breakpoint keeps the plain-string form.""" + client = OpenAIChatCompletionClient(api_key="test-api-key", model="test-model") + msgs = client._prepare_message_for_openai(Message(role="user", contents=[Content.from_text("hello")])) + assert msgs[0]["content"] == "hello" + + +def test_prepare_message_system_prompt_cache_breakpoint_keeps_parts() -> None: + """A system message with a breakpoint keeps typed content parts.""" + client = OpenAIChatCompletionClient(api_key="test-api-key", model="test-model") + content = Content.from_text( + "system prefix", + additional_properties={"prompt_cache_breakpoint": {"mode": "explicit"}}, + ) + msgs = client._prepare_message_for_openai(Message(role="system", contents=[content])) + assert isinstance(msgs[0]["content"], list) + assert msgs[0]["content"][0]["prompt_cache_breakpoint"] == {"mode": "explicit"} + + +def test_prepare_message_system_without_breakpoint_keeps_string_form() -> None: + """System messages without a breakpoint keep the joined-string form.""" + client = OpenAIChatCompletionClient(api_key="test-api-key", model="test-model") + msgs = client._prepare_message_for_openai(Message(role="system", contents=[Content.from_text("plain system")])) + assert msgs[0]["content"] == "plain system" + + +def test_prepare_content_for_openai_image_prompt_cache_breakpoint() -> None: + """An image part carries an explicit prompt cache breakpoint onto the request.""" + client = OpenAIChatCompletionClient(api_key="test-api-key", model="test-model") + image = Content.from_uri( + uri="https://example.com/x.png", + media_type="image/png", + additional_properties={"prompt_cache_breakpoint": {"mode": "explicit"}}, + ) + part = client._prepare_content_for_openai(image) + assert part["type"] == "image_url" + assert part["prompt_cache_breakpoint"] == {"mode": "explicit"} + + +def test_prepare_options_prompt_cache_options_passthrough() -> None: + """Request-level prompt_cache_options reaches the Chat Completions run options.""" + client = OpenAIChatCompletionClient(api_key="test-api-key", model="test-model") + run_options = client._prepare_options( + [Message(role="user", contents=[Content.from_text("hi")])], + {"model": "test-model", "prompt_cache_options": {"mode": "explicit", "ttl": "30m"}}, + ) + assert run_options["prompt_cache_options"] == {"mode": "explicit", "ttl": "30m"} + + +# endregion diff --git a/python/samples/02-agents/providers/openai/README.md b/python/samples/02-agents/providers/openai/README.md index a8ec0f6d1e4..7247833d003 100644 --- a/python/samples/02-agents/providers/openai/README.md +++ b/python/samples/02-agents/providers/openai/README.md @@ -22,6 +22,7 @@ This folder contains OpenAI provider samples for the generic clients in | [`client_basic.py`](client_basic.py) | Basic non-streaming and streaming responses sample with an explicit `gpt-5.4-nano` model and API key. | | [`client_image_analysis.py`](client_image_analysis.py) | Analyze images with the responses client. | | [`client_image_generation.py`](client_image_generation.py) | Generate images from text prompts. | +| [`client_prompt_caching.py`](client_prompt_caching.py) | Explicit prompt cache breakpoints and `prompt_cache_options` on GPT-5.6 models. | | [`client_reasoning.py`](client_reasoning.py) | Reasoning-focused sample for models such as `gpt-5`. | | [`client_streaming_image_generation.py`](client_streaming_image_generation.py) | Streaming image generation sample. | | [`client_verbosity.py`](client_verbosity.py) | GPT-5 `verbosity` option (`low`/`medium`/`high`) with default and per-call overrides. | diff --git a/python/samples/02-agents/providers/openai/client_prompt_caching.py b/python/samples/02-agents/providers/openai/client_prompt_caching.py new file mode 100644 index 00000000000..38afdaf93e6 --- /dev/null +++ b/python/samples/02-agents/providers/openai/client_prompt_caching.py @@ -0,0 +1,86 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio + +from agent_framework import Content, Message +from agent_framework.openai import OpenAIChatClient, OpenAIChatOptions +from dotenv import load_dotenv + +load_dotenv() + +""" +OpenAI Chat Client Prompt Caching Example + +Demonstrates explicit prompt cache breakpoints on GPT-5.6 and later models. Cache +writes are billed on these models, so marking exactly where a reusable prefix ends +lets you control what gets cached. + +Two knobs work together: + +- ``prompt_cache_options`` on ``OpenAIChatOptions`` sets the request-wide policy. + ``{"mode": "explicit"}`` disables the automatic breakpoint on the latest message, + so only the breakpoints you place are used for cache reads and writes. +- ``Content.additional_properties["prompt_cache_breakpoint"]`` marks the end of the + reusable prefix on a specific content part. + +The content before a breakpoint must be at least 1024 tokens long to be cached. +Running the same prefix twice shows the cache hit through +``usage_details["cache_read_input_token_count"]`` on later responses. + +Environment variables: + OPENAI_API_KEY — OpenAI API key + +See: https://developers.openai.com/api/docs/guides/prompt-caching#prompt-cache-breakpoints +""" + +# A stable block of context that is reused across requests, for example a product +# catalog, a policy document, or long system guidance. Repeated here to clear the +# 1024-token minimum a cache breakpoint requires. +STABLE_CONTEXT = ( + "You are a support assistant for the Contoso appliance store. " + "Always answer briefly, quote the relevant catalog section, and never invent " + "model numbers. If a question is out of scope, say so and point the customer " + "to support@contoso.example. " +) * 40 + + +def build_messages(question: str) -> list[Message]: + """Build a request with a cache breakpoint at the end of the stable prefix.""" + return [ + Message( + role="user", + contents=[ + Content.from_text( + STABLE_CONTEXT, + additional_properties={"prompt_cache_breakpoint": {"mode": "explicit"}}, + ) + ], + ), + Message(role="user", contents=[Content.from_text(question)]), + ] + + +async def main() -> None: + print("\033[92m=== OpenAI Chat Client Prompt Caching Example ===\033[0m\n") + + client = OpenAIChatClient[OpenAIChatOptions](model="gpt-5.6-luna") + options: OpenAIChatOptions = {"prompt_cache_options": {"mode": "explicit"}} + + questions = ["Do you sell refrigerators?", "What is the return policy contact?"] + for turn, question in enumerate(questions, start=1): + response = await client.get_response(build_messages(question), options=options) + usage = response.usage_details or {} + cached = usage.get("cache_read_input_token_count", 0) + print(f"Turn {turn}: {question}") + print(f" Answer: {response.text}") + print(f" Cached input tokens: {cached}\n") + if turn < len(questions): + # A freshly written cache entry becomes readable shortly after the request + # completes; the brief pause keeps the next turn from racing this one. + await asyncio.sleep(2) + + print("The first turn writes the prefix to the cache; later turns read it back.") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/uv.lock b/python/uv.lock index a0068be040b..88ab2296a38 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -874,7 +874,7 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "agent-framework-core", editable = "packages/core" }, - { name = "openai", specifier = ">=2.25.0,<3" }, + { name = "openai", specifier = ">=2.45.0,<3" }, ] [[package]]