diff --git a/py/pyproject.toml b/py/pyproject.toml index 0128bee9..9bd5a399 100644 --- a/py/pyproject.toml +++ b/py/pyproject.toml @@ -377,7 +377,7 @@ latest = "ai[openai]==0.5.2" "0.4.0" = "ai[openai]==0.4.0" [tool.braintrust.matrix.openai] -latest = "openai==3.8.0" +latest = "openai==3.13.0" "1.92.0" = "openai==1.92.0" "1.77.0" = "openai==1.77.0" "1.71.0" = "openai==1.71.0" diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_openai.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_openai.py index 67dd7357..00cbf44e 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_openai.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_openai.py @@ -8,18 +8,27 @@ from wrapt import FunctionWrapper -def _is_braintrust_wrapped() -> bool: - attr = inspect.getattr_static(openai.resources.chat.completions.Completions, "create", None) +def _is_braintrust_wrapped(resource) -> bool: + attr = inspect.getattr_static(resource, "create", None) return isinstance(attr, FunctionWrapper) +resources = [openai.resources.chat.completions.Completions] +try: + from openai.resources.beta.agents.sessions.sessions import Sessions + + resources.append(Sessions) +except ImportError: + pass + + # 1. Verify not patched initially -assert not _is_braintrust_wrapped() +assert all(not _is_braintrust_wrapped(resource) for resource in resources) # 2. Instrument results = auto_instrument() assert results.get("openai") == True -assert _is_braintrust_wrapped() +assert all(_is_braintrust_wrapped(resource) for resource in resources) # 3. Idempotent results2 = auto_instrument() diff --git a/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_agents_session_stream[async].yaml b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_agents_session_stream[async].yaml new file mode 100644 index 00000000..6b60634c --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_agents_session_stream[async].yaml @@ -0,0 +1,155 @@ +interactions: +- request: + body: '{"environment":{"type":"none"},"agent":{"model":"gpt-6-astra","instructions":"Answer + with only the requested number."},"input":"Reply with 25.","stream":true}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '158' + content-type: + - application/json + host: + - api.openai.com + openai-beta: + - agents=v1 + user-agent: + - AsyncOpenAI/Python 3.13.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 3.13.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/agents/sessions + response: + body: + string: 'event: agent.session.created + + data: {"type":"agent.session.created","event_id":"evt_59a75bd26f154cd08a190a511cb3a5a80369118dd4ae45dcaf","session":{"metadata":{},"id":"sess_04f1304d204a6379006aa441d997b88191a32759374fa62538","object":"agent.session","created_at":1789149657,"last_active_at":1789149657,"status":"in_progress","required_actions":[],"error":null,"agent":{"id":"agent_cf663404a9484214ad27b9a2464e03fea78653a0f3544816bd","name":null,"model":"gpt-6-astra","reasoning":{"effort":"medium","summary":null},"text":{"format":{"type":"text"},"verbosity":"medium"},"service_tier":"auto","instructions":"Answer + with only the requested number.","tools":[],"multi_agent":{"enabled":false,"max_concurrent_subagents":null}},"environment":{"type":"none"},"vault_ids":[],"usage":null}} + + + event: agent.session.turn.created + + data: {"type":"agent.session.turn.created","event_id":"evt_840c3c730c7a49009d0ca9bd8fe66718dac6c143efeb40c1b3","session_id":"sess_04f1304d204a6379006aa441d997b88191a32759374fa62538","turn_id":"turn_04f1304d204a6379006aa441dfcbfc819192643e3f3b4c97d3","turn":{"id":"turn_04f1304d204a6379006aa441dfcbfc819192643e3f3b4c97d3","object":"agent.session.turn","session_id":"sess_04f1304d204a6379006aa441d997b88191a32759374fa62538","agent_id":"agent_cf663404a9484214ad27b9a2464e03fea78653a0f3544816bd","subagent_id":null,"status":"queued","created_at":1789149661,"started_at":null,"completed_at":null,"error":null,"usage":null}} + + + event: agent.session.turn.item.added + + data: {"type":"agent.session.turn.item.added","event_id":"evt_87fbce7250214075abb78d6032e29945ba81cef327c14f3695","session_id":"sess_04f1304d204a6379006aa441d997b88191a32759374fa62538","turn_id":"turn_04f1304d204a6379006aa441dfcbfc819192643e3f3b4c97d3","output_index":null,"item":{"type":"message","id":"msg_d90d03c8e2ff600c2e32095b65697a5df0e58ca4744c0bc998","turn_id":"turn_04f1304d204a6379006aa441dfcbfc819192643e3f3b4c97d3","role":"user","content":[{"type":"input_text","text":"Reply + with 25."}],"status":"completed","phase":null}} + + + event: agent.session.in_progress + + data: {"type":"agent.session.in_progress","event_id":"evt_96779b9c47bc4252800ba02e0580135a239359f8289945e593","session":{"metadata":{},"id":"sess_04f1304d204a6379006aa441d997b88191a32759374fa62538","object":"agent.session","created_at":1789149657,"last_active_at":1789149657,"status":"in_progress","required_actions":[],"error":null,"agent":{"id":"agent_cf663404a9484214ad27b9a2464e03fea78653a0f3544816bd","name":null,"model":"gpt-6-astra","reasoning":{"effort":"medium","summary":null},"text":{"format":{"type":"text"},"verbosity":"medium"},"service_tier":"auto","instructions":"Answer + with only the requested number.","tools":[],"multi_agent":{"enabled":false,"max_concurrent_subagents":null}},"environment":{"type":"none"},"vault_ids":[],"usage":null}} + + + event: agent.session.turn.in_progress + + data: {"type":"agent.session.turn.in_progress","event_id":"evt_a60d7ad468b1425c906b8a52a1aff739a0d1c5dea25c4d5b80","session_id":"sess_04f1304d204a6379006aa441d997b88191a32759374fa62538","turn_id":"turn_04f1304d204a6379006aa441dfcbfc819192643e3f3b4c97d3","turn":{"id":"turn_04f1304d204a6379006aa441dfcbfc819192643e3f3b4c97d3","object":"agent.session.turn","session_id":"sess_04f1304d204a6379006aa441d997b88191a32759374fa62538","agent_id":"agent_cf663404a9484214ad27b9a2464e03fea78653a0f3544816bd","subagent_id":null,"status":"in_progress","created_at":1789149661,"started_at":1789149661,"completed_at":null,"error":null,"usage":null}} + + + event: agent.session.turn.item.added + + data: {"type":"agent.session.turn.item.added","event_id":"evt_90753d7413ff40999c49e58021d7b74c799ab60209204cc4b9","session_id":"sess_04f1304d204a6379006aa441d997b88191a32759374fa62538","turn_id":"turn_04f1304d204a6379006aa441dfcbfc819192643e3f3b4c97d3","output_index":0,"item":{"type":"message","id":"msg_01402b651ba1b38d016aa441df081087d1a8b8a2dcf19960b3","turn_id":"turn_04f1304d204a6379006aa441dfcbfc819192643e3f3b4c97d3","role":"assistant","content":[],"status":"in_progress","phase":"final_answer"}} + + + event: agent.session.turn.content_part.added + + data: {"type":"agent.session.turn.content_part.added","event_id":"evt_9f066441eb0344d798e4dccd13904314918428e5c5db458387","session_id":"sess_04f1304d204a6379006aa441d997b88191a32759374fa62538","turn_id":"turn_04f1304d204a6379006aa441dfcbfc819192643e3f3b4c97d3","item_id":"msg_01402b651ba1b38d016aa441df081087d1a8b8a2dcf19960b3","output_index":0,"content_index":0,"part":{"type":"output_text","text":""}} + + + event: agent.session.turn.output_text.delta + + data: {"type":"agent.session.turn.output_text.delta","event_id":"evt_306aebbed4ce42f79a451155a17036ccf08cdb7d7ff8479a96","session_id":"sess_04f1304d204a6379006aa441d997b88191a32759374fa62538","turn_id":"turn_04f1304d204a6379006aa441dfcbfc819192643e3f3b4c97d3","item_id":"msg_01402b651ba1b38d016aa441df081087d1a8b8a2dcf19960b3","output_index":0,"content_index":0,"delta":"25"} + + + event: agent.session.turn.output_text.done + + data: {"type":"agent.session.turn.output_text.done","event_id":"evt_7c2e923ce4404960a1b87304d6c2fb36d525f6859c4e4b9480","session_id":"sess_04f1304d204a6379006aa441d997b88191a32759374fa62538","turn_id":"turn_04f1304d204a6379006aa441dfcbfc819192643e3f3b4c97d3","item_id":"msg_01402b651ba1b38d016aa441df081087d1a8b8a2dcf19960b3","output_index":0,"content_index":0,"text":"25"} + + + event: agent.session.turn.content_part.done + + data: {"type":"agent.session.turn.content_part.done","event_id":"evt_6dd20767810040d4b4cb3075461a72968b3e923c03f84b7a83","session_id":"sess_04f1304d204a6379006aa441d997b88191a32759374fa62538","turn_id":"turn_04f1304d204a6379006aa441dfcbfc819192643e3f3b4c97d3","item_id":"msg_01402b651ba1b38d016aa441df081087d1a8b8a2dcf19960b3","output_index":0,"content_index":0,"part":{"type":"output_text","text":"25"}} + + + event: agent.session.turn.item.done + + data: {"type":"agent.session.turn.item.done","event_id":"evt_0b83e6e583a34450a0192047a7b2f5baa1b0cf4cebfb427d90","session_id":"sess_04f1304d204a6379006aa441d997b88191a32759374fa62538","turn_id":"turn_04f1304d204a6379006aa441dfcbfc819192643e3f3b4c97d3","output_index":0,"item":{"type":"message","id":"msg_01402b651ba1b38d016aa441df081087d1a8b8a2dcf19960b3","turn_id":"turn_04f1304d204a6379006aa441dfcbfc819192643e3f3b4c97d3","role":"assistant","status":"completed","content":[{"type":"output_text","text":"25"}],"phase":"final_answer"}} + + + event: agent.session.turn.completed + + data: {"type":"agent.session.turn.completed","event_id":"evt_e6be0ba2851342c7a4b895180a2bb0e7e62a70bda57248938f","session_id":"sess_04f1304d204a6379006aa441d997b88191a32759374fa62538","turn_id":"turn_04f1304d204a6379006aa441dfcbfc819192643e3f3b4c97d3","turn":{"id":"turn_04f1304d204a6379006aa441dfcbfc819192643e3f3b4c97d3","object":"agent.session.turn","session_id":"sess_04f1304d204a6379006aa441d997b88191a32759374fa62538","agent_id":"agent_cf663404a9484214ad27b9a2464e03fea78653a0f3544816bd","subagent_id":null,"status":"completed","created_at":1789149661,"started_at":1789149661,"completed_at":1789149664,"error":null,"usage":null},"usage":null} + + + event: agent.session.idle + + data: {"type":"agent.session.idle","event_id":"evt_727f3cd32758438ba96fa291752d5bf5c393775dc62744b5ba","session":{"metadata":{},"id":"sess_04f1304d204a6379006aa441d997b88191a32759374fa62538","object":"agent.session","created_at":1789149657,"last_active_at":1789149657,"status":"idle","required_actions":[],"error":null,"agent":{"id":"agent_cf663404a9484214ad27b9a2464e03fea78653a0f3544816bd","name":null,"model":"gpt-6-astra","reasoning":{"effort":"medium","summary":null},"text":{"format":{"type":"text"},"verbosity":"medium"},"service_tier":"auto","instructions":"Answer + with only the requested number.","tools":[],"multi_agent":{"enabled":false,"max_concurrent_subagents":null}},"environment":{"type":"none"},"vault_ids":[],"usage":null}} + + + ' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - x-request-id,openai-processing-ms,openai-version,openai-organization,openai-project + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cache-control: + - no-cache + cf-cache-status: + - DYNAMIC + cf-ray: + - a398932a8ff9a0c2-YYZ + connection: + - keep-alive + content-type: + - text/event-stream + date: + - Fri, 11 Sep 2026 18:00:58 GMT + openai-processing-ms: + - '2019' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - origin, access-control-request-method, access-control-request-headers + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-request-id: + - req_d2192778fb7b420396699ce543812d01 + status: + code: 201 + message: Created +version: 1 diff --git a/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_agents_session_stream[sync].yaml b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_agents_session_stream[sync].yaml new file mode 100644 index 00000000..26c0a69f --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_agents_session_stream[sync].yaml @@ -0,0 +1,181 @@ +interactions: +- request: + body: '{"environment":{"type":"openai_hosted"},"agent":{"model":"gpt-6-astra","instructions":"Run + the requested command, then answer with only its output."},"input":"Run `printf + 24` in the terminal.","stream":true}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '207' + content-type: + - application/json + host: + - api.openai.com + openai-beta: + - agents=v1 + user-agent: + - OpenAI/Python 3.13.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 3.13.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/agents/sessions + response: + body: + string: 'event: agent.session.created + + data: {"type":"agent.session.created","event_id":"evt_125f266c14464226b396d2150adbed5ba38f5fa1a4bb475384","session":{"metadata":{},"id":"sess_0c2c2a6c88586215006aa440a6d4e48191aa692519badcb6ab","object":"agent.session","created_at":1789149350,"last_active_at":1789149350,"status":"idle","required_actions":[],"error":null,"agent":{"id":"agent_dd743e0ac47b4c428b1910a0b603c9cdc603a4e4282a4e028d","name":null,"model":"gpt-6-astra","reasoning":{"effort":"medium","summary":null},"text":{"format":{"type":"text"},"verbosity":"medium"},"service_tier":"auto","instructions":"Run + the requested command, then answer with only its output.","tools":[],"multi_agent":{"enabled":false,"max_concurrent_subagents":null}},"environment":{"type":"openai_hosted","id":"ccarenv_b64_Y2NhcmVudl82YWE0NDBhNmRhNWM4MTkxYmQ0NmYxMGEyYzgzMGVlMw","packages":{"python":[],"system":[],"npm":[]},"network":{"access":"enabled","allowed_domains":[]},"capability_directories":[],"skills":[],"plugins":[],"files":[]},"vault_ids":[],"usage":null}} + + + event: agent.session.turn.created + + data: {"type":"agent.session.turn.created","event_id":"evt_cb74a913175d4a0b9b2262ab3051503db84214f777b741efb3","session_id":"sess_0c2c2a6c88586215006aa440a6d4e48191aa692519badcb6ab","turn_id":"turn_0c2c2a6c88586215006aa440b15da48191922c4badbe3e7d46","turn":{"id":"turn_0c2c2a6c88586215006aa440b15da48191922c4badbe3e7d46","object":"agent.session.turn","session_id":"sess_0c2c2a6c88586215006aa440a6d4e48191aa692519badcb6ab","agent_id":"agent_dd743e0ac47b4c428b1910a0b603c9cdc603a4e4282a4e028d","subagent_id":null,"status":"queued","created_at":1789149359,"started_at":null,"completed_at":null,"error":null,"usage":null}} + + + event: agent.session.turn.item.added + + data: {"type":"agent.session.turn.item.added","event_id":"evt_f0a1e97254b244e79302ac490370cd99be9c75987aa341e49f","session_id":"sess_0c2c2a6c88586215006aa440a6d4e48191aa692519badcb6ab","turn_id":"turn_0c2c2a6c88586215006aa440b15da48191922c4badbe3e7d46","output_index":null,"item":{"type":"message","id":"msg_9c3b4c3795cf12c91eb75916a5f9e2649252daf4d787774be0","turn_id":"turn_0c2c2a6c88586215006aa440b15da48191922c4badbe3e7d46","role":"user","content":[{"type":"input_text","text":"Run + `printf 24` in the terminal."}],"status":"completed","phase":null}} + + + event: agent.session.in_progress + + data: {"type":"agent.session.in_progress","event_id":"evt_0b8e7607fd4446e988e96952dc71ff19b3fe7611da2d45fba2","session":{"metadata":{},"id":"sess_0c2c2a6c88586215006aa440a6d4e48191aa692519badcb6ab","object":"agent.session","created_at":1789149350,"last_active_at":1789149350,"status":"in_progress","required_actions":[],"error":null,"agent":{"id":"agent_dd743e0ac47b4c428b1910a0b603c9cdc603a4e4282a4e028d","name":null,"model":"gpt-6-astra","reasoning":{"effort":"medium","summary":null},"text":{"format":{"type":"text"},"verbosity":"medium"},"service_tier":"auto","instructions":"Run + the requested command, then answer with only its output.","tools":[],"multi_agent":{"enabled":false,"max_concurrent_subagents":null}},"environment":{"type":"openai_hosted","id":"ccarenv_b64_Y2NhcmVudl82YWE0NDBhNmRhNWM4MTkxYmQ0NmYxMGEyYzgzMGVlMw","packages":{"python":[],"system":[],"npm":[]},"network":{"access":"enabled","allowed_domains":[]},"capability_directories":[],"skills":[],"plugins":[],"files":[]},"vault_ids":[],"usage":null}} + + + event: agent.session.turn.in_progress + + data: {"type":"agent.session.turn.in_progress","event_id":"evt_922a69ed00d64d8a918f3439b126beb51211507a37f14bf2b2","session_id":"sess_0c2c2a6c88586215006aa440a6d4e48191aa692519badcb6ab","turn_id":"turn_0c2c2a6c88586215006aa440b15da48191922c4badbe3e7d46","turn":{"id":"turn_0c2c2a6c88586215006aa440b15da48191922c4badbe3e7d46","object":"agent.session.turn","session_id":"sess_0c2c2a6c88586215006aa440a6d4e48191aa692519badcb6ab","agent_id":"agent_dd743e0ac47b4c428b1910a0b603c9cdc603a4e4282a4e028d","subagent_id":null,"status":"in_progress","created_at":1789149359,"started_at":1789149359,"completed_at":null,"error":null,"usage":null}} + + + : + + + event: agent.session.environment.ready + + data: {"type":"agent.session.environment.ready","event_id":"evt_d06069af7d0d446abf0acc2d8be74ad0e871eecd6c7b4c31b2","session_id":"sess_0c2c2a6c88586215006aa440a6d4e48191aa692519badcb6ab","turn_id":null,"environment":{"id":"ccarenv_b64_Y2NhcmVudl82YWE0NDBhNmRhNWM4MTkxYmQ0NmYxMGEyYzgzMGVlMw","type":"openai_hosted","status":"ready","error":null}} + + + event: agent.session.turn.item.added + + data: {"type":"agent.session.turn.item.added","event_id":"evt_614e4be5289946baaebdd8b29595b99669e84a60784f4afebf","session_id":"sess_0c2c2a6c88586215006aa440a6d4e48191aa692519badcb6ab","turn_id":"turn_0c2c2a6c88586215006aa440b15da48191922c4badbe3e7d46","output_index":0,"item":{"type":"command_execution","id":"call_dluR63csbEuWrfnAqgobFUDn","turn_id":"turn_0c2c2a6c88586215006aa440b15da48191922c4badbe3e7d46","command":"/bin/bash + -lc ''printf 24''","cwd":"/workspace","status":"in_progress","output":null,"exit_code":null,"duration_ms":null}} + + + event: agent.session.turn.item.done + + data: {"type":"agent.session.turn.item.done","event_id":"evt_401e5b626b504a86a58ff369b919ced9d4ff9d1f3b1b4af08d","session_id":"sess_0c2c2a6c88586215006aa440a6d4e48191aa692519badcb6ab","turn_id":"turn_0c2c2a6c88586215006aa440b15da48191922c4badbe3e7d46","output_index":0,"item":{"type":"command_execution","id":"call_dluR63csbEuWrfnAqgobFUDn","turn_id":"turn_0c2c2a6c88586215006aa440b15da48191922c4badbe3e7d46","command":"/bin/bash + -lc ''printf 24''","cwd":"/workspace","status":"completed","output":"24","exit_code":null,"duration_ms":null}} + + + event: agent.session.environment.connected + + data: {"type":"agent.session.environment.connected","event_id":"evt_77f470a5c89f406184eaa549659fae32cbe31aa292e249268b","session_id":"sess_0c2c2a6c88586215006aa440a6d4e48191aa692519badcb6ab","turn_id":null,"environment":{"id":"ccarenv_b64_Y2NhcmVudl82YWE0NDBhNmRhNWM4MTkxYmQ0NmYxMGEyYzgzMGVlMw","type":"openai_hosted","status":"connected","error":null}} + + + event: agent.session.turn.item.added + + data: {"type":"agent.session.turn.item.added","event_id":"evt_f3a5b28451c349c6a53049b2c3463c8f9989a854025e43f09e","session_id":"sess_0c2c2a6c88586215006aa440a6d4e48191aa692519badcb6ab","turn_id":"turn_0c2c2a6c88586215006aa440b15da48191922c4badbe3e7d46","output_index":1,"item":{"type":"message","id":"msg_0284123ca05485d1016aa440c8b79487d1bdf4525e4d5e70d3","turn_id":"turn_0c2c2a6c88586215006aa440b15da48191922c4badbe3e7d46","role":"assistant","content":[],"status":"in_progress","phase":"final_answer"}} + + + event: agent.session.turn.content_part.added + + data: {"type":"agent.session.turn.content_part.added","event_id":"evt_b96cffe2e63c45ae9a97816f104dd2641e44aa318fcd46bdaf","session_id":"sess_0c2c2a6c88586215006aa440a6d4e48191aa692519badcb6ab","turn_id":"turn_0c2c2a6c88586215006aa440b15da48191922c4badbe3e7d46","item_id":"msg_0284123ca05485d1016aa440c8b79487d1bdf4525e4d5e70d3","output_index":1,"content_index":0,"part":{"type":"output_text","text":""}} + + + event: agent.session.turn.output_text.delta + + data: {"type":"agent.session.turn.output_text.delta","event_id":"evt_494678af9762416aa1c2830b2e5455d48f0375012f544b8f8c","session_id":"sess_0c2c2a6c88586215006aa440a6d4e48191aa692519badcb6ab","turn_id":"turn_0c2c2a6c88586215006aa440b15da48191922c4badbe3e7d46","item_id":"msg_0284123ca05485d1016aa440c8b79487d1bdf4525e4d5e70d3","output_index":1,"content_index":0,"delta":"24"} + + + event: agent.session.turn.output_text.done + + data: {"type":"agent.session.turn.output_text.done","event_id":"evt_a3290bd50f4c440c838758c099919d5ccb0355d841fc4f64b7","session_id":"sess_0c2c2a6c88586215006aa440a6d4e48191aa692519badcb6ab","turn_id":"turn_0c2c2a6c88586215006aa440b15da48191922c4badbe3e7d46","item_id":"msg_0284123ca05485d1016aa440c8b79487d1bdf4525e4d5e70d3","output_index":1,"content_index":0,"text":"24"} + + + event: agent.session.turn.content_part.done + + data: {"type":"agent.session.turn.content_part.done","event_id":"evt_8eb0e1c3ce6e407a9bcb02ddc100600c6bc2d00f535b45d083","session_id":"sess_0c2c2a6c88586215006aa440a6d4e48191aa692519badcb6ab","turn_id":"turn_0c2c2a6c88586215006aa440b15da48191922c4badbe3e7d46","item_id":"msg_0284123ca05485d1016aa440c8b79487d1bdf4525e4d5e70d3","output_index":1,"content_index":0,"part":{"type":"output_text","text":"24"}} + + + event: agent.session.turn.item.done + + data: {"type":"agent.session.turn.item.done","event_id":"evt_fb5750359ae447cbabba02f608809014f99a587fa63f44649f","session_id":"sess_0c2c2a6c88586215006aa440a6d4e48191aa692519badcb6ab","turn_id":"turn_0c2c2a6c88586215006aa440b15da48191922c4badbe3e7d46","output_index":1,"item":{"type":"message","id":"msg_0284123ca05485d1016aa440c8b79487d1bdf4525e4d5e70d3","turn_id":"turn_0c2c2a6c88586215006aa440b15da48191922c4badbe3e7d46","role":"assistant","status":"completed","content":[{"type":"output_text","text":"24"}],"phase":"final_answer"}} + + + event: agent.session.turn.completed + + data: {"type":"agent.session.turn.completed","event_id":"evt_2f44ae53b2714e5787a864282f8246c1119d03c7672e484683","session_id":"sess_0c2c2a6c88586215006aa440a6d4e48191aa692519badcb6ab","turn_id":"turn_0c2c2a6c88586215006aa440b15da48191922c4badbe3e7d46","turn":{"id":"turn_0c2c2a6c88586215006aa440b15da48191922c4badbe3e7d46","object":"agent.session.turn","session_id":"sess_0c2c2a6c88586215006aa440a6d4e48191aa692519badcb6ab","agent_id":"agent_dd743e0ac47b4c428b1910a0b603c9cdc603a4e4282a4e028d","subagent_id":null,"status":"completed","created_at":1789149359,"started_at":1789149359,"completed_at":1789149386,"error":null,"usage":null},"usage":null} + + + event: agent.session.idle + + data: {"type":"agent.session.idle","event_id":"evt_5fcbaf9c335a463e99ba46594200f7aeb6cdf87eafef4f7a95","session":{"metadata":{},"id":"sess_0c2c2a6c88586215006aa440a6d4e48191aa692519badcb6ab","object":"agent.session","created_at":1789149350,"last_active_at":1789149350,"status":"idle","required_actions":[],"error":null,"agent":{"id":"agent_dd743e0ac47b4c428b1910a0b603c9cdc603a4e4282a4e028d","name":null,"model":"gpt-6-astra","reasoning":{"effort":"medium","summary":null},"text":{"format":{"type":"text"},"verbosity":"medium"},"service_tier":"auto","instructions":"Run + the requested command, then answer with only its output.","tools":[],"multi_agent":{"enabled":false,"max_concurrent_subagents":null}},"environment":{"type":"openai_hosted","id":"ccarenv_b64_Y2NhcmVudl82YWE0NDBhNmRhNWM4MTkxYmQ0NmYxMGEyYzgzMGVlMw","packages":{"python":[],"system":[],"npm":[]},"network":{"access":"enabled","allowed_domains":[]},"capability_directories":[],"skills":[],"plugins":[],"files":[]},"vault_ids":[],"usage":null}} + + + ' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - x-request-id,openai-processing-ms,openai-version,openai-organization,openai-project + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cache-control: + - no-cache + cf-cache-status: + - DYNAMIC + cf-ray: + - a3988ba06f0bbf5d-YYZ + connection: + - keep-alive + content-type: + - text/event-stream + date: + - Fri, 11 Sep 2026 17:55:59 GMT + openai-processing-ms: + - '10447' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - origin, access-control-request-method, access-control-request-headers + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-request-id: + - req_b6071ace19894c9c97f03396978b6937 + status: + code: 201 + message: Created +version: 1 diff --git a/py/src/braintrust/integrations/openai/integration.py b/py/src/braintrust/integrations/openai/integration.py index 5ecdc672..f5f1fd06 100644 --- a/py/src/braintrust/integrations/openai/integration.py +++ b/py/src/braintrust/integrations/openai/integration.py @@ -3,6 +3,7 @@ from braintrust.integrations.base import BaseIntegration from .patchers import ( + AgentsSessionsPatcher, AudioSpeechPatcher, AudioTranscriptionsPatcher, AudioTranslationsPatcher, @@ -20,6 +21,7 @@ class OpenAIIntegration(BaseIntegration): name = "openai" import_names = ("openai",) patchers = ( + AgentsSessionsPatcher, ChatCompletionsPatcher, EmbeddingsPatcher, ModerationsPatcher, diff --git a/py/src/braintrust/integrations/openai/patchers.py b/py/src/braintrust/integrations/openai/patchers.py index 47de9821..532e97c6 100644 --- a/py/src/braintrust/integrations/openai/patchers.py +++ b/py/src/braintrust/integrations/openai/patchers.py @@ -7,6 +7,7 @@ from wrapt import BoundFunctionWrapper, FunctionWrapper from .tracing import ( + _agents_session_create_wrapper, _audio_speech_create_wrapper, _audio_transcription_create_wrapper, _audio_translation_create_wrapper, @@ -86,6 +87,36 @@ def _make_method_patchers( return sync_patcher, async_patcher, instance_patcher +# --------------------------------------------------------------------------- +# Agents API sessions +# --------------------------------------------------------------------------- + +_agents_session_create_sync, _agents_session_create_async, _wrap_agents_session_create = _make_method_patchers( + name_prefix="openai.agents.sessions.create", + target_module="openai.resources.beta.agents.sessions.sessions", + sync_class="Sessions", + async_class="AsyncSessions", + method="create", + wrapper=_agents_session_create_wrapper, + wrap_name="openai.wrap.agents.sessions.create", +) + + +class AgentsSessionsPatcher(CompositeFunctionWrapperPatcher): + """Patch streamed OpenAI Agents API session creation for tracing.""" + + name = "openai.agents.sessions" + sub_patchers = ( + _agents_session_create_sync, + _agents_session_create_async, + ) + + +class _WrapAgentsSessions(CompositeFunctionWrapperPatcher): + name = "openai.wrap.agents.sessions" + sub_patchers = (_wrap_agents_session_create,) + + # --------------------------------------------------------------------------- # Chat completions # --------------------------------------------------------------------------- @@ -416,6 +447,7 @@ class _WrapResponsesRaw(CompositeFunctionWrapperPatcher): # --------------------------------------------------------------------------- _WRAP_TARGETS: tuple[tuple[str, type[CompositeFunctionWrapperPatcher]], ...] = ( + ("beta.agents.sessions", _WrapAgentsSessions), ("chat.completions", _WrapChatCompletions), ("embeddings", _WrapEmbeddings), ("moderations", _WrapModerations), diff --git a/py/src/braintrust/integrations/openai/test_openai.py b/py/src/braintrust/integrations/openai/test_openai.py index 0cd30842..118ec9c0 100644 --- a/py/src/braintrust/integrations/openai/test_openai.py +++ b/py/src/braintrust/integrations/openai/test_openai.py @@ -83,6 +83,16 @@ def _supports_response_web_search_tools() -> bool: return True +def _supports_agents_api() -> bool: + try: + from openai.resources.beta.agents.sessions.sessions import Sessions + + del Sessions + except ImportError: + return False + return True + + @pytest.mark.vcr def test_openai_chat_metrics(memory_logger): assert not memory_logger.pop() @@ -480,6 +490,74 @@ def test_openai_responses_web_search_tool_spans_stream(memory_logger): assert tool_span["metadata"]["status"] == web_search_call.status +@pytest.mark.parametrize("is_async", (False, True), ids=("sync", "async")) +@pytest.mark.vcr +def test_openai_agents_session_stream(memory_logger, is_async): + if not _supports_agents_api(): + pytest.skip("OpenAI Agents API is not available in this SDK version") + + input_text = "Reply with 25." if is_async else "Run `printf 24` in the terminal." + environment_type = "none" if is_async else "openai_hosted" + expected_text = "25" if is_async else "24" + instructions = ( + "Answer with only the requested number." + if is_async + else "Run the requested command, then answer with only its output." + ) + client = wrap_openai(AsyncOpenAI() if is_async else openai.OpenAI()) + sessions = getattr(client.beta, "agents").sessions + params = dict( + agent={ + "model": "gpt-6-astra", + "instructions": instructions, + }, + environment={"type": environment_type}, + input=input_text, + stream=True, + ) + + if is_async: + + async def collect_events(): + stream = await sessions.create(**params) + async with stream: + return [event async for event in stream] + + events = asyncio.run(collect_events()) + else: + with sessions.create(**params) as stream: + events = list(stream) + + completed = next( + event for event in events if event.type == "agent.session.turn.completed" and event.turn.subagent_id is None + ) + + spans = memory_logger.pop() + task_spans = _find_spans_by_type(spans, SpanTypeAttribute.TASK) + tool_spans = _find_spans_by_type(spans, SpanTypeAttribute.TOOL) + + assert len(task_spans) == 1 + task_span = task_spans[0] + assert task_span["span_attributes"]["name"] == "openai.agents.sessions.create" + assert task_span["input"] == input_text + assert expected_text in task_span["output"] + assert task_span["metadata"]["provider"] == "openai" + assert task_span["metadata"]["model"] == "gpt-6-astra" + assert task_span["metadata"]["environment_type"] == environment_type + assert task_span["metadata"]["session_id"] == completed.session_id + assert task_span["metadata"]["turn_id"] == completed.turn_id + assert task_span["metadata"]["status"] == "completed" + assert task_span["context"]["span_origin"]["instrumentation"]["name"] == "openai-auto" + assert task_span["metrics"]["time_to_first_token"] >= 0 + + if not is_async: + command_span = _find_span_by_name(tool_spans, "command_execution") + assert command_span["span_parents"] == [task_span["span_id"]] + assert "printf 24" in command_span["input"]["command"] + assert "24" in command_span["output"]["output"] + assert command_span["metadata"]["status"] == "completed" + + @pytest.mark.vcr def test_openai_embeddings(memory_logger): assert not memory_logger.pop() diff --git a/py/src/braintrust/integrations/openai/tracing.py b/py/src/braintrust/integrations/openai/tracing.py index faa1d4fc..b16a2092 100644 --- a/py/src/braintrust/integrations/openai/tracing.py +++ b/py/src/braintrust/integrations/openai/tracing.py @@ -548,6 +548,21 @@ async def call(): return ResponseWrapper(wrapped, None, "openai.responses.parse", return_raw=True).create(*args, **kwargs) +def _agents_session_create_wrapper(wrapped, instance, args, kwargs): + # Non-streaming session creation only submits detached work. The Agents API + # delivers a complete turn through this invocation only when stream=True. + if kwargs.get("stream") is not True: + return wrapped(*args, **kwargs) + + if _is_async_callable(wrapped): + + async def call(): + return await AgentSessionWrapper(None, wrapped).acreate(*args, **kwargs) + + return call() + return AgentSessionWrapper(wrapped, None).create(*args, **kwargs) + + # --------------------------------------------------------------------------- # Core tracing wrappers # --------------------------------------------------------------------------- @@ -1130,6 +1145,305 @@ def _log_response_tool_spans(output: Any, *, parent_export: str | None) -> None: tool_span.log(output=output_data) +_AGENT_TOOL_ITEM_INPUT_KEYS = { + "command_execution": ("command", "cwd"), + "mcp_call": ("arguments",), + "web_search_call": ("action",), + "create_subagent_call": ("content", "model", "reasoning_effort"), + "send_subagent_input_call": ("content", "recipient_agent_id"), + "resume_subagent_call": ("recipient_agent_id",), + "wait_for_subagents_call": ("recipient_agent_ids",), + "interrupt_subagent_call": ("recipient_agent_id",), + "close_subagent_call": ("recipient_agent_id",), +} + +_AGENT_TOOL_ITEM_OUTPUT_KEYS = { + "command_execution": ("output", "exit_code", "duration_ms"), + "mcp_call": ("output",), +} + +_AGENT_TURN_EVENTS = { + "agent.session.turn.created", + "agent.session.turn.in_progress", + "agent.session.turn.completed", + "agent.session.turn.failed", + "agent.session.turn.cancelled", +} + +_AGENT_TERMINAL_TURN_EVENTS = { + "agent.session.turn.completed", + "agent.session.turn.failed", + "agent.session.turn.cancelled", +} + + +def _normalize_agent_tools(tools: Any) -> Any: + """Normalize function definitions while leaving built-in tool config untouched.""" + if not isinstance(tools, (list, tuple)): + return tools + + normalized = None + for index, tool in enumerate(tools): + if not isinstance(tool, dict) or tool.get("type") != "function": + continue + if normalized is None: + normalized = list(tools) + normalized[index] = { + "type": "function", + "function": clean_nones( + { + "name": tool.get("name"), + "description": tool.get("description"), + "parameters": tool.get("parameters"), + } + ), + } + return normalized if normalized is not None else tools + + +def _agent_session_params(params: dict[str, Any]) -> dict[str, Any]: + metadata: dict[str, Any] = {"provider": "openai"} + agent = params.get("agent") + if isinstance(agent, dict): + metadata.update( + clean_nones( + { + key: agent.get(key) + for key in ("model", "instructions", "reasoning", "service_tier", "text", "multi_agent") + } + ) + ) + tools = agent.get("tools") + if tools: + metadata["tools"] = _normalize_agent_tools(tools) + + agent_id = params.get("agent_id") + if agent_id is not None and not _is_not_given(agent_id): + metadata["agent_id"] = agent_id + + metadata["environment_type"] = params["environment"]["type"] + + return { + "input": _process_attachments_in_input(params.get("input")), + "metadata": metadata, + } + + +def _agent_tool_span_name(item: Any) -> str: + server_label = getattr(item, "server_label", None) + name = getattr(item, "name", None) + if server_label and name: + return f"{server_label}.{name}" + return str(name or item.type) + + +def _agent_tool_span_data(item: Any, keys: tuple[str, ...]) -> Any: + values = clean_nones({key: getattr(item, key, None) for key in keys}) + if not values: + return None + if keys == ("arguments",): + return values["arguments"] + return values + + +def _agent_tool_span_metadata(item: Any) -> dict[str, Any]: + return clean_nones( + { + "tool_type": item.type, + "tool_id": item.id, + "call_id": getattr(item, "call_id", None), + "status": item.status, + "turn_id": item.turn_id, + "server_label": getattr(item, "server_label", None), + "agent_id": getattr(item, "agent_id", None), + "sender_agent_id": getattr(item, "sender_agent_id", None), + } + ) + + +def _agent_tool_span_error(item: Any) -> Any: + error = getattr(item, "error", None) + if error is not None: + return error + if item.status == "failed": + return "Agent tool call failed" + return None + + +class _AgentSessionTrace: + """Accumulate one directly streamed Agents API turn into a task span.""" + + def __init__(self, span: Span, start_time: float) -> None: + self.span = span + self.start_time = start_time + self.parent_export = span.export() + self.root_turn_id: str | None = None + self.tool_start_times: dict[str, float] = {} + self.first_token_logged = False + self.finished = False + + def observe(self, event: Any) -> None: + event_type = event.type + if event_type in {"agent.session.created", "agent.session.in_progress", "agent.session.idle"}: + self._observe_session(event.session) + + if event_type in _AGENT_TURN_EVENTS and event.turn.subagent_id is None: + self.root_turn_id = event.turn_id + if event_type in _AGENT_TERMINAL_TURN_EVENTS: + self._observe_terminal_turn(event) + + if event_type in {"agent.session.turn.output_text.delta", "agent.session.turn.output_text.done"}: + if event.turn_id == self.root_turn_id and not self.first_token_logged: + text = getattr(event, "delta", None) or getattr(event, "text", None) + if text: + self.span.log(metrics={"time_to_first_token": time.time() - self.start_time}) + self.first_token_logged = True + + if event_type == "agent.session.turn.item.added": + item = event.item + if item.type in _AGENT_TOOL_ITEM_INPUT_KEYS: + self.tool_start_times[item.id] = time.time() + + if event_type == "agent.session.turn.item.done": + self._observe_completed_item(event.item) + + if event_type == "error": + self.span.log(error=event.error) + elif event_type == "agent.session.failed": + self.span.log(error=event.session.error or "Agent session failed") + elif event_type == "agent.session.environment.failed": + self.span.log(error=event.environment.error or "Agent session environment failed") + + def _observe_session(self, session: Any) -> None: + self.span.log( + metadata=clean_nones( + { + "session_id": session.id, + "agent_id": session.agent.id, + "model": session.agent.model, + "agent_name": session.agent.name, + "environment_type": session.environment.type, + } + ) + ) + + def _observe_completed_item(self, item: Any) -> None: + if item.type == "message" and item.phase == "final_answer": + if item.turn_id == self.root_turn_id: + self.span.log(output="".join(part.text for part in item.content)) + return + + if item.type not in _AGENT_TOOL_ITEM_INPUT_KEYS: + return + + start_time = self.tool_start_times.pop(item.id, None) + with start_span( + name=_agent_tool_span_name(item), + type=SpanTypeAttribute.TOOL, + start_time=start_time, + set_current=False, + parent=self.parent_export, + input=_agent_tool_span_data(item, _AGENT_TOOL_ITEM_INPUT_KEYS[item.type]), + metadata=_agent_tool_span_metadata(item), + ) as tool_span: + error = _agent_tool_span_error(item) + if error is not None: + tool_span.log(error=error) + output = _agent_tool_span_data(item, _AGENT_TOOL_ITEM_OUTPUT_KEYS.get(item.type, ())) + if output is not None: + tool_span.log(output=output) + + def _observe_terminal_turn(self, event: Any) -> None: + turn = event.turn + self.span.log( + metadata=clean_nones( + { + "session_id": event.session_id, + "turn_id": event.turn_id, + "agent_id": turn.agent_id, + "status": turn.status, + } + ) + ) + self.span.log(metrics=_parse_metrics_from_usage(event.usage or turn.usage)) + if turn.status == "failed": + self.span.log(error=turn.error or "Agent turn failed") + elif turn.status == "cancelled": + self.span.log(error="Agent turn cancelled") + + def finish(self) -> None: + if self.finished: + return + self.finished = True + self.span.end() + + +class AgentSessionWrapper: + def __init__(self, create_fn: Callable[..., Any] | None, acreate_fn: Callable[..., Any] | None) -> None: + self.create_fn = create_fn + self.acreate_fn = acreate_fn + + def create(self, *args: Any, **kwargs: Any) -> Any: + params = _agent_session_params(kwargs) + span = start_span( + name="openai.agents.sessions.create", + span_attributes={"type": SpanTypeAttribute.TASK}, + **params, + ) + start_time = time.time() + try: + stream = self.create_fn(*args, **kwargs) + except Exception as error: + span.log(error=error) + span.end() + raise + + trace = _AgentSessionTrace(span, start_time) + + def gen(): + try: + for event in stream: + trace.observe(event) + yield event + except Exception as error: + span.log(error=error) + raise + finally: + trace.finish() + + return _TracedStream(stream, gen(), trace.finish) + + async def acreate(self, *args: Any, **kwargs: Any) -> Any: + params = _agent_session_params(kwargs) + span = start_span( + name="openai.agents.sessions.create", + span_attributes={"type": SpanTypeAttribute.TASK}, + **params, + ) + start_time = time.time() + try: + stream = await self.acreate_fn(*args, **kwargs) + except Exception as error: + span.log(error=error) + span.end() + raise + + trace = _AgentSessionTrace(span, start_time) + + async def gen(): + try: + async for event in stream: + trace.observe(event) + yield event + except Exception as error: + span.log(error=error) + raise + finally: + trace.finish() + + return _AsyncTracedStream(stream, gen(), trace.finish) + + class ResponseWrapper: def __init__( self,