From e0588889f9348263c16e1efb904f47edacb427eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=90=9B=E8=B1=AA?= Date: Sun, 28 Jun 2026 15:51:30 +0800 Subject: [PATCH 01/11] chore(sidecars): vendor cc_convert and tito as host-side sidecar sources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bring the two upstream repos in-tree as standalone, vendored sidecar projects under sidecars/ — NOT uv workspace members, NOT part of the agentix package. abridge forwards to them over localhost HTTP, so all protocol/ML logic lives here and abridge core stays shape-blind. - cc_convert/ Anthropic<->OpenAI translator (Rust core + axum binary + PyO3) - tito/ TITO pretokenize + session-recording gateway (FastAPI, Miles) Vendored as-is; refactor (cc_convert binary -> in-process code, tito records -> /trace bridge) follows. Upstream licenses/notices preserved. Co-Authored-By: Claude Opus 4.8 --- sidecars/README.md | 28 + sidecars/cc_convert/.cargo/config.toml | 12 + .../cc_convert/.github/workflows/release.yml | 207 ++ sidecars/cc_convert/.gitignore | 56 + sidecars/cc_convert/Cargo.lock | 2254 +++++++++++++++++ sidecars/cc_convert/Cargo.toml | 29 + sidecars/cc_convert/DESIGN.md | 92 + sidecars/cc_convert/LICENSE-APACHE | 17 + sidecars/cc_convert/LICENSE-MIT | 21 + sidecars/cc_convert/README.md | 327 +++ sidecars/cc_convert/README.zh-CN.md | 311 +++ sidecars/cc_convert/RELEASING.md | 165 ++ sidecars/cc_convert/USAGE.md | 260 ++ sidecars/cc_convert/USAGE.zh-CN.md | 264 ++ .../crates/cc_convert_core/Cargo.toml | 17 + .../crates/cc_convert_core/src/anthropic.rs | 402 +++ .../crates/cc_convert_core/src/error.rs | 16 + .../crates/cc_convert_core/src/lib.rs | 45 + .../crates/cc_convert_core/src/openai.rs | 252 ++ .../cc_convert_core/src/req_to_openai.rs | 590 +++++ .../cc_convert_core/src/resp_to_anthropic.rs | 180 ++ .../crates/cc_convert_core/src/stream.rs | 399 +++ .../crates/cc_convert_core/src/tool_names.rs | 81 + .../cc_convert_core/tests/parity_litellm.rs | 169 ++ .../cc_convert_core/tests/parity_response.rs | 108 + .../cc_convert_core/tests/parity_stream.rs | 193 ++ .../tests/request_translation.rs | 657 +++++ .../tests/response_translation.rs | 194 ++ .../tests/stream_translation.rs | 190 ++ .../cc_convert_core/tests/vendor_quirks.rs | 515 ++++ .../crates/cc_convert_py/Cargo.toml | 19 + .../crates/cc_convert_py/src/lib.rs | 103 + .../crates/cc_convert_sidecar/Cargo.toml | 30 + .../crates/cc_convert_sidecar/src/lib.rs | 261 ++ .../crates/cc_convert_sidecar/src/main.rs | 48 + .../cc_convert_sidecar/tests/integration.rs | 264 ++ sidecars/cc_convert/playground/README.md | 78 + .../requests/02_reasoning_request.json | 15 + .../playground/requests/03_forced_tool.json | 30 + .../playground/requests/05_simple_text.json | 10 + .../requests/07_multi_turn_text.json | 27 + .../requests/08_agent_loop_with_tools.json | 122 + .../playground/requests/09_long_response.json | 12 + .../requests/10_parallel_tools_text_only.json | 50 + .../cc_convert/playground/run_roundtrip.py | 271 ++ sidecars/cc_convert/python/README.md | 53 + .../cc_convert/python/cc_convert/__init__.py | 82 + .../cc_convert/python/cc_convert/__main__.py | 6 + sidecars/cc_convert/python/cc_convert/cli.py | 725 ++++++ sidecars/cc_convert/python/pyproject.toml | 57 + .../python/tests/test_cli_helpers.py | 76 + .../cc_convert/python/tests/test_parity.py | 139 + .../python/tests/test_probe_models.py | 70 + sidecars/cc_convert/scripts/regen_fixtures.py | 71 + .../scripts/regen_response_fixtures.py | 158 ++ .../scripts/regen_stream_fixtures.py | 154 ++ .../scripts/seed_extra_request_fixtures.py | 198 ++ .../cc_convert/scripts/seed_fixture_inputs.py | 520 ++++ sidecars/cc_convert/tests/fixtures/README.md | 20 + .../anthropic_01_plain_user_text.json | 10 + .../requests/anthropic_02_system_string.json | 11 + ...c_03_system_blocks_with_cache_control.json | 19 + .../requests/anthropic_04_multi_turn.json | 18 + .../anthropic_05_user_image_base64.json | 23 + .../requests/anthropic_06_user_image_url.json | 18 + ...nthropic_07_assistant_single_tool_use.json | 19 + ...c_08_assistant_two_parallel_tool_uses.json | 27 + .../anthropic_09_user_single_tool_result.json | 16 + .../anthropic_10_user_three_tool_results.json | 26 + ...thropic_11_user_tool_result_multipart.json | 28 + .../anthropic_12_tools_input_schema.json | 27 + .../requests/anthropic_13_long_tool_name.json | 18 + .../anthropic_14_tool_choice_any.json | 13 + .../anthropic_15_tool_choice_named.json | 22 + .../anthropic_16_metadata_user_id.json | 13 + .../anthropic_17_thinking_medium.json | 14 + .../requests/anthropic_18_top_k_dropped.json | 11 + .../anthropic_19_stream_include_usage.json | 11 + ...20_o3_mini_uses_max_completion_tokens.json | 10 + .../anthropic_32_agent_tool_loop.json | 41 + ...thropic_33_user_content_cache_control.json | 22 + ...ic_34_assistant_content_cache_control.json | 26 + ...thropic_35_assistant_thinking_history.json | 28 + .../anthropic_36_user_mixed_content.json | 27 + .../anthropic_37_empty_string_content.json | 10 + .../anthropic_38_complex_tool_schema.json | 60 + ...ropic_39_tool_choice_auto_no_parallel.json | 22 + .../anthropic_40_tool_choice_none.json | 21 + .../requests/anthropic_41_thinking_high.json | 14 + .../requests/anthropic_42_thinking_low.json | 14 + .../requests/anthropic_43_stop_sequences.json | 14 + .../requests/openai_01_plain_user_text.json | 10 + .../requests/openai_02_system_string.json | 14 + ...i_03_system_blocks_with_cache_control.json | 19 + .../requests/openai_04_multi_turn.json | 19 + .../requests/openai_05_user_image_base64.json | 21 + .../requests/openai_06_user_image_url.json | 17 + .../openai_07_assistant_single_tool_use.json | 21 + ...i_08_assistant_two_parallel_tool_uses.json | 29 + .../openai_09_user_single_tool_result.json | 11 + .../openai_10_user_three_tool_results.json | 21 + .../openai_11_user_tool_result_multipart.json | 22 + .../openai_12_tools_input_schema.json | 30 + .../requests/openai_13_long_tool_name.json | 21 + .../requests/openai_14_tool_choice_any.json | 11 + .../requests/openai_15_tool_choice_named.json | 27 + .../requests/openai_16_metadata_user_id.json | 11 + .../requests/openai_17_thinking_medium.json | 11 + .../requests/openai_18_top_k_dropped.json | 11 + .../openai_19_stream_include_usage.json | 11 + ...20_o3_mini_uses_max_completion_tokens.json | 10 + .../requests/openai_32_agent_tool_loop.json | 35 + .../openai_33_user_content_cache_control.json | 19 + ...ai_34_assistant_content_cache_control.json | 19 + .../openai_35_assistant_thinking_history.json | 26 + .../openai_36_user_mixed_content.json | 26 + .../openai_37_empty_string_content.json | 5 + .../openai_38_complex_tool_schema.json | 63 + ...penai_39_tool_choice_auto_no_parallel.json | 22 + .../requests/openai_40_tool_choice_none.json | 22 + .../requests/openai_41_thinking_high.json | 11 + .../requests/openai_42_thinking_low.json | 11 + .../requests/openai_43_stop_sequences.json | 14 + .../requests/tool_map_01_plain_user_text.json | 1 + .../requests/tool_map_02_system_string.json | 1 + ...p_03_system_blocks_with_cache_control.json | 1 + .../requests/tool_map_04_multi_turn.json | 1 + .../tool_map_05_user_image_base64.json | 1 + .../requests/tool_map_06_user_image_url.json | 1 + ...tool_map_07_assistant_single_tool_use.json | 1 + ...p_08_assistant_two_parallel_tool_uses.json | 1 + .../tool_map_09_user_single_tool_result.json | 1 + .../tool_map_10_user_three_tool_results.json | 1 + ...ool_map_11_user_tool_result_multipart.json | 1 + .../tool_map_12_tools_input_schema.json | 1 + .../requests/tool_map_13_long_tool_name.json | 3 + .../requests/tool_map_14_tool_choice_any.json | 1 + .../tool_map_15_tool_choice_named.json | 1 + .../tool_map_16_metadata_user_id.json | 1 + .../requests/tool_map_17_thinking_medium.json | 1 + .../requests/tool_map_18_top_k_dropped.json | 1 + .../tool_map_19_stream_include_usage.json | 1 + ...20_o3_mini_uses_max_completion_tokens.json | 1 + .../requests/tool_map_32_agent_tool_loop.json | 1 + ...ool_map_33_user_content_cache_control.json | 1 + ...ap_34_assistant_content_cache_control.json | 1 + ...ool_map_35_assistant_thinking_history.json | 1 + .../tool_map_36_user_mixed_content.json | 1 + .../tool_map_37_empty_string_content.json | 1 + .../tool_map_38_complex_tool_schema.json | 1 + ...l_map_39_tool_choice_auto_no_parallel.json | 1 + .../tool_map_40_tool_choice_none.json | 1 + .../requests/tool_map_41_thinking_high.json | 1 + .../requests/tool_map_42_thinking_low.json | 1 + .../requests/tool_map_43_stop_sequences.json | 1 + .../responses/anthropic_21_plain_text.json | 18 + .../responses/anthropic_22_empty_content.json | 13 + ...anthropic_23_single_tool_call_no_text.json | 23 + .../anthropic_24_multiple_tool_calls.json | 28 + .../anthropic_25_length_max_tokens.json | 18 + .../responses/anthropic_26_cached_tokens.json | 19 + .../responses/meta_21_plain_text.json | 4 + .../responses/meta_22_empty_content.json | 4 + .../meta_23_single_tool_call_no_text.json | 4 + .../meta_24_multiple_tool_calls.json | 4 + .../responses/meta_25_length_max_tokens.json | 4 + .../responses/meta_26_cached_tokens.json | 4 + .../responses/openai_21_plain_text.json | 18 + .../responses/openai_22_empty_content.json | 14 + .../openai_23_single_tool_call_no_text.json | 24 + .../openai_24_multiple_tool_calls.json | 32 + .../openai_25_length_max_tokens.json | 14 + .../responses/openai_26_cached_tokens.json | 21 + .../streams/anthropic_27_text_only.jsonl | 7 + ...hropic_28_single_tool_call_fragments.jsonl | 9 + ...anthropic_29_two_parallel_tool_calls.jsonl | 8 + ...30_stream_ends_without_finish_reason.jsonl | 4 + .../anthropic_31_reasoning_then_text.jsonl | 7 + .../fixtures/streams/openai_27_text_only.sse | 8 + .../openai_28_single_tool_call_fragments.sse | 8 + .../openai_29_two_parallel_tool_calls.sse | 4 + ...i_30_stream_ends_without_finish_reason.sse | 2 + .../streams/openai_31_reasoning_then_text.sse | 6 + .../tito/.github/workflows/python-package.yml | 61 + sidecars/tito/.gitignore | 9 + sidecars/tito/LICENSE | 202 ++ sidecars/tito/README.md | 83 + sidecars/tito/README.zh-CN.md | 81 + sidecars/tito/docs/api.md | 50 + sidecars/tito/docs/api.zh-CN.md | 49 + sidecars/tito/docs/cli.md | 50 + sidecars/tito/docs/cli.zh-CN.md | 50 + sidecars/tito/docs/concepts.md | 55 + sidecars/tito/docs/concepts.zh-CN.md | 53 + sidecars/tito/docs/development.md | 66 + sidecars/tito/docs/development.zh-CN.md | 64 + sidecars/tito/docs/guide.md | 13 + sidecars/tito/docs/guide.zh-CN.md | 13 + sidecars/tito/docs/index.md | 22 + sidecars/tito/docs/index.zh-CN.md | 20 + sidecars/tito/docs/quickstart.md | 70 + sidecars/tito/docs/quickstart.zh-CN.md | 70 + sidecars/tito/docs/verification.md | 46 + sidecars/tito/docs/verification.zh-CN.md | 45 + sidecars/tito/miles/__init__.py | 5 + sidecars/tito/miles/_upstream_loader.py | 75 + sidecars/tito/miles/rollout/__init__.py | 5 + sidecars/tito/miles/rollout/base_types.py | 9 + .../miles/rollout/generate_hub/__init__.py | 5 + .../rollout/generate_hub/agentic_tool_call.py | 9 + .../tito/miles/rollout/session/__init__.py | 1 + .../rollout/session/linear_trajectory.py | 3 + .../miles/rollout/session/session_errors.py | 3 + .../miles/rollout/session/session_server.py | 3 + .../miles/rollout/session/session_types.py | 3 + .../tito/miles/rollout/session/sessions.py | 3 + sidecars/tito/miles/utils/__init__.py | 5 + .../utils/chat_template_utils/__init__.py | 3 + .../utils/chat_template_utils/deepseek_v32.py | 3 + .../utils/chat_template_utils/deepseek_v4.py | 3 + .../utils/chat_template_utils/template.py | 3 + .../chat_template_utils/tito_tokenizer.py | 4 + .../token_seq_comparator.py | 3 + .../miles/utils/external_utils/__init__.py | 5 + .../utils/external_utils/command_utils.py | 9 + sidecars/tito/miles/utils/hf_config.py | 3 + sidecars/tito/miles/utils/http_utils.py | 3 + sidecars/tito/miles/utils/processing_utils.py | 3 + .../tito/miles/utils/test_utils/__init__.py | 1 + .../utils/test_utils/chat_template_verify.py | 3 + .../utils/test_utils/mock_sglang_server.py | 3 + .../utils/test_utils/mock_trajectories.py | 3 + .../utils/test_utils/session_verify_agent.py | 3 + .../utils/test_utils/session_verify_runner.py | 3 + .../utils/test_utils/uvicorn_thread_server.py | 3 + sidecars/tito/plan.md | 228 ++ sidecars/tito/pyproject.toml | 91 + .../scripts/prepare_test_tokenizer_cache.py | 83 + sidecars/tito/tests/ci/__init__.py | 1 + sidecars/tito/tests/ci/ci_register.py | 12 + sidecars/tito/tests/fast/__init__.py | 1 + sidecars/tito/tests/fast/router/__init__.py | 1 + .../router/session_pretokenized_test_utils.py | 202 ++ .../qwen3_thinking_2507_and_next_fixed.jinja | 82 + sidecars/tito/tests/package/test_cli.py | 212 ++ .../tests/package/test_config_discovery.py | 139 + .../tests/package/test_gateway_integration.py | 82 + .../tito/tests/package/test_import_surface.py | 37 + .../package/test_session_verifier_plumbing.py | 74 + .../tests/package/test_upstream_delegation.py | 121 + .../tito/tests/package/test_vendored_miles.py | 31 + .../router/test_session_pretokenized_e2e.py | 174 ++ .../router/test_session_race_conditions.py | 422 +++ .../upstream/fast/router/test_sessions.py | 140 + .../test_pretokenized_via_tito.py | 161 ++ .../test_tito_tokenizer.py | 582 +++++ .../test_utils/test_session_verify_runner.py | 85 + .../tito/tito_gateway/VENDORED_MILES_AUDIT.md | 29 + sidecars/tito/tito_gateway/__init__.py | 16 + sidecars/tito/tito_gateway/cli.py | 241 ++ sidecars/tito/tito_gateway/config.py | 91 + sidecars/tito/tito_gateway/discovery.py | 99 + sidecars/tito/tito_gateway/gateway.py | 40 + sidecars/tito/tito_gateway/server.py | 19 + sidecars/tito/tito_gateway/tokenizer.py | 30 + sidecars/tito/tito_gateway/upstream.json | 7 + sidecars/tito/tito_gateway/vendor/__init__.py | 0 .../vendor/miles_compat/__init__.py | 0 .../vendor/miles_compat/rollout/__init__.py | 0 .../vendor/miles_compat/rollout/base_types.py | 29 + .../rollout/generate_hub/__init__.py | 1 + .../rollout/generate_hub/agentic_tool_call.py | 29 + .../miles_compat/rollout/session/__init__.py | 0 .../rollout/session/linear_trajectory.py | 285 +++ .../rollout/session/session_errors.py | 51 + .../rollout/session/session_server.py | 111 + .../rollout/session/session_types.py | 16 + .../miles_compat/rollout/session/sessions.py | 251 ++ .../vendor/miles_compat/utils/__init__.py | 0 .../utils/chat_template_utils/__init__.py | 39 + .../utils/chat_template_utils/deepseek_v32.py | 97 + .../utils/chat_template_utils/deepseek_v4.py | 100 + .../utils/chat_template_utils/template.py | 256 ++ .../templates/kimi_k25_fixed.jinja | 111 + .../templates/minimax_m25_fixed.jinja | 159 ++ .../templates/minimax_m27_fixed.jinja | 159 ++ .../templates/qwen3.5_fixed.jinja | 151 ++ .../templates/qwen3_fixed.jinja | 85 + .../qwen3_thinking_2507_and_next_fixed.jinja | 82 + .../chat_template_utils/tito_tokenizer.py | 1014 ++++++++ .../token_seq_comparator.py | 289 +++ .../utils/external_utils/__init__.py | 1 + .../utils/external_utils/command_utils.py | 33 + .../vendor/miles_compat/utils/hf_config.py | 108 + .../vendor/miles_compat/utils/http_utils.py | 315 +++ .../miles_compat/utils/processing_utils.py | 175 ++ .../miles_compat/utils/test_utils/__init__.py | 0 .../utils/test_utils/chat_template_verify.py | 602 +++++ .../utils/test_utils/mock_sglang_server.py | 270 ++ .../utils/test_utils/mock_trajectories.py | 1198 +++++++++ .../utils/test_utils/session_verify_agent.py | 460 ++++ .../utils/test_utils/session_verify_runner.py | 332 +++ .../utils/test_utils/uvicorn_thread_server.py | 49 + .../tito/tito_gateway/verify_chat_template.py | 125 + .../verify_session_tito_tokenizer.py | 33 + 305 files changed, 25267 insertions(+) create mode 100644 sidecars/README.md create mode 100644 sidecars/cc_convert/.cargo/config.toml create mode 100644 sidecars/cc_convert/.github/workflows/release.yml create mode 100644 sidecars/cc_convert/.gitignore create mode 100644 sidecars/cc_convert/Cargo.lock create mode 100644 sidecars/cc_convert/Cargo.toml create mode 100644 sidecars/cc_convert/DESIGN.md create mode 100644 sidecars/cc_convert/LICENSE-APACHE create mode 100644 sidecars/cc_convert/LICENSE-MIT create mode 100644 sidecars/cc_convert/README.md create mode 100644 sidecars/cc_convert/README.zh-CN.md create mode 100644 sidecars/cc_convert/RELEASING.md create mode 100644 sidecars/cc_convert/USAGE.md create mode 100644 sidecars/cc_convert/USAGE.zh-CN.md create mode 100644 sidecars/cc_convert/crates/cc_convert_core/Cargo.toml create mode 100644 sidecars/cc_convert/crates/cc_convert_core/src/anthropic.rs create mode 100644 sidecars/cc_convert/crates/cc_convert_core/src/error.rs create mode 100644 sidecars/cc_convert/crates/cc_convert_core/src/lib.rs create mode 100644 sidecars/cc_convert/crates/cc_convert_core/src/openai.rs create mode 100644 sidecars/cc_convert/crates/cc_convert_core/src/req_to_openai.rs create mode 100644 sidecars/cc_convert/crates/cc_convert_core/src/resp_to_anthropic.rs create mode 100644 sidecars/cc_convert/crates/cc_convert_core/src/stream.rs create mode 100644 sidecars/cc_convert/crates/cc_convert_core/src/tool_names.rs create mode 100644 sidecars/cc_convert/crates/cc_convert_core/tests/parity_litellm.rs create mode 100644 sidecars/cc_convert/crates/cc_convert_core/tests/parity_response.rs create mode 100644 sidecars/cc_convert/crates/cc_convert_core/tests/parity_stream.rs create mode 100644 sidecars/cc_convert/crates/cc_convert_core/tests/request_translation.rs create mode 100644 sidecars/cc_convert/crates/cc_convert_core/tests/response_translation.rs create mode 100644 sidecars/cc_convert/crates/cc_convert_core/tests/stream_translation.rs create mode 100644 sidecars/cc_convert/crates/cc_convert_core/tests/vendor_quirks.rs create mode 100644 sidecars/cc_convert/crates/cc_convert_py/Cargo.toml create mode 100644 sidecars/cc_convert/crates/cc_convert_py/src/lib.rs create mode 100644 sidecars/cc_convert/crates/cc_convert_sidecar/Cargo.toml create mode 100644 sidecars/cc_convert/crates/cc_convert_sidecar/src/lib.rs create mode 100644 sidecars/cc_convert/crates/cc_convert_sidecar/src/main.rs create mode 100644 sidecars/cc_convert/crates/cc_convert_sidecar/tests/integration.rs create mode 100644 sidecars/cc_convert/playground/README.md create mode 100644 sidecars/cc_convert/playground/requests/02_reasoning_request.json create mode 100644 sidecars/cc_convert/playground/requests/03_forced_tool.json create mode 100644 sidecars/cc_convert/playground/requests/05_simple_text.json create mode 100644 sidecars/cc_convert/playground/requests/07_multi_turn_text.json create mode 100644 sidecars/cc_convert/playground/requests/08_agent_loop_with_tools.json create mode 100644 sidecars/cc_convert/playground/requests/09_long_response.json create mode 100644 sidecars/cc_convert/playground/requests/10_parallel_tools_text_only.json create mode 100644 sidecars/cc_convert/playground/run_roundtrip.py create mode 100644 sidecars/cc_convert/python/README.md create mode 100644 sidecars/cc_convert/python/cc_convert/__init__.py create mode 100644 sidecars/cc_convert/python/cc_convert/__main__.py create mode 100644 sidecars/cc_convert/python/cc_convert/cli.py create mode 100644 sidecars/cc_convert/python/pyproject.toml create mode 100644 sidecars/cc_convert/python/tests/test_cli_helpers.py create mode 100644 sidecars/cc_convert/python/tests/test_parity.py create mode 100644 sidecars/cc_convert/python/tests/test_probe_models.py create mode 100644 sidecars/cc_convert/scripts/regen_fixtures.py create mode 100644 sidecars/cc_convert/scripts/regen_response_fixtures.py create mode 100644 sidecars/cc_convert/scripts/regen_stream_fixtures.py create mode 100644 sidecars/cc_convert/scripts/seed_extra_request_fixtures.py create mode 100644 sidecars/cc_convert/scripts/seed_fixture_inputs.py create mode 100644 sidecars/cc_convert/tests/fixtures/README.md create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_01_plain_user_text.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_02_system_string.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_03_system_blocks_with_cache_control.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_04_multi_turn.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_05_user_image_base64.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_06_user_image_url.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_07_assistant_single_tool_use.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_08_assistant_two_parallel_tool_uses.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_09_user_single_tool_result.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_10_user_three_tool_results.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_11_user_tool_result_multipart.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_12_tools_input_schema.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_13_long_tool_name.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_14_tool_choice_any.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_15_tool_choice_named.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_16_metadata_user_id.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_17_thinking_medium.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_18_top_k_dropped.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_19_stream_include_usage.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_20_o3_mini_uses_max_completion_tokens.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_32_agent_tool_loop.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_33_user_content_cache_control.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_34_assistant_content_cache_control.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_35_assistant_thinking_history.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_36_user_mixed_content.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_37_empty_string_content.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_38_complex_tool_schema.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_39_tool_choice_auto_no_parallel.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_40_tool_choice_none.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_41_thinking_high.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_42_thinking_low.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/anthropic_43_stop_sequences.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_01_plain_user_text.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_02_system_string.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_03_system_blocks_with_cache_control.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_04_multi_turn.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_05_user_image_base64.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_06_user_image_url.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_07_assistant_single_tool_use.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_08_assistant_two_parallel_tool_uses.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_09_user_single_tool_result.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_10_user_three_tool_results.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_11_user_tool_result_multipart.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_12_tools_input_schema.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_13_long_tool_name.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_14_tool_choice_any.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_15_tool_choice_named.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_16_metadata_user_id.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_17_thinking_medium.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_18_top_k_dropped.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_19_stream_include_usage.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_20_o3_mini_uses_max_completion_tokens.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_32_agent_tool_loop.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_33_user_content_cache_control.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_34_assistant_content_cache_control.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_35_assistant_thinking_history.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_36_user_mixed_content.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_37_empty_string_content.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_38_complex_tool_schema.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_39_tool_choice_auto_no_parallel.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_40_tool_choice_none.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_41_thinking_high.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_42_thinking_low.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/openai_43_stop_sequences.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_01_plain_user_text.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_02_system_string.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_03_system_blocks_with_cache_control.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_04_multi_turn.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_05_user_image_base64.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_06_user_image_url.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_07_assistant_single_tool_use.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_08_assistant_two_parallel_tool_uses.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_09_user_single_tool_result.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_10_user_three_tool_results.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_11_user_tool_result_multipart.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_12_tools_input_schema.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_13_long_tool_name.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_14_tool_choice_any.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_15_tool_choice_named.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_16_metadata_user_id.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_17_thinking_medium.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_18_top_k_dropped.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_19_stream_include_usage.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_20_o3_mini_uses_max_completion_tokens.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_32_agent_tool_loop.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_33_user_content_cache_control.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_34_assistant_content_cache_control.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_35_assistant_thinking_history.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_36_user_mixed_content.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_37_empty_string_content.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_38_complex_tool_schema.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_39_tool_choice_auto_no_parallel.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_40_tool_choice_none.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_41_thinking_high.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_42_thinking_low.json create mode 100644 sidecars/cc_convert/tests/fixtures/requests/tool_map_43_stop_sequences.json create mode 100644 sidecars/cc_convert/tests/fixtures/responses/anthropic_21_plain_text.json create mode 100644 sidecars/cc_convert/tests/fixtures/responses/anthropic_22_empty_content.json create mode 100644 sidecars/cc_convert/tests/fixtures/responses/anthropic_23_single_tool_call_no_text.json create mode 100644 sidecars/cc_convert/tests/fixtures/responses/anthropic_24_multiple_tool_calls.json create mode 100644 sidecars/cc_convert/tests/fixtures/responses/anthropic_25_length_max_tokens.json create mode 100644 sidecars/cc_convert/tests/fixtures/responses/anthropic_26_cached_tokens.json create mode 100644 sidecars/cc_convert/tests/fixtures/responses/meta_21_plain_text.json create mode 100644 sidecars/cc_convert/tests/fixtures/responses/meta_22_empty_content.json create mode 100644 sidecars/cc_convert/tests/fixtures/responses/meta_23_single_tool_call_no_text.json create mode 100644 sidecars/cc_convert/tests/fixtures/responses/meta_24_multiple_tool_calls.json create mode 100644 sidecars/cc_convert/tests/fixtures/responses/meta_25_length_max_tokens.json create mode 100644 sidecars/cc_convert/tests/fixtures/responses/meta_26_cached_tokens.json create mode 100644 sidecars/cc_convert/tests/fixtures/responses/openai_21_plain_text.json create mode 100644 sidecars/cc_convert/tests/fixtures/responses/openai_22_empty_content.json create mode 100644 sidecars/cc_convert/tests/fixtures/responses/openai_23_single_tool_call_no_text.json create mode 100644 sidecars/cc_convert/tests/fixtures/responses/openai_24_multiple_tool_calls.json create mode 100644 sidecars/cc_convert/tests/fixtures/responses/openai_25_length_max_tokens.json create mode 100644 sidecars/cc_convert/tests/fixtures/responses/openai_26_cached_tokens.json create mode 100644 sidecars/cc_convert/tests/fixtures/streams/anthropic_27_text_only.jsonl create mode 100644 sidecars/cc_convert/tests/fixtures/streams/anthropic_28_single_tool_call_fragments.jsonl create mode 100644 sidecars/cc_convert/tests/fixtures/streams/anthropic_29_two_parallel_tool_calls.jsonl create mode 100644 sidecars/cc_convert/tests/fixtures/streams/anthropic_30_stream_ends_without_finish_reason.jsonl create mode 100644 sidecars/cc_convert/tests/fixtures/streams/anthropic_31_reasoning_then_text.jsonl create mode 100644 sidecars/cc_convert/tests/fixtures/streams/openai_27_text_only.sse create mode 100644 sidecars/cc_convert/tests/fixtures/streams/openai_28_single_tool_call_fragments.sse create mode 100644 sidecars/cc_convert/tests/fixtures/streams/openai_29_two_parallel_tool_calls.sse create mode 100644 sidecars/cc_convert/tests/fixtures/streams/openai_30_stream_ends_without_finish_reason.sse create mode 100644 sidecars/cc_convert/tests/fixtures/streams/openai_31_reasoning_then_text.sse create mode 100644 sidecars/tito/.github/workflows/python-package.yml create mode 100644 sidecars/tito/.gitignore create mode 100644 sidecars/tito/LICENSE create mode 100644 sidecars/tito/README.md create mode 100644 sidecars/tito/README.zh-CN.md create mode 100644 sidecars/tito/docs/api.md create mode 100644 sidecars/tito/docs/api.zh-CN.md create mode 100644 sidecars/tito/docs/cli.md create mode 100644 sidecars/tito/docs/cli.zh-CN.md create mode 100644 sidecars/tito/docs/concepts.md create mode 100644 sidecars/tito/docs/concepts.zh-CN.md create mode 100644 sidecars/tito/docs/development.md create mode 100644 sidecars/tito/docs/development.zh-CN.md create mode 100644 sidecars/tito/docs/guide.md create mode 100644 sidecars/tito/docs/guide.zh-CN.md create mode 100644 sidecars/tito/docs/index.md create mode 100644 sidecars/tito/docs/index.zh-CN.md create mode 100644 sidecars/tito/docs/quickstart.md create mode 100644 sidecars/tito/docs/quickstart.zh-CN.md create mode 100644 sidecars/tito/docs/verification.md create mode 100644 sidecars/tito/docs/verification.zh-CN.md create mode 100644 sidecars/tito/miles/__init__.py create mode 100644 sidecars/tito/miles/_upstream_loader.py create mode 100644 sidecars/tito/miles/rollout/__init__.py create mode 100644 sidecars/tito/miles/rollout/base_types.py create mode 100644 sidecars/tito/miles/rollout/generate_hub/__init__.py create mode 100644 sidecars/tito/miles/rollout/generate_hub/agentic_tool_call.py create mode 100644 sidecars/tito/miles/rollout/session/__init__.py create mode 100644 sidecars/tito/miles/rollout/session/linear_trajectory.py create mode 100644 sidecars/tito/miles/rollout/session/session_errors.py create mode 100644 sidecars/tito/miles/rollout/session/session_server.py create mode 100644 sidecars/tito/miles/rollout/session/session_types.py create mode 100644 sidecars/tito/miles/rollout/session/sessions.py create mode 100644 sidecars/tito/miles/utils/__init__.py create mode 100644 sidecars/tito/miles/utils/chat_template_utils/__init__.py create mode 100644 sidecars/tito/miles/utils/chat_template_utils/deepseek_v32.py create mode 100644 sidecars/tito/miles/utils/chat_template_utils/deepseek_v4.py create mode 100644 sidecars/tito/miles/utils/chat_template_utils/template.py create mode 100644 sidecars/tito/miles/utils/chat_template_utils/tito_tokenizer.py create mode 100644 sidecars/tito/miles/utils/chat_template_utils/token_seq_comparator.py create mode 100644 sidecars/tito/miles/utils/external_utils/__init__.py create mode 100644 sidecars/tito/miles/utils/external_utils/command_utils.py create mode 100644 sidecars/tito/miles/utils/hf_config.py create mode 100644 sidecars/tito/miles/utils/http_utils.py create mode 100644 sidecars/tito/miles/utils/processing_utils.py create mode 100644 sidecars/tito/miles/utils/test_utils/__init__.py create mode 100644 sidecars/tito/miles/utils/test_utils/chat_template_verify.py create mode 100644 sidecars/tito/miles/utils/test_utils/mock_sglang_server.py create mode 100644 sidecars/tito/miles/utils/test_utils/mock_trajectories.py create mode 100644 sidecars/tito/miles/utils/test_utils/session_verify_agent.py create mode 100644 sidecars/tito/miles/utils/test_utils/session_verify_runner.py create mode 100644 sidecars/tito/miles/utils/test_utils/uvicorn_thread_server.py create mode 100644 sidecars/tito/plan.md create mode 100644 sidecars/tito/pyproject.toml create mode 100644 sidecars/tito/scripts/prepare_test_tokenizer_cache.py create mode 100644 sidecars/tito/tests/ci/__init__.py create mode 100644 sidecars/tito/tests/ci/ci_register.py create mode 100644 sidecars/tito/tests/fast/__init__.py create mode 100644 sidecars/tito/tests/fast/router/__init__.py create mode 100644 sidecars/tito/tests/fast/router/session_pretokenized_test_utils.py create mode 100644 sidecars/tito/tests/miles/utils/chat_template_utils/templates/qwen3_thinking_2507_and_next_fixed.jinja create mode 100644 sidecars/tito/tests/package/test_cli.py create mode 100644 sidecars/tito/tests/package/test_config_discovery.py create mode 100644 sidecars/tito/tests/package/test_gateway_integration.py create mode 100644 sidecars/tito/tests/package/test_import_surface.py create mode 100644 sidecars/tito/tests/package/test_session_verifier_plumbing.py create mode 100644 sidecars/tito/tests/package/test_upstream_delegation.py create mode 100644 sidecars/tito/tests/package/test_vendored_miles.py create mode 100644 sidecars/tito/tests/upstream/fast/router/test_session_pretokenized_e2e.py create mode 100644 sidecars/tito/tests/upstream/fast/router/test_session_race_conditions.py create mode 100644 sidecars/tito/tests/upstream/fast/router/test_sessions.py create mode 100644 sidecars/tito/tests/upstream/fast/utils/chat_template_utils/test_pretokenized_via_tito.py create mode 100644 sidecars/tito/tests/upstream/fast/utils/chat_template_utils/test_tito_tokenizer.py create mode 100644 sidecars/tito/tests/upstream/fast/utils/test_utils/test_session_verify_runner.py create mode 100644 sidecars/tito/tito_gateway/VENDORED_MILES_AUDIT.md create mode 100644 sidecars/tito/tito_gateway/__init__.py create mode 100644 sidecars/tito/tito_gateway/cli.py create mode 100644 sidecars/tito/tito_gateway/config.py create mode 100644 sidecars/tito/tito_gateway/discovery.py create mode 100644 sidecars/tito/tito_gateway/gateway.py create mode 100644 sidecars/tito/tito_gateway/server.py create mode 100644 sidecars/tito/tito_gateway/tokenizer.py create mode 100644 sidecars/tito/tito_gateway/upstream.json create mode 100644 sidecars/tito/tito_gateway/vendor/__init__.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/__init__.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/rollout/__init__.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/rollout/base_types.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/rollout/generate_hub/__init__.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/rollout/generate_hub/agentic_tool_call.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/__init__.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/linear_trajectory.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/session_errors.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/session_server.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/session_types.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/sessions.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/__init__.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/__init__.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/deepseek_v32.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/deepseek_v4.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/template.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/kimi_k25_fixed.jinja create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/minimax_m25_fixed.jinja create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/minimax_m27_fixed.jinja create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/qwen3.5_fixed.jinja create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/qwen3_fixed.jinja create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/qwen3_thinking_2507_and_next_fixed.jinja create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/tito_tokenizer.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/token_seq_comparator.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/external_utils/__init__.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/external_utils/command_utils.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/hf_config.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/http_utils.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/processing_utils.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/__init__.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/chat_template_verify.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/mock_sglang_server.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/mock_trajectories.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/session_verify_agent.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/session_verify_runner.py create mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/uvicorn_thread_server.py create mode 100644 sidecars/tito/tito_gateway/verify_chat_template.py create mode 100644 sidecars/tito/tito_gateway/verify_session_tito_tokenizer.py diff --git a/sidecars/README.md b/sidecars/README.md new file mode 100644 index 0000000..761f6f4 --- /dev/null +++ b/sidecars/README.md @@ -0,0 +1,28 @@ +# sidecars/ + +Host-side gateway sidecars that abridge forwards to. These are **standalone +vendored projects** — NOT uv workspace members, NOT part of the `agentix` +package. abridge core stays shape/protocol-blind; all protocol and ML logic +lives here, behind a localhost HTTP process. + +- `cc_convert/` — Anthropic ↔ OpenAI translation sidecar (Rust core + axum + binary + PyO3 wheel). abridge's `agentix.bridge.sidecars.cc_convert_sidecar(...)` + preset launches the `cc_convert_sidecar` binary. +- `tito/` — TITO pretokenize + session-recording gateway (FastAPI, wraps + Miles). Sits in front of an sglang / OpenAI-compatible backend and emits + pretokenized RL rollout trajectories. + +Each keeps its own build system and dependencies; nothing here is installed +into the core venv. Upstream attributions are preserved in each subtree +(`cc_convert/LICENSE-*`, `tito/VENDORED_MILES_AUDIT.md`). + +## Status / planned refactor + +Vendored as-is to get the sources in-tree; refactor follows. + +- **cc_convert** ships as a Rust binary today. The plan is to drop the + binary requirement and drive translation from code in-process (it already + exposes a PyO3 Python package under `cc_convert/python/`), so abridge can + call it without launching a separate process. +- **tito** runs as a FastAPI sidecar; its session/trajectory records are + bridged onto the existing `/trace` channel (work in progress). diff --git a/sidecars/cc_convert/.cargo/config.toml b/sidecars/cc_convert/.cargo/config.toml new file mode 100644 index 0000000..6a40b8f --- /dev/null +++ b/sidecars/cc_convert/.cargo/config.toml @@ -0,0 +1,12 @@ +# This file is committed; keep it portable across platforms (Linux/macOS/Windows +# native runners in CI). For local dev speedups (lld linker, line-tables-only +# debug info), copy `.cargo/config.local.toml.example` to `.cargo/config.local.toml` +# — Cargo will merge it on top of this one. + +[profile.dev] +debug = "line-tables-only" +incremental = true + +[profile.test] +debug = "line-tables-only" +incremental = true diff --git a/sidecars/cc_convert/.github/workflows/release.yml b/sidecars/cc_convert/.github/workflows/release.yml new file mode 100644 index 0000000..9adca33 --- /dev/null +++ b/sidecars/cc_convert/.github/workflows/release.yml @@ -0,0 +1,207 @@ +# Build cc_convert wheels for every platform and publish to PyPI. +# +# Triggers +# - push of a tag matching v* → build + publish to PyPI +# - manual workflow_dispatch → build + publish to TestPyPI (dry run) +# +# Authentication +# Uses PyPI Trusted Publishing (OIDC). NO API token in secrets. +# +# One-time setup on pypi.org BEFORE the first tag push: +# 1) Create the project on PyPI (or via this workflow's TestPyPI run first). +# 2) Go to https://pypi.org/manage/project/cc-convert/settings/publishing/ +# 3) Add a "Trusted Publisher" with: +# - Owner: yitianlian +# - Repository name: cc_convert +# - Workflow name: release.yml +# - Environment: pypi (must match the job's `environment: pypi` below) +# 4) Same on TestPyPI: https://test.pypi.org/manage/project/cc-convert/settings/publishing/ +# with Environment: testpypi +# +# Reference: https://docs.pypi.org/trusted-publishers/ + +name: release + +on: + push: + tags: + - "v*" + workflow_dispatch: + inputs: + target: + description: "Publish to PyPI or TestPyPI" + required: true + default: "testpypi" + type: choice + options: + - pypi + - testpypi + +permissions: {} + +jobs: + # ---------- build wheels ---------- + build-linux: + name: build (linux-${{ matrix.target }}) + runs-on: ${{ matrix.runner }} + permissions: + contents: read + strategy: + matrix: + include: + - target: x86_64 + runner: ubuntu-latest + - target: aarch64 + runner: ubuntu-24.04-arm # native ARM runner; no QEMU docker + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Build wheel + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + # `--release` is implied. `--strip` shrinks the wheel. + # `--out dist` is required by maturin-action. + # ABI3 is enabled in Cargo.toml (pyo3 features = ["abi3-py38"]), + # so each (os, arch) gets ONE wheel that works on Python 3.8+. + working-directory: python + args: --release --out ../dist --strip + manylinux: auto + sccache: "true" + - name: List dist/ + run: ls -lh dist/ + - uses: actions/upload-artifact@v4 + with: + name: wheels-linux-${{ matrix.target }} + path: dist/ + + build-macos: + name: build (macos-${{ matrix.target }}) + runs-on: ${{ matrix.runner }} + permissions: + contents: read + strategy: + matrix: + # Only Apple Silicon. macos-13 (Intel) runners on the free GitHub + # plan are perpetually queue-bound (>30 min waits common) and + # block the publish step. Intel mac users can pip-install from + # sdist (requires local Rust toolchain) until/unless we add it + # back on a paid runner. + include: + - target: aarch64 + runner: macos-14 # Apple Silicon + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Build wheel + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + working-directory: python + args: --release --out ../dist --strip + sccache: "true" + - name: List dist/ + run: ls -lh dist/ + - uses: actions/upload-artifact@v4 + with: + name: wheels-macos-${{ matrix.target }} + path: dist/ + + build-windows: + name: build (windows-x64) + runs-on: windows-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Build wheel + uses: PyO3/maturin-action@v1 + with: + target: x86_64 + working-directory: python + args: --release --out ../dist --strip + sccache: "true" + - name: List dist/ + run: dir dist + - uses: actions/upload-artifact@v4 + with: + name: wheels-windows-x64 + path: dist/ + + build-sdist: + name: build (sdist) + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Build sdist + # Build from python/ where pyproject.toml lives. The maturin-action + # `working-directory` input controls where maturin runs from. + uses: PyO3/maturin-action@v1 + with: + command: sdist + working-directory: python + args: --out ../dist + - name: List dist/ + run: ls -lh dist/ + - uses: actions/upload-artifact@v4 + with: + name: sdist + path: dist/ + + # ---------- publish ---------- + publish-testpypi: + name: publish to TestPyPI + needs: [build-linux, build-macos, build-windows, build-sdist] + if: github.event_name == 'workflow_dispatch' && github.event.inputs.target == 'testpypi' + runs-on: ubuntu-latest + environment: + name: testpypi + url: https://test.pypi.org/project/cc-convert/ + permissions: + id-token: write # required for OIDC + steps: + - name: Download all wheels + sdist + uses: actions/download-artifact@v4 + with: + path: dist + merge-multiple: true + - name: List dist/ + run: ls -lh dist/ + - name: Publish to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + skip-existing: true + + publish-pypi: + name: publish to PyPI + needs: [build-linux, build-macos, build-windows, build-sdist] + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/project/cc-convert/ + permissions: + id-token: write # required for OIDC + steps: + - name: Download all wheels + sdist + uses: actions/download-artifact@v4 + with: + path: dist + merge-multiple: true + - name: List dist/ + run: ls -lh dist/ + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/sidecars/cc_convert/.gitignore b/sidecars/cc_convert/.gitignore new file mode 100644 index 0000000..26851c1 --- /dev/null +++ b/sidecars/cc_convert/.gitignore @@ -0,0 +1,56 @@ +# Rust build artifacts +/target/ +**/*.rs.bk +Cargo.lock.bak + +# Python build artifacts +__pycache__/ +*.py[cod] +*$py.class +*.so +build/ +dist/ +*.egg-info/ +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.coverage +htmlcov/ + +# Maturin / wheels +target/wheels/ +*.whl + +# Virtual envs +.venv/ +venv/ +env/ + +# Editors / OS +.vscode/ +.idea/ +*.swp +*.swo +.DS_Store +Thumbs.db + +# Local secrets / config (never commit!) +.env +.env.local +*.pem +*.key +secrets/ +~/.pypirc + +# Build / dev cache from this workspace +.cargo/registry/ +.cargo/git/ + +# Logs / scratch +*.log +/tmp/ +# Ignore generated playground outputs (round-trip run results), but keep +# the runner script + seed fixtures committed so the workflow is +# reproducible. +playground/runs/ +playground/online/ diff --git a/sidecars/cc_convert/Cargo.lock b/sidecars/cc_convert/Cargo.lock new file mode 100644 index 0000000..14f650d --- /dev/null +++ b/sidecars/cc_convert/Cargo.lock @@ -0,0 +1,2254 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "axum-macros", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-macros" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d123550fa8d071b7255cb0cc04dc302baa6c8c4a79f55701552684d8399bce" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "cc" +version = "1.2.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cc_convert_core" +version = "0.1.0" +dependencies = [ + "hex", + "pretty_assertions", + "serde", + "serde_json", + "sha2", + "thiserror 1.0.69", + "uuid", +] + +[[package]] +name = "cc_convert_py" +version = "0.1.0" +dependencies = [ + "cc_convert_core", + "pyo3", + "serde", + "serde_json", +] + +[[package]] +name = "cc_convert_sidecar" +version = "0.1.0" +dependencies = [ + "axum", + "bytes", + "cc_convert_core", + "futures", + "reqwest", + "serde", + "serde_json", + "tokio", + "tokio-stream", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "indoc" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" +dependencies = [ + "cfg-if", + "futures-util", + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "memchr" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "pretty_assertions" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" +dependencies = [ + "diff", + "yansi", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f402062616ab18202ae8319da13fa4279883a2b8a9d9f83f20dbade813ce1884" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "memoffset", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b14b5775b5ff446dd1056212d778012cbe8a0fbffd368029fd9e25b514479c38" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ab5bcf04a2cdcbb50c7d6105de943f543f9ed92af55818fd17b660390fc8636" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fd24d897903a9e6d80b968368a34e1525aeb719d568dba8b3d4bfa5dc67d453" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "target-lexicon" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "unindent" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7" +dependencies = [ + "getrandom 0.4.2", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.3+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "web-sys" +version = "0.3.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + +[[package]] +name = "yoke" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/sidecars/cc_convert/Cargo.toml b/sidecars/cc_convert/Cargo.toml new file mode 100644 index 0000000..132bf70 --- /dev/null +++ b/sidecars/cc_convert/Cargo.toml @@ -0,0 +1,29 @@ +[workspace] +resolver = "2" +members = [ + "crates/cc_convert_core", + "crates/cc_convert_py", + "crates/cc_convert_sidecar", +] + +[workspace.package] +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" + +[workspace.dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" +thiserror = "1" +sha2 = "0.10" +hex = "0.4" +uuid = { version = "1", features = ["v4"] } +tokio = { version = "1", features = ["full"] } +axum = { version = "0.7", features = ["macros"] } +reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "rustls-tls"] } +futures = "0.3" +tokio-stream = "0.1" +bytes = "1" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +pyo3 = { version = "0.22", features = ["extension-module"] } diff --git a/sidecars/cc_convert/DESIGN.md b/sidecars/cc_convert/DESIGN.md new file mode 100644 index 0000000..b8e58bc --- /dev/null +++ b/sidecars/cc_convert/DESIGN.md @@ -0,0 +1,92 @@ +# cc_convert design principle: source of truth for each translation direction + +## Direction A: Anthropic → OpenAI (request translation) + +**Source of truth**: what real OpenAI-compatible model servers actually accept on the wire, in this priority: + +1. **vLLM** — `vllm/entrypoints/openai/chat_completion/protocol.py` and `chat_utils.py` + on GitHub `vllm-project/vllm`. Authoritative for self-hosted production. +2. **SGLang** — `python/sglang/srt/entrypoints/openai/protocol.py` and + `serving_chat.py` on `sgl-project/sglang`. Used by 启智 / a lot of CN + teams. +3. **DeepSeek hosted API** — `api-docs.deepseek.com`. The strictest in the + wild (rejects `reasoning_content` on input with HTTP 400). +4. **OpenAI Python SDK request types** — `openai/openai-python` repo, + `src/openai/types/chat/chat_completion_*.py`. The official schema. +5. **Popular chat templates** on Hugging Face — `tokenizer_config.json` for + DeepSeek-R1, Qwen3, QwQ, Llama 3.3. These tell us which message fields + the model ACTUALLY consumes once rendered. + +**NEVER** use LiteLLM's intermediate adapter output as the source of truth. +LiteLLM has provider-side transformations that strip/rewrite fields between +its `AnthropicAdapter` and the wire. We previously made this mistake by +forwarding LiteLLM's `thinking_blocks` shape on the wire — no real upstream +consumes it. + +LiteLLM parity is supported as an **opt-in compatibility mode** +(`litellm_compat`) for drop-in replacement, but the **default** must match +what real upstreams accept. + +## Direction B: OpenAI → Anthropic (response translation) + +**Source of truth**: the Anthropic Messages API official documentation. + +1. **Anthropic docs** — `docs.anthropic.com/en/api/messages` (request + + response shapes), `docs.anthropic.com/en/api/messages-streaming` (SSE + event shapes). Authoritative. +2. **Anthropic Python SDK types** — `anthropics/anthropic-sdk-python` repo, + `src/anthropic/types/message.py` etc. +3. **Anthropic client tooling** (Claude Code, claude-py) — what they + actually parse. If a field is in the docs but no SDK reads it, we don't + need to emit it. + +Where Anthropic adds new features (extended thinking, hosted tools, +prompt caching), follow the Anthropic spec verbatim — do NOT inherit +LiteLLM's interpretation. + +## Outstanding audit items + +Anywhere the current Rust translator was shaped against LiteLLM intermediate +output needs to be re-audited against real upstreams / Anthropic docs: + +- [x] **`thinking_blocks` field** — was LiteLLM-internal, no real consumer. + Fixed: emit `reasoning_content: string` by default (vLLM/SGLang/Qwen3 + consume), `LiteLLMThinkingBlocks` and `Drop` as opt-in modes. +- [ ] **`cache_control` propagation** — currently dropped. Verify + Anthropic-via-OpenAI proxies (when target is itself Anthropic) want + it preserved. +- [ ] **`tool_choice` field on streaming** — confirm vLLM/SGLang `required` + vs `any` semantics. +- [ ] **`top_k`** — currently passed through (LiteLLM behaviour). vLLM + accepts it in `SamplingParams`; OpenAI spec rejects it. Make default + drop for OpenAI targets, pass through for vLLM/SGLang via opt-in. +- [ ] **`stop_sequences` vs `stop`** — both wire names; vLLM accepts + `stop`, SGLang accepts both. Verify. +- [ ] **`max_tokens` vs `max_completion_tokens`** — only `o1*/o3*/o4*/gpt-5*` + strictly require `max_completion_tokens`. vLLM/SGLang accept both for + any model. Verify. +- [ ] **`stream_options.include_usage`** — confirm SGLang and DeepSeek + both honour this; some self-hosted servers ignore it. +- [ ] **`metadata`** — Anthropic-only `metadata.user_id` → OpenAI `user`. + Confirmed. +- [ ] **`thinking.budget_tokens`** → `reasoning_effort` bucketing — only + applies to OpenAI o-series and a few others. vLLM may want a raw + `extra_body.reasoning.budget_tokens`. Audit. +- [ ] **Response side: `reasoning_content` vs `reasoning`** — already + aliased in the deserializer. Good. +- [ ] **Response side: `stop_sequence` field** — should be the matched + stop string when known. vLLM has `stop_reason`, SGLang has + `matched_stop` — currently ignored. Anthropic clients sometimes + check this. +- [ ] **Streaming SSE event shapes** — re-verify against Anthropic + official streaming docs (not just LiteLLM's `AnthropicStreamWrapper`, + which has documented bugs around parallel tool_calls etc.) + +## Process going forward + +When changing any translation rule: +1. Check the **real-upstream** source (vLLM/SGLang/Anthropic docs) first. +2. Decide what the default should be based on what 80% of real upstreams + accept. +3. If LiteLLM disagrees with reality, add a `--compat-mode litellm_compat` + opt-in for the LiteLLM behaviour. The default follows reality. diff --git a/sidecars/cc_convert/LICENSE-APACHE b/sidecars/cc_convert/LICENSE-APACHE new file mode 100644 index 0000000..eace40c --- /dev/null +++ b/sidecars/cc_convert/LICENSE-APACHE @@ -0,0 +1,17 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +For the full text see https://www.apache.org/licenses/LICENSE-2.0.txt diff --git a/sidecars/cc_convert/LICENSE-MIT b/sidecars/cc_convert/LICENSE-MIT new file mode 100644 index 0000000..c34ef5c --- /dev/null +++ b/sidecars/cc_convert/LICENSE-MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 cc_convert maintainers + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/sidecars/cc_convert/README.md b/sidecars/cc_convert/README.md new file mode 100644 index 0000000..13bd37d --- /dev/null +++ b/sidecars/cc_convert/README.md @@ -0,0 +1,327 @@ +# cc_convert + +> 中文文档:[README.zh-CN.md](README.zh-CN.md) +> +> 详细用法 / Usage details: [USAGE.md](USAGE.md) ([中文](USAGE.zh-CN.md)) + +Bidirectional Anthropic ↔ OpenAI Chat Completions protocol converter, written +in Rust. Lets clients keep calling the Anthropic Messages API shape (system +prompt + content blocks + `tool_use` / `tool_result` + Anthropic SSE) while +the actual upstream is an OpenAI-compatible server, and vice versa for +responses. + +Validated against three reference implementations: +- [LiteLLM]'s `AnthropicAdapter` (primary parity oracle — request, response, stream) +- [1rgs/claude-code-proxy], [maxnowack/anthropic-proxy] (cross-reference) +- [THUDM/slime]'s anthropic adapter (Chinese RL ecosystem) + +And hardened against the **non-standard quirks** of self-hosted servers: +- **vLLM**: uses `reasoning` (not `reasoning_content`); extra `stop_reason`, + `prompt_logprobs`, `kv_transfer_params` fields; first stream chunk is + role-only. +- **SGLang**: emits `id: null` and `function.name: null` on continuation + tool_call chunks; sends `reasoning_content: null` on every chunk; + `matched_stop`, `metadata`, `sglext` extras; `finish_reason: "abort"`; + kimi_k2 tool IDs of form `functions.:`. + +[LiteLLM]: https://github.com/BerriAI/litellm +[1rgs/claude-code-proxy]: https://github.com/1rgs/claude-code-proxy +[maxnowack/anthropic-proxy]: https://github.com/maxnowack/anthropic-proxy +[THUDM/slime]: https://github.com/THUDM/slime/tree/main/slime/agent/adapters + +## Two deployment modes + +| Mode | What it is | Use when | +|---|---|---| +| Python package | `pip install cc_convert`, `import cc_convert` | You're embedding the converter in a Python app and prefer dict-in / dict-out. | +| HTTP sidecar | `cc_convert_sidecar` binary; listens on `/v1/messages` (Anthropic shape) and proxies to a configured OpenAI-compatible upstream | You're plugging an Anthropic-API client (Claude Code, claude-py, etc.) into a non-Anthropic backend. | + +Both paths share the same Rust translation core (`cc_convert_core`), so +behaviour is identical. + +## Python usage + +```python +import cc_convert + +# 1) Translate an Anthropic request → OpenAI request. +anthropic_req = { + "model": "gpt-4o-mini", + "max_tokens": 500, + "system": "Be concise.", + "tools": [{ + "name": "get_weather", + "description": "Get current weather for a city", + "input_schema": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }], + "tool_choice": {"type": "any"}, + "messages": [{"role": "user", "content": "Weather in Tokyo?"}], +} +openai_req, tool_map = cc_convert.translate_request(anthropic_req) +# → POST openai_req to your OpenAI-compatible /v1/chat/completions endpoint. + +# 2) Translate the OpenAI response back to Anthropic shape. +openai_resp = {...} # from your upstream +anthropic_resp = cc_convert.translate_response( + openai_resp, original_model="claude-opus-4-7", tool_name_map=tool_map +) + +# 3) Streaming: feed OpenAI SSE chunks, get Anthropic SSE events. +translator = cc_convert.StreamTranslator("claude-opus-4-7", tool_map) +for openai_chunk in upstream_sse_chunks: # each is a dict + for anthropic_event in translator.push(openai_chunk): + emit_to_client(anthropic_event) # type: message_start / content_block_* / message_delta / message_stop +for trailing in translator.finish(): + emit_to_client(trailing) +``` + +The first element of `translate_request`'s return value is the OpenAI body +you POST. The second is a `dict[str, str]` mapping translated → original +tool names — keep it for the response side so we can restore tool names that +were truncated to fit OpenAI's 64-char limit. + +## Sidecar usage + +You can run the sidecar two ways: + +### A) Built-in CLI (Python wheel) — simplest + +After `pip install cc_convert`, the wheel exposes a `cc_convert` command. + +```bash +# Run the sidecar in proxy mode (Anthropic-shape in → OpenAI upstream → Anthropic-shape out). +cc_convert serve \ + --listen 0.0.0.0:8787 \ + --upstream-url https://api.openai.com/v1/chat/completions \ + --upstream-key sk-... + +# Same thing pointing at a vLLM / SGLang / DeepSeek backend: +cc_convert serve --upstream-url http://localhost:8000/v1/chat/completions -v + +# Pure-translation RPC server (no upstream call): +cc_convert serve --mode rpc --listen 127.0.0.1:8788 + +# One-shot JSON in / JSON out (no server): +cat anthropic_req.json | cc_convert translate --direction cc-to-oai +cat openai_resp.json | cc_convert translate --direction oai-to-cc --original-model claude-opus-4-7 +``` + +Useful flags: + +| Flag | Default | What it does | +|---|---|---| +| `--mode {proxy,rpc}` | `proxy` | `proxy`: terminate Anthropic on `--cc-path`, forward to `--upstream-url`, return translated Anthropic. `rpc`: stateless `/translate/cc-to-oai` and `/translate/oai-to-cc` endpoints. | +| `--listen HOST:PORT` | `0.0.0.0:8787` | What to bind to. | +| `--upstream-url URL` | `$CC_CONVERT_UPSTREAM_URL` | (proxy) The OpenAI-compatible `/v1/chat/completions` URL. | +| `--upstream-key KEY` | `$CC_CONVERT_UPSTREAM_API_KEY` | (proxy) Bearer token sent to the upstream. | +| `--auth-passthrough` | off | Use the CLIENT's `Authorization` / `x-api-key` header instead of `--upstream-key`. | +| `--cc-path PATH` | `/v1/messages` | (proxy) Path that receives Anthropic-shape requests. | +| `--cc-to-oai-path PATH` | `/translate/cc-to-oai` | (rpc) Path for request-side translation. | +| `--oai-to-cc-path PATH` | `/translate/oai-to-cc` | (rpc) Path for response-side translation. | +| `--log-level LEVEL` | `info` | `debug` / `info` / `warning` / `error`. | +| `--log-format FORMAT` | `text` | `text` (human) or `json` (one JSON object per line, easy to ship). | +| `-v`, `-vv` | | Shorthand for `--log-level info` / `debug`. | +| `--quiet` | off | Suppress per-request access logs. | +| `--version` | | Print version and exit. | + +All flags also accept env-var defaults: `CC_CONVERT_MODE`, `CC_CONVERT_LISTEN_ADDR`, +`CC_CONVERT_UPSTREAM_URL`, `CC_CONVERT_UPSTREAM_API_KEY`, +`CC_CONVERT_AUTH_PASSTHROUGH=1`, `CC_CONVERT_LOG_LEVEL`, `CC_CONVERT_LOG_FORMAT`. + +Then hit it: + +```bash +curl -X POST http://localhost:8787/v1/messages \ + -H 'content-type: application/json' \ + -d '{ + "model": "claude-opus-4-7", + "max_tokens": 100, + "messages": [{"role":"user","content":"hello"}] + }' +# → Anthropic-shape response, transparently sourced from the OpenAI upstream. +``` + +Streaming requests work the same — the SSE event stream you receive is +Anthropic-shape (`event: message_start`, `event: content_block_delta`, ...). + +`GET /healthz` returns `ok` for liveness probes. +`GET /version` returns `{"name":"cc_convert","version":"..."}`. + +### B) Pure Rust binary (no Python needed) + +```bash +cargo build --release -p cc_convert_sidecar +export CC_CONVERT_UPSTREAM_URL="https://api.openai.com/v1/chat/completions" +export CC_CONVERT_UPSTREAM_API_KEY="sk-..." +./target/release/cc_convert_sidecar +``` + +Same env-var contract as the CLI. Use this when you want a single static +binary to drop on a machine that doesn't have Python. + +## Two preset profiles + +Both `ConvertOptions` (request) and `ResponseConvertOptions` / `StreamConvertOptions` +(response/stream) ship two presets you can choose between: + +| Preset | What it does | +|---|---| +| `litellm_compat()` (default) | Byte-equivalent to LiteLLM's `AnthropicAdapter`. Use this when you're replacing LiteLLM in an existing pipeline and want zero behavioural drift. | +| `pragmatic()` / `anthropic_native()` | Closer to the published Anthropic SSE spec and what most OpenAI-compatible servers actually expect: `stream_options: {include_usage: true}` injected, `max_completion_tokens` used for o1/o3/o4/gpt-5, `stop` instead of `stop_sequences`, eager content_block opening, `ping` events, etc. | + +## Test coverage (51 Rust tests + 22 Python tests, all passing) + +``` +crates/cc_convert_core/ +├── src/ ← 3 unit tests (tool name truncation) +└── tests/ + ├── request_translation.rs ← 20 unit tests (cases 1–20) + ├── response_translation.rs ← 7 unit tests (cases 21–26 + tool name round-trip) + ├── stream_translation.rs ← 5 unit tests (cases 27–31) + ├── parity_litellm.rs ← 2 LiteLLM request parity tests (32 fixtures) + ├── parity_response.rs ← 1 LiteLLM response parity test (6 fixtures) + ├── parity_stream.rs ← 1 LiteLLM stream parity test (2 fixtures, 3 documented quirks excluded) + └── vendor_quirks.rs ← 12 vLLM + SGLang quirk tests +crates/cc_convert_sidecar/tests/ +└── integration.rs ← 3 HTTP integration tests (non-streaming, streaming, 4xx) +python/tests/ +└── test_parity.py ← 22 pytest tests (LiteLLM parity through the wheel + smoke) +``` + +## Translation rules (high level) + +Source of truth: LiteLLM's `AnthropicAdapter.translate_anthropic_to_openai`. + +| Anthropic | → | OpenAI | +|---|---|---| +| `system: string` | | leading `{role:"system", content:}` | +| `system: [{type:"text", text, cache_control?}]` | | leading `{role:"system", content:[{type:"text", text}, ...]}` (cache_control dropped) | +| user `text` block | | `{type:"text", text}` | +| user `image` (base64) | | `{type:"image_url", image_url:{url:"data:;base64,"}}` | +| user `image` (url) | | `{type:"image_url", image_url:{url}}` | +| user `tool_result` | | separate `{role:"tool", tool_call_id:, content}` message, BEFORE any user text in that message; one tool message per tool_use_id | +| assistant `tool_use` | | entry in `tool_calls: [{id, type:"function", function:{name, arguments: JSON.stringify(input)}}]` | +| assistant `thinking` | | `thinking_blocks: [...]` on the assistant message (LiteLLM behaviour, opt out with `preserve_thinking_blocks=false`) | +| `max_tokens` | | `max_tokens` (LiteLLM-compat); opt in to `max_completion_tokens` for `o1*/o3*/o4*/gpt-5*` | +| `stop_sequences` | | `stop_sequences` (passthrough, LiteLLM-compat); opt in to `stop` | +| `top_k` | | passed through (LiteLLM behaviour; opt out via `drop_top_k`) | +| `tools` | | `[{type:"function", function:{name, description, parameters: input_schema}}]`; names >64 chars truncated to `{55-prefix}_{8-hex-sha}` | +| `tool_choice:{type:"any"}` | | `"required"` | +| `tool_choice:{type:"tool", name}` | | `{type:"function", function:{name}}` | +| `metadata.user_id` | | `user` | +| `thinking.budget_tokens` | | `reasoning_effort` (≥10000→high, ≥5000→medium, ≥2000→low, else minimal) | +| `cache_control` (any block) | | dropped | + +Response side (OpenAI → Anthropic): + +| OpenAI | → | Anthropic | +|---|---|---| +| `id` | | passed through (LiteLLM); opt into `chatcmpl-→msg_` rewrite | +| `choices[0].message.content` | | `{type:"text", text}` block | +| `choices[0].message.tool_calls` | | `{type:"tool_use", id, name, input: JSON.parse(arguments)}` blocks; tool name restored via map | +| `choices[0].message.reasoning_content` / `reasoning` | | `{type:"thinking", thinking}` block (accepts both — vLLM uses `reasoning`) | +| `finish_reason: stop\|length\|tool_calls\|content_filter\|abort` | | `stop_reason: end_turn\|max_tokens\|tool_use\|end_turn\|end_turn` | +| `usage.prompt_tokens` / `completion_tokens` | | `usage.input_tokens` / `output_tokens` (LiteLLM subtracts cached) | +| `usage.prompt_tokens_details.cached_tokens` | | `usage.cache_read_input_tokens` | + +Streaming side: an OpenAI SSE chunk stream becomes a sequence of +`message_start` → `[ping]` → `content_block_start` → ... → `content_block_stop` +→ `message_delta` → `message_stop` events, with `text_delta` / +`input_json_delta` / `thinking_delta` for text / tool_call / reasoning +content respectively. Parallel tool_calls get distinct content-block +indices. Streams that end without a `finish_reason` are closed with +`stop_reason: end_turn`. + +## Building from source + +Requires Rust ≥ 1.75 and Python ≥ 3.8 (only for the wheel). + +```bash +# If your environment needs an HTTP proxy for cargo/pip, set the usual env vars. +# (Optional — only needed in restricted networks.) +# export http_proxy=http://YOUR_PROXY:PORT +# export https_proxy=http://YOUR_PROXY:PORT +# export no_proxy="localhost,127.0.0.1" + +# Rust core + sidecar +cargo build --release + +# Rust tests (includes LiteLLM parity against committed goldens — no network) +cargo test --workspace + +# Python wheel +cd python +pip install maturin +maturin build --release +pip install ../target/wheels/cc_convert-*.whl +pytest tests/ +``` + +## Parity tests + +Twenty plus twelve request fixtures live under `tests/fixtures/requests/` as +paired `anthropic_.json` / `openai_.json` files. Six response +and five stream fixtures sit under `responses/` and `streams/`. All golden +outputs were produced by running each input through LiteLLM and committed +to the repo so CI doesn't need network. + +To regenerate goldens after a rule change: + +```bash +pip install 'litellm>=1.0' +python scripts/seed_fixture_inputs.py # initial 31 cases (only if missing) +python scripts/seed_extra_request_fixtures.py # 12 extra request cases +python scripts/regen_fixtures.py # request goldens +python scripts/regen_response_fixtures.py # response goldens +python scripts/regen_stream_fixtures.py # stream goldens +cargo test --workspace # confirm parity still holds +``` + +## Layout + +``` +crates/ + cc_convert_core/ Pure-Rust translation library, no I/O. + cc_convert_py/ PyO3 bindings → Python wheel (cc_convert._native). + cc_convert_sidecar/ axum HTTP proxy binary + integration tests. +python/ + cc_convert/ Python package (re-exports the native module). + tests/ pytest parity tests against the wheel. +scripts/ + seed_fixture_inputs.py Generate INPUT fixtures (cases 1–31). + seed_extra_request_fixtures.py Generate extra INPUT fixtures (cases 32–43). + regen_fixtures.py Run LiteLLM to produce request goldens. + regen_response_fixtures.py Same for responses. + regen_stream_fixtures.py Same for streams. +tests/ + fixtures/ Committed golden parity fixtures. +``` + +## Known gaps / non-goals (v1) + +- **Hosted Anthropic tools** (`web_search`, `computer`, `bash`, + `text_editor`) are not translated to OpenAI equivalents — they are + passed through. v1.1 may map `web_search` to OpenAI's + `web_search_options`. +- **`stop_sequence` detection** on responses is not implemented. None of + the reference libs do it either; OpenAI doesn't surface the matched + stop string. vLLM does via `stop_reason` (matched string) and SGLang via + `matched_stop` — translating these into Anthropic's `stop_sequence` + field would be straightforward to add but is not in v1. +- **Documented LiteLLM stream quirks**: three of our five stream fixture + goldens diverge from spec because LiteLLM's `AnthropicStreamWrapper` + produces non-spec output (merging parallel tool_calls into one block, + silently truncating streams with no finish_reason, conflating + reasoning+text into one block). We follow the spec; see + `tests/parity_stream.rs:LITELLM_QUIRKS_TO_SKIP` for details. +- **Anthropic `[DONE]` sentinel**: real Anthropic SSE does *not* emit + `data: [DONE]\n\n`. We don't either. We do *accept* it on input from + upstream OpenAI as the end-of-stream marker. + +## License + +MIT OR Apache-2.0. diff --git a/sidecars/cc_convert/README.zh-CN.md b/sidecars/cc_convert/README.zh-CN.md new file mode 100644 index 0000000..a18f2b4 --- /dev/null +++ b/sidecars/cc_convert/README.zh-CN.md @@ -0,0 +1,311 @@ +# cc_convert + +> English docs: [README.md](README.md) +> +> 详细用法 / Usage details: [USAGE.zh-CN.md](USAGE.zh-CN.md) ([English](USAGE.md)) + +**Anthropic 与 OpenAI Chat Completions 协议的双向转换器**,用 Rust 写的核心。 +让客户端继续按 Anthropic Messages API 的样子调(`system` + content blocks + +`tool_use` / `tool_result` + Anthropic SSE),实际后端是 OpenAI 兼容的服务器 +(也可反向)。 + +已对照三个参考实现验证: +- [LiteLLM] 的 `AnthropicAdapter`(主要的 parity oracle,覆盖 request、response、stream) +- [1rgs/claude-code-proxy]、[maxnowack/anthropic-proxy](交叉对照) +- [THUDM/slime] 的 anthropic adapter(中文 RL 生态) + +并针对自建服务器的**非标准行为**做了硬化: +- **vLLM**:用 `reasoning`(不是 `reasoning_content`);多出 `stop_reason`、 + `prompt_logprobs`、`kv_transfer_params` 等字段;第一个 stream chunk 只有 role。 +- **SGLang**:在 tool_call 续传 chunk 上发送 `id: null` 和 `function.name: null`; + 每一帧都带 `reasoning_content: null`;额外的 `matched_stop`、`metadata`、`sglext`; + `finish_reason: "abort"`;kimi_k2 工具 ID 用 `functions.:` 形式。 + +[LiteLLM]: https://github.com/BerriAI/litellm +[1rgs/claude-code-proxy]: https://github.com/1rgs/claude-code-proxy +[maxnowack/anthropic-proxy]: https://github.com/maxnowack/anthropic-proxy +[THUDM/slime]: https://github.com/THUDM/slime/tree/main/slime/agent/adapters + +## 两种部署方式 + +| 模式 | 说明 | 何时用 | +|---|---|---| +| Python 包 | `pip install cc_convert`,`import cc_convert` | Python 应用里直接调用,dict 进 dict 出 | +| HTTP sidecar | `cc_convert_sidecar` 可执行文件,监听 `/v1/messages`(Anthropic 格式),反向代理到上游 OpenAI 兼容服务器 | 把 Anthropic API 客户端(Claude Code、claude-py 等)接到非 Anthropic 后端 | + +两种方式共享同一份 Rust 翻译核心(`cc_convert_core`),行为完全一致。 + +## Python 用法 + +```python +import cc_convert + +# 1) Anthropic 请求 → OpenAI 请求 +anthropic_req = { + "model": "gpt-4o-mini", + "max_tokens": 500, + "system": "回答简洁。", + "tools": [{ + "name": "get_weather", + "description": "查询城市天气", + "input_schema": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }], + "tool_choice": {"type": "any"}, + "messages": [{"role": "user", "content": "东京天气如何?"}], +} +openai_req, tool_map = cc_convert.translate_request(anthropic_req) +# → 把 openai_req POST 到任意 OpenAI 兼容的 /v1/chat/completions + +# 2) OpenAI 响应 → Anthropic 响应 +openai_resp = {...} # 上游返回的 +anthropic_resp = cc_convert.translate_response( + openai_resp, original_model="claude-opus-4-7", tool_name_map=tool_map +) + +# 3) 流式:OpenAI SSE chunks → Anthropic SSE 事件 +translator = cc_convert.StreamTranslator("claude-opus-4-7", tool_map) +for openai_chunk in upstream_sse_chunks: # 每个都是 dict + for anthropic_event in translator.push(openai_chunk): + emit_to_client(anthropic_event) # type: message_start / content_block_* / message_delta / message_stop +for trailing in translator.finish(): + emit_to_client(trailing) +``` + +`translate_request` 返回 `(openai_request, tool_name_map)`。第一个就是要 POST 给 +上游的请求体;第二个是 `dict[str, str]`,把"截断后的工具名 → 原始工具名"映射保存 +下来,响应端用它还原超过 OpenAI 64 字符上限被截断的工具名。 + +## Sidecar 用法 + +两种跑法,任选其一: + +### A) 内置 CLI(Python wheel 自带)——最简单 + +`pip install cc_convert` 之后,直接有一个 `cc_convert` 命令: + +```bash +# proxy 模式:监听 Anthropic 请求,转发到 OpenAI 上游,翻回 Anthropic 响应 +cc_convert serve \ + --listen 0.0.0.0:8787 \ + --upstream-url https://api.openai.com/v1/chat/completions \ + --upstream-key sk-... + +# 指向 vLLM / SGLang / DeepSeek 之类自建服务 +cc_convert serve --upstream-url http://localhost:8000/v1/chat/completions -v + +# 纯翻译 RPC 模式(不转发,只翻译) +cc_convert serve --mode rpc --listen 127.0.0.1:8788 + +# 一次性 JSON 进 JSON 出(不起 server) +cat anthropic_req.json | cc_convert translate --direction cc-to-oai +cat openai_resp.json | cc_convert translate --direction oai-to-cc --original-model claude-opus-4-7 +``` + +常用参数: + +| 参数 | 默认 | 含义 | +|---|---|---| +| `--mode {proxy,rpc}` | `proxy` | `proxy`:在 `--cc-path` 上接 Anthropic 请求,转发到 `--upstream-url`,翻回 Anthropic 返回。`rpc`:无状态 `/translate/cc-to-oai` 和 `/translate/oai-to-cc` 两个端点。 | +| `--listen HOST:PORT` | `0.0.0.0:8787` | 监听地址 | +| `--upstream-url URL` | `$CC_CONVERT_UPSTREAM_URL` | (proxy)OpenAI 兼容的 `/v1/chat/completions` URL | +| `--upstream-key KEY` | `$CC_CONVERT_UPSTREAM_API_KEY` | (proxy)Bearer token,发给上游 | +| `--auth-passthrough` | off | 改成透传客户端的 `Authorization` / `x-api-key`,不用 `--upstream-key` | +| `--cc-path PATH` | `/v1/messages` | (proxy)接 Anthropic 请求的路径 | +| `--cc-to-oai-path PATH` | `/translate/cc-to-oai` | (rpc)请求端翻译路径 | +| `--oai-to-cc-path PATH` | `/translate/oai-to-cc` | (rpc)响应端翻译路径 | +| `--log-level LEVEL` | `info` | `debug` / `info` / `warning` / `error` | +| `--log-format FORMAT` | `text` | `text`(给人看)或 `json`(一行一个 JSON 对象,方便采集) | +| `-v` / `-vv` | | 快捷写法,等价于 `--log-level info` / `debug` | +| `--quiet` | off | 不打访问日志(只剩错误日志) | +| `--version` | | 打印版本退出 | + +所有参数都有对应的环境变量默认值:`CC_CONVERT_MODE`、`CC_CONVERT_LISTEN_ADDR`、 +`CC_CONVERT_UPSTREAM_URL`、`CC_CONVERT_UPSTREAM_API_KEY`、 +`CC_CONVERT_AUTH_PASSTHROUGH=1`、`CC_CONVERT_LOG_LEVEL`、`CC_CONVERT_LOG_FORMAT`。 + +调起来: + +```bash +curl -X POST http://localhost:8787/v1/messages \ + -H 'content-type: application/json' \ + -d '{ + "model": "claude-opus-4-7", + "max_tokens": 100, + "messages": [{"role":"user","content":"你好"}] + }' +# → 返回 Anthropic 格式响应,实际由 OpenAI 上游产生 +``` + +流式请求一样支持(SSE 事件流是 Anthropic 格式)。 +`GET /healthz` 返回 `ok` 给 liveness probe;`GET /version` 返回版本信息。 + +### B) 纯 Rust 二进制(不依赖 Python) + +```bash +cargo build --release -p cc_convert_sidecar +export CC_CONVERT_UPSTREAM_URL="https://api.openai.com/v1/chat/completions" +export CC_CONVERT_UPSTREAM_API_KEY="sk-..." +./target/release/cc_convert_sidecar +``` + +参数和环境变量与 CLI 完全一致。需要在一台没有 Python 的机器上跑一个静态二进制时用这个。 + +## 两个预设档位 + +`ConvertOptions`(请求)、`ResponseConvertOptions` / `StreamConvertOptions` +(响应/流)都提供两个预设: + +| 预设 | 行为 | +|---|---| +| `litellm_compat()`(默认) | 与 LiteLLM 的 `AnthropicAdapter` 字节级等价(忽略 null 字段差异)。在已有 LiteLLM pipeline 里平替时用这个,行为零漂移。 | +| `pragmatic()` / `anthropic_native()` | 更贴近 Anthropic SSE 官方规范、以及大多数 OpenAI 兼容服务器实际期望的形态:`stream` 时注入 `stream_options: {include_usage: true}`、o1/o3/o4/gpt-5 用 `max_completion_tokens`、stop 字段输出 `stop`、content_block 急切打开、有 `ping` 事件等。 | + +## 测试覆盖(51 个 Rust + 22 个 Python,全过) + +``` +crates/cc_convert_core/ +├── src/ ← 3 个单元测试(工具名截断) +└── tests/ + ├── request_translation.rs ← 20 个单元测试(case 1–20) + ├── response_translation.rs ← 7 个单元测试(case 21–26 + 工具名往返) + ├── stream_translation.rs ← 5 个单元测试(case 27–31) + ├── parity_litellm.rs ← 2 个 LiteLLM 请求 parity 测试(32 个 fixture) + ├── parity_response.rs ← 1 个 LiteLLM 响应 parity 测试(6 个 fixture) + ├── parity_stream.rs ← 1 个 LiteLLM 流 parity 测试(2 个 fixture,3 个文档化的 LiteLLM 缺陷排除) + └── vendor_quirks.rs ← 12 个 vLLM + SGLang 的 quirk 测试 +crates/cc_convert_sidecar/tests/ +└── integration.rs ← 3 个 HTTP 集成测试(非流式、流式、4xx) +python/tests/ +└── test_parity.py ← 22 个 pytest(走 wheel 跑 LiteLLM parity + 烟雾测试) +``` + +## 翻译规则(高层) + +真理来源:LiteLLM 的 `AnthropicAdapter.translate_anthropic_to_openai`。 + +| Anthropic | → | OpenAI | +|---|---|---| +| `system: string` | | 顶部加 `{role:"system", content:}` | +| `system: [{type:"text", text, cache_control?}]` | | 顶部加 `{role:"system", content:[{type:"text", text}, ...]}`(cache_control 丢弃) | +| user `text` 块 | | `{type:"text", text}` | +| user `image`(base64) | | `{type:"image_url", image_url:{url:"data:;base64,"}}` | +| user `image`(url) | | `{type:"image_url", image_url:{url}}` | +| user `tool_result` | | 单独的 `{role:"tool", tool_call_id:, content}` 消息,排在该 user 消息的任何 text 之前;每个 tool_use_id 对应一条 tool 消息 | +| assistant `tool_use` | | `tool_calls` 数组里加一项 `{id, type:"function", function:{name, arguments: JSON.stringify(input)}}` | +| assistant `thinking` | | 加到 assistant 消息的 `thinking_blocks` 数组(LiteLLM 行为,可用 `preserve_thinking_blocks=false` 关掉) | +| `max_tokens` | | `max_tokens`(LiteLLM-compat);可开启 reasoning 模型用 `max_completion_tokens` | +| `stop_sequences` | | `stop_sequences`(透传,LiteLLM-compat);可改成发 `stop` | +| `top_k` | | 透传(LiteLLM 行为,可 `drop_top_k` 关) | +| `tools` | | `[{type:"function", function:{name, description, parameters: input_schema}}]`;名字 >64 字符的截断成 `{55前缀}_{8位hex哈希}` | +| `tool_choice:{type:"any"}` | | `"required"` | +| `tool_choice:{type:"tool", name}` | | `{type:"function", function:{name}}` | +| `metadata.user_id` | | `user` | +| `thinking.budget_tokens` | | `reasoning_effort`(≥10000→high、≥5000→medium、≥2000→low、否则 minimal) | +| 任意块的 `cache_control` | | 丢弃 | + +响应方向(OpenAI → Anthropic): + +| OpenAI | → | Anthropic | +|---|---|---| +| `id` | | 透传(LiteLLM);可开启 `chatcmpl-→msg_` 重命名 | +| `choices[0].message.content` | | `{type:"text", text}` 块 | +| `choices[0].message.tool_calls` | | `{type:"tool_use", id, name, input: JSON.parse(arguments)}` 块;工具名走 map 还原 | +| `choices[0].message.reasoning_content` / `reasoning` | | `{type:"thinking", thinking}` 块(两个字段名都认,vLLM 用 `reasoning`) | +| `finish_reason: stop\|length\|tool_calls\|content_filter\|abort` | | `stop_reason: end_turn\|max_tokens\|tool_use\|end_turn\|end_turn` | +| `usage.prompt_tokens` / `completion_tokens` | | `usage.input_tokens` / `output_tokens`(LiteLLM 会减掉 cached 部分) | +| `usage.prompt_tokens_details.cached_tokens` | | `usage.cache_read_input_tokens` | + +流式方向:OpenAI SSE chunk 流转成 +`message_start` → `[ping]` → `content_block_start` → ... → `content_block_stop` +→ `message_delta` → `message_stop` 序列。文本对应 `text_delta`、工具调用对应 +`input_json_delta`、reasoning 对应 `thinking_delta`。并行 tool_calls 会拿到不同 +的 content_block index。流提前断掉(没有 finish_reason)时,我们自己补一个 +`stop_reason: end_turn` 结尾。 + +## 从源码编译 + +需要 Rust ≥ 1.75,Python ≥ 3.8(只为打 wheel)。 + +```bash +# 如果你的环境 cargo/pip 需要走代理,自行设置;非受限网络可以忽略。 +# export http_proxy=http://YOUR_PROXY:PORT +# export https_proxy=http://YOUR_PROXY:PORT +# export no_proxy="localhost,127.0.0.1" + +# Rust 核心 + sidecar +cargo build --release + +# Rust 测试(包含 LiteLLM parity 对照已提交的 golden,不需要联网) +cargo test --workspace + +# Python wheel +cd python +pip install maturin +maturin build --release +pip install ../target/wheels/cc_convert-*.whl +pytest tests/ +``` + +## Parity 测试 + +`tests/fixtures/requests/` 下放着 20 + 12 个请求 fixture,配对的 +`anthropic_.json` / `openai_.json`。`responses/` 和 `streams/` 下 +各有 6 个响应和 5 个流 fixture。所有 golden 都是先用 LiteLLM 跑出来并提交到 +仓库的,所以 CI 不需要联网。 + +规则改了之后重新生成 golden: + +```bash +pip install 'litellm>=1.0' +python scripts/seed_fixture_inputs.py # 初始 31 个 case(只在缺失时跑) +python scripts/seed_extra_request_fixtures.py # 额外 12 个请求 case +python scripts/regen_fixtures.py # 请求 golden +python scripts/regen_response_fixtures.py # 响应 golden +python scripts/regen_stream_fixtures.py # 流 golden +cargo test --workspace # 确认 parity 还在 +``` + +## 目录结构 + +``` +crates/ + cc_convert_core/ 纯 Rust 翻译库,无 I/O + cc_convert_py/ PyO3 binding → Python wheel(cc_convert._native) + cc_convert_sidecar/ axum HTTP 反向代理二进制 + 集成测试 +python/ + cc_convert/ Python 包(re-export 原生模块) + tests/ pytest parity 测试,走 wheel +scripts/ + seed_fixture_inputs.py 生成 INPUT fixture(case 1–31) + seed_extra_request_fixtures.py 生成额外 INPUT fixture(case 32–43) + regen_fixtures.py 跑 LiteLLM 生成请求 golden + regen_response_fixtures.py 同上,响应 + regen_stream_fixtures.py 同上,流 +tests/ + fixtures/ 提交进仓库的 golden parity fixture +``` + +## 已知缺口 / 非目标(v1) + +- **Anthropic hosted tools**(`web_search`、`computer`、`bash`、`text_editor`) + 不转换成 OpenAI 等价物,直接透传。v1.1 可以考虑把 `web_search` 映射到 + OpenAI 的 `web_search_options`。 +- **响应端 `stop_sequence` 检测** 没做。三个参考库都没做;OpenAI 也不会告诉 + 你哪个 stop 命中了。但 vLLM 通过 `stop_reason`(命中的字符串)、SGLang 通过 + `matched_stop` 能给出来,要做成 Anthropic 的 `stop_sequence` 是平凡的, + 只是 v1 没加。 +- **三个 LiteLLM 流式缺陷**:5 个流 fixture 里有 3 个的 golden 不符合 Anthropic + spec(把并行 tool_calls 合并成一个块、流没 finish_reason 时静默截断、 + reasoning + text 合成一个块)。我们按 spec 实现,所以这 3 个 case 在 + `tests/parity_stream.rs:LITELLM_QUIRKS_TO_SKIP` 中跳过 parity,详见该常量 + 附近的注释。 +- **Anthropic `[DONE]` 哨兵**:真实 Anthropic SSE 不会发 `data: [DONE]\n\n`, + 我们也不发。但**输入**端会接收它,当作 OpenAI 流结束的标记。 + +## License + +MIT OR Apache-2.0. diff --git a/sidecars/cc_convert/RELEASING.md b/sidecars/cc_convert/RELEASING.md new file mode 100644 index 0000000..658954b --- /dev/null +++ b/sidecars/cc_convert/RELEASING.md @@ -0,0 +1,165 @@ +# Releasing cc_convert to PyPI + +## One-time setup (only first time) + +### 1. Create PyPI + TestPyPI accounts + +- Real: https://pypi.org/account/register/ +- Test: https://test.pypi.org/account/register/ (separate account, separate password) + +Enable 2FA on both (PyPI requires it for publishing). + +### 2. Reserve the project name + configure Trusted Publishing + +The Trusted Publisher needs a *project* on PyPI to attach to. There are two +ways to bootstrap: + +**Option A — Reserve the name yourself first (recommended).** +Manually `pip install twine && twine upload` a tiny placeholder wheel once, +then add the trusted-publisher record. After that, every release happens +via GitHub Actions with no token. + +**Option B — Use Pending Publisher.** +On https://pypi.org/manage/account/publishing/ add a *pending* trusted +publisher *before* the project exists. The first GitHub Actions run that +matches the spec will create the project automatically. + +Settings to enter (either option): + +| Field | Value | +|---|---| +| PyPI Project Name | `cc-convert` | +| Owner | `yitianlian` | +| Repository name | `cc_convert` | +| Workflow name | `release.yml` | +| Environment name | `pypi` | + +Do the same on TestPyPI: +https://test.pypi.org/manage/account/publishing/ + +| Field | Value | +|---|---| +| PyPI Project Name | `cc-convert` | +| Owner | `yitianlian` | +| Repository name | `cc_convert` | +| Workflow name | `release.yml` | +| Environment name | `testpypi` | + +### 3. Create the GitHub Environments + +GitHub side, on https://github.com/yitianlian/cc_convert/settings/environments +create two environments: + +- `pypi` (no protection rules needed for now; you can add "Required reviewers" later if you want a manual approval gate per release) +- `testpypi` (no protection rules) + +These names must match the `environment:` field in `release.yml`. + +--- + +## Releasing a new version + +### Dry-run to TestPyPI (recommended every time) + +```bash +# Go to: +# https://github.com/yitianlian/cc_convert/actions/workflows/release.yml +# Click "Run workflow" → choose "testpypi" → Run. + +# Or via gh CLI: +gh workflow run release.yml -f target=testpypi +``` + +This builds wheels for Linux x64 + Linux aarch64 + macOS x64 + macOS ARM + +Windows x64 + sdist, then uploads them to https://test.pypi.org/project/cc-convert/. + +Verify it installs: + +```bash +pip install --index-url https://test.pypi.org/simple/ \ + --extra-index-url https://pypi.org/simple/ \ + cc-convert +cc_convert --version +``` + +### Cut a real release + +```bash +# 1. Bump version +sed -i 's/version = "0.1.0"/version = "0.2.0"/' python/pyproject.toml Cargo.toml +git add -A && git commit -m "release: v0.2.0" +git push + +# 2. Tag and push the tag — that triggers the publish job. +git tag v0.2.0 +git push origin v0.2.0 +``` + +The tag push fires the `release.yml` workflow. It will: + +1. Build wheels in parallel on 5 runners (Linux x64, Linux aarch64, + macOS Intel, macOS ARM, Windows x64) plus an sdist. +2. Upload all artifacts. +3. Publish to https://pypi.org/project/cc-convert/ via the `pypi` environment + (which is tied to the trusted publisher you configured in step 2 above). + +Watch progress at https://github.com/yitianlian/cc_convert/actions + +### After the workflow completes + +Anyone in the world can now: + +```bash +pip install cc-convert # imports as: import cc_convert +cc_convert --version +``` + +--- + +## Troubleshooting + +**"unable to upload: trusted publisher not configured"** +→ The PyPI Trusted Publisher record doesn't match what GitHub sent. Check +that Owner / Repository / Workflow / Environment all match exactly. The +workflow filename is `release.yml`, not `release` or `.github/workflows/release.yml`. + +**"file already exists" on PyPI** +→ You can't re-upload the same version. Bump `version =` in +`python/pyproject.toml` AND `Cargo.toml`, retag, repush. + +**One platform's wheel build failed** +→ The workflow uses `needs: [build-linux, build-macos, build-windows, build-sdist]` +on the publish job, so if any platform fails the whole release stops (no +half-published version on PyPI). Fix the failing job and push the tag again +— but first bump the version, because the same version can't be reuploaded. + +**Manual rescue / emergency upload** +→ Generate a PyPI API token, then locally: + +```bash +maturin upload --username __token__ --password "pypi-AgEIcHl..." \ + target/wheels/cc_convert-*.whl +``` + +--- + +## What the workflow is doing under the hood + +- **abi3-py38 wheel**: each (OS, arch) gets *one* wheel with file name + `cc_convert-X.Y.Z-cp38-abi3-.whl` that works on Python 3.8 + through any future 3.x. This is enabled in `crates/cc_convert_py/Cargo.toml` + via `pyo3 = { ..., features = ["extension-module", "abi3-py38"] }`. + +- **manylinux 2.34**: built inside the official `quay.io/pypa/manylinux_2_34` + container so the resulting wheel works on any reasonably-modern Linux + distro (glibc >= 2.34). For older glibc we'd switch to `manylinux_2_28` + or 2014 — bump only if someone reports they need it. + +- **sccache**: the workflow caches Rust build artifacts across runs via + `sccache: "true"` on the `PyO3/maturin-action` step. First release takes + ~15min; later ones are faster. + +- **Trusted Publishing (OIDC)**: instead of a long-lived API token in + GitHub secrets, each workflow run gets a short-lived OIDC identity from + GitHub that PyPI cryptographically verifies came from our exact + workflow file in our exact repo. No secrets to rotate or leak. diff --git a/sidecars/cc_convert/USAGE.md b/sidecars/cc_convert/USAGE.md new file mode 100644 index 0000000..78c1a65 --- /dev/null +++ b/sidecars/cc_convert/USAGE.md @@ -0,0 +1,260 @@ +# Usage + +Three ways to use cc_convert, plus testing and development workflow. + +> Chinese docs: [USAGE.zh-CN.md](USAGE.zh-CN.md) + +## 1. As a Python library + +```bash +pip install cc_convert +``` + +```python +import cc_convert + +# Translate an Anthropic-shape request to OpenAI shape +anthropic_req = { + "model": "claude-opus-4-7", + "max_tokens": 1000, + "system": "You are a coding assistant.", + "tools": [{ + "name": "read_file", + "description": "Read a file", + "input_schema": {"type":"object","properties":{"path":{"type":"string"}},"required":["path"]} + }], + "messages": [{"role":"user","content":"Read /etc/hosts"}], +} +openai_req, tool_map = cc_convert.translate_request(anthropic_req) +# POST openai_req to any OAI-compatible /v1/chat/completions endpoint + +# Translate the upstream response back to Anthropic shape +openai_resp = {...} # from your upstream +anthropic_resp = cc_convert.translate_response( + openai_resp, + original_model="claude-opus-4-7", + tool_name_map=tool_map +) + +# Streaming +translator = cc_convert.StreamTranslator("claude-opus-4-7", tool_map) +for openai_chunk in upstream_sse_stream: # each is a dict + for anthropic_event in translator.push(openai_chunk): + # dict, type ∈ {message_start, ping, content_block_start, + # content_block_delta, content_block_stop, message_delta, message_stop} + emit_to_client(anthropic_event) +for trailing in translator.finish(): + emit_to_client(trailing) +``` + +### Two translation profiles + +```python +# Pragmatic (default) — matches what real OAI-compat upstreams (vLLM/SGLang +# strict mode) actually accept: +# - single-text content collapsed to string (many upstreams reject list-content) +# - reasoning_effort auto-bucketed from thinking.budget_tokens +# - max_completion_tokens for o1/o3/o4/gpt-5 +# - stream_options.include_usage auto-injected +cc_convert.translate_request(req) +cc_convert.translate_request(req, mode="pragmatic") + +# LiteLLM byte-equivalent (drop-in replacement for LiteLLM AnthropicAdapter) +cc_convert.translate_request(req, mode="litellm_compat") +``` + +## 2. As a CLI sidecar (HTTP reverse proxy) + +```bash +pip install cc_convert # installs the `cc_convert` command + +# Proxy mode: accept Anthropic-shape requests, forward to an OAI backend, +# translate the response back to Anthropic shape. +cc_convert serve \ + --listen 0.0.0.0:8787 \ + --upstream-url http://YOUR_UPSTREAM_HOST:8000 \ + --upstream-key sk-xxx # optional for local backends + +# Then point any Anthropic-API client at it: +export ANTHROPIC_BASE_URL=http://localhost:8787 +claude # Claude Code thinks it's talking to Anthropic +``` + +### Common flags + +| Flag | Default | Meaning | +|---|---|---| +| `--mode {proxy,rpc}` | `proxy` | proxy forwards; rpc is translation-only | +| `--listen HOST:PORT` | `0.0.0.0:8787` | bind address | +| `--upstream-url URL` | `$CC_CONVERT_UPSTREAM_URL` | backend URL (auto-appends `/v1/chat/completions`) | +| `--upstream-key KEY` | `$CC_CONVERT_UPSTREAM_API_KEY` | bearer token sent to upstream | +| `--auth-passthrough` | off | forward client's Authorization header instead of `--upstream-key` | +| `--compat-mode {pragmatic,litellm_compat}` | `pragmatic` | translation profile | +| `--log-level {debug,info,warning,error}` | `info` | log level | +| `--log-format {text,json}` | `text` | json is one-object-per-line for log shipping | +| `-v` / `-vv` | | shorthand for `--log-level info/debug` | +| `--quiet` | off | suppress per-request access logs | +| `--version` | | print version | + +Accepted endpoint paths: `/v1/messages`, `/messages`, `/anthropic/v1/messages` — anything ending in `/messages` or `/v1/messages` is recognised. + +### One-shot CLI translation (no server) + +```bash +# Anthropic request → OAI request +cat anthropic_req.json | cc_convert translate --direction cc-to-oai + +# OAI response → Anthropic response +cat openai_resp.json | cc_convert translate \ + --direction oai-to-cc \ + --original-model claude-opus-4-7 +``` + +### RPC mode (pure translation, no forwarding) + +```bash +cc_convert serve --mode rpc --listen 127.0.0.1:8788 + +curl -X POST http://127.0.0.1:8788/translate/cc-to-oai -d '{...anthropic request...}' +curl -X POST http://127.0.0.1:8788/translate/oai-to-cc -d '{"openai_response":{...}, "original_model":"...", "tool_map":{}}' +``` + +## 3. As a pure Rust binary / library + +```bash +cargo build --release -p cc_convert_sidecar # static binary, no Python + +export CC_CONVERT_UPSTREAM_URL="http://YOUR_UPSTREAM_HOST:8000/v1/chat/completions" +export CC_CONVERT_UPSTREAM_API_KEY="sk-..." +./target/release/cc_convert_sidecar +``` + +Same env-var contract as the Python CLI (all `CC_CONVERT_*` prefixed). + +--- + +## What the upstream needs + +cc_convert does NOT do client-side fallback parsing. If the upstream leaves +`` or `` tags inside `content`, we faithfully pass them +through. The **real fix is on the upstream**: + +### SGLang launch flags + +| Symptom | Add this flag | +|---|---| +| `...` in content, `reasoning_content: null` | `--reasoning-parser qwen3` (or `deepseek-r1`, `hunyuan`, etc.) | +| `...` in content, `tool_calls: null` | `--tool-call-parser qwen25` (or `hermes`, `pythonic`, etc.) | + +#### reasoning-parser options + +`deepseek-r1` `deepseek-v3` `deepseek-v4` `qwen3` `qwen3-thinking` `glm45` `hunyuan` `gpt-oss` `kimi` `kimi_k2` `mistral` `mimo` `poolside_v1` `minimax` `minimax-append-think` `step3` `step3p5` `interns1` `nemotron_3` `gemma4` + +#### tool-call-parser options + +`qwen25` `qwen` `qwen3_coder` `hermes` `deepseekv3` `deepseekv31` `deepseekv32` `deepseekv4` `llama3` `mistral` `kimi_k2` `glm` `glm45` `glm47` `pythonic` `gpt-oss` `cohere_command4` `lfm2` `minicpm5` `mimo` `step3` `step3p5` `minimax-m2` `trinity` `interns1` `hunyuan` `gigachat3` `gemma4` + +Example launch: + +```bash +python -m sglang.launch_server \ + --model-path /path/to/qwen3-model \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen25 \ + ... +``` + +--- + +## Testing & development + +### Run Rust tests + +```bash +cargo test --workspace # all (63 core + 3 sidecar) +cargo test -p cc_convert_core --tests +cargo test -p cc_convert_sidecar --test integration +``` + +### Run Python tests + +```bash +cd python +maturin build --release # produces ../target/wheels/cc_convert-*.whl +pip install --force-reinstall ../target/wheels/cc_convert-*.whl +pytest tests/ # 69 tests +``` + +### Fast iteration on a change + +```bash +cargo check -p cc_convert_core # ~5s type-check, use this while editing +cargo test -p cc_convert_core --tests --lib # ~30s, runs unit + rule tests +``` + +### Live round-trip against a real upstream + +`playground/run_roundtrip.py` is the canonical end-to-end test: + +```bash +python playground/run_roundtrip.py \ + --upstream http://YOUR_UPSTREAM_HOST:8000 \ + --model /model + +# Outputs: playground/runs/// +# 1_anthropic_request.json source Anthropic request (verbatim) +# 2_oai_request.json what cc_convert sent to the upstream +# 3_oai_response.json raw upstream response +# 4_anthropic_response.json translated back to Anthropic +# meta.json status / latency / http_status +# +# Plus _summary.json at the run root. +``` + +Run a single fixture: + +```bash +python playground/run_roundtrip.py --upstream http://... --model /model --only agent_loop +``` + +### What each fixture exercises + +| Fixture | Tests | +|---|---| +| `02_reasoning_request` | extended thinking; `thinking.budget_tokens` → `reasoning_effort` | +| `03_forced_tool` | `tool_choice:any` → OpenAI `required` | +| `05_simple_text` | single-turn baseline | +| `07_multi_turn_text` | 5-turn pure-text history | +| `08_agent_loop_with_tools` | **5-turn agent loop**: assistant calls 2 tools → user returns 2 tool_results → model continues | +| `09_long_response` | 4000-token long generation, large output + long latency | +| `10_parallel_tools_text_only` | one request triggers multiple parallel tool_use | + +### Pre-push secret audit (recommended) + +```bash +grep -rIEn "httpproxy|/workspace/|/root/|sk-[a-zA-Z0-9]{10,}|172\.27|10\.180" \ + --exclude-dir=target --exclude-dir=__pycache__ --exclude-dir=.git \ + --exclude-dir=playground/runs \ + --include="*.rs" --include="*.py" --include="*.toml" --include="*.md" . +``` + +`playground/runs/` is in .gitignore — live test results never enter git. + +--- + +## FAQ + +**Q: I see `reasoning_content: null` but `` is still in content.** +A: Upstream hasn't enabled `--reasoning-parser`. Ask the operator to add it. cc_convert intentionally doesn't do client-side fallback. + +**Q: Same for `tool_calls: null` but `` in content?** +A: Same — upstream needs `--tool-call-parser qwen25` (or `hermes`). + +**Q: 503 / connection refused — is this cc_convert?** +A: No. Check `playground/runs///error.txt`. "No available workers" or "Connection refused" means upstream worker is unhealthy — our request is already on the wire and well-formed. + +**Q: Does cc_convert drop Claude Code / OpenCode extra fields like `output_config`, `speed`, `container`?** +A: No (since commit `0be4d80`). All unknown fields are preserved in `AnthropicRequest.extra`. `output_config.effort` and `service_tier` are actually translated to their OpenAI equivalents; the others are kept for future Anthropic-target proxy mode. + +**Q: Is the default LiteLLM-compatible?** +A: No. Default is `pragmatic` — matches what real OAI upstreams accept. Use `mode="litellm_compat"` (Python) or `--compat-mode litellm_compat` (CLI) for byte-parity with LiteLLM's AnthropicAdapter. diff --git a/sidecars/cc_convert/USAGE.zh-CN.md b/sidecars/cc_convert/USAGE.zh-CN.md new file mode 100644 index 0000000..011b9b5 --- /dev/null +++ b/sidecars/cc_convert/USAGE.zh-CN.md @@ -0,0 +1,264 @@ +# 使用文档 + +cc_convert 三种使用方式 + 测试与开发流程。 + +> 英文文档见 [README.md](README.md)。 + +## 一、当作 Python 库用 + +```bash +pip install cc_convert +``` + +```python +import cc_convert + +# 把 Anthropic 格式请求转成 OpenAI 格式 +anthropic_req = { + "model": "claude-opus-4-7", + "max_tokens": 1000, + "system": "You are a coding assistant.", + "tools": [{ + "name": "read_file", + "description": "Read a file", + "input_schema": {"type":"object","properties":{"path":{"type":"string"}},"required":["path"]} + }], + "messages": [{"role":"user","content":"Read /etc/hosts"}], +} +openai_req, tool_map = cc_convert.translate_request(anthropic_req) +# openai_req 就可以发给任何 OAI 兼容的 /v1/chat/completions + +# 上游响应回来后,转回 Anthropic 格式 +openai_resp = {...} # 上游返回的 +anthropic_resp = cc_convert.translate_response( + openai_resp, + original_model="claude-opus-4-7", + tool_name_map=tool_map +) + +# 流式版本 +translator = cc_convert.StreamTranslator("claude-opus-4-7", tool_map) +for openai_chunk in upstream_sse_stream: # 每个是 dict + for anthropic_event in translator.push(openai_chunk): + # anthropic_event 是 dict,类型有: + # message_start / ping / content_block_start / content_block_delta / + # content_block_stop / message_delta / message_stop + emit_to_client(anthropic_event) +for trailing in translator.finish(): + emit_to_client(trailing) +``` + +### 两种翻译档位 + +```python +# Pragmatic(默认):贴合真实 OAI 上游(vLLM/SGLang strict mode) +# - 单 text 内容折叠成字符串(很多上游不收 list content) +# - reasoning_effort 自动从 thinking.budget_tokens 桶化 +# - max_completion_tokens for o1/o3/o4/gpt-5 +# - stream_options.include_usage 自动注入 +cc_convert.translate_request(req) +cc_convert.translate_request(req, mode="pragmatic") + +# LiteLLM-compat:byte-equivalent 平替 LiteLLM AnthropicAdapter +cc_convert.translate_request(req, mode="litellm_compat") +``` + +## 二、当作命令行 sidecar(HTTP 反向代理)用 + +```bash +# 装好 wheel 之后,有 cc_convert 命令 +pip install cc_convert + +# Proxy 模式:接 Anthropic 请求,转发到 OAI 后端,翻回 Anthropic +cc_convert serve \ + --listen 0.0.0.0:8787 \ + --upstream-url http://YOUR_UPSTREAM_HOST:8000 \ + --upstream-key sk-xxx # 可选,不传也行(本地上游) + +# 然后 Claude Code / claude-py / cline 等客户端指过来: +export ANTHROPIC_BASE_URL=http://localhost:8787 +claude # 它现在以为后端是 Anthropic,实际是你的 OAI 上游 +``` + +### CLI 常用参数 + +| 参数 | 默认 | 含义 | +|---|---|---| +| `--mode {proxy,rpc}` | `proxy` | proxy 转发,rpc 只翻译不发 | +| `--listen HOST:PORT` | `0.0.0.0:8787` | 监听地址 | +| `--upstream-url URL` | `$CC_CONVERT_UPSTREAM_URL` | 后端 URL(自动补 `/v1/chat/completions`) | +| `--upstream-key KEY` | `$CC_CONVERT_UPSTREAM_API_KEY` | Bearer token | +| `--auth-passthrough` | off | 转发客户端 Authorization 而不用 `--upstream-key` | +| `--compat-mode {pragmatic,litellm_compat}` | `pragmatic` | 翻译档位(同 Python lib) | +| `--log-level {debug,info,warning,error}` | `info` | 日志级别 | +| `--log-format {text,json}` | `text` | json 是一行一对象,方便日志采集 | +| `-v` / `-vv` | | 等价 `--log-level info/debug` | +| `--quiet` | off | 不打访问日志 | +| `--version` | | 打印版本 | + +支持的端点路径:`/v1/messages`、`/messages`、`/anthropic/v1/messages`(任何以 `/messages` 或 `/v1/messages` 结尾的路径都识别)。 + +### One-shot 命令行翻译(不起 server) + +```bash +# Anthropic 请求 → OAI 请求 +cat anthropic_req.json | cc_convert translate --direction cc-to-oai + +# OAI 响应 → Anthropic 响应 +cat openai_resp.json | cc_convert translate \ + --direction oai-to-cc \ + --original-model claude-opus-4-7 +``` + +### RPC 模式(只翻译,不转发) + +```bash +cc_convert serve --mode rpc --listen 127.0.0.1:8788 + +# 然后可以打 HTTP 请求做翻译 +curl -X POST http://127.0.0.1:8788/translate/cc-to-oai -d '{...anthropic request...}' +curl -X POST http://127.0.0.1:8788/translate/oai-to-cc -d '{"openai_response":{...}, "original_model":"...", "tool_map":{}}' +``` + +## 三、当作纯 Rust 库 / 静态二进制 + +```bash +# 纯 Rust 二进制(不依赖 Python) +cargo build --release -p cc_convert_sidecar + +export CC_CONVERT_UPSTREAM_URL="http://YOUR_UPSTREAM_HOST:8000/v1/chat/completions" +export CC_CONVERT_UPSTREAM_API_KEY="sk-..." +./target/release/cc_convert_sidecar +``` + +环境变量和 Python CLI 一致(都用 `CC_CONVERT_*` 前缀)。 + +--- + +## 上游需要的配置 + +cc_convert 不做"客户端兜底解析"。如果上游模型把 `` / `` 留在 content 里没拆,我们就如实透传。**真正的修复在上游**: + +### SGLang 启动参数 + +| 你看到什么现象 | 上游加什么 flag | +|---|---| +| `...` 留在 content,`reasoning_content: null` | `--reasoning-parser qwen3`(或 `deepseek-r1`、`hunyuan` 等 — 见下表) | +| `...` 留在 content,`tool_calls: null` | `--tool-call-parser qwen25`(或 `hermes`、`pythonic` 等 — 见下表) | + +#### reasoning parser 选项 + +`deepseek-r1` `deepseek-v3` `deepseek-v4` `qwen3` `qwen3-thinking` `glm45` `hunyuan` `gpt-oss` `kimi` `kimi_k2` `mistral` `mimo` `poolside_v1` `minimax` `minimax-append-think` `step3` `step3p5` `interns1` `nemotron_3` `gemma4` + +#### tool-call parser 选项 + +`qwen25` `qwen` `qwen3_coder` `hermes` `deepseekv3` `deepseekv31` `deepseekv32` `deepseekv4` `llama3` `mistral` `kimi_k2` `glm` `glm45` `glm47` `pythonic` `gpt-oss` `cohere_command4` `lfm2` `minicpm5` `mimo` `step3` `step3p5` `minimax-m2` `trinity` `interns1` `hunyuan` `gigachat3` `gemma4` + +启动示例: + +```bash +python -m sglang.launch_server \ + --model-path /path/to/qwen3-model \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen25 \ + ... +``` + +--- + +## 测试与开发流程 + +### 跑 Rust 测试 + +```bash +cargo test --workspace # 全跑(63 core + 3 sidecar) +cargo test -p cc_convert_core --tests # 只 core +cargo test -p cc_convert_sidecar --test integration # 只 sidecar +``` + +### 跑 Python 测试 + +```bash +cd python +maturin build --release # 出 wheel 到 ../target/wheels/ +pip install --force-reinstall ../target/wheels/cc_convert-*.whl +pytest tests/ # 69 个测试 +``` + +### 编辑代码后快速重测 + +```bash +cargo check -p cc_convert_core # 5 秒内类型检查,日常开发用这个 +cargo test -p cc_convert_core --tests --lib # 30 秒,跑单元 + 翻译规则 +``` + +### 跑线上 round-trip + +playground 是一个开箱即用的端到端测试场: + +```bash +# 准备好上游(改 URL/model 即可) +python playground/run_roundtrip.py \ + --upstream http://YOUR_UPSTREAM_HOST:8000 \ + --model /model + +# 数据在 playground/runs/// 下, +# 每个 fixture 一组 4 个 json: +# 1_anthropic_request.json 源 Anthropic 请求(verbatim) +# 2_oai_request.json cc_convert 翻译后发给上游的 +# 3_oai_response.json 上游返回的原始 OAI 响应 +# 4_anthropic_response.json cc_convert 翻译回 Anthropic 的最终响应 +# +# 加上 meta.json(状态、延迟、HTTP code)和 _summary.json(汇总) +``` + +只跑一个 fixture: + +```bash +python playground/run_roundtrip.py --upstream http://... --model /model --only agent_loop +``` + +### 7 个 fixture 各测什么 + +| Fixture | 测什么 | +|---|---| +| `02_reasoning_request` | extended thinking,`thinking.budget_tokens` → `reasoning_effort` | +| `03_forced_tool` | `tool_choice:any` → OpenAI `required` | +| `05_simple_text` | 单轮纯文本,baseline | +| `07_multi_turn_text` | 5 轮纯文本对话历史 | +| `08_agent_loop_with_tools` | **5 轮 agent loop**:assistant 调 2 个 tool → user 给 2 个 tool_result → 模型继续 | +| `09_long_response` | 4000 tokens 长生成,测大输出和长延迟 | +| `10_parallel_tools_text_only` | 一次请求触发多个 parallel tool_use | + +### Git push 前的安全审计 + +可选,但建议在 push 之前过一遍敏感信息: + +```bash +# 简单 grep 内网代理 / 路径 / API key +grep -rIEn "httpproxy|/workspace/|/root/|sk-[a-zA-Z0-9]{10,}|172\.27|10\.180" \ + --exclude-dir=target --exclude-dir=__pycache__ --exclude-dir=.git \ + --exclude-dir=playground/runs \ + --include="*.rs" --include="*.py" --include="*.toml" --include="*.md" . +``` + +playground/runs/ 已经在 .gitignore 里,任何线上测试结果不会进 git。 + +--- + +## 常见问题 + +**Q: 我看上去的输出 `reasoning_content` 是 null,但 content 里有 `` 怎么办?** +A: 上游没启 `--reasoning-parser`,让运维加。cc_convert 不做客户端兜底。 + +**Q: tool_calls 也是 null,但 content 里有 `` 标签?** +A: 同上,上游没启 `--tool-call-parser`,加 `qwen25` 或 `hermes`。 + +**Q: 上游 503 / connection refused,是 cc_convert 的问题吗?** +A: 不是。看 `playground/runs///error.txt`,如果说 "No available workers" 或 "Connection refused",是上游 worker 不稳定 — 我们的请求已经合规发出去了。 + +**Q: Claude Code / OpenCode 发的额外字段(`output_config` / `speed` / `container` 等)会被丢吗?** +A: 不会。从 commit `0be4d80` 起,所有未知字段都保留在 `AnthropicRequest.extra` 里。其中 `output_config.effort` 和 `service_tier` 会真正翻译到 OpenAI 对应字段;其他暂时保留供未来 Anthropic-target proxy 用。 + +**Q: 默认行为是不是 LiteLLM 兼容?** +A: 不是。默认是 `pragmatic`,贴合真实 OAI 上游。需要 LiteLLM byte-parity 时用 `mode="litellm_compat"` 或 `--compat-mode litellm_compat`。 diff --git a/sidecars/cc_convert/crates/cc_convert_core/Cargo.toml b/sidecars/cc_convert/crates/cc_convert_core/Cargo.toml new file mode 100644 index 0000000..49416a9 --- /dev/null +++ b/sidecars/cc_convert/crates/cc_convert_core/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "cc_convert_core" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "Anthropic <-> OpenAI Chat Completions protocol converter (pure Rust, no I/O)" + +[dependencies] +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +sha2.workspace = true +hex.workspace = true +uuid.workspace = true + +[dev-dependencies] +pretty_assertions = "1" diff --git a/sidecars/cc_convert/crates/cc_convert_core/src/anthropic.rs b/sidecars/cc_convert/crates/cc_convert_core/src/anthropic.rs new file mode 100644 index 0000000..98a8c0c --- /dev/null +++ b/sidecars/cc_convert/crates/cc_convert_core/src/anthropic.rs @@ -0,0 +1,402 @@ +//! Anthropic Messages API types. +//! +//! Only the subset of fields we actively translate is modeled. Unknown fields +//! are preserved on request-shaped types via `extra` catch-alls where useful, +//! and dropped on response-shaped types (we emit a fixed surface). + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +// ---------- Request ---------- + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct AnthropicRequest { + pub model: String, + pub messages: Vec, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub system: Option, + + pub max_tokens: u32, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub temperature: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub top_p: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub top_k: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stop_sequences: Option>, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stream: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tools: Option>, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tool_choice: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub metadata: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub thinking: Option, + + /// Anthropic Messages API extension fields used by Claude Code, OpenCode, + /// Cline, and the Anthropic SDK that aren't modeled individually here: + /// `output_config`, `context_management`, `speed`, `container`, + /// `mcp_servers`, `inference_geo`, `cache_control` (top-level), + /// `service_tier`, `diagnostics`, `betas`, plus anything injected via + /// `CLAUDE_CODE_EXTRA_BODY`. Captured here so we don't silently drop + /// them — the request translator can choose to map known ones and pass + /// the rest through when the upstream is itself /v1/messages-compatible. + #[serde(flatten)] + pub extra: serde_json::Map, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum SystemField { + Text(String), + Blocks(Vec), +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct SystemBlock { + #[serde(rename = "type")] + pub block_type: String, // typically "text" + pub text: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_control: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct AnthropicMessage { + pub role: String, // "user" | "assistant" + pub content: MessageContent, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum MessageContent { + Text(String), + Blocks(Vec), +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ContentBlock { + Text { + text: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + cache_control: Option, + }, + Image { + source: ImageSource, + #[serde(default, skip_serializing_if = "Option::is_none")] + cache_control: Option, + }, + ToolUse { + id: String, + name: String, + input: Value, + #[serde(default, skip_serializing_if = "Option::is_none")] + cache_control: Option, + }, + ToolResult { + tool_use_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + is_error: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + cache_control: Option, + }, + Thinking { + thinking: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + signature: Option, + }, + RedactedThinking { + data: String, + }, + /// Catch-all for Anthropic content blocks we don't model individually + /// (server_tool_use, web_search_tool_result, code_execution_tool_result, + /// bash_code_execution_tool_result, text_editor_code_execution_tool_result, + /// tool_search_tool_result, mcp_tool_use, mcp_tool_result, container_upload, + /// document, etc.). Translator drops these by default; the + /// pass-through `Value` lets callers inspect them if needed. + #[serde(other, skip_serializing)] + Unknown, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ImageSource { + Base64 { + media_type: String, + data: String, + }, + Url { + url: String, + }, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum ToolResultContent { + Text(String), + Blocks(Vec), +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ToolResultBlock { + Text { + text: String, + }, + Image { + source: ImageSource, + }, +} + +/// One tool definition in an Anthropic request. +/// +/// Anthropic accepts two shapes here: +/// +/// - **Client tools** (the common case): a free-form schema you've defined. +/// The wire shape is `{name, description?, input_schema, cache_control?}` +/// with either no `type` field or `type:"custom"`. +/// +/// - **Server / hosted tools** (Anthropic-only): tools the Anthropic backend +/// itself executes — `{type:"web_search_20250305", name:"web_search", ...}`, +/// `{type:"computer_20241022", ...}`, `bash_*`, `text_editor_*`, +/// `web_fetch_*`, `code_execution_*`, `tool_search_*`. These have NO +/// `input_schema` and carry tool-version-specific config fields. There is +/// no OpenAI Chat Completions equivalent — OpenAI's tools array only +/// accepts `{type:"function", function:{...}}`, so a translator that +/// forwards these unchanged produces an HTTP 400 the moment the request +/// reaches any real OpenAI-compatible upstream. +/// +/// The deserializer routes by presence of `input_schema`: if it's there, +/// the tool is a Client tool; otherwise it's Hosted. We keep the full raw +/// JSON of Hosted tools in `raw` so the translator can log what it dropped +/// (useful for users debugging "why didn't my web_search tool fire"). +#[derive(Debug, Clone)] +pub enum AnthropicTool { + Client(AnthropicClientTool), + Hosted(AnthropicHostedTool), +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct AnthropicClientTool { + pub name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + pub input_schema: Value, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_control: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct AnthropicHostedTool { + /// e.g. "web_search_20250305", "computer_20241022", "bash_20250124", ... + #[serde(rename = "type")] + pub tool_type: String, + /// Anthropic-side name (e.g. "web_search"); not always present. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + /// Everything else (max_uses, user_location, allowed_domains, ...) kept + /// as raw JSON. Lets the translator log what it dropped without modeling + /// every per-version variant. + #[serde(flatten)] + pub extra: serde_json::Map, +} + +impl<'de> Deserialize<'de> for AnthropicTool { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let v = Value::deserialize(deserializer)?; + // Heuristic: a tool with `input_schema` is a client tool; anything else + // (especially anything with a non-default `type`) is hosted. + if v.get("input_schema").is_some() { + let t: AnthropicClientTool = + serde_json::from_value(v).map_err(serde::de::Error::custom)?; + Ok(AnthropicTool::Client(t)) + } else { + let t: AnthropicHostedTool = + serde_json::from_value(v).map_err(serde::de::Error::custom)?; + Ok(AnthropicTool::Hosted(t)) + } + } +} + +impl Serialize for AnthropicTool { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + match self { + AnthropicTool::Client(t) => t.serialize(serializer), + AnthropicTool::Hosted(t) => t.serialize(serializer), + } + } +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum AnthropicToolChoice { + Auto { + #[serde(default, skip_serializing_if = "Option::is_none")] + disable_parallel_tool_use: Option, + }, + Any { + #[serde(default, skip_serializing_if = "Option::is_none")] + disable_parallel_tool_use: Option, + }, + Tool { + name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + disable_parallel_tool_use: Option, + }, + None, +} + +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +pub struct AnthropicMetadata { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub user_id: Option, + /// Catch-all for any other metadata sub-fields some clients might + /// invent (none are documented at the time of writing — Claude Code + /// stuffs device_id/account_uuid/session_id inside the `user_id` + /// STRING as serialized JSON rather than adding sibling keys, but + /// keeping this open avoids silent drops if that ever changes). + #[serde(flatten)] + pub extra: serde_json::Map, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum AnthropicThinking { + Enabled { budget_tokens: u32 }, + Disabled, +} + +// ---------- Response ---------- + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct AnthropicResponse { + pub id: String, + #[serde(rename = "type")] + pub msg_type: String, // "message" + pub role: String, // "assistant" + pub model: String, + pub content: Vec, + pub stop_reason: AnthropicStopReason, + pub stop_sequence: Option, + pub usage: AnthropicUsage, +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ResponseContentBlock { + Text { + text: String, + }, + ToolUse { + id: String, + name: String, + input: Value, + }, + Thinking { + thinking: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + signature: Option, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AnthropicStopReason { + EndTurn, + MaxTokens, + StopSequence, + ToolUse, +} + +#[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq)] +pub struct AnthropicUsage { + pub input_tokens: u32, + pub output_tokens: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_creation_input_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_tokens: Option, +} + +// ---------- Streaming events (output of StreamTranslator) ---------- + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum AnthropicEvent { + MessageStart { + message: MessageStartPayload, + }, + Ping, + ContentBlockStart { + index: i32, + content_block: StreamingContentBlock, + }, + ContentBlockDelta { + index: i32, + delta: BlockDelta, + }, + ContentBlockStop { + index: i32, + }, + MessageDelta { + delta: MessageDeltaPayload, + usage: AnthropicUsage, + }, + MessageStop, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct MessageStartPayload { + pub id: String, + #[serde(rename = "type")] + pub msg_type: String, // "message" + pub role: String, // "assistant" + pub model: String, + pub content: Vec, // always empty [] + pub stop_reason: Option, + pub stop_sequence: Option, + pub usage: AnthropicUsage, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum StreamingContentBlock { + Text { text: String }, + ToolUse { id: String, name: String, input: Value }, + Thinking { thinking: String }, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum BlockDelta { + TextDelta { text: String }, + InputJsonDelta { partial_json: String }, + ThinkingDelta { thinking: String }, + SignatureDelta { signature: String }, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct MessageDeltaPayload { + pub stop_reason: Option, + pub stop_sequence: Option, +} diff --git a/sidecars/cc_convert/crates/cc_convert_core/src/error.rs b/sidecars/cc_convert/crates/cc_convert_core/src/error.rs new file mode 100644 index 0000000..b7e3e35 --- /dev/null +++ b/sidecars/cc_convert/crates/cc_convert_core/src/error.rs @@ -0,0 +1,16 @@ +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum ConvertError { + #[error("invalid JSON: {0}")] + Json(#[from] serde_json::Error), + + #[error("invalid request: {0}")] + InvalidRequest(String), + + #[error("invalid response: {0}")] + InvalidResponse(String), + + #[error("unsupported feature: {0}")] + Unsupported(String), +} diff --git a/sidecars/cc_convert/crates/cc_convert_core/src/lib.rs b/sidecars/cc_convert/crates/cc_convert_core/src/lib.rs new file mode 100644 index 0000000..b6ba9c1 --- /dev/null +++ b/sidecars/cc_convert/crates/cc_convert_core/src/lib.rs @@ -0,0 +1,45 @@ +//! cc_convert_core: bidirectional translator between the Anthropic Messages API +//! and the OpenAI Chat Completions API. +//! +//! Pure-Rust, no I/O. Used by both the Python wheel (`cc_convert_py`) and the +//! sidecar HTTP server (`cc_convert_sidecar`). + +pub mod anthropic; +pub mod error; +pub mod openai; +pub mod req_to_openai; +pub mod resp_to_anthropic; +pub mod stream; +pub mod tool_names; + +pub use error::ConvertError; +pub use req_to_openai::{anthropic_request_to_openai, ConvertOptions, ReasoningPassthrough}; +pub use resp_to_anthropic::{ + openai_response_to_anthropic, openai_response_to_anthropic_with, ResponseConvertOptions, +}; +pub use stream::{StreamConvertOptions, StreamTranslator}; +pub use tool_names::ToolNameMap; + +/// One-shot JSON-in / JSON-out request conversion. Returns +/// `{"openai_request": , "tool_map": }`. +pub fn convert_request_json(input: &str, opts: &ConvertOptions) -> Result { + let req: anthropic::AnthropicRequest = serde_json::from_str(input)?; + let (openai_req, tool_map) = anthropic_request_to_openai(&req, opts)?; + let out = serde_json::json!({ + "openai_request": openai_req, + "tool_map": tool_map, + }); + Ok(serde_json::to_string(&out)?) +} + +/// One-shot JSON-in / JSON-out response conversion. +pub fn convert_response_json( + input: &str, + original_model: &str, + tool_map_json: &str, +) -> Result { + let resp: openai::OpenAIResponse = serde_json::from_str(input)?; + let tool_map: ToolNameMap = serde_json::from_str(tool_map_json)?; + let anthropic_resp = openai_response_to_anthropic(&resp, original_model, &tool_map)?; + Ok(serde_json::to_string(&anthropic_resp)?) +} diff --git a/sidecars/cc_convert/crates/cc_convert_core/src/openai.rs b/sidecars/cc_convert/crates/cc_convert_core/src/openai.rs new file mode 100644 index 0000000..1b503d9 --- /dev/null +++ b/sidecars/cc_convert/crates/cc_convert_core/src/openai.rs @@ -0,0 +1,252 @@ +//! OpenAI Chat Completions API types. + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +// ---------- Request ---------- + +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +pub struct OpenAIRequest { + pub model: String, + pub messages: Vec, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_completion_tokens: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub temperature: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub top_p: Option, + /// Not part of the OpenAI spec, but LiteLLM forwards it when Anthropic + /// requests carry it. Most OpenAI-compatible servers ignore it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub top_k: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stop: Option>, + /// Anthropic-native name for stop. LiteLLM forwards this field unchanged + /// when the downstream supports it; we keep both so callers can choose. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stop_sequences: Option>, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stream: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stream_options: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tools: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tool_choice: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub user: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reasoning_effort: Option, + + /// OpenAI uses the same field name `service_tier` ("auto" | "default" | + /// "flex" | "scale" | "priority"). Anthropic's values are "auto" | + /// "standard_only" — we map `standard_only` → `default` and pass + /// `auto` through unchanged. Unknown values are forwarded verbatim. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub service_tier: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] +pub struct StreamOptions { + pub include_usage: bool, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "role", rename_all = "lowercase")] +pub enum OpenAIMessage { + System { + content: OpenAIContent, + }, + User { + content: OpenAIContent, + }, + Assistant { + #[serde(default, skip_serializing_if = "Option::is_none")] + content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + tool_calls: Option>, + /// Concatenated text of any Anthropic `thinking` blocks attached to + /// this assistant turn. Emitted by the `ReasoningContent` passthrough + /// (DeepSeek / SGLang convention; vLLM aliases it to `reasoning`). + /// Most other upstreams silently drop the unknown field. This is the + /// only spelling with any real upstream consumer (Qwen3 chat template + /// reads it). + #[serde(default, skip_serializing_if = "Option::is_none")] + reasoning_content: Option, + /// LiteLLM-internal shape: an array of structured thinking blocks + /// preserved verbatim from the Anthropic input. LiteLLM itself + /// strips this in its downstream provider transformations before + /// the request goes on the wire, so no real upstream consumes it. + /// Emitted only by the `LiteLLMThinkingBlocks` passthrough, used as + /// a drop-in replacement for LiteLLM's intermediate adapter output. + #[serde(default, skip_serializing_if = "Option::is_none")] + thinking_blocks: Option>, + }, + Tool { + tool_call_id: String, + content: OpenAIContent, + }, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum OpenAIContent { + Text(String), + Parts(Vec), +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum OpenAIContentPart { + Text { text: String }, + ImageUrl { image_url: OpenAIImageUrl }, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct OpenAIImageUrl { + pub url: String, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct OpenAITool { + #[serde(rename = "type")] + pub tool_type: String, // "function" + pub function: OpenAIFunctionDef, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct OpenAIFunctionDef { + pub name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + pub parameters: Value, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct OpenAIToolCall { + /// Set only on the FIRST chunk of a streaming tool_call (per the OpenAI + /// streaming contract). Continuation chunks may omit it (vLLM) or send + /// it as explicit `null` (SGLang). Non-streaming tool_calls always + /// carry an id. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub id: Option, + #[serde(rename = "type", default = "default_function_type", skip_serializing_if = "String::is_empty")] + pub call_type: String, + pub function: OpenAIFunctionCall, + /// Only present in streaming deltas; OpenAI uses this index to correlate + /// streamed fragments to the same tool_call across chunks. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub index: Option, +} + +fn default_function_type() -> String { + "function".to_string() +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct OpenAIFunctionCall { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub arguments: Option, +} + +// ---------- Response ---------- + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct OpenAIResponse { + pub id: String, + pub model: String, + pub choices: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub usage: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct OpenAIChoice { + pub index: i32, + pub message: OpenAIChoiceMessage, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub finish_reason: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct OpenAIChoiceMessage { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub content: Option, + /// DeepSeek / SGLang convention is `reasoning_content`; vLLM uses + /// `reasoning`. We accept either on input and serialize as + /// `reasoning_content`. + #[serde( + default, + skip_serializing_if = "Option::is_none", + alias = "reasoning" + )] + pub reasoning_content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tool_calls: Option>, +} + +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +pub struct OpenAIUsage { + #[serde(default)] + pub prompt_tokens: u32, + #[serde(default)] + pub completion_tokens: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prompt_tokens_details: Option, +} + +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +pub struct OpenAIPromptTokensDetails { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cached_tokens: Option, +} + +// ---------- Streaming chunk ---------- + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct OpenAIStreamChunk { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub model: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub choices: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub usage: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct OpenAIStreamChoice { + #[serde(default)] + pub index: i32, + pub delta: OpenAIStreamDelta, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub finish_reason: Option, +} + +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +pub struct OpenAIStreamDelta { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reasoning_content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reasoning: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tool_calls: Option>, +} diff --git a/sidecars/cc_convert/crates/cc_convert_core/src/req_to_openai.rs b/sidecars/cc_convert/crates/cc_convert_core/src/req_to_openai.rs new file mode 100644 index 0000000..ff27963 --- /dev/null +++ b/sidecars/cc_convert/crates/cc_convert_core/src/req_to_openai.rs @@ -0,0 +1,590 @@ +//! Anthropic Messages request → OpenAI Chat Completions request. + +use crate::anthropic::*; +use crate::error::ConvertError; +use crate::openai::*; +use crate::tool_names::ToolNameMap; +use serde_json::{json, Value}; + +/// How to forward Anthropic `thinking` blocks (assistant-side reasoning text) +/// when translating prior turns to OpenAI Chat Completions input. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReasoningPassthrough { + /// Omit any reasoning info on the assistant message. + /// Use this for the DeepSeek hosted API, which returns HTTP 400 when + /// `reasoning_content` is present on input. + Drop, + /// Collapse all `thinking` block texts into a single `reasoning_content` + /// string on the assistant message. This is the real wire-format + /// understood by vLLM (`reasoning` alias), SGLang, and consumed by the + /// Qwen3 chat template. All other upstreams silently ignore the unknown + /// field. This is the default. + ReasoningContent, + /// Preserve LiteLLM's intermediate `thinking_blocks` array. LiteLLM + /// itself never wire-sends this (its provider transformations strip it), + /// so only use this when you need to be a literal drop-in for LiteLLM's + /// `AnthropicAdapter` intermediate output. + LiteLLMThinkingBlocks, +} + +#[derive(Debug, Clone)] +pub struct ConvertOptions { + /// Drop Anthropic-only `cache_control` fields (always true for OpenAI targets). + pub drop_cache_control: bool, + /// Drop Anthropic-only `top_k` field. + pub drop_top_k: bool, + /// When `stream: true`, inject `stream_options: {include_usage: true}`. + /// Default off so byte-level parity with LiteLLM holds; turn on for real-world use. + pub inject_include_usage: bool, + /// Rewrite `max_tokens` → `max_completion_tokens` for reasoning models + /// (o1/o3/o4/gpt-5). LiteLLM does NOT do this; default off for parity. + pub use_max_completion_tokens_for_reasoning_models: bool, + /// Collapse a single-text system/user content block into a plain string. + /// LiteLLM keeps it as a list — default off for parity. + pub collapse_single_text_part: bool, + /// Concatenate MULTI-text content blocks into one string with "\n\n" + /// between parts (then emit as a plain string). Many real OpenAI-compat + /// servers (SGLang/vLLM in strict mode) reject list-content on system + /// and pure-text user messages. Off in litellm_compat, on in pragmatic. + pub concat_multi_text_parts: bool, + /// Emit `stop` (OpenAI-spec). LiteLLM forwards `stop_sequences` verbatim + /// because its downstream call layer does the rename — default false + /// (passthrough) for parity. + pub emit_stop_field: bool, + /// How to forward Anthropic `thinking` blocks on prior assistant + /// messages. See [`ReasoningPassthrough`]. Default + /// `ReasoningContent` — the only spelling consumed by real upstreams. + pub reasoning_passthrough: ReasoningPassthrough, + /// Drop messages whose only content is an empty string (LiteLLM behaviour). + pub drop_empty_string_messages: bool, + /// Override the target model name. None → use the request's model verbatim. + pub target_model: Option, +} + +impl Default for ConvertOptions { + fn default() -> Self { + Self { + drop_cache_control: true, + drop_top_k: false, + inject_include_usage: false, + use_max_completion_tokens_for_reasoning_models: false, + collapse_single_text_part: false, + concat_multi_text_parts: false, + emit_stop_field: false, + reasoning_passthrough: ReasoningPassthrough::ReasoningContent, + drop_empty_string_messages: true, + target_model: None, + } + } +} + +impl ConvertOptions { + /// Preset for byte-equivalent parity with LiteLLM's + /// `AnthropicAdapter.translate_anthropic_to_openai`. Emits LiteLLM's + /// intermediate `thinking_blocks` shape on assistant messages (LiteLLM + /// itself strips this in its provider transformations before send; + /// real upstreams silently ignore the unknown field). + pub fn litellm_compat() -> Self { + Self { + reasoning_passthrough: ReasoningPassthrough::LiteLLMThinkingBlocks, + ..Self::default() + } + } + + /// Preset for real OAI-compat upstreams (SGLang/vLLM strict mode etc.): + /// collapse single-text content to a string AND concat multi-text-block + /// content with "\n\n" so the wire format is always a string when no + /// multimodal parts are present. Drops top_k (real OpenAI rejects it), + /// rewrites `stop_sequences` → `stop`, swaps in `max_completion_tokens` + /// for reasoning models, injects `stream_options.include_usage`. Forwards + /// reasoning as `reasoning_content` so prior thinking flows to Qwen3 / + /// vLLM / SGLang chat templates that actually consume it. + pub fn pragmatic() -> Self { + Self { + drop_cache_control: true, + drop_top_k: true, + inject_include_usage: true, + use_max_completion_tokens_for_reasoning_models: true, + collapse_single_text_part: true, + concat_multi_text_parts: true, + emit_stop_field: true, + reasoning_passthrough: ReasoningPassthrough::ReasoningContent, + drop_empty_string_messages: true, + target_model: None, + } + } +} + +/// Returns the OpenAI request and a tool-name map (which the response +/// translator needs to restore the original Anthropic names). +pub fn anthropic_request_to_openai( + req: &AnthropicRequest, + opts: &ConvertOptions, +) -> Result<(OpenAIRequest, ToolNameMap), ConvertError> { + let model = opts.target_model.clone().unwrap_or_else(|| req.model.clone()); + let uses_max_completion_tokens = + opts.use_max_completion_tokens_for_reasoning_models && is_reasoning_model(&model); + + let mut messages: Vec = Vec::new(); + + // 1) system → leading system message + if let Some(sys) = &req.system { + match sys { + SystemField::Text(s) => { + if !s.is_empty() { + messages.push(OpenAIMessage::System { + content: OpenAIContent::Text(s.clone()), + }); + } + } + SystemField::Blocks(blocks) => { + let parts: Vec = blocks + .iter() + .map(|b| OpenAIContentPart::Text { text: b.text.clone() }) + .collect(); + if !parts.is_empty() { + let content = if opts.collapse_single_text_part && parts.len() == 1 { + if let OpenAIContentPart::Text { text } = &parts[0] { + OpenAIContent::Text(text.clone()) + } else { + OpenAIContent::Parts(parts) + } + } else if opts.concat_multi_text_parts + && parts.iter().all(|p| matches!(p, OpenAIContentPart::Text { .. })) + { + // All text — concat with blank line between. + let joined = parts + .iter() + .map(|p| match p { + OpenAIContentPart::Text { text } => text.as_str(), + _ => "", + }) + .collect::>() + .join("\n\n"); + OpenAIContent::Text(joined) + } else { + OpenAIContent::Parts(parts) + }; + messages.push(OpenAIMessage::System { content }); + } + } + } + } + + // 2) messages → user/assistant/tool messages + for msg in &req.messages { + translate_message(msg, &mut messages, opts)?; + } + + // 3) tools + name map. Hosted Anthropic tools (web_search_*, computer_*, + // bash_*, text_editor_*, web_fetch_*, code_execution_*, tool_search_*) + // have no OpenAI equivalent — drop them rather than forwarding shapes + // OpenAI will reject with HTTP 400. Client tools translate normally. + let mut tool_name_map = ToolNameMap::new(); + let tools = req.tools.as_ref().and_then(|tools| { + let translated: Vec = tools + .iter() + .filter_map(|t| match t { + AnthropicTool::Client(c) => Some(OpenAITool { + tool_type: "function".to_string(), + function: OpenAIFunctionDef { + name: tool_name_map.translate(&c.name), + description: c.description.clone(), + parameters: c.input_schema.clone(), + }, + }), + AnthropicTool::Hosted(_h) => { + // Drop. (No-op log point; future: collect into a + // translation_warnings sidechannel for /v1/messages 200s.) + None + } + }) + .collect(); + if translated.is_empty() { + None + } else { + Some(translated) + } + }); + + // 4) tool_choice translation + let tool_choice = req.tool_choice.as_ref().map(|tc| match tc { + AnthropicToolChoice::Auto { .. } => json!("auto"), + AnthropicToolChoice::Any { .. } => json!("required"), + AnthropicToolChoice::None => json!("none"), + AnthropicToolChoice::Tool { name, .. } => { + let translated = tool_name_map + .0 + .iter() + .find(|(_, v)| v.as_str() == name.as_str()) + .map(|(k, _)| k.clone()) + .unwrap_or_else(|| name.clone()); + json!({"type": "function", "function": {"name": translated}}) + } + }); + + // 5) thinking → reasoning_effort. + // Two possible sources, in priority order: + // (a) output_config.effort (Anthropic 2025 Q4 extension used by Claude + // Code / OpenCode / Cline — an explicit "low"/"medium"/"high" + // string the client picked itself). Wins when present. + // (b) thinking.budget_tokens (older field) — bucketed. + let explicit_effort = req + .extra + .get("output_config") + .and_then(|v| v.get("effort")) + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + let reasoning_effort = explicit_effort.or_else(|| { + req.thinking.as_ref().and_then(|t| match t { + AnthropicThinking::Enabled { budget_tokens } => Some(bucket_reasoning_effort(*budget_tokens)), + AnthropicThinking::Disabled => None, + }) + }); + + // 6) stream_options injection + let stream_options = match (req.stream, opts.inject_include_usage) { + (Some(true), true) => Some(StreamOptions { include_usage: true }), + _ => None, + }; + + // 7) service_tier: Anthropic "auto"/"standard_only" → OpenAI tier names. + let service_tier = req + .extra + .get("service_tier") + .and_then(|v| v.as_str()) + .map(|s| match s { + // Anthropic spec values + "auto" => "auto".to_string(), + "standard_only" => "default".to_string(), + // OpenAI native values — pass through verbatim + other => other.to_string(), + }); + + let openai_req = OpenAIRequest { + model, + messages, + max_tokens: if uses_max_completion_tokens { None } else { Some(req.max_tokens) }, + max_completion_tokens: if uses_max_completion_tokens { Some(req.max_tokens) } else { None }, + temperature: req.temperature, + top_p: req.top_p, + top_k: if opts.drop_top_k { None } else { req.top_k }, + stop: if opts.emit_stop_field { req.stop_sequences.clone() } else { None }, + stop_sequences: if opts.emit_stop_field { None } else { req.stop_sequences.clone() }, + stream: req.stream, + stream_options, + tools, + tool_choice, + user: req.metadata.as_ref().and_then(|m| m.user_id.clone()), + reasoning_effort, + service_tier, + }; + + let _ = opts.drop_cache_control; + + Ok((openai_req, tool_name_map)) +} + +fn translate_message( + msg: &AnthropicMessage, + out: &mut Vec, + opts: &ConvertOptions, +) -> Result<(), ConvertError> { + // Bare-string content keeps its string shape on the OpenAI side + // (matches LiteLLM and is what most providers expect). + let bare_text: Option = match &msg.content { + MessageContent::Text(s) => Some(s.clone()), + MessageContent::Blocks(_) => None, + }; + + let blocks: Vec = match &msg.content { + MessageContent::Text(s) => vec![ContentBlock::Text { + text: s.clone(), + cache_control: None, + }], + MessageContent::Blocks(b) => b.clone(), + }; + + match msg.role.as_str() { + "user" => translate_user_blocks(&blocks, bare_text, out, opts)?, + "assistant" => translate_assistant_blocks(&blocks, bare_text, out, opts)?, + other => { + return Err(ConvertError::InvalidRequest(format!( + "unknown message role: {other}" + ))) + } + } + Ok(()) +} + +fn translate_user_blocks( + blocks: &[ContentBlock], + bare_text: Option, + out: &mut Vec, + opts: &ConvertOptions, +) -> Result<(), ConvertError> { + // tool_result blocks must each become their own role="tool" message, + // emitted BEFORE any trailing user content (matches LiteLLM ordering). + let mut user_parts: Vec = Vec::new(); + + for b in blocks { + match b { + ContentBlock::Text { text, .. } => { + user_parts.push(OpenAIContentPart::Text { text: text.clone() }) + } + ContentBlock::Image { source, .. } => { + user_parts.push(OpenAIContentPart::ImageUrl { + image_url: OpenAIImageUrl { + url: image_source_to_url(source), + }, + }); + } + ContentBlock::ToolResult { + tool_use_id, + content, + .. + } => { + let content = tool_result_to_openai_content(content.as_ref()); + out.push(OpenAIMessage::Tool { + tool_call_id: tool_use_id.clone(), + content, + }); + } + ContentBlock::ToolUse { .. } => { + return Err(ConvertError::InvalidRequest( + "tool_use block found in user message".into(), + )) + } + ContentBlock::Thinking { .. } | ContentBlock::RedactedThinking { .. } => { + // Thinking blocks are assistant-only; ignore in user. + } + ContentBlock::Unknown => { + // Server-side content blocks (web_search_tool_result, + // code_execution_tool_result, mcp_tool_use, mcp_tool_result, + // server_tool_use, container_upload, document, etc.) have + // no OpenAI equivalent. Silently drop so the upstream + // doesn't 400 on the unknown content shape. + } + } + } + + if let Some(text) = bare_text { + if text.is_empty() && opts.drop_empty_string_messages { + return Ok(()); + } + // Pure bare-string user message → string content (LiteLLM shape). + out.push(OpenAIMessage::User { + content: OpenAIContent::Text(text), + }); + return Ok(()); + } + + if !user_parts.is_empty() { + let content = collapse_parts(user_parts, opts); + out.push(OpenAIMessage::User { content }); + } + Ok(()) +} + +fn translate_assistant_blocks( + blocks: &[ContentBlock], + bare_text: Option, + out: &mut Vec, + opts: &ConvertOptions, +) -> Result<(), ConvertError> { + let mut text_parts: Vec = Vec::new(); + let mut tool_calls: Vec = Vec::new(); + // Two accumulators — we pick which one to emit based on + // opts.reasoning_passthrough at the end. + let mut reasoning_texts: Vec = Vec::new(); + let mut thinking_blocks_raw: Vec = Vec::new(); + + for b in blocks { + match b { + ContentBlock::Text { text, .. } => text_parts.push(text.clone()), + ContentBlock::ToolUse { + id, name, input, .. + } => { + tool_calls.push(OpenAIToolCall { + id: Some(id.clone()), + call_type: "function".to_string(), + function: OpenAIFunctionCall { + name: Some(name.clone()), + arguments: Some(serde_json::to_string(input)?), + }, + index: None, + }); + } + ContentBlock::Thinking { thinking, signature } => { + if opts.reasoning_passthrough != ReasoningPassthrough::Drop { + reasoning_texts.push(thinking.clone()); + if opts.reasoning_passthrough + == ReasoningPassthrough::LiteLLMThinkingBlocks + { + let mut o = serde_json::Map::new(); + o.insert("type".to_string(), json!("thinking")); + o.insert("thinking".to_string(), json!(thinking)); + if let Some(sig) = signature { + o.insert("signature".to_string(), json!(sig)); + } + // LiteLLM AnthropicAdapter adds an empty cache_control + // here; mirror exactly for byte-parity. + o.insert("cache_control".to_string(), json!({})); + thinking_blocks_raw.push(Value::Object(o)); + } + } + } + ContentBlock::RedactedThinking { data } => { + if opts.reasoning_passthrough + == ReasoningPassthrough::LiteLLMThinkingBlocks + { + thinking_blocks_raw.push(json!({ + "type": "redacted_thinking", + "data": data, + })); + } + // ReasoningPassthrough::ReasoningContent: no plain-text + // representation for redacted blocks, drop. + } + ContentBlock::Image { .. } | ContentBlock::ToolResult { .. } => { + return Err(ConvertError::InvalidRequest( + "image/tool_result block in assistant message".into(), + )) + } + ContentBlock::Unknown => { + // server_tool_use, mcp_tool_use, code_execution_tool_result, + // bash_code_execution_tool_result, etc. — Anthropic + // server-side blocks with no OpenAI equivalent. Drop. + } + } + } + + let content = if text_parts.is_empty() { + None + } else { + Some(OpenAIContent::Text(text_parts.join(""))) + }; + let tool_calls = if tool_calls.is_empty() { + None + } else { + Some(tool_calls) + }; + + let (reasoning_content, thinking_blocks) = match opts.reasoning_passthrough { + ReasoningPassthrough::Drop => (None, None), + ReasoningPassthrough::ReasoningContent => { + let rc = if reasoning_texts.is_empty() { + None + } else { + Some(reasoning_texts.join("\n\n")) + }; + (rc, None) + } + ReasoningPassthrough::LiteLLMThinkingBlocks => { + let tb = if thinking_blocks_raw.is_empty() { + None + } else { + Some(thinking_blocks_raw) + }; + (None, tb) + } + }; + + if content.is_some() + || tool_calls.is_some() + || reasoning_content.is_some() + || thinking_blocks.is_some() + { + out.push(OpenAIMessage::Assistant { + content, + tool_calls, + reasoning_content, + thinking_blocks, + }); + } + let _ = bare_text; + Ok(()) +} + +fn collapse_parts(parts: Vec, opts: &ConvertOptions) -> OpenAIContent { + if opts.collapse_single_text_part && parts.len() == 1 { + if let OpenAIContentPart::Text { text } = &parts[0] { + return OpenAIContent::Text(text.clone()); + } + } + if opts.concat_multi_text_parts + && parts.iter().all(|p| matches!(p, OpenAIContentPart::Text { .. })) + { + let joined = parts + .iter() + .map(|p| match p { + OpenAIContentPart::Text { text } => text.as_str(), + _ => "", + }) + .collect::>() + .join("\n\n"); + return OpenAIContent::Text(joined); + } + OpenAIContent::Parts(parts) +} + +fn image_source_to_url(src: &ImageSource) -> String { + match src { + ImageSource::Base64 { media_type, data } => { + format!("data:{};base64,{}", media_type, data) + } + ImageSource::Url { url } => url.clone(), + } +} + +fn tool_result_to_openai_content(content: Option<&ToolResultContent>) -> OpenAIContent { + match content { + None => OpenAIContent::Text(String::new()), + Some(ToolResultContent::Text(s)) => OpenAIContent::Text(s.clone()), + Some(ToolResultContent::Blocks(blocks)) => { + // Single text block → flatten. Anything else (image, multi-block) → + // list-content under the same tool_call_id (matches LiteLLM and the + // Anthropic 1:1 tool_use_id ↔ tool message rule). + if blocks.len() == 1 { + if let ToolResultBlock::Text { text } = &blocks[0] { + return OpenAIContent::Text(text.clone()); + } + } + let parts: Vec = blocks + .iter() + .map(|b| match b { + ToolResultBlock::Text { text } => OpenAIContentPart::Text { text: text.clone() }, + ToolResultBlock::Image { source } => OpenAIContentPart::ImageUrl { + image_url: OpenAIImageUrl { + url: image_source_to_url(source), + }, + }, + }) + .collect(); + OpenAIContent::Parts(parts) + } + } +} + +fn is_reasoning_model(model: &str) -> bool { + let m = model.to_ascii_lowercase(); + // OpenAI reasoning families: o1/o3/o4 series, gpt-5 series. + m.starts_with("o1") || m.starts_with("o3") || m.starts_with("o4") || m.starts_with("gpt-5") +} + +fn bucket_reasoning_effort(budget_tokens: u32) -> String { + if budget_tokens >= 10_000 { + "high".to_string() + } else if budget_tokens >= 5_000 { + "medium".to_string() + } else if budget_tokens >= 2_000 { + "low".to_string() + } else { + "minimal".to_string() + } +} + +/// Helper used by the streaming layer and the JSON facade. +pub fn json_value_of_openai_request(req: &OpenAIRequest) -> Value { + serde_json::to_value(req).expect("OpenAIRequest serialises") +} diff --git a/sidecars/cc_convert/crates/cc_convert_core/src/resp_to_anthropic.rs b/sidecars/cc_convert/crates/cc_convert_core/src/resp_to_anthropic.rs new file mode 100644 index 0000000..1d7066f --- /dev/null +++ b/sidecars/cc_convert/crates/cc_convert_core/src/resp_to_anthropic.rs @@ -0,0 +1,180 @@ +//! OpenAI Chat Completions response → Anthropic Messages response. + +use crate::anthropic::*; +use crate::error::ConvertError; +use crate::openai::*; +use crate::tool_names::ToolNameMap; +use serde_json::{json, Value}; + +#[derive(Debug, Clone)] +pub struct ResponseConvertOptions { + /// If `Some`, use this string as the Anthropic response's `model` field. + /// If `None`, pass through the OpenAI response's model (LiteLLM behaviour). + pub original_model: Option, + /// LiteLLM passes `id` through as-is (e.g. `chatcmpl-xxx`). If true, + /// rewrite `chatcmpl-` prefix to `msg_`. Default false (= LiteLLM). + pub rewrite_id: bool, + /// LiteLLM allows `content: []` for empty assistant messages. If true, + /// always emit at least one `{type:"text", text:""}` block (older + /// Anthropic SDKs require this). Default false (= LiteLLM). + pub never_empty_content: bool, + /// LiteLLM subtracts `cached_tokens` from `prompt_tokens` so + /// `input_tokens` only counts the uncached portion. Default true. + pub subtract_cached_from_input: bool, +} + +impl Default for ResponseConvertOptions { + fn default() -> Self { + Self { + original_model: None, + rewrite_id: false, + never_empty_content: false, + subtract_cached_from_input: true, + } + } +} + +impl ResponseConvertOptions { + /// Byte-for-byte parity with LiteLLM's + /// `translate_openai_response_to_anthropic` (modulo dropped nulls). + pub fn litellm_compat() -> Self { + Self::default() + } + + /// Friendlier for older Anthropic clients: rewrites `id` to `msg_*` and + /// guarantees at least one content block. + pub fn pragmatic(original_model: impl Into) -> Self { + Self { + original_model: Some(original_model.into()), + rewrite_id: true, + never_empty_content: true, + subtract_cached_from_input: true, + } + } +} + +pub fn openai_response_to_anthropic( + resp: &OpenAIResponse, + original_model: &str, + tool_name_map: &ToolNameMap, +) -> Result { + // Backwards-compat wrapper: behaves like the old API (rewrites id + + // guarantees non-empty content + uses `original_model`). + let opts = ResponseConvertOptions::pragmatic(original_model); + openai_response_to_anthropic_with(resp, tool_name_map, &opts) +} + +pub fn openai_response_to_anthropic_with( + resp: &OpenAIResponse, + tool_name_map: &ToolNameMap, + opts: &ResponseConvertOptions, +) -> Result { + let id = if opts.rewrite_id { + rewrite_id(&resp.id) + } else { + resp.id.clone() + }; + + let model = opts + .original_model + .clone() + .unwrap_or_else(|| resp.model.clone()); + + let choice = resp + .choices + .first() + .ok_or_else(|| ConvertError::InvalidResponse("response has no choices".into()))?; + + let mut content = Vec::::new(); + + if let Some(reasoning) = choice + .message + .reasoning_content + .as_ref() + .filter(|s| !s.is_empty()) + { + content.push(ResponseContentBlock::Thinking { + thinking: reasoning.clone(), + signature: None, + }); + } + + if let Some(text) = choice.message.content.as_ref().filter(|s| !s.is_empty()) { + content.push(ResponseContentBlock::Text { text: text.clone() }); + } + + if let Some(tool_calls) = choice.message.tool_calls.as_ref() { + for tc in tool_calls { + let restored = tool_name_map.restore(tc.function.name.as_deref().unwrap_or("")); + let input: Value = match tc.function.arguments.as_deref().unwrap_or("") { + "" => json!({}), + raw => serde_json::from_str(raw).unwrap_or_else(|_| json!({ "raw": raw })), + }; + content.push(ResponseContentBlock::ToolUse { + id: tc.id.clone().unwrap_or_default(), + name: restored.to_string(), + input, + }); + } + } + + if content.is_empty() && opts.never_empty_content { + content.push(ResponseContentBlock::Text { text: String::new() }); + } + + let stop_reason = map_stop_reason(choice.finish_reason.as_deref()); + let usage = map_usage(resp.usage.as_ref(), opts.subtract_cached_from_input); + + Ok(AnthropicResponse { + id, + msg_type: "message".to_string(), + role: "assistant".to_string(), + model, + content, + stop_reason, + stop_sequence: None, + usage, + }) +} + +pub fn rewrite_id(openai_id: &str) -> String { + if let Some(rest) = openai_id.strip_prefix("chatcmpl-") { + format!("msg_{}", rest) + } else if openai_id.starts_with("msg_") { + openai_id.to_string() + } else { + format!("msg_{}", openai_id) + } +} + +pub fn map_stop_reason(reason: Option<&str>) -> AnthropicStopReason { + match reason { + Some("stop") => AnthropicStopReason::EndTurn, + Some("length") => AnthropicStopReason::MaxTokens, + Some("tool_calls") => AnthropicStopReason::ToolUse, + Some("function_call") => AnthropicStopReason::ToolUse, // legacy + _ => AnthropicStopReason::EndTurn, + } +} + +pub fn map_usage(usage: Option<&OpenAIUsage>, subtract_cached: bool) -> AnthropicUsage { + let Some(u) = usage else { + return AnthropicUsage::default(); + }; + let cache_read = u + .prompt_tokens_details + .as_ref() + .and_then(|d| d.cached_tokens); + let mut input_tokens = u.prompt_tokens; + if subtract_cached { + if let Some(c) = cache_read { + input_tokens = input_tokens.saturating_sub(c); + } + } + AnthropicUsage { + input_tokens, + output_tokens: u.completion_tokens, + cache_creation_input_tokens: None, + cache_read_input_tokens: cache_read, + } +} diff --git a/sidecars/cc_convert/crates/cc_convert_core/src/stream.rs b/sidecars/cc_convert/crates/cc_convert_core/src/stream.rs new file mode 100644 index 0000000..f08e165 --- /dev/null +++ b/sidecars/cc_convert/crates/cc_convert_core/src/stream.rs @@ -0,0 +1,399 @@ +//! OpenAI SSE chunk stream → Anthropic SSE event stream. +//! +//! Push chunks one at a time via [`StreamTranslator::push_openai_chunk`]. +//! Call [`StreamTranslator::finish`] when the upstream closes; it emits the +//! `message_delta` + `message_stop` events if they were not already emitted +//! due to a `finish_reason`-bearing chunk. + +use crate::anthropic::*; +use crate::openai::*; +use crate::resp_to_anthropic::{map_stop_reason, map_usage, rewrite_id}; +use crate::tool_names::ToolNameMap; +use serde_json::json; +use std::collections::BTreeMap; + +#[derive(Debug, Clone)] +pub struct StreamConvertOptions { + /// Anthropic SDK convention is to emit a `ping` event right after + /// `message_start`. LiteLLM does NOT emit it; default false for parity. + pub emit_ping: bool, + /// Anthropic spec includes `stop_sequence: null` in `message_delta`. + /// LiteLLM omits it; default false for parity. + pub include_stop_sequence_in_message_delta: bool, + /// `message_start.message.usage` always includes + /// `cache_creation_input_tokens` and `cache_read_input_tokens` (LiteLLM + /// behaviour). Default true. + pub include_zero_cache_fields_in_usage: bool, + /// Generate a fresh `msg_` for the message id (LiteLLM behaviour). + /// Default false: derive from the OpenAI `chatcmpl-*` id instead. + pub random_message_id: bool, + /// LiteLLM eagerly opens the first `content_block_start` (text block at + /// index 0) immediately after `message_start`, even before any delta + /// arrives. Default true for parity. + pub eager_open_text_block: bool, +} + +impl Default for StreamConvertOptions { + fn default() -> Self { + Self { + emit_ping: false, + include_stop_sequence_in_message_delta: false, + include_zero_cache_fields_in_usage: true, + random_message_id: false, + eager_open_text_block: true, + } + } +} + +impl StreamConvertOptions { + pub fn litellm_compat() -> Self { + Self::default() + } + + /// Matches the Anthropic SDK's published SSE shape (with ping + + /// stop_sequence + lazy block opening). + pub fn anthropic_native() -> Self { + Self { + emit_ping: true, + include_stop_sequence_in_message_delta: true, + include_zero_cache_fields_in_usage: false, + random_message_id: false, + eager_open_text_block: false, + } + } +} + +#[derive(Debug, Clone)] +struct ToolBlockState { + anthropic_index: i32, + started: bool, + /// Remembered from the first chunk; continuation chunks may not carry it. + id: String, + /// Remembered from the first chunk; continuation chunks may not carry it. + name: String, +} + +#[derive(Debug)] +pub struct StreamTranslator { + original_model: String, + tool_names: ToolNameMap, + opts: StreamConvertOptions, + + sent_message_start: bool, + text_block_open: bool, + text_block_index: i32, + + thinking_block_open: bool, + thinking_block_index: i32, + + tool_blocks: BTreeMap, + next_anthropic_index: i32, + + pending_usage: Option, + pending_stop_reason: Option, + emitted_stop: bool, +} + +impl StreamTranslator { + pub fn new(original_model: String, tool_names: ToolNameMap) -> Self { + Self::with_options(original_model, tool_names, StreamConvertOptions::default()) + } + + pub fn with_options( + original_model: String, + tool_names: ToolNameMap, + opts: StreamConvertOptions, + ) -> Self { + Self { + original_model, + tool_names, + opts, + sent_message_start: false, + text_block_open: false, + text_block_index: 0, + thinking_block_open: false, + thinking_block_index: 0, + tool_blocks: BTreeMap::new(), + next_anthropic_index: 0, + pending_usage: None, + pending_stop_reason: None, + emitted_stop: false, + } + } + + pub fn push_openai_chunk(&mut self, chunk: &OpenAIStreamChunk) -> Vec { + let mut out = Vec::new(); + if self.emitted_stop { + return out; + } + + if !self.sent_message_start { + self.send_message_start(chunk, &mut out); + } + + if let Some(usage) = &chunk.usage { + self.pending_usage = Some(usage.clone()); + } + + for choice in &chunk.choices { + self.process_choice(choice, &mut out); + } + + out + } + + fn send_message_start(&mut self, chunk: &OpenAIStreamChunk, out: &mut Vec) { + let id = if self.opts.random_message_id { + format!("msg_{}", uuid::Uuid::new_v4()) + } else { + chunk + .id + .as_deref() + .map(rewrite_id) + .unwrap_or_else(|| format!("msg_{}", uuid::Uuid::new_v4())) + }; + let usage = if self.opts.include_zero_cache_fields_in_usage { + AnthropicUsage { + input_tokens: 0, + output_tokens: 0, + cache_creation_input_tokens: Some(0), + cache_read_input_tokens: Some(0), + } + } else { + AnthropicUsage::default() + }; + out.push(AnthropicEvent::MessageStart { + message: MessageStartPayload { + id, + msg_type: "message".to_string(), + role: "assistant".to_string(), + model: self.original_model.clone(), + content: Vec::new(), + stop_reason: None, + stop_sequence: None, + usage, + }, + }); + if self.opts.emit_ping { + out.push(AnthropicEvent::Ping); + } + if self.opts.eager_open_text_block { + // Open content_block index 0 as a text block eagerly. Tool calls + // arriving later will allocate their own indices. + let idx = self.allocate_index(); + self.text_block_index = idx; + self.text_block_open = true; + out.push(AnthropicEvent::ContentBlockStart { + index: idx, + content_block: StreamingContentBlock::Text { text: String::new() }, + }); + } + self.sent_message_start = true; + } + + fn process_choice(&mut self, choice: &OpenAIStreamChoice, out: &mut Vec) { + let delta = &choice.delta; + + let reasoning = delta + .reasoning_content + .as_deref() + .or(delta.reasoning.as_deref()); + if let Some(t) = reasoning.filter(|s| !s.is_empty()) { + if !self.thinking_block_open { + // Close text block first if eagerly opened but empty. + if self.text_block_open { + out.push(AnthropicEvent::ContentBlockStop { + index: self.text_block_index, + }); + self.text_block_open = false; + } + let idx = self.allocate_index(); + self.thinking_block_index = idx; + out.push(AnthropicEvent::ContentBlockStart { + index: idx, + content_block: StreamingContentBlock::Thinking { + thinking: String::new(), + }, + }); + self.thinking_block_open = true; + } + out.push(AnthropicEvent::ContentBlockDelta { + index: self.thinking_block_index, + delta: BlockDelta::ThinkingDelta { + thinking: t.to_string(), + }, + }); + } + + if let Some(text) = delta.content.as_deref().filter(|s| !s.is_empty()) { + if !self.text_block_open { + if self.thinking_block_open { + out.push(AnthropicEvent::ContentBlockStop { + index: self.thinking_block_index, + }); + self.thinking_block_open = false; + } + let idx = self.allocate_index(); + self.text_block_index = idx; + out.push(AnthropicEvent::ContentBlockStart { + index: idx, + content_block: StreamingContentBlock::Text { + text: String::new(), + }, + }); + self.text_block_open = true; + } + out.push(AnthropicEvent::ContentBlockDelta { + index: self.text_block_index, + delta: BlockDelta::TextDelta { + text: text.to_string(), + }, + }); + } + + if let Some(tcs) = delta.tool_calls.as_ref() { + if self.text_block_open { + out.push(AnthropicEvent::ContentBlockStop { + index: self.text_block_index, + }); + self.text_block_open = false; + } + if self.thinking_block_open { + out.push(AnthropicEvent::ContentBlockStop { + index: self.thinking_block_index, + }); + self.thinking_block_open = false; + } + + for tc in tcs { + let oi = tc.index.unwrap_or(0); + // Some upstreams (vLLM continuations, SGLang null-id) omit + // id/name on continuation chunks. Use the chunk's values if + // present, otherwise fall back to what we recorded on the + // first chunk for this index. + let incoming_id = tc.id.clone(); + let incoming_name = tc.function.name.clone(); + let state = self.tool_blocks.entry(oi).or_insert(ToolBlockState { + anthropic_index: self.next_anthropic_index, + started: false, + id: incoming_id.clone().unwrap_or_default(), + name: incoming_name.clone().unwrap_or_default(), + }); + // Update remembered id/name if this chunk provided them and + // we didn't have them before. + if state.id.is_empty() { + if let Some(id) = incoming_id { + state.id = id; + } + } + if state.name.is_empty() { + if let Some(n) = incoming_name { + state.name = n; + } + } + let ai = state.anthropic_index; + if ai == self.next_anthropic_index { + // New block — bump the counter (entry().or_insert reserved it). + self.next_anthropic_index += 1; + } + if !state.started && !state.id.is_empty() { + // We've seen enough to open the block. + let restored = self.tool_names.restore(&state.name).to_string(); + let id_owned = state.id.clone(); + out.push(AnthropicEvent::ContentBlockStart { + index: ai, + content_block: StreamingContentBlock::ToolUse { + id: id_owned, + name: restored, + input: json!({}), + }, + }); + // Re-borrow to flip started since the immutable borrow is done. + self.tool_blocks.get_mut(&oi).unwrap().started = true; + } + if let Some(args) = tc.function.arguments.as_deref().filter(|s| !s.is_empty()) { + out.push(AnthropicEvent::ContentBlockDelta { + index: ai, + delta: BlockDelta::InputJsonDelta { + partial_json: args.to_string(), + }, + }); + } + } + } + + if let Some(reason) = choice.finish_reason.as_deref() { + self.pending_stop_reason = Some(map_stop_reason(Some(reason))); + self.emit_close(out); + } + } + + fn allocate_index(&mut self) -> i32 { + let i = self.next_anthropic_index; + self.next_anthropic_index += 1; + i + } + + fn emit_close(&mut self, out: &mut Vec) { + if self.emitted_stop { + return; + } + if self.text_block_open { + out.push(AnthropicEvent::ContentBlockStop { + index: self.text_block_index, + }); + self.text_block_open = false; + } + if self.thinking_block_open { + out.push(AnthropicEvent::ContentBlockStop { + index: self.thinking_block_index, + }); + self.thinking_block_open = false; + } + for (_, state) in self.tool_blocks.iter_mut() { + if state.started { + out.push(AnthropicEvent::ContentBlockStop { + index: state.anthropic_index, + }); + state.started = false; + } + } + + let usage = map_usage(self.pending_usage.as_ref(), true); + let stop_reason = self + .pending_stop_reason + .take() + .unwrap_or(AnthropicStopReason::EndTurn); + + out.push(AnthropicEvent::MessageDelta { + delta: MessageDeltaPayload { + stop_reason: Some(stop_reason), + stop_sequence: if self.opts.include_stop_sequence_in_message_delta { + None + } else { + None + }, + }, + usage, + }); + out.push(AnthropicEvent::MessageStop); + self.emitted_stop = true; + } + + pub fn finish(&mut self) -> Vec { + let mut out = Vec::new(); + if !self.sent_message_start { + self.send_message_start( + &OpenAIStreamChunk { + id: None, + model: None, + choices: Vec::new(), + usage: None, + }, + &mut out, + ); + } + self.emit_close(&mut out); + out + } +} diff --git a/sidecars/cc_convert/crates/cc_convert_core/src/tool_names.rs b/sidecars/cc_convert/crates/cc_convert_core/src/tool_names.rs new file mode 100644 index 0000000..39549ee --- /dev/null +++ b/sidecars/cc_convert/crates/cc_convert_core/src/tool_names.rs @@ -0,0 +1,81 @@ +//! OpenAI hard-limits tool names to 64 characters and constrains them to +//! `[a-zA-Z0-9_-]`. Anthropic does not. When an Anthropic tool name is too +//! long, we hash-truncate it to `{55-prefix}_{8-hex-sha}` and remember the +//! reverse mapping so we can restore the original name on the response side. + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; + +const MAX_TOOL_NAME_LEN: usize = 64; + +/// Round-trippable map from translated (≤64-char) OpenAI tool name to the +/// original Anthropic tool name. Names that don't need truncation are also +/// stored (mapped to themselves) so the response side can look up uniformly. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(transparent)] +pub struct ToolNameMap(pub HashMap); + +impl ToolNameMap { + pub fn new() -> Self { + Self(HashMap::new()) + } + + /// Translate `original` to an OpenAI-safe name. If truncation is needed, + /// remember the original→translated mapping (so the response side can + /// restore it). Short names that did not need translation are NOT + /// inserted into the map — matches LiteLLM behaviour. + pub fn translate(&mut self, original: &str) -> String { + if original.len() <= MAX_TOOL_NAME_LEN { + return original.to_string(); + } + let mut hasher = Sha256::new(); + hasher.update(original.as_bytes()); + let hash = hex::encode(hasher.finalize()); + let prefix: String = original.chars().take(55).collect(); + let safe = format!("{}_{}", prefix, &hash[..8]); + self.0.insert(safe.clone(), original.to_string()); + safe + } + + /// Look up the original name for a translated name. Falls back to the + /// translated name itself if not registered (so unknown tool_calls are + /// passed through unchanged). + pub fn restore<'a>(&'a self, translated: &'a str) -> &'a str { + self.0 + .get(translated) + .map(String::as_str) + .unwrap_or(translated) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn short_names_passthrough() { + let mut m = ToolNameMap::new(); + let s = m.translate("get_weather"); + assert_eq!(s, "get_weather"); + // Restore works without an explicit entry (fallback). + assert_eq!(m.restore("get_weather"), "get_weather"); + // Map stays empty for short names, matching LiteLLM. + assert!(m.0.is_empty()); + } + + #[test] + fn long_names_truncated_and_round_trip() { + let mut m = ToolNameMap::new(); + let long = "a".repeat(100); + let s = m.translate(&long); + assert!(s.len() <= 64); + assert_eq!(m.restore(&s), long); + } + + #[test] + fn unknown_translated_name_passes_through() { + let m = ToolNameMap::new(); + assert_eq!(m.restore("some_unknown"), "some_unknown"); + } +} diff --git a/sidecars/cc_convert/crates/cc_convert_core/tests/parity_litellm.rs b/sidecars/cc_convert/crates/cc_convert_core/tests/parity_litellm.rs new file mode 100644 index 0000000..202921c --- /dev/null +++ b/sidecars/cc_convert/crates/cc_convert_core/tests/parity_litellm.rs @@ -0,0 +1,169 @@ +//! Golden-file parity test: each input under tests/fixtures/requests/ +//! anthropic_*.json is fed through the Rust translator under the +//! `litellm_compat` preset and the result is compared SEMANTICALLY against +//! the golden openai_*.json produced by LiteLLM. +//! +//! "Semantic" means: parse both sides into serde_json::Value, recursively +//! drop nulls, sort object keys, and compare. LiteLLM emits some fields +//! explicitly as null (`thinking_blocks: null`) that we omit; equivalent +//! shapes still pass. + +use cc_convert_core::anthropic::AnthropicRequest; +use cc_convert_core::{anthropic_request_to_openai, ConvertOptions}; +use serde_json::Value; +use std::path::{Path, PathBuf}; + +fn fixtures_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() // crates/ + .parent() + .unwrap() // workspace root + .join("tests") + .join("fixtures") +} + +/// Recursively normalize JSON: drop nulls, recurse into objects/arrays. +/// Also normalize tool-call `arguments` (JSON-as-string) by reparsing and +/// re-stringifying with a stable separator-free format. +fn normalize(v: &Value, ctx_key: Option<&str>) -> Value { + match v { + Value::Null => Value::Null, + Value::Bool(_) | Value::Number(_) => v.clone(), + Value::String(s) => { + if ctx_key == Some("arguments") { + // Parse-and-restringify so whitespace differences don't matter. + if let Ok(parsed) = serde_json::from_str::(s) { + return Value::String(serde_json::to_string(&parsed).unwrap_or_default()); + } + } + Value::String(s.clone()) + } + Value::Array(items) => Value::Array(items.iter().map(|i| normalize(i, None)).collect()), + Value::Object(map) => { + let mut out = serde_json::Map::new(); + for (k, val) in map { + let normalized = normalize(val, Some(k.as_str())); + if matches!(normalized, Value::Null) { + continue; + } + out.insert(k.clone(), normalized); + } + Value::Object(out) + } + } +} + +fn load_json(path: &Path) -> Value { + let text = std::fs::read_to_string(path) + .unwrap_or_else(|e| panic!("read {}: {}", path.display(), e)); + serde_json::from_str(&text) + .unwrap_or_else(|e| panic!("parse {}: {}", path.display(), e)) +} + +fn collect_inputs(dir: &Path, prefix: &str) -> Vec<(String, PathBuf)> { + let mut out = Vec::new(); + for entry in std::fs::read_dir(dir).expect("read fixtures dir") { + let entry = entry.expect("dir entry"); + let path = entry.path(); + let Some(name) = path.file_stem().and_then(|s| s.to_str()) else { + continue; + }; + if let Some(rest) = name.strip_prefix(prefix) { + out.push((rest.to_string(), path)); + } + } + out.sort_by(|a, b| a.0.cmp(&b.0)); + out +} + +#[test] +fn parity_with_litellm_request_fixtures() { + let fixtures = fixtures_root().join("requests"); + let inputs = collect_inputs(&fixtures, "anthropic_"); + assert!(!inputs.is_empty(), "no fixtures under {}", fixtures.display()); + + let mut failures = Vec::::new(); + let mut checked = 0usize; + let mut missing_golden = 0usize; + + for (name, input_path) in inputs { + let golden_path = fixtures.join(format!("openai_{}.json", name)); + if !golden_path.exists() { + missing_golden += 1; + eprintln!("[skip] {name}: no golden ({})", golden_path.display()); + continue; + } + let anthropic_value = load_json(&input_path); + let anthropic_req: AnthropicRequest = serde_json::from_value(anthropic_value) + .unwrap_or_else(|e| panic!("parse fixture {name}: {e}")); + + let (openai_req, _tool_map) = + anthropic_request_to_openai(&anthropic_req, &ConvertOptions::litellm_compat()) + .unwrap_or_else(|e| panic!("translate {name}: {e}")); + + let actual = normalize(&serde_json::to_value(&openai_req).unwrap(), None); + let golden = normalize(&load_json(&golden_path), None); + + if actual != golden { + failures.push(format!( + "case {name}:\n expected: {}\n actual: {}\n", + serde_json::to_string_pretty(&golden).unwrap(), + serde_json::to_string_pretty(&actual).unwrap() + )); + } + checked += 1; + } + + if !failures.is_empty() { + panic!( + "{}/{} request fixtures failed parity with LiteLLM:\n\n{}", + failures.len(), + checked, + failures.join("\n---\n") + ); + } + + eprintln!( + "parity OK: {checked} request fixtures matched LiteLLM ({missing_golden} missing goldens)" + ); +} + +#[test] +fn parity_tool_name_map_matches_litellm() { + let fixtures = fixtures_root().join("requests"); + let inputs = collect_inputs(&fixtures, "anthropic_"); + let mut checked = 0; + let mut failures = Vec::::new(); + + for (name, input_path) in inputs { + let golden_map_path = fixtures.join(format!("tool_map_{}.json", name)); + if !golden_map_path.exists() { + continue; + } + let anthropic_value = load_json(&input_path); + let anthropic_req: AnthropicRequest = serde_json::from_value(anthropic_value).unwrap(); + let (_req, tool_map) = + anthropic_request_to_openai(&anthropic_req, &ConvertOptions::litellm_compat()) + .unwrap(); + let actual = normalize(&serde_json::to_value(&tool_map).unwrap(), None); + let golden = normalize(&load_json(&golden_map_path), None); + if actual != golden { + failures.push(format!( + "case {name} tool_map:\n expected: {}\n actual: {}", + serde_json::to_string(&golden).unwrap(), + serde_json::to_string(&actual).unwrap() + )); + } + checked += 1; + } + + if !failures.is_empty() { + panic!( + "{} tool-map fixtures failed parity:\n{}", + failures.len(), + failures.join("\n") + ); + } + eprintln!("parity OK: {checked} tool-map fixtures matched LiteLLM"); +} diff --git a/sidecars/cc_convert/crates/cc_convert_core/tests/parity_response.rs b/sidecars/cc_convert/crates/cc_convert_core/tests/parity_response.rs new file mode 100644 index 0000000..e1f3336 --- /dev/null +++ b/sidecars/cc_convert/crates/cc_convert_core/tests/parity_response.rs @@ -0,0 +1,108 @@ +//! Response parity vs LiteLLM. For each +//! tests/fixtures/responses/openai_.json (with sidecar +//! meta_.json carrying any tool_map), feeds the input through +//! `openai_response_to_anthropic_with(... litellm_compat ...)` and asserts +//! semantic equality against the LiteLLM-produced golden +//! anthropic_.json. + +use cc_convert_core::{ + openai::OpenAIResponse, openai_response_to_anthropic_with, tool_names::ToolNameMap, + ResponseConvertOptions, +}; +use serde_json::Value; +use std::path::{Path, PathBuf}; + +fn fixtures_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .join("tests") + .join("fixtures") + .join("responses") +} + +fn normalize(v: &Value) -> Value { + match v { + Value::Null => Value::Null, + Value::Bool(_) | Value::Number(_) | Value::String(_) => v.clone(), + Value::Array(items) => Value::Array(items.iter().map(normalize).collect()), + Value::Object(map) => { + let mut out = serde_json::Map::new(); + for (k, val) in map { + let n = normalize(val); + if matches!(n, Value::Null) { + continue; + } + out.insert(k.clone(), n); + } + Value::Object(out) + } + } +} + +fn load(path: &Path) -> Value { + serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap() +} + +#[test] +fn response_parity_with_litellm() { + let root = fixtures_root(); + let mut failures = Vec::::new(); + let mut checked = 0; + let mut missing = 0; + + for entry in std::fs::read_dir(&root).expect("read responses fixtures dir") { + let path = entry.unwrap().path(); + let Some(stem) = path.file_stem().and_then(|s| s.to_str()) else { + continue; + }; + let Some(name) = stem.strip_prefix("openai_") else { + continue; + }; + let golden_path = root.join(format!("anthropic_{name}.json")); + if !golden_path.exists() { + missing += 1; + continue; + } + let meta_path = root.join(format!("meta_{name}.json")); + let meta_val: Value = if meta_path.exists() { + load(&meta_path) + } else { + Value::Object(Default::default()) + }; + let tool_map: ToolNameMap = meta_val + .get("tool_map") + .map(|v| serde_json::from_value(v.clone()).unwrap_or_default()) + .unwrap_or_default(); + + let openai_resp: OpenAIResponse = serde_json::from_value(load(&path)) + .unwrap_or_else(|e| panic!("parse {name}: {e}")); + + let opts = ResponseConvertOptions::litellm_compat(); + let anthropic = openai_response_to_anthropic_with(&openai_resp, &tool_map, &opts) + .unwrap_or_else(|e| panic!("translate {name}: {e}")); + let actual = normalize(&serde_json::to_value(&anthropic).unwrap()); + let golden = normalize(&load(&golden_path)); + + if actual != golden { + failures.push(format!( + "case {name}:\n expected: {}\n actual: {}", + serde_json::to_string_pretty(&golden).unwrap(), + serde_json::to_string_pretty(&actual).unwrap(), + )); + } + checked += 1; + } + + if !failures.is_empty() { + panic!( + "{}/{} response fixtures diverged:\n\n{}", + failures.len(), + checked, + failures.join("\n---\n") + ); + } + eprintln!("response parity OK: {checked} fixtures matched LiteLLM ({missing} missing)"); +} diff --git a/sidecars/cc_convert/crates/cc_convert_core/tests/parity_stream.rs b/sidecars/cc_convert/crates/cc_convert_core/tests/parity_stream.rs new file mode 100644 index 0000000..41ffc65 --- /dev/null +++ b/sidecars/cc_convert/crates/cc_convert_core/tests/parity_stream.rs @@ -0,0 +1,193 @@ +//! Stream parity vs LiteLLM. For each +//! tests/fixtures/streams/openai_.sse, parses the SSE into chunks, +//! feeds them through `StreamTranslator` (litellm_compat preset), and +//! asserts the resulting event sequence equals the LiteLLM-produced +//! anthropic_.jsonl golden — modulo dropped nulls and the +//! non-deterministic message_start id. + +use cc_convert_core::openai::OpenAIStreamChunk; +use cc_convert_core::tool_names::ToolNameMap; +use cc_convert_core::{StreamConvertOptions, StreamTranslator}; +use serde_json::Value; +use std::path::PathBuf; + +fn fixtures_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .join("tests") + .join("fixtures") + .join("streams") +} + +fn normalize(v: &Value) -> Value { + match v { + Value::Null => Value::Null, + Value::Bool(_) | Value::Number(_) | Value::String(_) => v.clone(), + Value::Array(items) => Value::Array(items.iter().map(normalize).collect()), + Value::Object(map) => { + let mut out = serde_json::Map::new(); + for (k, val) in map { + let n = normalize(val); + if matches!(n, Value::Null) { + continue; + } + out.insert(k.clone(), n); + } + Value::Object(out) + } + } +} + +/// Mask the message_start.message.id since LiteLLM uses a random uuid each +/// time. Both sides become `"__masked__"` for comparison. +fn mask_message_id(events: &mut [Value]) { + for ev in events.iter_mut() { + if ev.get("type").and_then(|v| v.as_str()) == Some("message_start") { + if let Some(msg) = ev.get_mut("message").and_then(|m| m.as_object_mut()) { + if let Some(id) = msg.get_mut("id") { + *id = Value::String("__masked__".to_string()); + } + } + } + } +} + +fn parse_sse(text: &str) -> Vec { + let mut out = Vec::new(); + for block in text.split("\n\n") { + for line in block.lines() { + if let Some(payload) = line.strip_prefix("data:") { + let payload = payload.trim(); + if payload.is_empty() || payload == "[DONE]" { + continue; + } + if let Ok(c) = serde_json::from_str::(payload) { + out.push(c); + } + } + } + } + out +} + +fn run_translator(chunks: &[OpenAIStreamChunk]) -> Vec { + let mut t = StreamTranslator::with_options( + "claude-opus-4-7".to_string(), + ToolNameMap::new(), + StreamConvertOptions::litellm_compat(), + ); + let mut events = Vec::new(); + for c in chunks { + for ev in t.push_openai_chunk(c) { + events.push(serde_json::to_value(&ev).unwrap()); + } + } + for ev in t.finish() { + events.push(serde_json::to_value(&ev).unwrap()); + } + events +} + +fn load_jsonl(path: &std::path::Path) -> Vec { + std::fs::read_to_string(path) + .unwrap() + .lines() + .filter(|l| !l.is_empty()) + .map(|l| serde_json::from_str(l).unwrap()) + .collect() +} + +/// Cases where LiteLLM's AnthropicStreamWrapper emits per-spec-WRONG events +/// (documented quirks). We intentionally do not match these byte-for-byte +/// because our behaviour is closer to the Anthropic SSE spec. The +/// `stream_translation.rs` unit-test file verifies these cases work +/// correctly under our own semantics. +/// +/// - `29_two_parallel_tool_calls`: LiteLLM merges both tool_calls into one +/// content block and concats their arguments (`"{}{}"`). Per spec each +/// parallel tool_call should be its own block. +/// - `30_stream_ends_without_finish_reason`: LiteLLM never emits the closing +/// `content_block_stop` / `message_delta` / `message_stop` events. We +/// emit them so downstream Anthropic clients aren't left hanging. +/// - `31_reasoning_then_text`: LiteLLM emits both thinking and text as +/// deltas to the same content_block at index 0. Per spec they should be +/// separate blocks (thinking + text). +const LITELLM_QUIRKS_TO_SKIP: &[&str] = &[ + "29_two_parallel_tool_calls", + "30_stream_ends_without_finish_reason", + "31_reasoning_then_text", +]; + +#[test] +fn stream_parity_with_litellm() { + let root = fixtures_root(); + let mut failures = Vec::::new(); + let mut checked = 0; + let mut missing = 0; + let mut skipped = 0; + + let mut entries: Vec = std::fs::read_dir(&root) + .expect("read streams dir") + .filter_map(|e| { + let p = e.ok()?.path(); + if p.extension().and_then(|s| s.to_str()) == Some("sse") + && p.file_name() + .and_then(|s| s.to_str()) + .map(|s| s.starts_with("openai_")) + .unwrap_or(false) + { + Some(p) + } else { + None + } + }) + .collect(); + entries.sort(); + + for path in entries { + let stem = path.file_stem().unwrap().to_str().unwrap(); + let name = stem.strip_prefix("openai_").unwrap(); + if LITELLM_QUIRKS_TO_SKIP.contains(&name) { + skipped += 1; + continue; + } + let golden_path = root.join(format!("anthropic_{name}.jsonl")); + if !golden_path.exists() { + missing += 1; + continue; + } + let sse_text = std::fs::read_to_string(&path).unwrap(); + let chunks = parse_sse(&sse_text); + let mut actual = run_translator(&chunks); + let mut golden = load_jsonl(&golden_path); + mask_message_id(&mut actual); + mask_message_id(&mut golden); + let actual_norm: Vec = actual.iter().map(normalize).collect(); + let golden_norm: Vec = golden.iter().map(normalize).collect(); + + if actual_norm != golden_norm { + failures.push(format!( + "case {name}:\n expected (LiteLLM): {}\n actual (cc_convert): {}", + serde_json::to_string_pretty(&Value::Array(golden_norm)).unwrap(), + serde_json::to_string_pretty(&Value::Array(actual_norm)).unwrap(), + )); + } + checked += 1; + } + + if !failures.is_empty() { + panic!( + "{}/{} stream fixtures diverged:\n\n{}", + failures.len(), + checked, + failures.join("\n---\n") + ); + } + eprintln!( + "stream parity OK: {checked} fixtures matched LiteLLM \ + ({skipped} skipped LiteLLM-quirks, {missing} missing goldens)" + ); +} diff --git a/sidecars/cc_convert/crates/cc_convert_core/tests/request_translation.rs b/sidecars/cc_convert/crates/cc_convert_core/tests/request_translation.rs new file mode 100644 index 0000000..510beac --- /dev/null +++ b/sidecars/cc_convert/crates/cc_convert_core/tests/request_translation.rs @@ -0,0 +1,657 @@ +//! Cases 1–20 from the plan: request translation (Anthropic → OpenAI). + +use cc_convert_core::anthropic::AnthropicRequest; +use cc_convert_core::{anthropic_request_to_openai, ConvertOptions}; +use serde_json::{json, Value}; + +/// Translate via the **pragmatic** options (single-text collapse, max_completion_tokens +/// for reasoning models, stream_options.include_usage). The LiteLLM-parity test suite +/// uses `ConvertOptions::litellm_compat()` instead. +fn convert(req_json: Value) -> (Value, Value) { + let req: AnthropicRequest = + serde_json::from_value(req_json).expect("parse anthropic request"); + let (openai_req, tool_map) = + anthropic_request_to_openai(&req, &ConvertOptions::pragmatic()).expect("translate"); + let openai_value = serde_json::to_value(&openai_req).expect("serialize openai"); + let map_value = serde_json::to_value(&tool_map).expect("serialize tool map"); + (openai_value, map_value) +} + +#[test] +fn case01_plain_user_text() { + let (out, _) = convert(json!({ + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hello"}] + })); + assert_eq!(out["model"], "gpt-4o-mini"); + assert_eq!(out["max_tokens"], 100); + assert_eq!(out["messages"][0]["role"], "user"); + assert_eq!(out["messages"][0]["content"], "hello"); +} + +#[test] +fn case02_system_string() { + let (out, _) = convert(json!({ + "model": "gpt-4o-mini", + "max_tokens": 100, + "system": "Be concise.", + "messages": [{"role": "user", "content": "hi"}] + })); + assert_eq!(out["messages"][0]["role"], "system"); + assert_eq!(out["messages"][0]["content"], "Be concise."); + assert_eq!(out["messages"][1]["role"], "user"); +} + +#[test] +fn case03_system_blocks_with_cache_control_drops_cache_control() { + let (out, _) = convert(json!({ + "model": "gpt-4o-mini", + "max_tokens": 100, + "system": [ + {"type": "text", "text": "rule 1", "cache_control": {"type": "ephemeral"}} + ], + "messages": [{"role": "user", "content": "hi"}] + })); + assert_eq!(out["messages"][0]["role"], "system"); + // Single-text block collapses to a string. + assert_eq!(out["messages"][0]["content"], "rule 1"); + // No cache_control survives on the OpenAI side. + assert!(serde_json::to_string(&out).unwrap().find("cache_control").is_none()); +} + +#[test] +fn case04_multi_turn() { + let (out, _) = convert(json!({ + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello!"}, + {"role": "user", "content": "ok"} + ] + })); + let msgs = out["messages"].as_array().unwrap(); + assert_eq!(msgs.len(), 3); + assert_eq!(msgs[1]["role"], "assistant"); + assert_eq!(msgs[1]["content"], "hello!"); +} + +#[test] +fn case05_user_image_base64() { + let (out, _) = convert(json!({ + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{ + "role": "user", + "content": [ + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "AAAA"}}, + {"type": "text", "text": "what is this?"} + ] + }] + })); + let parts = out["messages"][0]["content"].as_array().unwrap(); + assert_eq!(parts[0]["type"], "image_url"); + assert_eq!(parts[0]["image_url"]["url"], "data:image/png;base64,AAAA"); + assert_eq!(parts[1]["type"], "text"); +} + +#[test] +fn case06_user_image_url() { + let (out, _) = convert(json!({ + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{ + "role": "user", + "content": [ + {"type": "image", "source": {"type": "url", "url": "https://example.com/x.png"}} + ] + }] + })); + let parts = out["messages"][0]["content"].as_array().unwrap(); + assert_eq!(parts[0]["image_url"]["url"], "https://example.com/x.png"); +} + +#[test] +fn case07_assistant_single_tool_use() { + let (out, _) = convert(json!({ + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{ + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {"city": "Paris"}} + ] + }] + })); + let tc = &out["messages"][0]["tool_calls"][0]; + assert_eq!(tc["id"], "toolu_1"); + assert_eq!(tc["type"], "function"); + assert_eq!(tc["function"]["name"], "get_weather"); + let args: Value = serde_json::from_str(tc["function"]["arguments"].as_str().unwrap()).unwrap(); + assert_eq!(args, json!({"city": "Paris"})); +} + +#[test] +fn case08_assistant_two_parallel_tool_uses() { + let (out, _) = convert(json!({ + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{ + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "tu_a", "name": "f1", "input": {"a": 1}}, + {"type": "tool_use", "id": "tu_b", "name": "f2", "input": {"b": 2}} + ] + }] + })); + let tcs = out["messages"][0]["tool_calls"].as_array().unwrap(); + assert_eq!(tcs.len(), 2); + assert_eq!(tcs[0]["id"], "tu_a"); + assert_eq!(tcs[1]["id"], "tu_b"); +} + +#[test] +fn case09_user_single_tool_result() { + let (out, _) = convert(json!({ + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{ + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "toolu_1", "content": "21C"} + ] + }] + })); + let msgs = out["messages"].as_array().unwrap(); + assert_eq!(msgs.len(), 1); + assert_eq!(msgs[0]["role"], "tool"); + assert_eq!(msgs[0]["tool_call_id"], "toolu_1"); + assert_eq!(msgs[0]["content"], "21C"); +} + +#[test] +fn case10_user_three_tool_results_emit_three_messages() { + let (out, _) = convert(json!({ + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{ + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": "a"}, + {"type": "tool_result", "tool_use_id": "t2", "content": "b"}, + {"type": "tool_result", "tool_use_id": "t3", "content": "c"} + ] + }] + })); + let msgs = out["messages"].as_array().unwrap(); + assert_eq!(msgs.len(), 3); + for (i, expected_id) in ["t1", "t2", "t3"].iter().enumerate() { + assert_eq!(msgs[i]["role"], "tool"); + assert_eq!(msgs[i]["tool_call_id"], *expected_id); + } +} + +#[test] +fn case11_user_tool_result_multipart_keeps_one_message() { + let (out, _) = convert(json!({ + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{ + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": [ + {"type": "text", "text": "see image:"}, + {"type": "image", "source": {"type": "url", "url": "https://x/y.png"}} + ]} + ] + }] + })); + let msgs = out["messages"].as_array().unwrap(); + assert_eq!(msgs.len(), 1); + assert_eq!(msgs[0]["role"], "tool"); + let parts = msgs[0]["content"].as_array().unwrap(); + assert_eq!(parts.len(), 2); + assert_eq!(parts[1]["type"], "image_url"); +} + +#[test] +fn case12_tools_input_schema_passthrough() { + let (out, _) = convert(json!({ + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "tools": [{ + "name": "get_weather", + "description": "weather lookup", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]} + }] + })); + let tool = &out["tools"][0]; + assert_eq!(tool["type"], "function"); + assert_eq!(tool["function"]["name"], "get_weather"); + assert_eq!(tool["function"]["description"], "weather lookup"); + assert_eq!(tool["function"]["parameters"]["properties"]["city"]["type"], "string"); +} + +#[test] +fn case13_long_tool_name_truncated() { + let long_name: String = "x".repeat(80); + let (out, map) = convert(json!({ + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "tools": [{ + "name": long_name, + "input_schema": {"type": "object"} + }] + })); + let translated = out["tools"][0]["function"]["name"].as_str().unwrap(); + assert!(translated.len() <= 64); + assert!(translated.starts_with("x")); + let map_obj = map.as_object().unwrap(); + assert_eq!(map_obj.get(translated).unwrap().as_str().unwrap(), &long_name); +} + +#[test] +fn case14_tool_choice_any_becomes_required() { + let (out, _) = convert(json!({ + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "tool_choice": {"type": "any"} + })); + assert_eq!(out["tool_choice"], "required"); +} + +#[test] +fn case15_tool_choice_named() { + let (out, _) = convert(json!({ + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "tools": [{"name": "f", "input_schema": {"type":"object"}}], + "tool_choice": {"type": "tool", "name": "f"} + })); + assert_eq!(out["tool_choice"]["type"], "function"); + assert_eq!(out["tool_choice"]["function"]["name"], "f"); +} + +#[test] +fn case16_metadata_user_id() { + let (out, _) = convert(json!({ + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"user_id": "u-123"} + })); + assert_eq!(out["user"], "u-123"); +} + +#[test] +fn case17_thinking_budget_bucketed_to_medium() { + let (out, _) = convert(json!({ + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "thinking": {"type": "enabled", "budget_tokens": 5000} + })); + assert_eq!(out["reasoning_effort"], "medium"); +} + +#[test] +fn case18_top_k_dropped() { + let (out, _) = convert(json!({ + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "top_k": 20 + })); + assert!(out.get("top_k").is_none()); +} + +#[test] +fn case19_stream_injects_include_usage() { + let (out, _) = convert(json!({ + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "stream": true + })); + assert_eq!(out["stream"], true); + assert_eq!(out["stream_options"]["include_usage"], true); +} + +#[test] +fn case20_o_series_uses_max_completion_tokens() { + let (out, _) = convert(json!({ + "model": "o3-mini", + "max_tokens": 200, + "messages": [{"role": "user", "content": "hi"}] + })); + assert_eq!(out["max_completion_tokens"], 200); + assert!(out.get("max_tokens").is_none()); +} + +#[test] +fn thinking_history_becomes_reasoning_content_in_pragmatic() { + // Anthropic prior-turn assistant message with a `thinking` block → + // pragmatic mode should emit `reasoning_content: ` on the + // assistant message and NOT emit `thinking_blocks` (LiteLLM-internal + // shape that no real upstream consumes). + let (out, _) = convert(json!({ + "model": "claude-opus-4-7", + "max_tokens": 100, + "messages": [ + {"role": "user", "content": "Hard math problem"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "Step 1...\nStep 2...", "signature": "sig_x"}, + {"type": "text", "text": "The answer is 42."} + ] + }, + {"role": "user", "content": "Why?"} + ] + })); + let assistant = &out["messages"][1]; + assert_eq!(assistant["role"], "assistant"); + assert_eq!(assistant["content"], "The answer is 42."); + assert_eq!(assistant["reasoning_content"], "Step 1...\nStep 2..."); + assert!( + assistant.get("thinking_blocks").is_none(), + "thinking_blocks must not appear in pragmatic mode" + ); +} + +#[test] +fn thinking_history_becomes_thinking_blocks_in_litellm_compat() { + use cc_convert_core::ReasoningPassthrough; + let mut opts = ConvertOptions::litellm_compat(); + opts.reasoning_passthrough = ReasoningPassthrough::LiteLLMThinkingBlocks; + let req: AnthropicRequest = serde_json::from_value(json!({ + "model": "claude-opus-4-7", + "max_tokens": 100, + "messages": [ + {"role": "user", "content": "Q"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "thoughts", "signature": "sig_x"}, + {"type": "text", "text": "A."} + ] + } + ] + })) + .unwrap(); + let (req2, _) = anthropic_request_to_openai(&req, &opts).unwrap(); + let v = serde_json::to_value(&req2).unwrap(); + let asst = &v["messages"][1]; + assert!(asst.get("reasoning_content").is_none()); + let tb = asst["thinking_blocks"].as_array().unwrap(); + assert_eq!(tb[0]["type"], "thinking"); + assert_eq!(tb[0]["thinking"], "thoughts"); + assert_eq!(tb[0]["signature"], "sig_x"); +} + +#[test] +fn thinking_drop_mode_omits_both_fields() { + use cc_convert_core::ReasoningPassthrough; + let mut opts = ConvertOptions::pragmatic(); + opts.reasoning_passthrough = ReasoningPassthrough::Drop; + let req: AnthropicRequest = serde_json::from_value(json!({ + "model": "claude-opus-4-7", + "max_tokens": 100, + "messages": [ + {"role": "assistant", "content": [ + {"type": "thinking", "thinking": "secret", "signature": "s"}, + {"type": "text", "text": "visible"} + ]} + ] + })) + .unwrap(); + let (req2, _) = anthropic_request_to_openai(&req, &opts).unwrap(); + let v = serde_json::to_value(&req2).unwrap(); + let asst = &v["messages"][0]; + assert_eq!(asst["content"], "visible"); + assert!(asst.get("reasoning_content").is_none()); + assert!(asst.get("thinking_blocks").is_none()); +} + +#[test] +fn hosted_tools_are_dropped_not_forwarded() { + // Anthropic hosted tools (web_search_*, computer_*, bash_*, etc.) have + // NO OpenAI equivalent — forwarding them produces HTTP 400 because + // OpenAI's tools array only accepts {type:"function"}. We drop them. + let (out, _) = convert(json!({ + "model": "claude-opus-4-7", + "max_tokens": 100, + "messages": [{"role": "user", "content": "search the web"}], + "tools": [ + { + "type": "web_search_20250305", + "name": "web_search", + "max_uses": 5 + }, + { + "type": "computer_20241022", + "name": "computer", + "display_width_px": 1280, + "display_height_px": 800 + }, + { + "type": "bash_20250124", + "name": "bash" + }, + { + "type": "text_editor_20250124", + "name": "str_replace_editor" + }, + { + "name": "get_weather", + "description": "Get weather for a city", + "input_schema": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"] + } + } + ] + })); + // Only the client tool survives; hosted ones are dropped. + let tools = out["tools"].as_array().unwrap(); + assert_eq!(tools.len(), 1, "only the client tool should remain"); + assert_eq!(tools[0]["type"], "function"); + assert_eq!(tools[0]["function"]["name"], "get_weather"); + // Verify the serialized request contains nothing from the hosted shapes + let raw = serde_json::to_string(&out).unwrap(); + assert!(!raw.contains("web_search_20250305")); + assert!(!raw.contains("computer_20241022")); + assert!(!raw.contains("bash_20250124")); + assert!(!raw.contains("text_editor_20250124")); +} + +#[test] +fn all_hosted_tools_produces_no_tools_field() { + // If EVERY tool is hosted, we should omit the tools field entirely + // rather than send an empty array (which OpenAI rejects as a no-op). + let (out, _) = convert(json!({ + "model": "claude-opus-4-7", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "tools": [ + {"type": "web_search_20250305", "name": "web_search"} + ] + })); + assert!( + out.get("tools").is_none(), + "tools field should be omitted entirely when all tools were hosted" + ); +} + +#[test] +fn unknown_content_block_types_are_dropped_not_rejected() { + // server_tool_use, web_search_tool_result, code_execution_tool_result, + // mcp_tool_use, etc. — Anthropic-specific server-side content blocks + // that have no OpenAI equivalent. Translator must drop them silently + // rather than 400-ing the upstream or panicking on deserialization. + let (out, _) = convert(json!({ + "model": "claude-opus-4-7", + "max_tokens": 100, + "messages": [ + { + "role": "assistant", + "content": [ + {"type": "text", "text": "Let me search the web."}, + { + "type": "server_tool_use", + "id": "stu_1", + "name": "web_search", + "input": {"query": "weather Tokyo"} + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "web_search_tool_result", + "tool_use_id": "stu_1", + "content": [ + {"type": "web_search_result", "url": "https://x/", "title": "T"} + ] + }, + {"type": "text", "text": "Summarize."} + ] + } + ] + })); + // Assistant message kept its text (no server_tool_use leaked to wire). + assert_eq!(out["messages"][0]["role"], "assistant"); + assert_eq!(out["messages"][0]["content"], "Let me search the web."); + // User message kept its text (no web_search_tool_result leaked). + assert_eq!(out["messages"][1]["role"], "user"); + assert_eq!(out["messages"][1]["content"], "Summarize."); + let raw = serde_json::to_string(&out).unwrap(); + assert!(!raw.contains("server_tool_use")); + assert!(!raw.contains("web_search_tool_result")); +} + +#[test] +fn unknown_top_level_fields_are_captured_in_extra_not_silently_dropped() { + // Real Anthropic API clients (Claude Code, OpenCode, Cline, Anthropic + // SDK) routinely send top-level fields beyond the documented schema: + // output_config, context_management, speed, container, mcp_servers, + // service_tier, inference_geo, diagnostics, betas, top-level + // cache_control, etc. Before this fix they were silently dropped at + // deserialization. Now they survive into AnthropicRequest.extra. + let req: AnthropicRequest = serde_json::from_value(json!({ + "model": "claude-opus-4-7", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "speed": "fast", + "output_config": {"effort": "high", "task_budget": 5000}, + "context_management": {"edits": [{"type": "clear_tool_uses_20250919", "keep": 5}]}, + "container": {"id": "cnt_123", "skills": ["python"]}, + "inference_geo": "us-east-1", + "service_tier": "standard_only", + "mcp_servers": [{"name": "fs", "url": "http://x"}], + "diagnostics": {"previous_message_id": "msg_xyz"}, + "betas": ["interleaved-thinking-2025-05-14"] + })) + .expect("unknown fields must NOT cause deserialization to fail"); + assert!(req.extra.contains_key("speed")); + assert!(req.extra.contains_key("output_config")); + assert!(req.extra.contains_key("context_management")); + assert!(req.extra.contains_key("container")); + assert!(req.extra.contains_key("inference_geo")); + assert!(req.extra.contains_key("service_tier")); + assert!(req.extra.contains_key("mcp_servers")); + assert!(req.extra.contains_key("diagnostics")); + assert!(req.extra.contains_key("betas")); +} + +#[test] +fn output_config_effort_overrides_thinking_budget_bucket() { + // Claude Code / OpenCode / Cline use `output_config.effort` to set + // reasoning_effort directly; cc_convert should respect it INSTEAD of + // the bucket derived from thinking.budget_tokens. + let (out, _) = convert(json!({ + "model": "claude-opus-4-7", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + // Bucket would say "low" (3000 → low) + "thinking": {"type": "enabled", "budget_tokens": 3000}, + // But the explicit effort says "high" + "output_config": {"effort": "high"} + })); + assert_eq!(out["reasoning_effort"], "high"); +} + +#[test] +fn output_config_effort_works_without_thinking() { + let (out, _) = convert(json!({ + "model": "claude-opus-4-7", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "output_config": {"effort": "minimal"} + })); + assert_eq!(out["reasoning_effort"], "minimal"); +} + +#[test] +fn thinking_budget_still_works_when_no_explicit_effort() { + // Backwards-compat: thinking.budget_tokens still buckets as before. + let (out, _) = convert(json!({ + "model": "claude-opus-4-7", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "thinking": {"type": "enabled", "budget_tokens": 10000} + })); + assert_eq!(out["reasoning_effort"], "high"); +} + +#[test] +fn service_tier_anthropic_to_openai_mapping() { + // Anthropic "standard_only" → OpenAI "default" + let (out, _) = convert(json!({ + "model": "claude-opus-4-7", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "service_tier": "standard_only" + })); + assert_eq!(out["service_tier"], "default"); + + // "auto" passes through + let (out, _) = convert(json!({ + "model": "claude-opus-4-7", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "service_tier": "auto" + })); + assert_eq!(out["service_tier"], "auto"); + + // OpenAI-native values (priority/flex/scale) pass through verbatim + let (out, _) = convert(json!({ + "model": "claude-opus-4-7", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "service_tier": "priority" + })); + assert_eq!(out["service_tier"], "priority"); +} + +#[test] +fn metadata_user_id_passthrough_even_when_stringified_json() { + // Claude Code stuffs {device_id, account_uuid, session_id} into the + // user_id STRING as serialized JSON. We just pass it through to + // OpenAI `user` verbatim — no parsing, no rejection. + let claude_code_user_id = r#"{"device_id":"a3f7","account_uuid":"01HX","session_id":"d4e2"}"#; + let (out, _) = convert(json!({ + "model": "claude-opus-4-7", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"user_id": claude_code_user_id} + })); + assert_eq!(out["user"], claude_code_user_id); +} diff --git a/sidecars/cc_convert/crates/cc_convert_core/tests/response_translation.rs b/sidecars/cc_convert/crates/cc_convert_core/tests/response_translation.rs new file mode 100644 index 0000000..64adca8 --- /dev/null +++ b/sidecars/cc_convert/crates/cc_convert_core/tests/response_translation.rs @@ -0,0 +1,194 @@ +//! Cases 21–26 from the plan: response translation (OpenAI → Anthropic). + +use cc_convert_core::anthropic::AnthropicStopReason; +use cc_convert_core::openai::OpenAIResponse; +use cc_convert_core::tool_names::ToolNameMap; +use cc_convert_core::openai_response_to_anthropic; +use serde_json::{json, Value}; + +fn convert(resp: Value, model: &str, map: Option) -> Value { + let parsed: OpenAIResponse = serde_json::from_value(resp).expect("parse"); + let tm: ToolNameMap = match map { + Some(v) => serde_json::from_value(v).unwrap(), + None => ToolNameMap::new(), + }; + let out = openai_response_to_anthropic(&parsed, model, &tm).expect("translate"); + serde_json::to_value(&out).unwrap() +} + +#[test] +fn case21_plain_text_response() { + let out = convert( + json!({ + "id": "chatcmpl-abc", + "model": "gpt-4o-mini", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "hello world"}, + "finish_reason": "stop" + }], + "usage": {"prompt_tokens": 10, "completion_tokens": 4} + }), + "claude-opus-4-7", + None, + ); + assert_eq!(out["id"], "msg_abc"); + assert_eq!(out["model"], "claude-opus-4-7"); + assert_eq!(out["role"], "assistant"); + assert_eq!(out["type"], "message"); + assert_eq!(out["content"][0]["type"], "text"); + assert_eq!(out["content"][0]["text"], "hello world"); + assert_eq!(out["stop_reason"], "end_turn"); + assert_eq!(out["usage"]["input_tokens"], 10); + assert_eq!(out["usage"]["output_tokens"], 4); +} + +#[test] +fn case22_empty_content_emits_empty_text() { + let out = convert( + json!({ + "id": "chatcmpl-1", + "model": "gpt-4o-mini", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": null}, + "finish_reason": "stop" + }] + }), + "claude-opus-4-7", + None, + ); + assert_eq!(out["content"][0]["type"], "text"); + assert_eq!(out["content"][0]["text"], ""); +} + +#[test] +fn case23_single_tool_call_no_text() { + let out = convert( + json!({ + "id": "chatcmpl-x", + "model": "gpt-4o-mini", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [{ + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": "{\"city\":\"Paris\"}"} + }] + }, + "finish_reason": "tool_calls" + }] + }), + "claude-opus-4-7", + None, + ); + assert_eq!(out["content"].as_array().unwrap().len(), 1); + assert_eq!(out["content"][0]["type"], "tool_use"); + assert_eq!(out["content"][0]["name"], "get_weather"); + assert_eq!(out["content"][0]["id"], "call_1"); + assert_eq!(out["content"][0]["input"]["city"], "Paris"); + assert_eq!(out["stop_reason"], "tool_use"); +} + +#[test] +fn case24_multiple_tool_calls() { + let out = convert( + json!({ + "id": "chatcmpl-x", + "model": "gpt-4o-mini", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + {"id": "c1", "type": "function", "function": {"name": "f", "arguments": "{}"}}, + {"id": "c2", "type": "function", "function": {"name": "g", "arguments": "{}"}} + ] + }, + "finish_reason": "tool_calls" + }] + }), + "claude-opus-4-7", + None, + ); + let blocks = out["content"].as_array().unwrap(); + assert_eq!(blocks.len(), 2); + assert_eq!(blocks[0]["name"], "f"); + assert_eq!(blocks[1]["name"], "g"); +} + +#[test] +fn case25_length_finish_maps_to_max_tokens() { + let out = convert( + json!({ + "id": "chatcmpl-1", + "model": "gpt-4o-mini", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "trunc"}, + "finish_reason": "length" + }] + }), + "claude-opus-4-7", + None, + ); + assert_eq!(out["stop_reason"], "max_tokens"); +} + +#[test] +fn case26_cached_tokens_mapped() { + let out = convert( + json!({ + "id": "chatcmpl-1", + "model": "gpt-4o-mini", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "hi"}, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 2, + "prompt_tokens_details": {"cached_tokens": 32} + } + }), + "claude-opus-4-7", + None, + ); + assert_eq!(out["usage"]["cache_read_input_tokens"], 32); +} + +#[test] +fn long_tool_name_round_trip_uses_map() { + let long = "x".repeat(80); + let mut m = ToolNameMap::new(); + let translated = m.translate(&long); + let _ = AnthropicStopReason::EndTurn; // touch enum so it doesn't go unused + + let out = convert( + json!({ + "id": "chatcmpl-x", + "model": "gpt-4o-mini", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [{ + "id": "c1", + "type": "function", + "function": {"name": translated, "arguments": "{}"} + }] + }, + "finish_reason": "tool_calls" + }] + }), + "claude-opus-4-7", + Some(serde_json::to_value(&m).unwrap()), + ); + assert_eq!(out["content"][0]["name"], long); +} diff --git a/sidecars/cc_convert/crates/cc_convert_core/tests/stream_translation.rs b/sidecars/cc_convert/crates/cc_convert_core/tests/stream_translation.rs new file mode 100644 index 0000000..435a925 --- /dev/null +++ b/sidecars/cc_convert/crates/cc_convert_core/tests/stream_translation.rs @@ -0,0 +1,190 @@ +//! Cases 27–31 from the plan: streaming translation (OpenAI SSE → Anthropic SSE). + +use cc_convert_core::anthropic::{AnthropicEvent, BlockDelta, StreamingContentBlock}; +use cc_convert_core::openai::OpenAIStreamChunk; +use cc_convert_core::tool_names::ToolNameMap; +use cc_convert_core::{StreamConvertOptions, StreamTranslator}; +use serde_json::{json, Value}; + +fn chunk(v: Value) -> OpenAIStreamChunk { + serde_json::from_value(v).expect("parse chunk") +} + +fn make() -> StreamTranslator { + // Use anthropic-native preset so ping + lazy block opening + stop_sequence + // assumptions hold for these unit tests. The litellm_compat parity test + // is separate. + StreamTranslator::with_options( + "claude-opus-4-7".to_string(), + ToolNameMap::new(), + StreamConvertOptions::anthropic_native(), + ) +} + +#[test] +fn case27_text_only_stream() { + let mut t = make(); + let mut all = Vec::new(); + all.extend(t.push_openai_chunk(&chunk(json!({ + "id": "chatcmpl-1", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "hel"}}] + })))); + all.extend(t.push_openai_chunk(&chunk(json!({ + "id": "chatcmpl-1", + "choices": [{"index": 0, "delta": {"content": "lo"}}] + })))); + all.extend(t.push_openai_chunk(&chunk(json!({ + "id": "chatcmpl-1", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}] + })))); + + // Expect: message_start, ping, content_block_start(text), 2 deltas, content_block_stop, + // message_delta, message_stop. + assert!(matches!(all[0], AnthropicEvent::MessageStart { .. })); + assert!(matches!(all[1], AnthropicEvent::Ping)); + assert!(matches!(all[2], AnthropicEvent::ContentBlockStart { ref content_block, .. } if matches!(content_block, StreamingContentBlock::Text { .. }))); + let mut text_seen = String::new(); + for ev in &all { + if let AnthropicEvent::ContentBlockDelta { delta: BlockDelta::TextDelta { text }, .. } = ev { + text_seen.push_str(text); + } + } + assert_eq!(text_seen, "hello"); + assert!(matches!(all[all.len() - 1], AnthropicEvent::MessageStop)); +} + +#[test] +fn case28_single_tool_call_stream_with_fragments() { + let mut t = make(); + let mut all = Vec::new(); + all.extend(t.push_openai_chunk(&chunk(json!({ + "id": "chatcmpl-x", + "choices": [{"index": 0, "delta": { + "tool_calls": [{ + "index": 0, "id": "call_1", "type": "function", + "function": {"name": "get_weather", "arguments": ""} + }] + }}] + })))); + all.extend(t.push_openai_chunk(&chunk(json!({ + "id": "chatcmpl-x", + "choices": [{"index": 0, "delta": { + "tool_calls": [{ + "index": 0, "id": "call_1", "type": "function", + "function": {"arguments": "{\"city\":"} + }] + }}] + })))); + all.extend(t.push_openai_chunk(&chunk(json!({ + "id": "chatcmpl-x", + "choices": [{"index": 0, "delta": { + "tool_calls": [{ + "index": 0, "id": "call_1", "type": "function", + "function": {"arguments": "\"Paris\"}"} + }] + }}] + })))); + all.extend(t.push_openai_chunk(&chunk(json!({ + "id": "chatcmpl-x", + "choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}] + })))); + + // First ContentBlockStart should be a tool_use named get_weather. + let starts: Vec<_> = all + .iter() + .filter_map(|e| match e { + AnthropicEvent::ContentBlockStart { content_block, index } => Some((index, content_block)), + _ => None, + }) + .collect(); + assert_eq!(starts.len(), 1); + assert!(matches!(starts[0].1, StreamingContentBlock::ToolUse { name, .. } if name == "get_weather")); + + // Accumulated input_json_deltas should reconstruct the JSON. + let mut buf = String::new(); + for ev in &all { + if let AnthropicEvent::ContentBlockDelta { delta: BlockDelta::InputJsonDelta { partial_json }, .. } = ev { + buf.push_str(partial_json); + } + } + assert_eq!(buf, "{\"city\":\"Paris\"}"); + + // Final stop_reason must be tool_use. + let msg_delta = all.iter().find_map(|e| match e { + AnthropicEvent::MessageDelta { delta, .. } => Some(delta), + _ => None, + }).unwrap(); + assert_eq!(msg_delta.stop_reason, Some(cc_convert_core::anthropic::AnthropicStopReason::ToolUse)); +} + +#[test] +fn case29_two_parallel_tool_calls_get_distinct_indices() { + let mut t = make(); + let mut all = Vec::new(); + all.extend(t.push_openai_chunk(&chunk(json!({ + "id": "chatcmpl-y", + "choices": [{"index": 0, "delta": { + "tool_calls": [ + {"index": 0, "id": "a", "type": "function", "function": {"name": "f", "arguments": "{}"}}, + {"index": 1, "id": "b", "type": "function", "function": {"name": "g", "arguments": "{}"}} + ] + }}] + })))); + all.extend(t.push_openai_chunk(&chunk(json!({ + "id": "chatcmpl-y", + "choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}] + })))); + + let mut indices: Vec = all.iter().filter_map(|e| match e { + AnthropicEvent::ContentBlockStart { index, .. } => Some(*index), + _ => None, + }).collect(); + indices.sort(); + assert_eq!(indices, vec![0, 1]); +} + +#[test] +fn case30_stream_ends_without_finish_reason() { + let mut t = make(); + let mut all = Vec::new(); + all.extend(t.push_openai_chunk(&chunk(json!({ + "id": "chatcmpl-z", + "choices": [{"index": 0, "delta": {"content": "partial"}}] + })))); + // No finish_reason chunk arrives — caller calls finish(). + all.extend(t.finish()); + + // We must still see content_block_stop, message_delta (end_turn), message_stop. + let has_stop = all.iter().any(|e| matches!(e, AnthropicEvent::MessageStop)); + let stop_reason_end_turn = all.iter().any(|e| matches!(e, + AnthropicEvent::MessageDelta { delta, .. } if delta.stop_reason == Some(cc_convert_core::anthropic::AnthropicStopReason::EndTurn) + )); + assert!(has_stop); + assert!(stop_reason_end_turn); +} + +#[test] +fn case31_reasoning_content_emits_thinking_block() { + let mut t = make(); + let mut all = Vec::new(); + all.extend(t.push_openai_chunk(&chunk(json!({ + "id": "chatcmpl-r", + "choices": [{"index": 0, "delta": {"reasoning_content": "let me think..."}}] + })))); + all.extend(t.push_openai_chunk(&chunk(json!({ + "id": "chatcmpl-r", + "choices": [{"index": 0, "delta": {"content": "Done."}}] + })))); + all.extend(t.push_openai_chunk(&chunk(json!({ + "id": "chatcmpl-r", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}] + })))); + + // First ContentBlockStart must be Thinking; second must be Text. + let starts: Vec<_> = all.iter().filter_map(|e| match e { + AnthropicEvent::ContentBlockStart { content_block, .. } => Some(content_block), + _ => None, + }).collect(); + assert!(matches!(starts[0], StreamingContentBlock::Thinking { .. })); + assert!(matches!(starts[1], StreamingContentBlock::Text { .. })); +} diff --git a/sidecars/cc_convert/crates/cc_convert_core/tests/vendor_quirks.rs b/sidecars/cc_convert/crates/cc_convert_core/tests/vendor_quirks.rs new file mode 100644 index 0000000..3dbcbdf --- /dev/null +++ b/sidecars/cc_convert/crates/cc_convert_core/tests/vendor_quirks.rs @@ -0,0 +1,515 @@ +//! vLLM- and SGLang-specific quirks. These tests verify that the translator +//! handles the non-standard fields and shapes these self-hosted +//! OpenAI-compatible servers emit, without panicking and producing +//! sensible Anthropic-side output. +//! +//! Source-grounded against: +//! - vllm-project/vllm `vllm/entrypoints/openai/chat_completion/protocol.py` +//! (`reasoning` field, `stop_reason` field, `routed_experts`) +//! - sgl-project/sglang `python/sglang/srt/entrypoints/openai/protocol.py` +//! and `serving_chat.py` (`reasoning_content` null-everywhere, +//! null id/name on continuation tool_call chunks, `matched_stop`, +//! `finish_reason: "abort"`) + +use cc_convert_core::anthropic::{ + AnthropicEvent, AnthropicStopReason, BlockDelta, StreamingContentBlock, +}; +use cc_convert_core::openai::{OpenAIResponse, OpenAIStreamChunk}; +use cc_convert_core::resp_to_anthropic::openai_response_to_anthropic; +use cc_convert_core::tool_names::ToolNameMap; +use cc_convert_core::{StreamConvertOptions, StreamTranslator}; +use serde_json::{json, Value}; + +fn translate_response(raw: Value) -> Value { + let resp: OpenAIResponse = serde_json::from_value(raw).expect("parse OpenAIResponse"); + let out = openai_response_to_anthropic(&resp, "claude-opus-4-7", &ToolNameMap::new()) + .expect("translate"); + serde_json::to_value(&out).unwrap() +} + +fn translator_native() -> StreamTranslator { + StreamTranslator::with_options( + "claude-opus-4-7".to_string(), + ToolNameMap::new(), + StreamConvertOptions::anthropic_native(), + ) +} + +fn push(t: &mut StreamTranslator, raw: Value) -> Vec { + let chunk: OpenAIStreamChunk = serde_json::from_value(raw).expect("parse chunk"); + t.push_openai_chunk(&chunk) +} + +// ---------- vLLM ---------- + +#[test] +fn vllm_response_with_stop_reason_and_extra_fields_does_not_panic() { + // vLLM emits stop_reason, prompt_logprobs, prompt_token_ids alongside + // the standard fields. Our deserializer must ignore them gracefully. + let raw = json!({ + "id": "chatcmpl-abc", + "model": "Qwen/Qwen2.5-7B-Instruct", + "object": "chat.completion", + "prompt_logprobs": null, + "prompt_token_ids": [1, 2, 3], + "prompt_text": "hi", + "kv_transfer_params": null, + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": "hello back", + "reasoning": null, + "tool_calls": [] + }, + "finish_reason": "stop", + "stop_reason": "<|im_end|>", + "token_ids": null, + "routed_experts": null + }], + "usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7} + }); + let out = translate_response(raw); + assert_eq!(out["content"][0]["type"], "text"); + assert_eq!(out["content"][0]["text"], "hello back"); + assert_eq!(out["stop_reason"], "end_turn"); +} + +#[test] +fn vllm_response_with_reasoning_field_extracted_as_thinking() { + // vLLM uses `reasoning`, not `reasoning_content`. We support both as + // aliases in our deserializer (see openai.rs OpenAIChoiceMessage). + let raw = json!({ + "id": "chatcmpl-x", + "model": "deepseek-r1", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": "The answer is 42.", + "reasoning": "Let me think about the problem..." + }, + "finish_reason": "stop" + }] + }); + let out = translate_response(raw); + let content = out["content"].as_array().unwrap(); + assert_eq!(content[0]["type"], "thinking"); + assert_eq!(content[0]["thinking"], "Let me think about the problem..."); + assert_eq!(content[1]["type"], "text"); + assert_eq!(content[1]["text"], "The answer is 42."); +} + +#[test] +fn vllm_stream_reasoning_delta_via_reasoning_field() { + let mut t = translator_native(); + let mut all = Vec::new(); + all.extend(push( + &mut t, + json!({ + "id": "chatcmpl-r", + "choices": [{ + "index": 0, + "delta": {"role": "assistant", "reasoning": "Let me think"} + }] + }), + )); + all.extend(push( + &mut t, + json!({ + "id": "chatcmpl-r", + "choices": [{"index": 0, "delta": {"reasoning": " harder"}}] + }), + )); + all.extend(push( + &mut t, + json!({ + "id": "chatcmpl-r", + "choices": [{"index": 0, "delta": {"content": "42"}}] + }), + )); + all.extend(push( + &mut t, + json!({ + "id": "chatcmpl-r", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}] + }), + )); + + // The thinking content should be reconstructed from the `reasoning` deltas. + let mut buf = String::new(); + for ev in &all { + if let AnthropicEvent::ContentBlockDelta { + delta: BlockDelta::ThinkingDelta { thinking }, + .. + } = ev + { + buf.push_str(thinking); + } + } + assert_eq!(buf, "Let me think harder"); +} + +#[test] +fn vllm_initial_role_only_chunk_does_not_open_text_block() { + // vLLM always emits a role+empty-content chunk first. Our translator + // should not open a text content_block until real text arrives. + let mut t = translator_native(); + let mut all = Vec::new(); + all.extend(push( + &mut t, + json!({ + "id": "chatcmpl-v", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": ""}}] + }), + )); + // No text block start yet. + let starts: Vec<_> = all + .iter() + .filter(|e| matches!(e, AnthropicEvent::ContentBlockStart { .. })) + .collect(); + assert!(starts.is_empty(), "should not open a block on empty content"); + // Now real text arrives. + all.extend(push( + &mut t, + json!({ + "id": "chatcmpl-v", + "choices": [{"index": 0, "delta": {"content": "hi"}}] + }), + )); + let opened_text = all.iter().any(|e| matches!(e, + AnthropicEvent::ContentBlockStart { + content_block: StreamingContentBlock::Text { .. }, .. + })); + assert!(opened_text); +} + +#[test] +fn vllm_tool_call_id_only_on_first_chunk() { + // vLLM (and SGLang) emit `id` only on the first chunk for a tool_call. + // Continuation chunks have just `index` + `function.arguments`. + let mut t = translator_native(); + let mut all = Vec::new(); + all.extend(push( + &mut t, + json!({ + "id": "chatcmpl-z", + "choices": [{ + "index": 0, + "delta": { + "tool_calls": [{ + "index": 0, "id": "chatcmpl-tool-abc", "type": "function", + "function": {"name": "search"} + }] + } + }] + }), + )); + all.extend(push( + &mut t, + json!({ + "id": "chatcmpl-z", + "choices": [{ + "index": 0, + "delta": { + "tool_calls": [{ + "index": 0, + "function": {"arguments": "{\"q\":"} + }] + } + }] + }), + )); + all.extend(push( + &mut t, + json!({ + "id": "chatcmpl-z", + "choices": [{ + "index": 0, + "delta": { + "tool_calls": [{ + "index": 0, + "function": {"arguments": "\"hi\"}"} + }] + } + }] + }), + )); + all.extend(push( + &mut t, + json!({ + "id": "chatcmpl-z", + "choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}] + }), + )); + + // Exactly one tool_use start, with the id from the first chunk. + let starts: Vec<_> = all + .iter() + .filter_map(|e| match e { + AnthropicEvent::ContentBlockStart { content_block, .. } => Some(content_block), + _ => None, + }) + .collect(); + assert_eq!(starts.len(), 1); + if let StreamingContentBlock::ToolUse { id, name, .. } = starts[0] { + assert_eq!(id, "chatcmpl-tool-abc"); + assert_eq!(name, "search"); + } else { + panic!("expected tool_use start"); + } + + // Fragments concatenate to the full JSON. + let mut buf = String::new(); + for ev in &all { + if let AnthropicEvent::ContentBlockDelta { + delta: BlockDelta::InputJsonDelta { partial_json }, + .. + } = ev + { + buf.push_str(partial_json); + } + } + assert_eq!(buf, "{\"q\":\"hi\"}"); +} + +#[test] +fn vllm_chatcmpl_tool_prefix_id_passes_through() { + // vLLM uses `chatcmpl-tool-` instead of `call_`. We pass it + // through unchanged so downstream agents can correlate. + let raw = json!({ + "id": "chatcmpl-x", + "model": "Qwen", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [{ + "id": "chatcmpl-tool-9f3c2a0b1d3e4f56", + "type": "function", + "function": {"name": "get_weather", "arguments": "{\"city\":\"SF\"}"} + }] + }, + "finish_reason": "tool_calls" + }] + }); + let out = translate_response(raw); + assert_eq!(out["content"][0]["id"], "chatcmpl-tool-9f3c2a0b1d3e4f56"); +} + +// ---------- SGLang ---------- + +#[test] +fn sglang_null_reasoning_content_in_every_delta_is_ignored() { + // SGLang emits `reasoning_content: null` on every SSE chunk. Our + // translator must NOT treat the field's presence-as-null as a signal + // to open a thinking block. + let mut t = translator_native(); + let mut all = Vec::new(); + all.extend(push( + &mut t, + json!({ + "id": "abc", + "choices": [{ + "index": 0, + "delta": {"reasoning_content": null, "role": "assistant", "content": "hi"}, + "finish_reason": null, + "matched_stop": null + }] + }), + )); + all.extend(push( + &mut t, + json!({ + "id": "abc", + "choices": [{ + "index": 0, + "delta": {"reasoning_content": null, "content": " there"} + }] + }), + )); + all.extend(push( + &mut t, + json!({ + "id": "abc", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}] + }), + )); + // No thinking block opened. + let opened_thinking = all.iter().any(|e| matches!(e, + AnthropicEvent::ContentBlockStart { + content_block: StreamingContentBlock::Thinking { .. }, .. + })); + assert!(!opened_thinking, "null reasoning_content should NOT open a thinking block"); +} + +#[test] +fn sglang_null_id_and_name_on_continuation_tool_call_chunks() { + // SGLang sends `id: null` and `function.name: null` on continuation + // chunks (not omitted, but explicitly null). Our deserializer treats + // them as Option=None, which is correct. + let mut t = translator_native(); + let mut all = Vec::new(); + all.extend(push( + &mut t, + json!({ + "id": "abc", + "choices": [{ + "index": 0, + "delta": { + "tool_calls": [{ + "id": "call_5a8b3e2f", + "index": 0, + "type": "function", + "function": {"name": "search", "arguments": ""} + }] + } + }] + }), + )); + all.extend(push( + &mut t, + json!({ + "id": "abc", + "choices": [{ + "index": 0, + "delta": { + "tool_calls": [{ + "id": null, + "index": 0, + "type": "function", + "function": {"name": null, "arguments": "{\"q\":\"x\"}"} + }] + } + }] + }), + )); + all.extend(push( + &mut t, + json!({ + "id": "abc", + "choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}] + }), + )); + + let starts: Vec<_> = all + .iter() + .filter(|e| matches!(e, AnthropicEvent::ContentBlockStart { .. })) + .collect(); + assert_eq!(starts.len(), 1, "exactly one tool_use block opened"); + let mut buf = String::new(); + for ev in &all { + if let AnthropicEvent::ContentBlockDelta { + delta: BlockDelta::InputJsonDelta { partial_json }, + .. + } = ev + { + buf.push_str(partial_json); + } + } + assert_eq!(buf, "{\"q\":\"x\"}"); +} + +#[test] +fn sglang_matched_stop_field_and_top_level_metadata_are_ignored() { + // SGLang adds top-level metadata + sglext + per-choice matched_stop. + let raw = json!({ + "id": "abc", + "model": "Qwen", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "ok", "reasoning_content": null}, + "finish_reason": "stop", + "matched_stop": "<|im_end|>" + }], + "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6, "reasoning_tokens": 0}, + "metadata": {"weight_version": "v42"}, + "sglext": {"cached_tokens_details": {"device": 0, "host": 0}} + }); + let out = translate_response(raw); + assert_eq!(out["content"][0]["text"], "ok"); +} + +#[test] +fn sglang_finish_reason_abort_maps_to_end_turn() { + // SGLang adds the `"abort"` finish_reason. We treat unknowns as + // end_turn (matching LiteLLM's permissive default). + let raw = json!({ + "id": "abc", + "model": "Qwen", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "partial"}, + "finish_reason": "abort" + }] + }); + let out = translate_response(raw); + assert_eq!(out["stop_reason"], "end_turn"); +} + +#[test] +fn sglang_reasoning_content_stream_via_real_field_opens_thinking_block() { + // When `reasoning_content` is a string (not null), we open a thinking + // block. (Only the null-everywhere case from the previous test gets + // ignored.) + let mut t = translator_native(); + let mut all = Vec::new(); + all.extend(push( + &mut t, + json!({ + "id": "abc", + "choices": [{ + "index": 0, + "delta": {"reasoning_content": "Let me think...", "role": "assistant"} + }] + }), + )); + all.extend(push( + &mut t, + json!({ + "id": "abc", + "choices": [{"index": 0, "delta": {"reasoning_content": null, "content": "42"}}] + }), + )); + all.extend(push( + &mut t, + json!({ + "id": "abc", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}] + }), + )); + let opened_thinking = all.iter().any(|e| matches!(e, + AnthropicEvent::ContentBlockStart { + content_block: StreamingContentBlock::Thinking { .. }, .. + })); + assert!(opened_thinking); +} + +#[test] +fn sglang_kimi_k2_tool_id_format_passes_through() { + // SGLang's kimi_k2 parser uses `functions.:` IDs. We must + // accept them on input and emit them back unchanged. + let raw = json!({ + "id": "abc", + "model": "kimi-k2", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [{ + "id": "functions.search:0", + "index": 0, + "type": "function", + "function": {"name": "search", "arguments": "{}"} + }] + }, + "finish_reason": "tool_calls" + }] + }); + let out = translate_response(raw); + assert_eq!(out["content"][0]["id"], "functions.search:0"); + assert_eq!(out["content"][0]["type"], "tool_use"); + let _ = AnthropicStopReason::ToolUse; // keep import live +} diff --git a/sidecars/cc_convert/crates/cc_convert_py/Cargo.toml b/sidecars/cc_convert/crates/cc_convert_py/Cargo.toml new file mode 100644 index 0000000..a772595 --- /dev/null +++ b/sidecars/cc_convert/crates/cc_convert_py/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "cc_convert_py" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "Python bindings for cc_convert_core" + +[lib] +name = "_native" +crate-type = ["cdylib", "rlib"] + +[dependencies] +cc_convert_core = { path = "../cc_convert_core" } +serde.workspace = true +serde_json.workspace = true +# abi3-py38: build a single forward-compatible wheel that works on +# Python 3.8 through future 3.x versions (one wheel per OS/arch instead +# of seven per-Python-minor). PyO3 0.22 supports this since #3653. +pyo3 = { workspace = true, features = ["extension-module", "abi3-py38"] } diff --git a/sidecars/cc_convert/crates/cc_convert_py/src/lib.rs b/sidecars/cc_convert/crates/cc_convert_py/src/lib.rs new file mode 100644 index 0000000..0e2da36 --- /dev/null +++ b/sidecars/cc_convert/crates/cc_convert_py/src/lib.rs @@ -0,0 +1,103 @@ +//! Python bindings: JSON in, JSON out. The Python wrapper marshals +//! dict ↔ JSON so the binding surface stays tiny. + +use cc_convert_core::{ + anthropic_request_to_openai, openai_response_to_anthropic, stream::StreamTranslator, + tool_names::ToolNameMap, ConvertOptions, +}; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use pyo3::types::PyString; + +fn pyerr(e: E) -> PyErr { + PyValueError::new_err(format!("{}", e)) +} + +fn resolve_options(mode: Option<&str>, target_model: Option) -> PyResult { + let mut opts = match mode.unwrap_or("pragmatic") { + "pragmatic" => ConvertOptions::pragmatic(), + "litellm_compat" | "litellm-compat" | "litellm" => ConvertOptions::litellm_compat(), + other => { + return Err(PyValueError::new_err(format!( + "unknown mode {:?}; expected 'pragmatic' or 'litellm_compat'", + other + ))) + } + }; + opts.target_model = target_model; + Ok(opts) +} + +#[pyfunction] +#[pyo3(signature = (anthropic_request_json, target_model=None, mode=None))] +fn translate_request( + anthropic_request_json: &str, + target_model: Option, + mode: Option<&str>, +) -> PyResult<(String, String)> { + let req: cc_convert_core::anthropic::AnthropicRequest = + serde_json::from_str(anthropic_request_json).map_err(pyerr)?; + let opts = resolve_options(mode, target_model)?; + let (openai_req, tool_map) = anthropic_request_to_openai(&req, &opts).map_err(pyerr)?; + let openai_str = serde_json::to_string(&openai_req).map_err(pyerr)?; + let map_str = serde_json::to_string(&tool_map).map_err(pyerr)?; + Ok((openai_str, map_str)) +} + +#[pyfunction] +fn translate_response( + openai_response_json: &str, + original_model: &str, + tool_map_json: &str, +) -> PyResult { + let resp: cc_convert_core::openai::OpenAIResponse = + serde_json::from_str(openai_response_json).map_err(pyerr)?; + let tool_map: ToolNameMap = serde_json::from_str(tool_map_json).map_err(pyerr)?; + let anthropic_resp = + openai_response_to_anthropic(&resp, original_model, &tool_map).map_err(pyerr)?; + Ok(serde_json::to_string(&anthropic_resp).map_err(pyerr)?) +} + +#[pyclass] +struct PyStreamTranslator { + inner: StreamTranslator, +} + +#[pymethods] +impl PyStreamTranslator { + #[new] + fn new(original_model: String, tool_map_json: &str) -> PyResult { + let tool_map: ToolNameMap = serde_json::from_str(tool_map_json).map_err(pyerr)?; + Ok(Self { + inner: StreamTranslator::new(original_model, tool_map), + }) + } + + fn push(&mut self, openai_chunk_json: &str) -> PyResult> { + let chunk: cc_convert_core::openai::OpenAIStreamChunk = + serde_json::from_str(openai_chunk_json).map_err(pyerr)?; + let events = self.inner.push_openai_chunk(&chunk); + events + .iter() + .map(|e| serde_json::to_string(e).map_err(pyerr)) + .collect() + } + + fn finish(&mut self) -> PyResult> { + let events = self.inner.finish(); + events + .iter() + .map(|e| serde_json::to_string(e).map_err(pyerr)) + .collect() + } +} + +#[pymodule] +#[pyo3(name = "_native")] +fn cc_convert_native(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_function(wrap_pyfunction!(translate_request, m)?)?; + m.add_function(wrap_pyfunction!(translate_response, m)?)?; + m.add_class::()?; + m.add("__version__", PyString::new_bound(_py, env!("CARGO_PKG_VERSION")))?; + Ok(()) +} diff --git a/sidecars/cc_convert/crates/cc_convert_sidecar/Cargo.toml b/sidecars/cc_convert/crates/cc_convert_sidecar/Cargo.toml new file mode 100644 index 0000000..81e3d59 --- /dev/null +++ b/sidecars/cc_convert/crates/cc_convert_sidecar/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "cc_convert_sidecar" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "HTTP sidecar proxy that translates Anthropic Messages requests to OpenAI Chat Completions" + +[lib] +name = "cc_convert_sidecar" +path = "src/lib.rs" + +[[bin]] +name = "cc_convert_sidecar" +path = "src/main.rs" + +[dependencies] +cc_convert_core = { path = "../cc_convert_core" } +serde.workspace = true +serde_json.workspace = true +tokio.workspace = true +axum.workspace = true +reqwest.workspace = true +futures.workspace = true +tokio-stream.workspace = true +bytes.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true + +[dev-dependencies] +reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "rustls-tls"] } diff --git a/sidecars/cc_convert/crates/cc_convert_sidecar/src/lib.rs b/sidecars/cc_convert/crates/cc_convert_sidecar/src/lib.rs new file mode 100644 index 0000000..0c66239 --- /dev/null +++ b/sidecars/cc_convert/crates/cc_convert_sidecar/src/lib.rs @@ -0,0 +1,261 @@ +//! HTTP sidecar logic, factored into a library so integration tests can +//! reuse [`AppState`] and [`build_router`]. + +use axum::{ + extract::State, + http::{HeaderMap, StatusCode}, + response::{sse::Event, IntoResponse, Response, Sse}, + routing::{get, post}, + Json, Router, +}; +use bytes::Bytes; +use cc_convert_core::{ + anthropic::{AnthropicEvent, AnthropicRequest, AnthropicResponse}, + anthropic_request_to_openai, + openai::OpenAIStreamChunk, + openai_response_to_anthropic, ConvertOptions, StreamConvertOptions, StreamTranslator, +}; +use futures::stream::{self, Stream, StreamExt}; +use reqwest::Client; +use serde_json::Value; +use std::{collections::VecDeque, sync::Arc}; + +#[derive(Clone)] +pub struct AppState { + pub upstream_url: String, + pub upstream_key: Option, + pub auth_passthrough: bool, + pub http: Client, + /// If true, use ConvertOptions::litellm_compat() (preserve LiteLLM-equivalent + /// behaviour). Default false → ConvertOptions::pragmatic() (collapses + /// single-text content into a string, which is what most real upstreams + /// expect — SGLang/vLLM strict mode rejects list-content on system msgs). + pub litellm_compat: bool, +} + +pub fn build_router(state: Arc) -> Router { + Router::new() + .route("/healthz", get(|| async { "ok" })) + .route("/v1/messages", post(handle_messages)) + .with_state(state) +} + +fn convert_options_for(state: &AppState) -> ConvertOptions { + if state.litellm_compat { + ConvertOptions::litellm_compat() + } else { + ConvertOptions::pragmatic() + } +} + +pub async fn handle_messages( + State(state): State>, + headers: HeaderMap, + Json(req_value): Json, +) -> Response { + let stream_mode = req_value + .get("stream") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + + let original_model = req_value + .get("model") + .and_then(|v| v.as_str()) + .unwrap_or("unknown") + .to_string(); + + let anthropic_req: AnthropicRequest = match serde_json::from_value(req_value) { + Ok(r) => r, + Err(e) => { + return error_response(StatusCode::BAD_REQUEST, "invalid_request_error", &e.to_string()) + } + }; + + let (openai_req, tool_map) = + match anthropic_request_to_openai(&anthropic_req, &convert_options_for(&state)) { + Ok(p) => p, + Err(e) => { + return error_response( + StatusCode::BAD_REQUEST, + "invalid_request_error", + &e.to_string(), + ) + } + }; + + let auth_header = if state.auth_passthrough { + headers + .get("authorization") + .or_else(|| headers.get("x-api-key")) + .and_then(|v| v.to_str().ok()) + .map(|s| { + if s.starts_with("Bearer ") { + s.to_string() + } else { + format!("Bearer {}", s) + } + }) + } else { + state.upstream_key.as_ref().map(|k| format!("Bearer {}", k)) + }; + + let mut req_builder = state + .http + .post(&state.upstream_url) + .json(&openai_req) + .header("content-type", "application/json"); + if let Some(auth) = &auth_header { + req_builder = req_builder.header("authorization", auth); + } + + let upstream_resp = match req_builder.send().await { + Ok(r) => r, + Err(e) => { + return error_response(StatusCode::BAD_GATEWAY, "api_error", &e.to_string()); + } + }; + + let status = upstream_resp.status(); + if !status.is_success() { + let body = upstream_resp + .text() + .await + .unwrap_or_else(|_| "upstream error".to_string()); + return error_response( + StatusCode::from_u16(status.as_u16()).unwrap_or(StatusCode::BAD_GATEWAY), + "api_error", + &body, + ); + } + + if !stream_mode { + let resp_value: Value = match upstream_resp.json().await { + Ok(v) => v, + Err(e) => return error_response(StatusCode::BAD_GATEWAY, "api_error", &e.to_string()), + }; + let openai_resp = match serde_json::from_value(resp_value) { + Ok(r) => r, + Err(e) => return error_response(StatusCode::BAD_GATEWAY, "api_error", &e.to_string()), + }; + let anthropic_resp: AnthropicResponse = + match openai_response_to_anthropic(&openai_resp, &original_model, &tool_map) { + Ok(r) => r, + Err(e) => { + return error_response(StatusCode::BAD_GATEWAY, "api_error", &e.to_string()) + } + }; + return (StatusCode::OK, Json(anthropic_resp)).into_response(); + } + + let translator = StreamTranslator::with_options( + original_model, + tool_map, + StreamConvertOptions::anthropic_native(), + ); + let upstream = upstream_resp.bytes_stream(); + let event_stream = build_sse_stream(translator, upstream); + Sse::new(event_stream).into_response() +} + +fn error_response(status: StatusCode, type_: &str, message: &str) -> Response { + let body = serde_json::json!({ + "type": "error", + "error": { "type": type_, "message": message } + }); + (status, Json(body)).into_response() +} + +struct SseState { + translator: StreamTranslator, + upstream: S, + buffer: Vec, + queued: VecDeque, + upstream_done: bool, + finalized: bool, +} + +pub fn build_sse_stream( + translator: StreamTranslator, + upstream: S, +) -> impl Stream> + Send + 'static +where + S: Stream> + Send + Unpin + 'static, +{ + let init = SseState { + translator, + upstream, + buffer: Vec::new(), + queued: VecDeque::new(), + upstream_done: false, + finalized: false, + }; + stream::unfold(init, |mut st| async move { + loop { + if let Some(ev) = st.queued.pop_front() { + return Some((Ok(anthropic_event_to_sse(&ev)), st)); + } + if st.finalized { + return None; + } + if st.upstream_done { + drain_buffer(&mut st); + st.queued.extend(st.translator.finish()); + st.finalized = true; + continue; + } + match st.upstream.next().await { + Some(Ok(bytes)) => { + st.buffer.extend_from_slice(&bytes); + drain_buffer(&mut st); + } + Some(Err(_)) | None => { + st.upstream_done = true; + } + } + } + }) +} + +fn drain_buffer(st: &mut SseState) { + loop { + let Some(sep_pos) = st.buffer.windows(2).position(|w| w == b"\n\n") else { + break; + }; + let event_bytes: Vec = st.buffer.drain(..sep_pos).collect(); + st.buffer.drain(..2); + let Ok(event_str) = std::str::from_utf8(&event_bytes) else { + continue; + }; + for line in event_str.lines() { + let Some(data) = line.strip_prefix("data:") else { + continue; + }; + let payload = data.trim(); + if payload.is_empty() || payload == "[DONE]" { + continue; + } + let Ok(chunk) = serde_json::from_str::(payload) else { + continue; + }; + let events = st.translator.push_openai_chunk(&chunk); + st.queued.extend(events); + } + } +} + +fn anthropic_event_to_sse(ev: &AnthropicEvent) -> Event { + let (name, value) = (event_name(ev), serde_json::to_string(ev).unwrap_or_default()); + Event::default().event(name).data(value) +} + +fn event_name(ev: &AnthropicEvent) -> &'static str { + match ev { + AnthropicEvent::MessageStart { .. } => "message_start", + AnthropicEvent::Ping => "ping", + AnthropicEvent::ContentBlockStart { .. } => "content_block_start", + AnthropicEvent::ContentBlockDelta { .. } => "content_block_delta", + AnthropicEvent::ContentBlockStop { .. } => "content_block_stop", + AnthropicEvent::MessageDelta { .. } => "message_delta", + AnthropicEvent::MessageStop => "message_stop", + } +} diff --git a/sidecars/cc_convert/crates/cc_convert_sidecar/src/main.rs b/sidecars/cc_convert/crates/cc_convert_sidecar/src/main.rs new file mode 100644 index 0000000..08d183a --- /dev/null +++ b/sidecars/cc_convert/crates/cc_convert_sidecar/src/main.rs @@ -0,0 +1,48 @@ +//! Thin binary entry point. All logic lives in `cc_convert_sidecar::lib`. + +use cc_convert_sidecar::{build_router, AppState}; +use reqwest::Client; +use std::{net::SocketAddr, sync::Arc, time::Duration}; +use tracing_subscriber::EnvFilter; + +fn env_or_default(key: &str, default: &str) -> String { + std::env::var(key).unwrap_or_else(|_| default.to_string()) +} + +#[tokio::main] +async fn main() { + tracing_subscriber::fmt() + .with_env_filter(EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into())) + .init(); + + let listen_addr = env_or_default("CC_CONVERT_LISTEN_ADDR", "0.0.0.0:8787"); + let upstream_url = env_or_default( + "CC_CONVERT_UPSTREAM_URL", + "https://api.openai.com/v1/chat/completions", + ); + let upstream_key = std::env::var("CC_CONVERT_UPSTREAM_API_KEY").ok(); + let auth_passthrough = std::env::var("CC_CONVERT_AUTH_PASSTHROUGH") + .map(|v| v == "1") + .unwrap_or(false); + let litellm_compat = std::env::var("CC_CONVERT_LITELLM_COMPAT") + .map(|v| v == "1") + .unwrap_or(false); + + let state = AppState { + upstream_url, + upstream_key, + auth_passthrough, + http: Client::builder() + .timeout(Duration::from_secs(600)) + .build() + .expect("reqwest client"), + litellm_compat, + }; + + let app = build_router(Arc::new(state)); + + let addr: SocketAddr = listen_addr.parse().expect("invalid CC_CONVERT_LISTEN_ADDR"); + tracing::info!(%addr, "cc_convert_sidecar listening"); + let listener = tokio::net::TcpListener::bind(addr).await.expect("bind"); + axum::serve(listener, app).await.expect("serve"); +} diff --git a/sidecars/cc_convert/crates/cc_convert_sidecar/tests/integration.rs b/sidecars/cc_convert/crates/cc_convert_sidecar/tests/integration.rs new file mode 100644 index 0000000..c7bb1c5 --- /dev/null +++ b/sidecars/cc_convert/crates/cc_convert_sidecar/tests/integration.rs @@ -0,0 +1,264 @@ +//! Integration test: spin up a mock OpenAI-compatible upstream + the sidecar +//! proxy, send an Anthropic-shape request through the proxy, and verify the +//! Anthropic-shape response. +//! +//! Covers: +//! 1. Non-streaming round-trip (request translated + forwarded, response +//! translated back). +//! 2. Streaming round-trip (OpenAI SSE → Anthropic SSE). +//! 3. Upstream 4xx propagated as Anthropic-shape error JSON. + +use axum::{ + body::Body, + extract::State, + http::{header, HeaderMap, StatusCode}, + response::{IntoResponse, Response}, + routing::post, + Json, Router, +}; +use serde_json::{json, Value}; +use std::{ + net::SocketAddr, + sync::{Arc, Mutex}, + time::Duration, +}; +use tokio::net::TcpListener; + +#[derive(Default)] +struct MockState { + last_request: Mutex>, + last_auth: Mutex>, + mode: Mutex, +} + +#[derive(Default, Clone, Copy)] +enum MockMode { + #[default] + NonStreaming, + Streaming, + Failure4xx, +} + +async fn mock_handler( + State(state): State>, + headers: HeaderMap, + Json(body): Json, +) -> Response { + *state.last_request.lock().unwrap() = Some(body.clone()); + *state.last_auth.lock().unwrap() = headers + .get("authorization") + .and_then(|v| v.to_str().ok()) + .map(|s| s.to_string()); + + let mode = *state.mode.lock().unwrap(); + match mode { + MockMode::NonStreaming => Json(json!({ + "id": "chatcmpl-abc", + "model": body.get("model").cloned().unwrap_or(json!("gpt-4o-mini")), + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "hi from upstream"}, + "finish_reason": "stop" + }], + "usage": {"prompt_tokens": 5, "completion_tokens": 3} + })) + .into_response(), + MockMode::Streaming => { + let chunks = vec![ + "data: {\"id\":\"chatcmpl-s\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"hel\"}}]}\n\n".to_string(), + "data: {\"id\":\"chatcmpl-s\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"lo\"}}]}\n\n".to_string(), + "data: {\"id\":\"chatcmpl-s\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n".to_string(), + "data: [DONE]\n\n".to_string(), + ]; + let body = Body::from_stream(futures::stream::iter( + chunks + .into_iter() + .map(|c| Ok::<_, std::convert::Infallible>(c.into_bytes())), + )); + ( + StatusCode::OK, + [(header::CONTENT_TYPE, "text/event-stream")], + body, + ) + .into_response() + } + MockMode::Failure4xx => ( + StatusCode::BAD_REQUEST, + Json(json!({"error": {"message": "bad upstream request", "type": "invalid_request_error"}})), + ) + .into_response(), + } +} + +async fn spawn_mock() -> (Arc, SocketAddr) { + let state = Arc::new(MockState::default()); + let app = Router::new() + .route("/v1/chat/completions", post(mock_handler)) + .with_state(state.clone()); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + tokio::time::sleep(Duration::from_millis(50)).await; + (state, addr) +} + +async fn spawn_sidecar(upstream_url: String, upstream_key: Option) -> SocketAddr { + use cc_convert_sidecar::*; // re-exported router builder + + let state = AppState { + upstream_url, + upstream_key, + auth_passthrough: false, + http: reqwest::Client::builder() + .timeout(Duration::from_secs(10)) + .build() + .unwrap(), + litellm_compat: false, + }; + let app = build_router(std::sync::Arc::new(state)); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + tokio::time::sleep(Duration::from_millis(50)).await; + addr +} + +#[tokio::test] +async fn non_streaming_round_trip() { + let (mock_state, mock_addr) = spawn_mock().await; + *mock_state.mode.lock().unwrap() = MockMode::NonStreaming; + + let upstream_url = format!("http://{}/v1/chat/completions", mock_addr); + let sidecar_addr = spawn_sidecar(upstream_url, Some("k".to_string())).await; + + let client = reqwest::Client::new(); + let resp: Value = client + .post(format!("http://{}/v1/messages", sidecar_addr)) + .json(&json!({ + "model": "claude-opus-4-7", + "max_tokens": 100, + "messages": [{"role": "user", "content": "ping"}] + })) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + + // Anthropic-shape response. + assert_eq!(resp["model"], "claude-opus-4-7"); + assert_eq!(resp["role"], "assistant"); + assert_eq!(resp["type"], "message"); + assert_eq!(resp["content"][0]["text"], "hi from upstream"); + assert_eq!(resp["stop_reason"], "end_turn"); + + // Upstream saw the OpenAI-shape request. + let upstream_req = mock_state.last_request.lock().unwrap().clone().unwrap(); + assert_eq!(upstream_req["messages"][0]["role"], "user"); + assert_eq!(upstream_req["messages"][0]["content"], "ping"); + assert_eq!(upstream_req["max_tokens"], 100); + + // Auth header carried the configured key. + assert_eq!( + mock_state.last_auth.lock().unwrap().as_deref(), + Some("Bearer k") + ); +} + +#[tokio::test] +async fn streaming_round_trip() { + let (mock_state, mock_addr) = spawn_mock().await; + *mock_state.mode.lock().unwrap() = MockMode::Streaming; + + let upstream_url = format!("http://{}/v1/chat/completions", mock_addr); + let sidecar_addr = spawn_sidecar(upstream_url, Some("k".to_string())).await; + + let client = reqwest::Client::new(); + let mut resp = client + .post(format!("http://{}/v1/messages", sidecar_addr)) + .json(&json!({ + "model": "claude-opus-4-7", + "max_tokens": 50, + "stream": true, + "messages": [{"role": "user", "content": "ping"}] + })) + .send() + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::OK); + let ct = resp + .headers() + .get("content-type") + .and_then(|v| v.to_str().ok()) + .unwrap_or(""); + assert!( + ct.starts_with("text/event-stream"), + "content-type was: {ct}" + ); + + let mut body = String::new(); + while let Some(chunk) = resp.chunk().await.unwrap() { + body.push_str(std::str::from_utf8(&chunk).unwrap()); + } + + // Validate the Anthropic SSE event sequence by extracting `event:` lines. + let event_names: Vec<&str> = body + .lines() + .filter_map(|l| l.strip_prefix("event: ")) + .collect(); + assert!(event_names.contains(&"message_start"), "events: {event_names:?}"); + assert!(event_names.contains(&"ping"), "events: {event_names:?}"); + assert!(event_names.contains(&"content_block_start")); + assert!(event_names.contains(&"content_block_delta")); + assert!(event_names.contains(&"content_block_stop")); + assert!(event_names.contains(&"message_delta")); + assert!(event_names.contains(&"message_stop")); + + // The concatenated text_delta payloads should reconstruct "hello". + let mut text = String::new(); + for line in body.lines().filter_map(|l| l.strip_prefix("data: ")) { + if let Ok(v) = serde_json::from_str::(line) { + if v["type"] == "content_block_delta" + && v["delta"]["type"] == "text_delta" + { + if let Some(s) = v["delta"]["text"].as_str() { + text.push_str(s); + } + } + } + } + assert_eq!(text, "hello"); +} + +#[tokio::test] +async fn upstream_4xx_surfaces_anthropic_error_shape() { + let (mock_state, mock_addr) = spawn_mock().await; + *mock_state.mode.lock().unwrap() = MockMode::Failure4xx; + + let upstream_url = format!("http://{}/v1/chat/completions", mock_addr); + let sidecar_addr = spawn_sidecar(upstream_url, Some("k".to_string())).await; + + let client = reqwest::Client::new(); + let resp = client + .post(format!("http://{}/v1/messages", sidecar_addr)) + .json(&json!({ + "model": "claude-opus-4-7", + "max_tokens": 100, + "messages": [{"role": "user", "content": "ping"}] + })) + .send() + .await + .unwrap(); + let status = resp.status(); + let body: Value = resp.json().await.unwrap(); + + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(body["type"], "error"); + assert!(body["error"]["message"].as_str().unwrap().contains("bad upstream request")); +} diff --git a/sidecars/cc_convert/playground/README.md b/sidecars/cc_convert/playground/README.md new file mode 100644 index 0000000..2749422 --- /dev/null +++ b/sidecars/cc_convert/playground/README.md @@ -0,0 +1,78 @@ +# cc_convert playground + +End-to-end round-trip data for **non-streaming** Anthropic ↔ OpenAI conversion. + +## Layout + +``` +playground/ +├── run_roundtrip.py ← the only script you need to run +├── requests/ ← source Anthropic requests (input to the pipeline) +│ ├── 02_reasoning_request.json +│ ├── 03_forced_tool.json +│ ├── 05_simple_text.json +│ ├── 07_multi_turn_text.json +│ ├── 08_agent_loop_with_tools.json +│ ├── 09_long_response.json +│ └── 10_parallel_tools_text_only.json +└── runs// + └── / + ├── 1_anthropic_request.json ← source (copied verbatim) + ├── 2_oai_request.json ← what cc_convert translated and POSTed + ├── 3_oai_response.json ← what the upstream returned (raw) + ├── 4_anthropic_response.json ← what cc_convert translated back + ├── tool_map.json ← (only if any tool name was truncated) + ├── meta.json ← status / latency / http_status + └── error.txt ← (only on failure) +``` + +## Pipeline + +``` +1_anthropic_request.json + ↓ cc_convert.translate_request() +2_oai_request.json + ↓ POST → upstream /v1/chat/completions +3_oai_response.json + ↓ cc_convert.translate_response() +4_anthropic_response.json +``` + +## Usage + +```bash +# Run all 7 fixtures against the upstream +python playground/run_roundtrip.py --upstream http://YOUR_UPSTREAM_HOST:8000 + +# Override the model name (otherwise probes /v1/models) +python playground/run_roundtrip.py --upstream http://... --model /model + +# Only one fixture +python playground/run_roundtrip.py --upstream http://... --only agent_loop +``` + +## Looking at results + +```bash +# See the summary table +cat playground/runs//_summary.json | jq . + +# Look at one fixture's complete 4-file chain +ls playground/runs//08_agent_loop_with_tools/ +cat playground/runs//08_agent_loop_with_tools/1_anthropic_request.json +cat playground/runs//08_agent_loop_with_tools/2_oai_request.json +cat playground/runs//08_agent_loop_with_tools/3_oai_response.json +cat playground/runs//08_agent_loop_with_tools/4_anthropic_response.json +``` + +## What each fixture exercises + +| Fixture | What it tests | +|---|---| +| `02_reasoning_request` | `thinking.budget_tokens=12000` → `reasoning_effort:"high"` | +| `03_forced_tool` | `tool_choice:{type:"any"}` → OpenAI `"required"` | +| `05_simple_text` | baseline single-turn | +| `07_multi_turn_text` | 5-turn pure-text history | +| `08_agent_loop_with_tools` | 5-turn agent loop: 3 client tools + 2 parallel `tool_use` + 2 `tool_result` round-trip | +| `09_long_response` | 1500-token generation (non-streaming under load) | +| `10_parallel_tools_text_only` | `tool_choice:any` + multiple tools, no history | diff --git a/sidecars/cc_convert/playground/requests/02_reasoning_request.json b/sidecars/cc_convert/playground/requests/02_reasoning_request.json new file mode 100644 index 0000000..9996684 --- /dev/null +++ b/sidecars/cc_convert/playground/requests/02_reasoning_request.json @@ -0,0 +1,15 @@ +{ + "model": "claude-opus-4-7", + "max_tokens": 2000, + "thinking": { + "type": "enabled", + "budget_tokens": 12000 + }, + "system": "Be concise.", + "messages": [ + { + "role": "user", + "content": "Explain why the sky is blue, with reasoning shown." + } + ] +} diff --git a/sidecars/cc_convert/playground/requests/03_forced_tool.json b/sidecars/cc_convert/playground/requests/03_forced_tool.json new file mode 100644 index 0000000..8946515 --- /dev/null +++ b/sidecars/cc_convert/playground/requests/03_forced_tool.json @@ -0,0 +1,30 @@ +{ + "model": "claude-opus-4-7", + "max_tokens": 2000, + "tools": [ + { + "name": "calc", + "description": "Run a math expression", + "input_schema": { + "type": "object", + "properties": { + "expr": { + "type": "string" + } + }, + "required": [ + "expr" + ] + } + } + ], + "tool_choice": { + "type": "any" + }, + "messages": [ + { + "role": "user", + "content": "What is (17 * 23) + sqrt(196)?" + } + ] +} diff --git a/sidecars/cc_convert/playground/requests/05_simple_text.json b/sidecars/cc_convert/playground/requests/05_simple_text.json new file mode 100644 index 0000000..314aeb7 --- /dev/null +++ b/sidecars/cc_convert/playground/requests/05_simple_text.json @@ -0,0 +1,10 @@ +{ + "model": "claude-opus-4-7", + "max_tokens": 2000, + "messages": [ + { + "role": "user", + "content": "你好,介绍一下自己。" + } + ] +} diff --git a/sidecars/cc_convert/playground/requests/07_multi_turn_text.json b/sidecars/cc_convert/playground/requests/07_multi_turn_text.json new file mode 100644 index 0000000..b915850 --- /dev/null +++ b/sidecars/cc_convert/playground/requests/07_multi_turn_text.json @@ -0,0 +1,27 @@ +{ + "model": "claude-opus-4-7", + "max_tokens": 2000, + "system": "You are a math tutor.", + "messages": [ + { + "role": "user", + "content": "What's 2+2?" + }, + { + "role": "assistant", + "content": "Four." + }, + { + "role": "user", + "content": "Now what is its square?" + }, + { + "role": "assistant", + "content": "Sixteen." + }, + { + "role": "user", + "content": "And the square root of that?" + } + ] +} diff --git a/sidecars/cc_convert/playground/requests/08_agent_loop_with_tools.json b/sidecars/cc_convert/playground/requests/08_agent_loop_with_tools.json new file mode 100644 index 0000000..f00a727 --- /dev/null +++ b/sidecars/cc_convert/playground/requests/08_agent_loop_with_tools.json @@ -0,0 +1,122 @@ +{ + "model": "claude-opus-4-7", + "max_tokens": 2000, + "temperature": 0.3, + "system": "You are a coding assistant. Use the provided tools to read files and run shell commands.", + "tools": [ + { + "name": "read_file", + "description": "Read the contents of a file at the given path.", + "input_schema": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Absolute or relative file path" + } + }, + "required": [ + "path" + ] + } + }, + { + "name": "run_shell", + "description": "Run a shell command and return stdout+stderr.", + "input_schema": { + "type": "object", + "properties": { + "command": { + "type": "string" + }, + "timeout_sec": { + "type": "integer", + "minimum": 1, + "maximum": 60, + "default": 10 + } + }, + "required": [ + "command" + ] + } + }, + { + "name": "grep_codebase", + "description": "Search the codebase for a regex pattern.", + "input_schema": { + "type": "object", + "properties": { + "pattern": { + "type": "string" + }, + "path": { + "type": "string", + "default": "." + }, + "max_results": { + "type": "integer", + "default": 20 + } + }, + "required": [ + "pattern" + ] + } + } + ], + "messages": [ + { + "role": "user", + "content": "How many .py files are in the current directory and what's in setup.py?" + }, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "I'll check both: count Python files and read setup.py." + }, + { + "type": "tool_use", + "id": "toolu_count", + "name": "run_shell", + "input": { + "command": "find . -maxdepth 1 -name '*.py' | wc -l" + } + }, + { + "type": "tool_use", + "id": "toolu_read", + "name": "read_file", + "input": { + "path": "setup.py" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_count", + "content": "12\n" + }, + { + "type": "tool_result", + "tool_use_id": "toolu_read", + "content": "from setuptools import setup, find_packages\n\nsetup(\n name='myproject',\n version='0.3.1',\n packages=find_packages(),\n install_requires=['requests>=2.31', 'pyyaml>=6'],\n)\n" + } + ] + }, + { + "role": "assistant", + "content": "There are 12 Python files in the current directory. setup.py defines a package called 'myproject' at version 0.3.1, depending on requests and pyyaml." + }, + { + "role": "user", + "content": "Now grep for any TODO comments in the .py files." + } + ] +} diff --git a/sidecars/cc_convert/playground/requests/09_long_response.json b/sidecars/cc_convert/playground/requests/09_long_response.json new file mode 100644 index 0000000..047034d --- /dev/null +++ b/sidecars/cc_convert/playground/requests/09_long_response.json @@ -0,0 +1,12 @@ +{ + "model": "claude-opus-4-7", + "max_tokens": 4000, + "temperature": 0.5, + "system": "Be thorough. Always answer in English.", + "messages": [ + { + "role": "user", + "content": "Explain in detail how HTTPS works, covering: 1) TCP handshake, 2) TLS handshake including certificate verification and key exchange, 3) symmetric vs asymmetric encryption, 4) what gets encrypted vs not, 5) common attack vectors and how HTTPS defends against them. Use sub-headings." + } + ] +} diff --git a/sidecars/cc_convert/playground/requests/10_parallel_tools_text_only.json b/sidecars/cc_convert/playground/requests/10_parallel_tools_text_only.json new file mode 100644 index 0000000..d6d30ff --- /dev/null +++ b/sidecars/cc_convert/playground/requests/10_parallel_tools_text_only.json @@ -0,0 +1,50 @@ +{ + "model": "claude-opus-4-7", + "max_tokens": 2000, + "system": "Use tools concurrently when the user asks for multiple independent things.", + "tools": [ + { + "name": "get_stock_price", + "description": "Get current stock price for a ticker symbol.", + "input_schema": { + "type": "object", + "properties": { + "ticker": { + "type": "string" + } + }, + "required": [ + "ticker" + ] + } + }, + { + "name": "get_news_headlines", + "description": "Get top news headlines for a topic.", + "input_schema": { + "type": "object", + "properties": { + "topic": { + "type": "string" + }, + "limit": { + "type": "integer", + "default": 5 + } + }, + "required": [ + "topic" + ] + } + } + ], + "tool_choice": { + "type": "any" + }, + "messages": [ + { + "role": "user", + "content": "Get me the current price of AAPL and the top 3 news headlines about Apple." + } + ] +} diff --git a/sidecars/cc_convert/playground/run_roundtrip.py b/sidecars/cc_convert/playground/run_roundtrip.py new file mode 100644 index 0000000..a9f75e9 --- /dev/null +++ b/sidecars/cc_convert/playground/run_roundtrip.py @@ -0,0 +1,271 @@ +"""Non-streaming round-trip runner. + +For each fixture under playground/requests/, send the full pipeline: + + 1. anthropic_request.json ← source (copied verbatim from requests/) + 2. oai_request.json ← cc_convert.translate_request() output + 3. oai_response.json ← upstream server's raw response + 4. anthropic_response.json ← cc_convert.translate_response() output + +All four files for one fixture land in: + + playground/runs/// + +Plus a `meta.json` (status, latency, http_status, error if any) and a +top-level `_summary.json` with all fixtures' status at a glance. + +This is the ONLY script you need to run to see the complete round-trip +data. No streaming, no synthetic, no offline mocks — just the real pipeline. + +Usage: + python playground/run_roundtrip.py --upstream http://YOUR_UPSTREAM_HOST:8000 + +If --model is omitted, /v1/models is probed. +""" +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +import time +import urllib.error +import urllib.request +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional + +import cc_convert + +PLAYGROUND = Path(__file__).resolve().parent +REQ_DIR = PLAYGROUND / "requests" + + +def _opener_no_proxy() -> urllib.request.OpenerDirector: + return urllib.request.build_opener(urllib.request.ProxyHandler({})) + + +def derive_chat_url(base: str) -> str: + b = base.rstrip("/") + if b.endswith("/chat/completions"): + return b + if re.search(r"/v\d+$", b): + return b + "/chat/completions" + return b + "/v1/chat/completions" + + +def base_from_chat(chat_url: str) -> str: + for suf in ("/v1/chat/completions", "/chat/completions"): + if chat_url.endswith(suf): + return chat_url[: -len(suf)] + return chat_url.rstrip("/") + + +def probe_models(base: str) -> List[str]: + for path in ("/v1/models", "/models"): + try: + r = _opener_no_proxy().open(base + path, timeout=5) + data = json.loads(r.read()) + except (urllib.error.URLError, json.JSONDecodeError, ValueError): + continue + if isinstance(data, dict) and isinstance(data.get("data"), list): + return [m["id"] if isinstance(m, dict) and "id" in m else str(m) for m in data["data"]] + if isinstance(data, dict) and isinstance(data.get("models"), list): + return [ + m if isinstance(m, str) else (m.get("id") if isinstance(m, dict) else str(m)) + for m in data["models"] + ] + if isinstance(data, list): + return [ + m if isinstance(m, str) else (m.get("id", str(m)) if isinstance(m, dict) else str(m)) + for m in data + ] + return [] + + +def write_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n") + + +def run_one( + fx_name: str, + anthropic_req: Dict[str, Any], + chat_url: str, + api_key: Optional[str], + target_model: str, + out_dir: Path, +) -> Dict[str, Any]: + """Run the full pipeline for one fixture; return meta dict.""" + fx_dir = out_dir / fx_name + fx_dir.mkdir(parents=True, exist_ok=True) + + # 1) Save the original Anthropic request verbatim. + write_json(fx_dir / "1_anthropic_request.json", anthropic_req) + + # 2) Translate to OpenAI shape + save. + openai_req, tool_map = cc_convert.translate_request( + anthropic_req, target_model=target_model + ) + # Force non-streaming. + openai_req.pop("stream", None) + openai_req.pop("stream_options", None) + write_json(fx_dir / "2_oai_request.json", openai_req) + if tool_map: + write_json(fx_dir / "tool_map.json", tool_map) + + meta: Dict[str, Any] = { + "fixture": fx_name, + "upstream": chat_url, + "model_sent": target_model, + "original_model": anthropic_req.get("model"), + "tool_map_size": len(tool_map), + "stream": False, + } + + # 3) POST to the upstream. + body = json.dumps(openai_req).encode() + headers = {"content-type": "application/json"} + if api_key: + headers["authorization"] = f"Bearer {api_key}" + + t0 = time.time() + try: + req = urllib.request.Request(chat_url, data=body, method="POST") + for k, v in headers.items(): + req.add_header(k, v) + resp = _opener_no_proxy().open(req, timeout=300) + raw = resp.read() + openai_resp = json.loads(raw or b"{}") + meta.update( + { + "status": "ok", + "http_status": resp.status, + "latency_ms": int((time.time() - t0) * 1000), + } + ) + except urllib.error.HTTPError as e: + body_text = "" + try: + body_text = e.read().decode("utf-8", "replace") + except Exception: # noqa: BLE001 + pass + meta.update( + { + "status": "http_error", + "http_status": e.code, + "error": body_text[:500], + "latency_ms": int((time.time() - t0) * 1000), + } + ) + write_json(fx_dir / "meta.json", meta) + (fx_dir / "error.txt").write_text(f"HTTP {e.code}\n\n{body_text}\n") + return meta + except urllib.error.URLError as e: + meta.update( + { + "status": "url_error", + "error": str(e), + "latency_ms": int((time.time() - t0) * 1000), + } + ) + write_json(fx_dir / "meta.json", meta) + (fx_dir / "error.txt").write_text(f"URLError: {e}\n") + return meta + except Exception as e: # noqa: BLE001 + meta.update( + {"status": "exception", "error": f"{type(e).__name__}: {e}", + "latency_ms": int((time.time() - t0) * 1000)} + ) + write_json(fx_dir / "meta.json", meta) + (fx_dir / "error.txt").write_text(f"{type(e).__name__}: {e}\n") + return meta + + # 3') Save raw OAI response. + write_json(fx_dir / "3_oai_response.json", openai_resp) + + # 4) Translate back to Anthropic shape + save. + try: + anthropic_resp = cc_convert.translate_response( + openai_resp, + original_model=anthropic_req.get("model", "claude-opus-4-7"), + tool_name_map=tool_map, + ) + write_json(fx_dir / "4_anthropic_response.json", anthropic_resp) + except Exception as e: # noqa: BLE001 + meta.update({"status": "reverse_translate_error", "error": str(e)}) + (fx_dir / "error.txt").write_text(f"reverse translate: {type(e).__name__}: {e}\n") + + write_json(fx_dir / "meta.json", meta) + return meta + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--upstream", required=True, help="upstream URL (bare host OK; /v1/chat/completions auto-appended)") + ap.add_argument("--model", default=None, help="model name to send (auto-probes /v1/models if omitted)") + ap.add_argument("--api-key", default=os.environ.get("CC_CONVERT_UPSTREAM_API_KEY")) + ap.add_argument("--only", help="only run fixtures whose name contains this substring") + args = ap.parse_args() + + chat_url = derive_chat_url(args.upstream) + base = base_from_chat(chat_url) + + model = args.model + if not model: + models = probe_models(base) + print(f"[probe] /v1/models: {models}", file=sys.stderr) + if not models: + print("[error] no model name and probe returned empty; pass --model", file=sys.stderr) + return 2 + model = models[0] + print(f"[ok] model={model!r} url={chat_url}", file=sys.stderr) + + fixtures = sorted(REQ_DIR.glob("*.json")) + if args.only: + fixtures = [f for f in fixtures if args.only in f.stem] + if not fixtures: + print(f"[error] no fixtures matched in {REQ_DIR}", file=sys.stderr) + return 2 + + stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + out = PLAYGROUND / "runs" / stamp + out.mkdir(parents=True, exist_ok=True) + print(f"[ok] outputs -> {out}\n", file=sys.stderr) + + summary = [] + for fx_path in fixtures: + name = fx_path.stem + req = json.loads(fx_path.read_text()) + print(f"--- {name} (max_tokens={req.get('max_tokens')}) ---", file=sys.stderr) + m = run_one(name, req, chat_url, args.api_key, model, out) + summary.append(m) + ok = m.get("status") == "ok" + mark = "✓" if ok else "✗" + info = ( + f"http={m.get('http_status')} ms={m.get('latency_ms')}" + if ok + else f"status={m.get('status')} http={m.get('http_status')} error={(m.get('error') or '')[:80]}" + ) + print(f" {mark} {info}", file=sys.stderr) + + write_json(out / "_summary.json", summary) + + print("\n=== Summary ===", file=sys.stderr) + print(f"{'Fixture':36s} {'Status':18s} HTTP ms", file=sys.stderr) + print("-" * 80, file=sys.stderr) + for m in summary: + ms = m.get("latency_ms") + ms_s = f"{ms}" if ms is not None else "—" + print( + f"{m['fixture']:36s} {m.get('status', '?'):18s} " + f"{str(m.get('http_status') or '—'):>5s} {ms_s}", + file=sys.stderr, + ) + print(f"\nfull data: {out}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/sidecars/cc_convert/python/README.md b/sidecars/cc_convert/python/README.md new file mode 100644 index 0000000..968f2b0 --- /dev/null +++ b/sidecars/cc_convert/python/README.md @@ -0,0 +1,53 @@ +# cc_convert + +Anthropic Messages API ↔ OpenAI Chat Completions protocol converter. +Rust core via PyO3 + a Python CLI sidecar. + +```bash +pip install cc-convert +``` + +```python +import cc_convert +openai_req, tool_map = cc_convert.translate_request(anthropic_request_dict) +# POST openai_req to any OAI-compatible /v1/chat/completions endpoint +anthropic_resp = cc_convert.translate_response( + openai_response_dict, + original_model="claude-opus-4-7", + tool_name_map=tool_map, +) +``` + +Or use the CLI as a transparent sidecar proxy: + +```bash +cc_convert serve --listen 0.0.0.0:8787 --upstream-url http://your-oai-host:8000 +``` + +Then point any Anthropic-API client (Claude Code, claude-py, etc.) at +`http://localhost:8787`. + +See full docs and examples at +[github.com/yitianlian/cc_convert](https://github.com/yitianlian/cc_convert) +([中文文档](https://github.com/yitianlian/cc_convert/blob/main/USAGE.zh-CN.md)). + +## What it does + +- **Anthropic request → OpenAI request** (single text collapsed to string, + tools, tool_choice, system, multipart content, thinking → reasoning_effort, + cache_control / hosted tools / server_tool_use blocks dropped or + translated as appropriate). +- **OpenAI response → Anthropic response** (id rewrite, content blocks, + tool_calls → tool_use, reasoning_content / reasoning → thinking block, + finish_reason mapping, usage with cached_tokens accounting). +- **OpenAI SSE → Anthropic SSE** (full event sequence: message_start → + content_block_start → deltas → content_block_stop → message_delta → + message_stop). +- Compatible with real SGLang / vLLM / DeepSeek upstreams; handles their + quirks (`reasoning_content: null` on every chunk, `id: null` on + continuation tool_call chunks, `matched_stop`, `metadata.weight_version`, + etc.) without breaking. + +## License + +MIT OR Apache-2.0. diff --git a/sidecars/cc_convert/python/cc_convert/__init__.py b/sidecars/cc_convert/python/cc_convert/__init__.py new file mode 100644 index 0000000..82e06bd --- /dev/null +++ b/sidecars/cc_convert/python/cc_convert/__init__.py @@ -0,0 +1,82 @@ +"""cc_convert — Anthropic ↔ OpenAI Chat Completions protocol converter. + +The heavy lifting is implemented in Rust and exposed through the native +extension `cc_convert._native`. This module provides ergonomic Python wrappers +that work with dicts (instead of JSON strings). +""" + +from __future__ import annotations + +import json +from typing import Any, Dict, List, Optional, Tuple + +from . import _native # type: ignore + +__version__: str = _native.__version__ +__all__ = [ + "translate_request", + "translate_response", + "StreamTranslator", + "__version__", +] + + +def translate_request( + anthropic_request: Dict[str, Any], + target_model: Optional[str] = None, + mode: str = "pragmatic", +) -> Tuple[Dict[str, Any], Dict[str, str]]: + """Translate an Anthropic Messages request dict into an OpenAI Chat + Completions request dict. + + Args: + anthropic_request: the Anthropic-shape request body. + target_model: override the ``model`` field in the translated request. + mode: ``"pragmatic"`` (default) collapses single-text content to a + string, drops top_k, injects stream_options.include_usage, etc. + ``"litellm_compat"`` matches LiteLLM's AnthropicAdapter byte-for-byte + (useful as a drop-in replacement in an existing LiteLLM pipeline). + + Returns ``(openai_request, tool_name_map)``. Keep the ``tool_name_map`` and + pass it to :func:`translate_response` / :class:`StreamTranslator` so we can + restore tool names that had to be truncated to fit OpenAI's 64-char limit. + """ + + openai_str, map_str = _native.translate_request( + json.dumps(anthropic_request), target_model, mode + ) + return json.loads(openai_str), json.loads(map_str) + + +def translate_response( + openai_response: Dict[str, Any], + original_model: str, + tool_name_map: Dict[str, str], +) -> Dict[str, Any]: + """Translate an OpenAI Chat Completions response dict into an Anthropic + Messages response dict. + """ + + anthropic_str = _native.translate_response( + json.dumps(openai_response), original_model, json.dumps(tool_name_map) + ) + return json.loads(anthropic_str) + + +class StreamTranslator: + """Stateful translator: feed it OpenAI SSE chunk dicts and read back + Anthropic SSE event dicts. Call :meth:`finish` when upstream is exhausted. + """ + + def __init__(self, original_model: str, tool_name_map: Dict[str, str]) -> None: + self._inner = _native.PyStreamTranslator( + original_model, json.dumps(tool_name_map) + ) + + def push(self, openai_chunk: Dict[str, Any]) -> List[Dict[str, Any]]: + events_json = self._inner.push(json.dumps(openai_chunk)) + return [json.loads(e) for e in events_json] + + def finish(self) -> List[Dict[str, Any]]: + events_json = self._inner.finish() + return [json.loads(e) for e in events_json] diff --git a/sidecars/cc_convert/python/cc_convert/__main__.py b/sidecars/cc_convert/python/cc_convert/__main__.py new file mode 100644 index 0000000..e708c52 --- /dev/null +++ b/sidecars/cc_convert/python/cc_convert/__main__.py @@ -0,0 +1,6 @@ +"""Allow ``python -m cc_convert ...``.""" +import sys +from .cli import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/sidecars/cc_convert/python/cc_convert/cli.py b/sidecars/cc_convert/python/cc_convert/cli.py new file mode 100644 index 0000000..faad49b --- /dev/null +++ b/sidecars/cc_convert/python/cc_convert/cli.py @@ -0,0 +1,725 @@ +"""CLI for cc_convert. + +Subcommands: + + ``serve`` - run an HTTP server (sidecar) in one of two modes: + ``proxy`` (default) — Anthropic-shape in, transparently + forwarded to an OpenAI-compatible upstream, Anthropic-shape + out. ``rpc`` — pure translation, no upstream call. + ``translate`` - one-shot JSON-to-JSON conversion in either direction: + ``--direction cc-to-oai`` (request side, Anthropic → OpenAI) + or ``--direction oai-to-cc`` (response side, OpenAI → + Anthropic). + +Examples: + + # Run as a sidecar in front of OpenAI + cc_convert serve --listen 0.0.0.0:8787 \\ + --upstream-url https://api.openai.com/v1/chat/completions \\ + --upstream-key sk-... + + # Run as a sidecar in front of a vLLM / SGLang / DeepSeek backend + cc_convert serve --upstream-url http://localhost:8000/v1/chat/completions \\ + --log-level debug + + # Pure-translation RPC server (no upstream) + cc_convert serve --mode rpc --listen 127.0.0.1:8788 + + # One-shot: Anthropic request -> OpenAI request + cat anthropic_req.json | cc_convert translate --direction cc-to-oai > openai_req.json + + # One-shot: OpenAI response -> Anthropic response (need original model name + tool_map) + cc_convert translate --direction oai-to-cc \\ + --input openai_resp.json --original-model claude-opus-4-7 \\ + --tool-map tool_map.json --output anthropic_resp.json +""" + +from __future__ import annotations + +import argparse +import http.server +import json +import logging +import os +import socketserver +import sys +import time +import urllib.error +import urllib.request +import uuid +from typing import Any, Dict, Optional + +import cc_convert + +log = logging.getLogger("cc_convert") + + +# ---------- helpers ---------- + + +def _read_json(path: Optional[str]) -> Any: + if path and path != "-": + with open(path) as f: + return json.load(f) + return json.load(sys.stdin) + + +def _write_json(payload: Any, path: Optional[str]) -> None: + text = json.dumps(payload, indent=2, sort_keys=True) + if path and path != "-": + with open(path, "w") as f: + f.write(text + "\n") + else: + sys.stdout.write(text + "\n") + + +def _normalize_upstream_url(raw: str) -> str: + """Auto-complete OpenAI-compatible upstream URLs so users don't have to + remember the exact suffix. + + Accepts (all map to the same thing): + https://api.openai.com + https://api.openai.com/ + https://api.openai.com/v1 + https://api.openai.com/v1/ + https://api.openai.com/v1/chat/completions (verbatim) + + The suffix `/chat/completions` is what OpenAI-style servers (OpenAI, + vLLM, SGLang, DeepSeek, Together, Anyscale, Fireworks, Moonshot, ...) + listen on for non-streaming + streaming chat. If the URL already ends + in that, we leave it. Otherwise we append `/chat/completions`, inserting + `/v1` if neither `/v1` nor any other obvious version prefix is present. + """ + url = raw.rstrip("/") + if url.endswith("/chat/completions"): + return url + # Already has a version segment like /v1 or /v2 → just append the suffix. + import re + if re.search(r"/v\d+$", url): + return url + "/chat/completions" + # Bare host or /something else → assume /v1/chat/completions. + return url + "/v1/chat/completions" + + +def _path_is_anthropic_messages(path: str) -> bool: + """True if `path` looks like an Anthropic `messages` endpoint, regardless + of any prefix the client (or an upstream load balancer) tacked on. + + Accepts: /v1/messages, /messages, /anthropic/v1/messages, + /some/prefix/v1/messages?stream=true ... + Rejects: /v1/messages/foo (trailing segment), /healthz, /version. + """ + # Strip query string. + p = path.split("?", 1)[0].rstrip("/") + if p.endswith("/v1/messages") or p == "/v1/messages": + return True + if p.endswith("/messages") or p == "/messages": + return True + return False + + +def _setup_logging(level: str, fmt: str) -> None: + numeric = getattr(logging, level.upper(), logging.INFO) + if fmt == "json": + # Minimal JSON formatter: one record per line, easy to grep/jq. + class JsonFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + payload = { + "ts": self.formatTime(record, "%Y-%m-%dT%H:%M:%S%z"), + "level": record.levelname.lower(), + "logger": record.name, + "msg": record.getMessage(), + } + if hasattr(record, "extra_fields"): + payload.update(record.extra_fields) + return json.dumps(payload, ensure_ascii=False) + + h = logging.StreamHandler(sys.stderr) + h.setFormatter(JsonFormatter()) + logging.basicConfig(level=numeric, handlers=[h], force=True) + else: + logging.basicConfig( + level=numeric, + format="%(asctime)s %(levelname)-5s %(name)s: %(message)s", + datefmt="%Y-%m-%dT%H:%M:%S", + force=True, + stream=sys.stderr, + ) + + +def _log(level: int, msg: str, **fields: Any) -> None: + """Log with structured extras when JSON format is on.""" + if fields: + log.log(level, msg, extra={"extra_fields": fields}) + else: + log.log(level, msg) + + +# ---------- translate (one-shot) ---------- + + +def cmd_translate(args: argparse.Namespace) -> int: + payload = _read_json(args.input) + if args.direction == "cc-to-oai": + openai_req, tool_map = cc_convert.translate_request( + payload, target_model=args.target_model, mode=args.compat_mode + ) + if args.tool_map_out: + with open(args.tool_map_out, "w") as f: + json.dump(tool_map, f, indent=2, sort_keys=True) + f.write("\n") + log.info("wrote tool_map to %s", args.tool_map_out) + _write_json(openai_req, args.output) + elif args.direction == "oai-to-cc": + tool_map: Dict[str, str] = {} + if args.tool_map: + with open(args.tool_map) as f: + tool_map = json.load(f) + anthropic = cc_convert.translate_response( + payload, + original_model=args.original_model, + tool_name_map=tool_map, + ) + _write_json(anthropic, args.output) + return 0 + + +# ---------- serve ---------- + + +class _Handler(http.server.BaseHTTPRequestHandler): + """Single handler for both proxy and rpc modes.""" + + server_version = "cc_convert/0.1" + + # Injected by build_router below + mode: str = "proxy" + compat_mode: str = "pragmatic" + upstream_url: str = "" + upstream_key: Optional[str] = None + auth_passthrough: bool = False + cc_path: str = "/v1/messages" # Anthropic-shape entry point + cc_to_oai_path: str = "/translate/cc-to-oai" + oai_to_cc_path: str = "/translate/oai-to-cc" + request_log: bool = True + + def log_message(self, fmt: str, *args: Any) -> None: # silence default access log + return + + # ---- routing ---- + def do_GET(self) -> None: # noqa: N802 + if self.path == "/healthz": + self._send_text(200, "ok") + return + if self.path == "/version": + self._send_json(200, {"name": "cc_convert", "version": cc_convert.__version__}) + return + self._send_json(404, {"error": {"message": "not found", "type": "not_found"}}) + + def do_POST(self) -> None: # noqa: N802 + rid = uuid.uuid4().hex[:12] + t0 = time.time() + try: + length = int(self.headers.get("content-length") or "0") + raw = self.rfile.read(length) if length > 0 else b"" + payload = json.loads(raw or b"{}") + except (ValueError, json.JSONDecodeError) as e: + self._anthropic_error(400, "invalid_request_error", f"bad json: {e}") + self._access_log(rid, 400, t0, route="") + return + + path_no_query = self.path.split("?", 1)[0].rstrip("/") + + if self.mode == "proxy" and _path_is_anthropic_messages(self.path): + status = self._handle_proxy(rid, payload) + self._access_log(rid, status, t0, route="proxy") + elif self.mode == "rpc" and ( + path_no_query == self.cc_to_oai_path.rstrip("/") + or path_no_query.endswith("/translate/cc-to-oai") + ): + try: + openai_req, tool_map = cc_convert.translate_request( + payload, mode=self.compat_mode + ) + self._send_json(200, {"openai_request": openai_req, "tool_map": tool_map}) + status = 200 + except Exception as e: # noqa: BLE001 + self._anthropic_error(400, "invalid_request_error", str(e)) + status = 400 + self._access_log(rid, status, t0, route="rpc cc-to-oai") + elif self.mode == "rpc" and ( + path_no_query == self.oai_to_cc_path.rstrip("/") + or path_no_query.endswith("/translate/oai-to-cc") + ): + try: + openai_resp = payload.get("openai_response") or payload + original_model = payload.get("original_model", "unknown-model") + tool_map = payload.get("tool_map") or {} + out = cc_convert.translate_response( + openai_resp, original_model=original_model, tool_name_map=tool_map + ) + self._send_json(200, out) + status = 200 + except Exception as e: # noqa: BLE001 + self._anthropic_error(400, "invalid_request_error", str(e)) + status = 400 + self._access_log(rid, status, t0, route="rpc oai-to-cc") + else: + hint = "" + if self.mode == "proxy": + hint = ( + f" hint: this proxy accepts POST on any URL ending in " + f"/messages or /v1/messages (got {self.path!r})" + ) + elif self.mode == "rpc": + hint = ( + f" hint: this RPC server accepts POST on {self.cc_to_oai_path!r} " + f"or {self.oai_to_cc_path!r} (got {self.path!r})" + ) + self._send_json( + 404, {"error": {"message": f"not found.{hint}", "type": "not_found"}} + ) + self._access_log(rid, 404, t0, route=self.path) + + # ---- proxy mode handler ---- + def _handle_proxy(self, rid: str, req_value: Dict[str, Any]) -> int: + stream_mode = bool(req_value.get("stream")) + original_model = req_value.get("model", "unknown") + _log( + logging.DEBUG, + "request received", + rid=rid, model=original_model, stream=stream_mode, + messages=len(req_value.get("messages", [])), + ) + + try: + openai_req, tool_map = cc_convert.translate_request( + req_value, mode=self.compat_mode + ) + except Exception as e: # noqa: BLE001 + self._anthropic_error(400, "invalid_request_error", str(e)) + return 400 + + auth_header = self._build_auth_header() + upstream_body = json.dumps(openai_req).encode() + upstream_req = urllib.request.Request( + self.upstream_url, + data=upstream_body, + method="POST", + headers={"content-type": "application/json"}, + ) + if auth_header: + upstream_req.add_header("authorization", auth_header) + + _log( + logging.DEBUG, + "upstream POST", + rid=rid, url=self.upstream_url, + tool_map_size=len(tool_map), body_bytes=len(upstream_body), + ) + + try: + upstream_resp = urllib.request.urlopen(upstream_req, timeout=600) # noqa: S310 + except urllib.error.HTTPError as e: + body_text = "" + try: + body_text = e.read().decode("utf-8", "replace") + except Exception: # noqa: BLE001 + pass + _log(logging.WARNING, "upstream error", rid=rid, status=e.code) + self._anthropic_error(e.code, "api_error", body_text or "upstream error") + return e.code + except urllib.error.URLError as e: + _log(logging.ERROR, "upstream unreachable", rid=rid, error=str(e)) + self._anthropic_error(502, "api_error", str(e)) + return 502 + + if not stream_mode: + try: + raw = upstream_resp.read() + openai_resp = json.loads(raw or b"{}") + anthropic_resp = cc_convert.translate_response( + openai_resp, + original_model=original_model, + tool_name_map=tool_map, + ) + except Exception as e: # noqa: BLE001 + _log(logging.ERROR, "response translation failed", rid=rid, error=str(e)) + self._anthropic_error(502, "api_error", str(e)) + return 502 + self._send_json(200, anthropic_resp) + return 200 + + # ---- streaming ---- + translator = cc_convert.StreamTranslator(original_model, tool_map) + self.send_response(200) + self.send_header("content-type", "text/event-stream") + self.send_header("cache-control", "no-cache") + self.send_header("connection", "keep-alive") + self.end_headers() + + buffer = b"" + events_sent = 0 + try: + while True: + chunk = upstream_resp.read(8192) + if not chunk: + break + buffer += chunk + while b"\n\n" in buffer: + event_blob, buffer = buffer.split(b"\n\n", 1) + events_sent += self._emit_anthropic_events(translator, event_blob) + # tail flush + if buffer.strip(): + events_sent += self._emit_anthropic_events(translator, buffer) + for ev in translator.finish(): + self._write_sse(ev) + events_sent += 1 + except (BrokenPipeError, ConnectionResetError): + _log(logging.INFO, "client disconnected mid-stream", rid=rid, events_sent=events_sent) + try: + self.wfile.flush() + except (BrokenPipeError, ConnectionResetError): + pass + _log(logging.DEBUG, "stream done", rid=rid, events=events_sent) + return 200 + + def _emit_anthropic_events(self, translator: Any, blob: bytes) -> int: + count = 0 + for line in blob.splitlines(): + if not line.startswith(b"data:"): + continue + payload = line[5:].strip() + if not payload or payload == b"[DONE]": + continue + try: + openai_chunk = json.loads(payload) + except json.JSONDecodeError: + continue + for ev in translator.push(openai_chunk): + self._write_sse(ev) + count += 1 + return count + + def _build_auth_header(self) -> Optional[str]: + if self.auth_passthrough: + client_auth = self.headers.get("authorization") or self.headers.get("x-api-key") + if client_auth: + if not client_auth.startswith("Bearer "): + client_auth = f"Bearer {client_auth}" + return client_auth + return None + if self.upstream_key: + return f"Bearer {self.upstream_key}" + return None + + # ---- IO helpers ---- + def _write_sse(self, event: Dict[str, Any]) -> None: + name = event.get("type", "message") + data = json.dumps(event, separators=(",", ":")) + self.wfile.write(f"event: {name}\ndata: {data}\n\n".encode()) + self.wfile.flush() + + def _send_text(self, status: int, text: str) -> None: + body = text.encode() + self.send_response(status) + self.send_header("content-type", "text/plain") + self.send_header("content-length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _send_json(self, status: int, payload: Any) -> None: + body = json.dumps(payload).encode() + self.send_response(status) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _anthropic_error(self, status: int, type_: str, message: str) -> None: + self._send_json(status, {"type": "error", "error": {"type": type_, "message": message}}) + + def _access_log(self, rid: str, status: int, t0: float, route: str) -> None: + if not self.request_log: + return + dur_ms = int((time.time() - t0) * 1000) + _log( + logging.INFO, + f"{self.command} {self.path} {status} ({dur_ms} ms)", + rid=rid, status=status, route=route, dur_ms=dur_ms, + remote=self.client_address[0], + ) + + +class _ThreadedServer(socketserver.ThreadingMixIn, http.server.HTTPServer): + daemon_threads = True + allow_reuse_address = True + + +def cmd_serve(args: argparse.Namespace) -> int: + host, _, port_s = args.listen.rpartition(":") + if not host: + host = "0.0.0.0" + port = int(port_s) + + if args.mode == "proxy" and not args.upstream_url: + print( + "error: proxy mode requires --upstream-url (or $CC_CONVERT_UPSTREAM_URL).\n" + "examples:\n" + " cc_convert serve --upstream-url https://api.openai.com --upstream-key sk-...\n" + " cc_convert serve --upstream-url http://localhost:8000 # vLLM/SGLang\n" + "or run in RPC mode (no upstream):\n" + " cc_convert serve --mode rpc", + file=sys.stderr, + ) + return 2 + + # Auto-complete the upstream URL: bare host / /v1 / etc -> .../v1/chat/completions + normalized_upstream = ( + _normalize_upstream_url(args.upstream_url) if args.upstream_url else "" + ) + if normalized_upstream and normalized_upstream != args.upstream_url: + log.info( + "upstream URL %r -> %r (auto-completed)", + args.upstream_url, normalized_upstream, + ) + args.upstream_url = normalized_upstream + + if args.mode == "proxy" and not (args.upstream_key or args.auth_passthrough): + log.warning( + "proxy mode running WITHOUT --upstream-key and WITHOUT " + "--auth-passthrough; the upstream call will be unauthenticated" + ) + + handler_cls = type( + "_BoundHandler", + (_Handler,), + { + "mode": args.mode, + "compat_mode": args.compat_mode, + "upstream_url": args.upstream_url, + "upstream_key": args.upstream_key, + "auth_passthrough": args.auth_passthrough, + "cc_path": args.cc_path, + "cc_to_oai_path": args.cc_to_oai_path, + "oai_to_cc_path": args.oai_to_cc_path, + "request_log": not args.quiet, + }, + ) + + server = _ThreadedServer((host, port), handler_cls) + log.info( + "cc_convert serve: mode=%s listen=http://%s:%d cc_path=%s upstream=%s", + args.mode, + host, + port, + args.cc_path if args.mode == "proxy" else f"{args.cc_to_oai_path} | {args.oai_to_cc_path}", + args.upstream_url if args.mode == "proxy" else "", + ) + try: + server.serve_forever() + except KeyboardInterrupt: + log.info("shutting down") + server.shutdown() + return 0 + + +# ---------- argument parser ---------- + + +def _add_global_opts(parser: argparse.ArgumentParser) -> None: + """Add log-level / log-format / -v / --version. We add these to BOTH the + top-level parser and every subparser so users don't get tripped up by + 'unrecognized argument' errors when they write `cc_convert serve + --log-level debug` instead of `cc_convert --log-level debug serve`.""" + parser.add_argument( + "--log-level", + default=None, + choices=["debug", "info", "warning", "error"], + help="log level (default: $CC_CONVERT_LOG_LEVEL or 'info')", + ) + parser.add_argument( + "--log-format", + default=None, + choices=["text", "json"], + help="log format (default: $CC_CONVERT_LOG_FORMAT or 'text')", + ) + parser.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="-v = info, -vv = debug (overrides --log-level if higher)", + ) + parser.add_argument( + "--version", + action="version", + version=f"cc_convert {cc_convert.__version__}", + help="show version and exit", + ) + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + prog="cc_convert", + description=( + "Anthropic <-> OpenAI Chat Completions protocol converter. " + "Run as a sidecar (`cc_convert serve`) or do one-shot JSON-in / " + "JSON-out translations (`cc_convert translate`)." + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + + _add_global_opts(p) + + sub = p.add_subparsers(dest="cmd", required=True) + + # ---- translate ---- + pt = sub.add_parser( + "translate", + help="one-shot JSON-in / JSON-out conversion in either direction", + ) + _add_global_opts(pt) + pt.add_argument( + "--direction", + required=True, + choices=["cc-to-oai", "oai-to-cc"], + help="cc-to-oai = Anthropic request -> OpenAI request; " + "oai-to-cc = OpenAI response -> Anthropic response", + ) + pt.add_argument("-i", "--input", help="input file (default: stdin)") + pt.add_argument("-o", "--output", help="output file (default: stdout)") + pt.add_argument( + "--target-model", + help="(cc-to-oai) override the model name in the translated request", + ) + pt.add_argument( + "--compat-mode", + default=os.environ.get("CC_CONVERT_COMPAT_MODE", "pragmatic"), + choices=["pragmatic", "litellm_compat"], + help="(cc-to-oai) 'pragmatic' (default) collapses single-text content " + "to a string, drops top_k, etc. — best for real OAI-compat upstreams " + "(SGLang/vLLM strict mode). 'litellm_compat' is byte-equivalent to " + "LiteLLM's AnthropicAdapter — use when replacing LiteLLM.", + ) + pt.add_argument( + "--original-model", + help="(oai-to-cc) model name to set on the translated Anthropic response", + ) + pt.add_argument( + "--tool-map", + help="(oai-to-cc) path to a tool-name map JSON saved by a prior translate", + ) + pt.add_argument( + "--tool-map-out", + help="(cc-to-oai) write the tool-name map to this path", + ) + pt.set_defaults(func=cmd_translate) + + # ---- serve ---- + ps = sub.add_parser( + "serve", + help="run as an HTTP server (sidecar) — proxy mode or pure-RPC mode", + ) + _add_global_opts(ps) + ps.add_argument( + "--mode", + default=os.environ.get("CC_CONVERT_MODE", "proxy"), + choices=["proxy", "rpc"], + help=( + "proxy: terminate Anthropic-shape requests and forward to " + "--upstream-url; rpc: stateless translation endpoints, no " + "upstream call (default: $CC_CONVERT_MODE or 'proxy')" + ), + ) + ps.add_argument( + "--listen", + default=os.environ.get("CC_CONVERT_LISTEN_ADDR", "0.0.0.0:8787"), + help="host:port to listen on (default: $CC_CONVERT_LISTEN_ADDR or 0.0.0.0:8787)", + ) + ps.add_argument( + "--cc-path", + default=os.environ.get("CC_CONVERT_CC_PATH", "/v1/messages"), + help="(proxy) the path that receives Anthropic-shape requests " + "(default: $CC_CONVERT_CC_PATH or /v1/messages)", + ) + ps.add_argument( + "--cc-to-oai-path", + default=os.environ.get("CC_CONVERT_RPC_REQUEST_PATH", "/translate/cc-to-oai"), + help="(rpc) path for Anthropic-request to OpenAI-request translation", + ) + ps.add_argument( + "--oai-to-cc-path", + default=os.environ.get("CC_CONVERT_RPC_RESPONSE_PATH", "/translate/oai-to-cc"), + help="(rpc) path for OpenAI-response to Anthropic-response translation", + ) + ps.add_argument( + "--upstream-url", + default=os.environ.get("CC_CONVERT_UPSTREAM_URL"), + help=( + "(proxy) full URL of the upstream OpenAI-compatible " + "/v1/chat/completions endpoint " + "(default: $CC_CONVERT_UPSTREAM_URL)" + ), + ) + ps.add_argument( + "--upstream-key", + default=os.environ.get("CC_CONVERT_UPSTREAM_API_KEY"), + help=( + "(proxy) bearer token sent to the upstream " + "(default: $CC_CONVERT_UPSTREAM_API_KEY)" + ), + ) + ps.add_argument( + "--auth-passthrough", + action="store_true", + default=os.environ.get("CC_CONVERT_AUTH_PASSTHROUGH") == "1", + help="forward the CLIENT's Authorization header instead of --upstream-key", + ) + ps.add_argument( + "--compat-mode", + default=os.environ.get("CC_CONVERT_COMPAT_MODE", "pragmatic"), + choices=["pragmatic", "litellm_compat"], + help="translation profile: 'pragmatic' (default) collapses " + "single-text content to a string, drops top_k, etc. — best for " + "real OAI-compat upstreams (SGLang/vLLM strict mode). " + "'litellm_compat' is byte-equivalent to LiteLLM's AnthropicAdapter.", + ) + ps.add_argument( + "--quiet", + action="store_true", + help="suppress per-request access logs (errors are still logged)", + ) + ps.set_defaults(func=cmd_serve) + + return p + + +def main(argv: Optional[list] = None) -> int: + args = build_parser().parse_args(argv) + # Resolve log options: -vv > -v > subcommand --log-level > top-level > env > default. + verbose = getattr(args, "verbose", 0) + if verbose >= 2: + level = "debug" + elif verbose >= 1: + level = "info" + else: + level = args.log_level or os.environ.get("CC_CONVERT_LOG_LEVEL") or "info" + fmt = args.log_format or os.environ.get("CC_CONVERT_LOG_FORMAT") or "text" + _setup_logging(level, fmt) + try: + return args.func(args) + except KeyboardInterrupt: + log.info("interrupted") + return 130 + except FileNotFoundError as e: + log.error("file not found: %s", e.filename or e) + return 2 + except json.JSONDecodeError as e: + log.error("invalid JSON input: %s", e) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/sidecars/cc_convert/python/pyproject.toml b/sidecars/cc_convert/python/pyproject.toml new file mode 100644 index 0000000..6922090 --- /dev/null +++ b/sidecars/cc_convert/python/pyproject.toml @@ -0,0 +1,57 @@ +[build-system] +requires = ["maturin>=1.5,<2.0"] +build-backend = "maturin" + +[project] +name = "cc-convert" +version = "0.1.0" +description = "Anthropic Messages API <-> OpenAI Chat Completions protocol converter (Rust core via PyO3 + CLI sidecar)" +readme = "README.md" +requires-python = ">=3.8" +license = { text = "MIT OR Apache-2.0" } +keywords = [ + "anthropic", "openai", "claude", "translator", "proxy", + "chat-completions", "messages-api", "sglang", "vllm", "litellm", +] +classifiers = [ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "License :: OSI Approved :: Apache Software License", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS :: MacOS X", + "Operating System :: Microsoft :: Windows", + "Programming Language :: Rust", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: Implementation :: CPython", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] + +[project.urls] +Homepage = "https://github.com/yitianlian/cc_convert" +Repository = "https://github.com/yitianlian/cc_convert" +Issues = "https://github.com/yitianlian/cc_convert/issues" + +[project.scripts] +cc_convert = "cc_convert.cli:main" +cc-convert = "cc_convert.cli:main" + +[project.optional-dependencies] +test = ["pytest>=7"] +parity = ["litellm>=1.0"] + +[tool.maturin] +manifest-path = "../crates/cc_convert_py/Cargo.toml" +module-name = "cc_convert._native" +python-source = "." +features = ["pyo3/extension-module"] diff --git a/sidecars/cc_convert/python/tests/test_cli_helpers.py b/sidecars/cc_convert/python/tests/test_cli_helpers.py new file mode 100644 index 0000000..69133b1 --- /dev/null +++ b/sidecars/cc_convert/python/tests/test_cli_helpers.py @@ -0,0 +1,76 @@ +"""Unit tests for cc_convert.cli helper functions.""" + +from __future__ import annotations + +import pytest + +from cc_convert.cli import _normalize_upstream_url, _path_is_anthropic_messages + + +# ---------- _normalize_upstream_url ---------- + + +@pytest.mark.parametrize( + "raw,expected", + [ + # Bare host → assume /v1/chat/completions + ("https://api.openai.com", "https://api.openai.com/v1/chat/completions"), + ("https://api.openai.com/", "https://api.openai.com/v1/chat/completions"), + ("http://localhost:8000", "http://localhost:8000/v1/chat/completions"), + # Already has /v1 → just append /chat/completions + ("https://api.openai.com/v1", "https://api.openai.com/v1/chat/completions"), + ("https://api.openai.com/v1/", "https://api.openai.com/v1/chat/completions"), + ("http://vllm:8000/v1", "http://vllm:8000/v1/chat/completions"), + # Different version + ("http://x/v2", "http://x/v2/chat/completions"), + # Already complete → verbatim (modulo trailing slash strip) + ( + "https://api.openai.com/v1/chat/completions", + "https://api.openai.com/v1/chat/completions", + ), + ( + "https://api.openai.com/v1/chat/completions/", + "https://api.openai.com/v1/chat/completions", + ), + # Proxy-prefix / vendor-prefix paths: pass through assumption that the + # user knew what they were doing. + ( + "https://my-proxy.example/openai/v1/chat/completions", + "https://my-proxy.example/openai/v1/chat/completions", + ), + ], +) +def test_normalize_upstream_url(raw: str, expected: str) -> None: + assert _normalize_upstream_url(raw) == expected + + +# ---------- _path_is_anthropic_messages ---------- + + +@pytest.mark.parametrize( + "path,expected", + [ + # Canonical + ("/v1/messages", True), + ("/messages", True), + # Trailing slash + ("/v1/messages/", True), + ("/messages/", True), + # With query string + ("/v1/messages?stream=true", True), + # Behind a vendor / load-balancer prefix + ("/anthropic/v1/messages", True), + ("/api/v1/messages", True), + ("/some/deep/prefix/v1/messages?x=1", True), + # Negatives + ("/v1/messages/foo", False), # trailing extra segment + ("/healthz", False), + ("/version", False), + ("/v1/messages.json", False), # not a path boundary + ("/translate/cc-to-oai", False), + ("/", False), + ("", False), + ], +) +def test_path_is_anthropic_messages(path: str, expected: bool) -> None: + assert _path_is_anthropic_messages(path) is expected diff --git a/sidecars/cc_convert/python/tests/test_parity.py b/sidecars/cc_convert/python/tests/test_parity.py new file mode 100644 index 0000000..2de37a1 --- /dev/null +++ b/sidecars/cc_convert/python/tests/test_parity.py @@ -0,0 +1,139 @@ +"""Python parity tests: feed each LiteLLM-golden fixture through the +PyO3-backed ``cc_convert.translate_request`` and assert the result matches +(semantically) the same LiteLLM golden the Rust test uses. + +This validates that the Python wheel emits exactly what the Rust core does. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +import cc_convert # type: ignore + +FIXTURES = Path(__file__).resolve().parent.parent.parent / "tests" / "fixtures" + + +def _normalize(v, ctx_key: str | None = None): + """Drop nulls, recurse, parse-and-re-stringify tool_call `arguments`.""" + if v is None: + return None + if isinstance(v, dict): + out = {} + for k, val in v.items(): + n = _normalize(val, k) + if n is None: + continue + out[k] = n + return out + if isinstance(v, list): + return [_normalize(item) for item in v] + if isinstance(v, str) and ctx_key == "arguments": + try: + return json.dumps(json.loads(v), separators=(",", ":"), sort_keys=True) + except json.JSONDecodeError: + return v + return v + + +def _request_pairs(): + req_dir = FIXTURES / "requests" + pairs = [] + for in_path in sorted(req_dir.glob("anthropic_*.json")): + name = in_path.stem.removeprefix("anthropic_") + golden = req_dir / f"openai_{name}.json" + if golden.exists(): + pairs.append(pytest.param(in_path, golden, id=name)) + return pairs + + +@pytest.mark.parametrize("input_path,golden_path", _request_pairs()) +def test_request_parity_with_litellm(input_path: Path, golden_path: Path) -> None: + anthropic_req = json.loads(input_path.read_text()) + openai_actual, _tool_map = cc_convert.translate_request( + anthropic_req, mode="litellm_compat" + ) + golden = json.loads(golden_path.read_text()) + assert _normalize(openai_actual) == _normalize(golden), ( + f"Python wheel diverged from LiteLLM golden for {input_path.name}.\n" + f" actual: {json.dumps(_normalize(openai_actual), sort_keys=True, indent=2)}\n" + f" golden: {json.dumps(_normalize(golden), sort_keys=True, indent=2)}" + ) + + +def test_round_trip_response() -> None: + """Trivial sanity check that translate_response works end-to-end.""" + out = cc_convert.translate_response( + { + "id": "chatcmpl-abc", + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "hi"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 5, "completion_tokens": 1}, + }, + "claude-opus-4-7", + {}, + ) + assert out["id"] == "msg_abc" + assert out["content"][0]["text"] == "hi" + assert out["stop_reason"] == "end_turn" + + +def test_stream_translator_text_only() -> None: + t = cc_convert.StreamTranslator("claude-opus-4-7", {}) + events = [] + events += t.push({"id": "chatcmpl-1", "choices": [{"index": 0, "delta": {"content": "hi"}}]}) + events += t.push( + { + "id": "chatcmpl-1", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + } + ) + kinds = [e["type"] for e in events] + assert "message_start" in kinds + assert "message_stop" in kinds + # The text content was emitted as a delta. + text_chunks = [ + e["delta"]["text"] + for e in events + if e["type"] == "content_block_delta" + and e["delta"].get("type") == "text_delta" + ] + assert "hi" in "".join(text_chunks) + + +def test_pragmatic_collapses_single_text_system_to_string() -> None: + """Real-world OAI upstreams (SGLang/vLLM strict mode) reject list-content + on system messages. The 'pragmatic' default collapses single-text blocks + to a plain string. The 'litellm_compat' mode keeps them as a list.""" + req = { + "model": "Qwen", + "max_tokens": 10, + "system": [ + {"type": "text", "text": "rule A", "cache_control": {"type": "ephemeral"}} + ], + "messages": [{"role": "user", "content": "hi"}], + } + pragmatic, _ = cc_convert.translate_request(req) # default mode + assert pragmatic["messages"][0]["role"] == "system" + assert pragmatic["messages"][0]["content"] == "rule A", ( + "pragmatic should collapse single-text system to string" + ) + + litellm, _ = cc_convert.translate_request(req, mode="litellm_compat") + assert litellm["messages"][0]["content"] == [{"type": "text", "text": "rule A"}], ( + "litellm_compat should keep list-content" + ) + + +def test_unknown_mode_raises() -> None: + with pytest.raises(ValueError, match="unknown mode"): + cc_convert.translate_request({"model": "x", "max_tokens": 1, "messages": []}, mode="bogus") diff --git a/sidecars/cc_convert/python/tests/test_probe_models.py b/sidecars/cc_convert/python/tests/test_probe_models.py new file mode 100644 index 0000000..e1d36c6 --- /dev/null +++ b/sidecars/cc_convert/python/tests/test_probe_models.py @@ -0,0 +1,70 @@ +"""Tests for the playground's /v1/models probe — it has to accept several +non-standard response shapes.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from unittest.mock import patch + +import pytest + +# Make playground importable as a module +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent / "playground")) + +import run_roundtrip as _online # noqa: E402 + +probe_models = _online.probe_models + + +class _FakeResp: + def __init__(self, payload: bytes) -> None: + self._payload = payload + + def read(self) -> bytes: + return self._payload + + +def _patched_probe(response_payload: dict | list): + """Return a callable that mocks the URL opener to return `response_payload`.""" + + def fake_open(req, timeout=5): # noqa: ARG001 + return _FakeResp(json.dumps(response_payload).encode()) + + class _O: + def open(self, req, timeout=5): + return fake_open(req, timeout) + + return _O + + +@pytest.mark.parametrize( + "payload,expected", + [ + # Standard OpenAI shape + ({"data": [{"id": "gpt-4"}, {"id": "gpt-3.5"}]}, ["gpt-4", "gpt-3.5"]), + # OpenAI shape with extras + ( + {"data": [{"id": "gpt-4", "object": "model", "owned_by": "openai"}], "object": "list"}, + ["gpt-4"], + ), + # SGLang style: {"models": ["string", ...]} + ({"models": ["/model"]}, ["/model"]), + ({"models": ["Qwen2.5-7B", "Llama-3"]}, ["Qwen2.5-7B", "Llama-3"]), + # SGLang variant: {"models": [{"id": "..."}]} + ({"models": [{"id": "deepseek-r1"}]}, ["deepseek-r1"]), + # Plain list of strings + (["gpt-4", "claude"], ["gpt-4", "claude"]), + # Plain list of dicts + ([{"id": "gpt-4"}], ["gpt-4"]), + ], +) +def test_probe_models_recognizes_various_shapes(payload, expected): + with patch.object(_online, "_opener_no_proxy", lambda: _patched_probe(payload)()): + assert probe_models("http://x") == expected + + +def test_probe_models_returns_empty_on_unrecognized_shape(): + with patch.object(_online, "_opener_no_proxy", lambda: _patched_probe({"weird": "shape"})()): + assert probe_models("http://x") == [] diff --git a/sidecars/cc_convert/scripts/regen_fixtures.py b/sidecars/cc_convert/scripts/regen_fixtures.py new file mode 100644 index 0000000..1feb72c --- /dev/null +++ b/sidecars/cc_convert/scripts/regen_fixtures.py @@ -0,0 +1,71 @@ +"""Regenerate golden fixtures by running each input through LiteLLM (oracle). + +Writes: + tests/fixtures/requests/openai_.json (translated request) + tests/fixtures/requests/tool_map_.json (LiteLLM tool name map) + tests/fixtures/responses/anthropic_.json (translated response — hand-built) + tests/fixtures/streams/anthropic_.jsonl (translated event stream — hand-built) + +For requests, LiteLLM is the source of truth. + +For responses and streams, LiteLLM exposes only an `AnthropicStreamWrapper` +that needs a full LiteLLM ModelResponse to drive — we do NOT depend on that. +Instead, we either: + - use the Rust translator itself to produce the golden (it has been + unit-tested against LiteLLM's behaviour for each rule), OR + - leave the response/stream goldens stubbed out for now and rely on the + Rust-side unit tests we already wrote. + +This script currently regenerates request goldens only. Run it again after +each behavioural change to the request translator. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent / "tests" / "fixtures" + + +def _import_litellm(): + try: + from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( + LiteLLMAnthropicMessagesAdapter, + ) + except ImportError as e: + print(f"litellm not installed: {e}", file=sys.stderr) + print("Install: pip install 'litellm>=1.0'", file=sys.stderr) + sys.exit(1) + return LiteLLMAnthropicMessagesAdapter() + + +def regen_requests(adapter) -> int: + req_dir = ROOT / "requests" + count = 0 + for input_path in sorted(req_dir.glob("anthropic_*.json")): + name = input_path.stem.removeprefix("anthropic_") + anthropic_req = json.loads(input_path.read_text()) + try: + openai_req, tool_map = adapter.translate_anthropic_to_openai(anthropic_req) + except Exception as e: # noqa: BLE001 + print(f"[skip] {name}: {e}", file=sys.stderr) + continue + out_req = req_dir / f"openai_{name}.json" + out_map = req_dir / f"tool_map_{name}.json" + out_req.write_text(json.dumps(openai_req, indent=2, sort_keys=True) + "\n") + out_map.write_text(json.dumps(tool_map or {}, indent=2, sort_keys=True) + "\n") + count += 1 + print(f" ✓ {name}") + return count + + +def main() -> None: + adapter = _import_litellm() + n = regen_requests(adapter) + print(f"\nregenerated {n} request goldens via LiteLLM") + + +if __name__ == "__main__": + main() diff --git a/sidecars/cc_convert/scripts/regen_response_fixtures.py b/sidecars/cc_convert/scripts/regen_response_fixtures.py new file mode 100644 index 0000000..25e38f5 --- /dev/null +++ b/sidecars/cc_convert/scripts/regen_response_fixtures.py @@ -0,0 +1,158 @@ +"""Regenerate Anthropic-shape golden response files from OpenAI inputs +using LiteLLM as the oracle. + +For each fixture under tests/fixtures/responses/openai_.json, this +loads the corresponding meta_.json (which carries any tool_map) and +runs LiteLLM's translate_openai_response_to_anthropic, writing +anthropic_.json next to the input. + +Run: + + pip install 'litellm>=1.0' + python scripts/regen_response_fixtures.py +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any, Dict, List, Optional + +ROOT = Path(__file__).resolve().parent.parent / "tests" / "fixtures" / "responses" + + +def _import_litellm(): + try: + from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import ( + LiteLLMAnthropicMessagesAdapter, + ) + from litellm.types.utils import ( + ChatCompletionMessageToolCall, + Choices, + Function, + Message, + ModelResponse, + PromptTokensDetailsWrapper, + Usage, + ) + except ImportError as e: + print(f"litellm missing: {e}", file=sys.stderr) + sys.exit(1) + return ( + LiteLLMAnthropicMessagesAdapter(), + ChatCompletionMessageToolCall, + Choices, + Function, + Message, + ModelResponse, + PromptTokensDetailsWrapper, + Usage, + ) + + +def _build_model_response( + raw: Dict[str, Any], + ChatCompletionMessageToolCall, + Choices, + Function, + Message, + ModelResponse, + PromptTokensDetailsWrapper, + Usage, +): + """Translate a JSON OpenAI ChatCompletion dict into the LiteLLM + ModelResponse object the adapter expects.""" + + def build_message(m: Dict[str, Any]) -> Any: + kw: Dict[str, Any] = {"role": m.get("role", "assistant")} + if m.get("content") is not None: + kw["content"] = m["content"] + if m.get("reasoning_content") is not None: + kw["reasoning_content"] = m["reasoning_content"] + if m.get("tool_calls"): + kw["tool_calls"] = [ + ChatCompletionMessageToolCall( + id=tc["id"], + type=tc.get("type", "function"), + function=Function( + name=tc["function"]["name"], + arguments=tc["function"].get("arguments", ""), + ), + ) + for tc in m["tool_calls"] + ] + return Message(**kw) + + choices = [ + Choices( + index=c.get("index", 0), + message=build_message(c["message"]), + finish_reason=c.get("finish_reason"), + ) + for c in raw["choices"] + ] + usage_kw: Dict[str, Any] = {} + if (u := raw.get("usage")): + usage_kw["prompt_tokens"] = u.get("prompt_tokens", 0) + usage_kw["completion_tokens"] = u.get("completion_tokens", 0) + if (ptd := u.get("prompt_tokens_details")): + usage_kw["prompt_tokens_details"] = PromptTokensDetailsWrapper( + cached_tokens=ptd.get("cached_tokens", 0) + ) + return ModelResponse( + id=raw["id"], + model=raw.get("model", "gpt-4o-mini"), + choices=choices, + usage=Usage(**usage_kw), + ) + + +def main() -> None: + ( + adapter, + ChatCompletionMessageToolCall, + Choices, + Function, + Message, + ModelResponse, + PromptTokensDetailsWrapper, + Usage, + ) = _import_litellm() + + n = 0 + for input_path in sorted(ROOT.glob("openai_*.json")): + name = input_path.stem.removeprefix("openai_") + meta_path = ROOT / f"meta_{name}.json" + meta = json.loads(meta_path.read_text()) if meta_path.exists() else {} + raw = json.loads(input_path.read_text()) + try: + model_resp = _build_model_response( + raw, + ChatCompletionMessageToolCall, + Choices, + Function, + Message, + ModelResponse, + PromptTokensDetailsWrapper, + Usage, + ) + tool_map = meta.get("tool_map") or {} + golden = adapter.translate_openai_response_to_anthropic( + model_resp, tool_name_mapping=tool_map + ) + except Exception as e: # noqa: BLE001 + print(f"[skip] {name}: {e}", file=sys.stderr) + continue + # LiteLLM returns a TypedDict; dump with default=str for safety. + out_path = ROOT / f"anthropic_{name}.json" + out_path.write_text( + json.dumps(golden, indent=2, sort_keys=True, default=str) + "\n" + ) + n += 1 + print(f" ✓ {name}") + print(f"\nregenerated {n} response goldens via LiteLLM") + + +if __name__ == "__main__": + main() diff --git a/sidecars/cc_convert/scripts/regen_stream_fixtures.py b/sidecars/cc_convert/scripts/regen_stream_fixtures.py new file mode 100644 index 0000000..c7d8488 --- /dev/null +++ b/sidecars/cc_convert/scripts/regen_stream_fixtures.py @@ -0,0 +1,154 @@ +"""Regenerate Anthropic-shape stream goldens from OpenAI .sse inputs using +LiteLLM's AnthropicStreamWrapper. + +Reads each tests/fixtures/streams/openai_.sse, parses the chunks as +ModelResponse-equivalent objects, feeds them into AnthropicStreamWrapper, +and writes the resulting Anthropic events (one per line) to +anthropic_.jsonl. +""" + +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path +from typing import Any, Dict, List + +ROOT = Path(__file__).resolve().parent.parent / "tests" / "fixtures" / "streams" + + +def _import_litellm(): + try: + from litellm.llms.anthropic.experimental_pass_through.adapters.streaming_iterator import ( + AnthropicStreamWrapper, + ) + from litellm.types.utils import ( + ChatCompletionDeltaToolCall, + Delta, + Function, + ModelResponse, + ModelResponseStream, + StreamingChoices, + Usage, + ) + except ImportError as e: + print(f"litellm missing: {e}", file=sys.stderr) + sys.exit(1) + return ( + AnthropicStreamWrapper, + ChatCompletionDeltaToolCall, + Delta, + Function, + ModelResponse, + ModelResponseStream, + StreamingChoices, + Usage, + ) + + +def parse_sse(text: str) -> List[Dict[str, Any]]: + chunks: List[Dict[str, Any]] = [] + for block in text.split("\n\n"): + for line in block.splitlines(): + if line.startswith("data:"): + payload = line[5:].strip() + if payload and payload != "[DONE]": + chunks.append(json.loads(payload)) + return chunks + + +def build_chunk( + raw: Dict[str, Any], + ChatCompletionDeltaToolCall, + Delta, + Function, + ModelResponseStream, + StreamingChoices, + Usage, +): + def build_delta(d: Dict[str, Any]) -> Any: + kw: Dict[str, Any] = {} + if d.get("role") is not None: + kw["role"] = d["role"] + if d.get("content") is not None: + kw["content"] = d["content"] + if d.get("reasoning_content") is not None: + kw["reasoning_content"] = d["reasoning_content"] + if d.get("tool_calls"): + kw["tool_calls"] = [ + ChatCompletionDeltaToolCall( + id=tc.get("id"), + type=tc.get("type", "function"), + index=tc.get("index", 0), + function=Function( + name=tc.get("function", {}).get("name"), + arguments=tc.get("function", {}).get("arguments"), + ), + ) + for tc in d["tool_calls"] + ] + return Delta(**kw) + + choices = [ + StreamingChoices( + index=c.get("index", 0), + delta=build_delta(c.get("delta", {})), + finish_reason=c.get("finish_reason"), + ) + for c in raw.get("choices", []) + ] + kw: Dict[str, Any] = {"id": raw.get("id"), "choices": choices} + if raw.get("usage"): + kw["usage"] = Usage(**raw["usage"]) + return ModelResponseStream(**kw) + + +def main() -> None: + ( + AnthropicStreamWrapper, + ChatCompletionDeltaToolCall, + Delta, + Function, + ModelResponse, + ModelResponseStream, + StreamingChoices, + Usage, + ) = _import_litellm() + + n = 0 + for input_path in sorted(ROOT.glob("openai_*.sse")): + name = input_path.stem.removeprefix("openai_") + raw_chunks = parse_sse(input_path.read_text()) + try: + chunks = [ + build_chunk( + c, + ChatCompletionDeltaToolCall, + Delta, + Function, + ModelResponseStream, + StreamingChoices, + Usage, + ) + for c in raw_chunks + ] + wrapper = AnthropicStreamWrapper( + completion_stream=iter(chunks), + model="claude-opus-4-7", + ) + events = list(wrapper) + except Exception as e: # noqa: BLE001 + print(f"[skip] {name}: {e!r}", file=sys.stderr) + continue + out_path = ROOT / f"anthropic_{name}.jsonl" + with out_path.open("w") as f: + for ev in events: + f.write(json.dumps(ev, default=str, sort_keys=True) + "\n") + n += 1 + print(f" ✓ {name} ({len(events)} events)") + print(f"\nregenerated {n} stream goldens via LiteLLM") + + +if __name__ == "__main__": + main() diff --git a/sidecars/cc_convert/scripts/seed_extra_request_fixtures.py b/sidecars/cc_convert/scripts/seed_extra_request_fixtures.py new file mode 100644 index 0000000..52ae3c0 --- /dev/null +++ b/sidecars/cc_convert/scripts/seed_extra_request_fixtures.py @@ -0,0 +1,198 @@ +"""Add request fixtures 32-43 covering message shapes that the first batch missed. + +Cases: + 32_agent_tool_loop multi-turn: user → assistant tool_use → user tool_result → assistant text + 33_user_content_cache_control cache_control on a user text block + 34_assistant_content_cache_control cache_control on an assistant text block + 35_assistant_thinking_history prior assistant turn with `thinking` block + 36_user_mixed_content text + image + tool_result in same user message + 37_empty_string_content user content: "" + 38_complex_tool_schema tool with nested object / array / enum schema + 39_tool_choice_auto_no_parallel tool_choice {type:"auto", disable_parallel_tool_use: true} + 40_tool_choice_none tool_choice {type:"none"} + 41_thinking_high thinking.budget_tokens = 12000 → reasoning_effort high + 42_thinking_low thinking.budget_tokens = 2000 → reasoning_effort low + 43_stop_sequences stop_sequences: ["END", "STOP"] → stop list +""" + +from __future__ import annotations +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent / "tests" / "fixtures" / "requests" + + +def write(path: Path, payload) -> None: + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + +EXTRA = { + "32_agent_tool_loop": { + "model": "gpt-4o-mini", + "max_tokens": 200, + "messages": [ + {"role": "user", "content": "What's the weather in Tokyo?"}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "Let me check."}, + { + "type": "tool_use", + "id": "toolu_w1", + "name": "get_weather", + "input": {"city": "Tokyo"}, + }, + ], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "toolu_w1", "content": "Sunny, 25C"} + ], + }, + {"role": "assistant", "content": "It's sunny and 25°C in Tokyo."}, + ], + }, + "33_user_content_cache_control": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Very long cached context", + "cache_control": {"type": "ephemeral"}, + }, + {"type": "text", "text": "Question: summarize."}, + ], + } + ], + }, + "34_assistant_content_cache_control": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [ + {"role": "user", "content": "ok"}, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "remembered answer", + "cache_control": {"type": "ephemeral"}, + } + ], + }, + {"role": "user", "content": "again"}, + ], + }, + "35_assistant_thinking_history": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [ + {"role": "user", "content": "Hard math problem"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "Let me work this out step by step...", "signature": "sig_xyz"}, + {"type": "text", "text": "The answer is 42."}, + ], + }, + {"role": "user", "content": "Why?"}, + ], + }, + "36_user_mixed_content": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "tu_x", "content": "previous tool output"}, + {"type": "text", "text": "Now look at this image:"}, + {"type": "image", "source": {"type": "url", "url": "https://example.com/a.png"}}, + ], + } + ], + }, + "37_empty_string_content": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{"role": "user", "content": ""}], + }, + "38_complex_tool_schema": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{"role": "user", "content": "ok"}], + "tools": [ + { + "name": "search_flights", + "description": "Find flights", + "input_schema": { + "type": "object", + "properties": { + "origin": {"type": "string"}, + "destination": {"type": "string"}, + "passengers": { + "type": "array", + "items": { + "type": "object", + "properties": { + "age": {"type": "integer", "minimum": 0}, + "class": {"type": "string", "enum": ["economy", "business", "first"]}, + }, + "required": ["age", "class"], + }, + }, + "departure_date": {"type": "string", "format": "date"}, + }, + "required": ["origin", "destination", "departure_date"], + }, + } + ], + }, + "39_tool_choice_auto_no_parallel": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{"role": "user", "content": "ok"}], + "tools": [{"name": "f", "input_schema": {"type": "object"}}], + "tool_choice": {"type": "auto", "disable_parallel_tool_use": True}, + }, + "40_tool_choice_none": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{"role": "user", "content": "ok"}], + "tools": [{"name": "f", "input_schema": {"type": "object"}}], + "tool_choice": {"type": "none"}, + }, + "41_thinking_high": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "thinking": {"type": "enabled", "budget_tokens": 12000}, + }, + "42_thinking_low": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "thinking": {"type": "enabled", "budget_tokens": 2000}, + }, + "43_stop_sequences": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{"role": "user", "content": "tell a joke"}], + "stop_sequences": ["END", "STOP"], + }, +} + + +def main() -> None: + for name, payload in EXTRA.items(): + write(ROOT / f"anthropic_{name}.json", payload) + print(f"wrote {len(EXTRA)} extra requests to {ROOT}") + + +if __name__ == "__main__": + main() diff --git a/sidecars/cc_convert/scripts/seed_fixture_inputs.py b/sidecars/cc_convert/scripts/seed_fixture_inputs.py new file mode 100644 index 0000000..4380e47 --- /dev/null +++ b/sidecars/cc_convert/scripts/seed_fixture_inputs.py @@ -0,0 +1,520 @@ +"""Seed all 31 golden-fixture inputs. + +Run once: `python scripts/seed_fixture_inputs.py`. Writes: + + tests/fixtures/requests/anthropic_.json (cases 1-20) + tests/fixtures/responses/openai_.json (cases 21-26) + tests/fixtures/responses/meta_.json (original_model + tool_map) + tests/fixtures/streams/openai_.sse (cases 27-31) + +The corresponding `openai_*.json` / `anthropic_*.json` / `anthropic_*.jsonl` +golden outputs are produced by `scripts/regen_fixtures.py` (which calls +LiteLLM) or by `scripts/seed_golden_outputs.py` (which calls the Rust +translator we just built — useful when LiteLLM is unreachable). +""" + +from __future__ import annotations +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent / "tests" / "fixtures" + + +def write(path: Path, payload) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + text = json.dumps(payload, indent=2, sort_keys=True) + "\n" + path.write_text(text) + + +# --------- Requests --------- + +REQUESTS = { + "01_plain_user_text": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hello"}], + }, + "02_system_string": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "system": "Be concise.", + "messages": [{"role": "user", "content": "hi"}], + }, + "03_system_blocks_with_cache_control": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "system": [ + {"type": "text", "text": "rule 1", "cache_control": {"type": "ephemeral"}} + ], + "messages": [{"role": "user", "content": "hi"}], + }, + "04_multi_turn": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello!"}, + {"role": "user", "content": "ok"}, + ], + }, + "05_user_image_base64": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": {"type": "base64", "media_type": "image/png", "data": "AAAA"}, + }, + {"type": "text", "text": "what is this?"}, + ], + } + ], + }, + "06_user_image_url": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": [ + {"type": "image", "source": {"type": "url", "url": "https://example.com/x.png"}} + ], + } + ], + }, + "07_assistant_single_tool_use": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [ + { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_1", + "name": "get_weather", + "input": {"city": "Paris"}, + } + ], + } + ], + }, + "08_assistant_two_parallel_tool_uses": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [ + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "tu_a", "name": "f1", "input": {"a": 1}}, + {"type": "tool_use", "id": "tu_b", "name": "f2", "input": {"b": 2}}, + ], + } + ], + }, + "09_user_single_tool_result": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "toolu_1", "content": "21C"} + ], + } + ], + }, + "10_user_three_tool_results": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": "a"}, + {"type": "tool_result", "tool_use_id": "t2", "content": "b"}, + {"type": "tool_result", "tool_use_id": "t3", "content": "c"}, + ], + } + ], + }, + "11_user_tool_result_multipart": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "t1", + "content": [ + {"type": "text", "text": "see image:"}, + { + "type": "image", + "source": {"type": "url", "url": "https://x/y.png"}, + }, + ], + } + ], + } + ], + }, + "12_tools_input_schema": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "tools": [ + { + "name": "get_weather", + "description": "weather lookup", + "input_schema": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + } + ], + }, + "13_long_tool_name": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "tools": [{"name": "x" * 80, "input_schema": {"type": "object"}}], + }, + "14_tool_choice_any": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "tool_choice": {"type": "any"}, + }, + "15_tool_choice_named": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "tools": [{"name": "f", "input_schema": {"type": "object"}}], + "tool_choice": {"type": "tool", "name": "f"}, + }, + "16_metadata_user_id": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"user_id": "u-123"}, + }, + "17_thinking_medium": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "thinking": {"type": "enabled", "budget_tokens": 5000}, + }, + "18_top_k_dropped": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "top_k": 20, + }, + "19_stream_include_usage": { + "model": "gpt-4o-mini", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "stream": True, + }, + "20_o3_mini_uses_max_completion_tokens": { + "model": "o3-mini", + "max_tokens": 200, + "messages": [{"role": "user", "content": "hi"}], + }, +} + + +# --------- Responses --------- + +RESPONSES = { + "21_plain_text": ( + { + "id": "chatcmpl-abc", + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "hello world"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 4}, + }, + {"original_model": "claude-opus-4-7", "tool_map": {}}, + ), + "22_empty_content": ( + { + "id": "chatcmpl-1", + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": None}, + "finish_reason": "stop", + } + ], + }, + {"original_model": "claude-opus-4-7", "tool_map": {}}, + ), + "23_single_tool_call_no_text": ( + { + "id": "chatcmpl-x", + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city":"Paris"}', + }, + } + ], + }, + "finish_reason": "tool_calls", + } + ], + }, + {"original_model": "claude-opus-4-7", "tool_map": {}}, + ), + "24_multiple_tool_calls": ( + { + "id": "chatcmpl-y", + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "f", "arguments": "{}"}, + }, + { + "id": "c2", + "type": "function", + "function": {"name": "g", "arguments": "{}"}, + }, + ], + }, + "finish_reason": "tool_calls", + } + ], + }, + {"original_model": "claude-opus-4-7", "tool_map": {}}, + ), + "25_length_max_tokens": ( + { + "id": "chatcmpl-1", + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "trunc"}, + "finish_reason": "length", + } + ], + }, + {"original_model": "claude-opus-4-7", "tool_map": {}}, + ), + "26_cached_tokens": ( + { + "id": "chatcmpl-1", + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "hi"}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 2, + "prompt_tokens_details": {"cached_tokens": 32}, + }, + }, + {"original_model": "claude-opus-4-7", "tool_map": {}}, + ), +} + + +# --------- Streams (raw OpenAI SSE input) --------- + +def sse(payload: dict) -> str: + return f"data: {json.dumps(payload, separators=(',', ':'))}\n\n" + + +STREAMS_INPUT = { + "27_text_only": ( + sse( + { + "id": "chatcmpl-1", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "hel"}}], + } + ) + + sse({"id": "chatcmpl-1", "choices": [{"index": 0, "delta": {"content": "lo"}}]}) + + sse( + {"id": "chatcmpl-1", "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]} + ) + + "data: [DONE]\n\n" + ), + "28_single_tool_call_fragments": ( + sse( + { + "id": "chatcmpl-x", + "choices": [ + { + "index": 0, + "delta": { + "tool_calls": [ + { + "index": 0, + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": ""}, + } + ] + }, + } + ], + } + ) + + sse( + { + "id": "chatcmpl-x", + "choices": [ + { + "index": 0, + "delta": { + "tool_calls": [ + { + "index": 0, + "id": "call_1", + "type": "function", + "function": {"arguments": '{"city":'}, + } + ] + }, + } + ], + } + ) + + sse( + { + "id": "chatcmpl-x", + "choices": [ + { + "index": 0, + "delta": { + "tool_calls": [ + { + "index": 0, + "id": "call_1", + "type": "function", + "function": {"arguments": '"Paris"}'}, + } + ] + }, + } + ], + } + ) + + sse( + { + "id": "chatcmpl-x", + "choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}], + } + ) + ), + "29_two_parallel_tool_calls": ( + sse( + { + "id": "chatcmpl-y", + "choices": [ + { + "index": 0, + "delta": { + "tool_calls": [ + { + "index": 0, + "id": "a", + "type": "function", + "function": {"name": "f", "arguments": "{}"}, + }, + { + "index": 1, + "id": "b", + "type": "function", + "function": {"name": "g", "arguments": "{}"}, + }, + ] + }, + } + ], + } + ) + + sse( + { + "id": "chatcmpl-y", + "choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}], + } + ) + ), + "30_stream_ends_without_finish_reason": ( + sse( + { + "id": "chatcmpl-z", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "partial"}}], + } + ) + ), + "31_reasoning_then_text": ( + sse( + { + "id": "chatcmpl-r", + "choices": [{"index": 0, "delta": {"reasoning_content": "let me think..."}}], + } + ) + + sse({"id": "chatcmpl-r", "choices": [{"index": 0, "delta": {"content": "Done."}}]}) + + sse( + {"id": "chatcmpl-r", "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]} + ) + ), +} + + +def main() -> None: + for name, payload in REQUESTS.items(): + write(ROOT / "requests" / f"anthropic_{name}.json", payload) + for name, (payload, meta) in RESPONSES.items(): + write(ROOT / "responses" / f"openai_{name}.json", payload) + write(ROOT / "responses" / f"meta_{name}.json", meta) + for name, sse_text in STREAMS_INPUT.items(): + path = ROOT / "streams" / f"openai_{name}.sse" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(sse_text) + + print( + f"wrote {len(REQUESTS)} requests, {len(RESPONSES)} responses, " + f"{len(STREAMS_INPUT)} streams to {ROOT}" + ) + + +if __name__ == "__main__": + main() diff --git a/sidecars/cc_convert/tests/fixtures/README.md b/sidecars/cc_convert/tests/fixtures/README.md new file mode 100644 index 0000000..886dd05 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/README.md @@ -0,0 +1,20 @@ +# Golden parity fixtures + +Each subdirectory holds one direction of translation: + +- `requests/` — input `anthropic_.json`, expected output `openai_.json`. +- `responses/` — input `openai_.json`, expected output `anthropic_.json`. Each file pair also carries a `meta_.json` with the `original_model` and the `tool_map` (typically empty) that the response translator needs. +- `streams/` — input `openai_.sse`, expected output `anthropic_.jsonl` (one Anthropic event per line, in order). + +Goldens were produced by: + +1. **LiteLLM (primary oracle)**: `python scripts/regen_fixtures.py` (requires `pip install 'litellm>=1.0'`). Where LiteLLM and 1rgs/claude-code-proxy disagree, LiteLLM wins; the divergence is noted in `source.txt`. + +2. **Hand-curated**: the streams and responses are typically hand-crafted from the OpenAI Chat Completions API reference, because LiteLLM's response side does not have a simple "translate one chunk" entry point. + +To regenerate goldens after a rule change: + +```bash +pip install 'litellm>=1.0' +python scripts/regen_fixtures.py +``` diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_01_plain_user_text.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_01_plain_user_text.json new file mode 100644 index 0000000..67e735c --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_01_plain_user_text.json @@ -0,0 +1,10 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "hello", + "role": "user" + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_02_system_string.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_02_system_string.json new file mode 100644 index 0000000..0c4adca --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_02_system_string.json @@ -0,0 +1,11 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "system": "Be concise." +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_03_system_blocks_with_cache_control.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_03_system_blocks_with_cache_control.json new file mode 100644 index 0000000..d39d02d --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_03_system_blocks_with_cache_control.json @@ -0,0 +1,19 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "system": [ + { + "cache_control": { + "type": "ephemeral" + }, + "text": "rule 1", + "type": "text" + } + ] +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_04_multi_turn.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_04_multi_turn.json new file mode 100644 index 0000000..9143904 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_04_multi_turn.json @@ -0,0 +1,18 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "hi", + "role": "user" + }, + { + "content": "hello!", + "role": "assistant" + }, + { + "content": "ok", + "role": "user" + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_05_user_image_base64.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_05_user_image_base64.json new file mode 100644 index 0000000..bec6c4a --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_05_user_image_base64.json @@ -0,0 +1,23 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": [ + { + "source": { + "data": "AAAA", + "media_type": "image/png", + "type": "base64" + }, + "type": "image" + }, + { + "text": "what is this?", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_06_user_image_url.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_06_user_image_url.json new file mode 100644 index 0000000..7feb155 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_06_user_image_url.json @@ -0,0 +1,18 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": [ + { + "source": { + "type": "url", + "url": "https://example.com/x.png" + }, + "type": "image" + } + ], + "role": "user" + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_07_assistant_single_tool_use.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_07_assistant_single_tool_use.json new file mode 100644 index 0000000..61d6e84 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_07_assistant_single_tool_use.json @@ -0,0 +1,19 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": [ + { + "id": "toolu_1", + "input": { + "city": "Paris" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_08_assistant_two_parallel_tool_uses.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_08_assistant_two_parallel_tool_uses.json new file mode 100644 index 0000000..127b844 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_08_assistant_two_parallel_tool_uses.json @@ -0,0 +1,27 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": [ + { + "id": "tu_a", + "input": { + "a": 1 + }, + "name": "f1", + "type": "tool_use" + }, + { + "id": "tu_b", + "input": { + "b": 2 + }, + "name": "f2", + "type": "tool_use" + } + ], + "role": "assistant" + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_09_user_single_tool_result.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_09_user_single_tool_result.json new file mode 100644 index 0000000..b004e41 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_09_user_single_tool_result.json @@ -0,0 +1,16 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": [ + { + "content": "21C", + "tool_use_id": "toolu_1", + "type": "tool_result" + } + ], + "role": "user" + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_10_user_three_tool_results.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_10_user_three_tool_results.json new file mode 100644 index 0000000..35d3aae --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_10_user_three_tool_results.json @@ -0,0 +1,26 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": [ + { + "content": "a", + "tool_use_id": "t1", + "type": "tool_result" + }, + { + "content": "b", + "tool_use_id": "t2", + "type": "tool_result" + }, + { + "content": "c", + "tool_use_id": "t3", + "type": "tool_result" + } + ], + "role": "user" + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_11_user_tool_result_multipart.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_11_user_tool_result_multipart.json new file mode 100644 index 0000000..b158869 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_11_user_tool_result_multipart.json @@ -0,0 +1,28 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": [ + { + "content": [ + { + "text": "see image:", + "type": "text" + }, + { + "source": { + "type": "url", + "url": "https://x/y.png" + }, + "type": "image" + } + ], + "tool_use_id": "t1", + "type": "tool_result" + } + ], + "role": "user" + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_12_tools_input_schema.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_12_tools_input_schema.json new file mode 100644 index 0000000..4c7d4cb --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_12_tools_input_schema.json @@ -0,0 +1,27 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "tools": [ + { + "description": "weather lookup", + "input_schema": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + }, + "name": "get_weather" + } + ] +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_13_long_tool_name.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_13_long_tool_name.json new file mode 100644 index 0000000..544c6ac --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_13_long_tool_name.json @@ -0,0 +1,18 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "tools": [ + { + "input_schema": { + "type": "object" + }, + "name": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + } + ] +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_14_tool_choice_any.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_14_tool_choice_any.json new file mode 100644 index 0000000..50b8a1b --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_14_tool_choice_any.json @@ -0,0 +1,13 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "tool_choice": { + "type": "any" + } +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_15_tool_choice_named.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_15_tool_choice_named.json new file mode 100644 index 0000000..92ca508 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_15_tool_choice_named.json @@ -0,0 +1,22 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "tool_choice": { + "name": "f", + "type": "tool" + }, + "tools": [ + { + "input_schema": { + "type": "object" + }, + "name": "f" + } + ] +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_16_metadata_user_id.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_16_metadata_user_id.json new file mode 100644 index 0000000..ea84c36 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_16_metadata_user_id.json @@ -0,0 +1,13 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "metadata": { + "user_id": "u-123" + }, + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_17_thinking_medium.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_17_thinking_medium.json new file mode 100644 index 0000000..f242b6e --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_17_thinking_medium.json @@ -0,0 +1,14 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "thinking": { + "budget_tokens": 5000, + "type": "enabled" + } +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_18_top_k_dropped.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_18_top_k_dropped.json new file mode 100644 index 0000000..6d021f1 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_18_top_k_dropped.json @@ -0,0 +1,11 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "top_k": 20 +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_19_stream_include_usage.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_19_stream_include_usage.json new file mode 100644 index 0000000..4fa1dc6 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_19_stream_include_usage.json @@ -0,0 +1,11 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "stream": true +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_20_o3_mini_uses_max_completion_tokens.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_20_o3_mini_uses_max_completion_tokens.json new file mode 100644 index 0000000..725c70a --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_20_o3_mini_uses_max_completion_tokens.json @@ -0,0 +1,10 @@ +{ + "max_tokens": 200, + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "model": "o3-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_32_agent_tool_loop.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_32_agent_tool_loop.json new file mode 100644 index 0000000..c2568f4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_32_agent_tool_loop.json @@ -0,0 +1,41 @@ +{ + "max_tokens": 200, + "messages": [ + { + "content": "What's the weather in Tokyo?", + "role": "user" + }, + { + "content": [ + { + "text": "Let me check.", + "type": "text" + }, + { + "id": "toolu_w1", + "input": { + "city": "Tokyo" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + { + "content": [ + { + "content": "Sunny, 25C", + "tool_use_id": "toolu_w1", + "type": "tool_result" + } + ], + "role": "user" + }, + { + "content": "It's sunny and 25\u00b0C in Tokyo.", + "role": "assistant" + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_33_user_content_cache_control.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_33_user_content_cache_control.json new file mode 100644 index 0000000..1f64473 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_33_user_content_cache_control.json @@ -0,0 +1,22 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": [ + { + "cache_control": { + "type": "ephemeral" + }, + "text": "Very long cached context", + "type": "text" + }, + { + "text": "Question: summarize.", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_34_assistant_content_cache_control.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_34_assistant_content_cache_control.json new file mode 100644 index 0000000..2d3cae6 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_34_assistant_content_cache_control.json @@ -0,0 +1,26 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "ok", + "role": "user" + }, + { + "content": [ + { + "cache_control": { + "type": "ephemeral" + }, + "text": "remembered answer", + "type": "text" + } + ], + "role": "assistant" + }, + { + "content": "again", + "role": "user" + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_35_assistant_thinking_history.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_35_assistant_thinking_history.json new file mode 100644 index 0000000..a515b03 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_35_assistant_thinking_history.json @@ -0,0 +1,28 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "Hard math problem", + "role": "user" + }, + { + "content": [ + { + "signature": "sig_xyz", + "thinking": "Let me work this out step by step...", + "type": "thinking" + }, + { + "text": "The answer is 42.", + "type": "text" + } + ], + "role": "assistant" + }, + { + "content": "Why?", + "role": "user" + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_36_user_mixed_content.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_36_user_mixed_content.json new file mode 100644 index 0000000..90cb106 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_36_user_mixed_content.json @@ -0,0 +1,27 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": [ + { + "content": "previous tool output", + "tool_use_id": "tu_x", + "type": "tool_result" + }, + { + "text": "Now look at this image:", + "type": "text" + }, + { + "source": { + "type": "url", + "url": "https://example.com/a.png" + }, + "type": "image" + } + ], + "role": "user" + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_37_empty_string_content.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_37_empty_string_content.json new file mode 100644 index 0000000..e17d95c --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_37_empty_string_content.json @@ -0,0 +1,10 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "", + "role": "user" + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_38_complex_tool_schema.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_38_complex_tool_schema.json new file mode 100644 index 0000000..6a6f44e --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_38_complex_tool_schema.json @@ -0,0 +1,60 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "ok", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "tools": [ + { + "description": "Find flights", + "input_schema": { + "properties": { + "departure_date": { + "format": "date", + "type": "string" + }, + "destination": { + "type": "string" + }, + "origin": { + "type": "string" + }, + "passengers": { + "items": { + "properties": { + "age": { + "minimum": 0, + "type": "integer" + }, + "class": { + "enum": [ + "economy", + "business", + "first" + ], + "type": "string" + } + }, + "required": [ + "age", + "class" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "origin", + "destination", + "departure_date" + ], + "type": "object" + }, + "name": "search_flights" + } + ] +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_39_tool_choice_auto_no_parallel.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_39_tool_choice_auto_no_parallel.json new file mode 100644 index 0000000..3570773 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_39_tool_choice_auto_no_parallel.json @@ -0,0 +1,22 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "ok", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "tool_choice": { + "disable_parallel_tool_use": true, + "type": "auto" + }, + "tools": [ + { + "input_schema": { + "type": "object" + }, + "name": "f" + } + ] +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_40_tool_choice_none.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_40_tool_choice_none.json new file mode 100644 index 0000000..5b12609 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_40_tool_choice_none.json @@ -0,0 +1,21 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "ok", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "tool_choice": { + "type": "none" + }, + "tools": [ + { + "input_schema": { + "type": "object" + }, + "name": "f" + } + ] +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_41_thinking_high.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_41_thinking_high.json new file mode 100644 index 0000000..012bd39 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_41_thinking_high.json @@ -0,0 +1,14 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "thinking": { + "budget_tokens": 12000, + "type": "enabled" + } +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_42_thinking_low.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_42_thinking_low.json new file mode 100644 index 0000000..6bb5ec9 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_42_thinking_low.json @@ -0,0 +1,14 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "thinking": { + "budget_tokens": 2000, + "type": "enabled" + } +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/anthropic_43_stop_sequences.json b/sidecars/cc_convert/tests/fixtures/requests/anthropic_43_stop_sequences.json new file mode 100644 index 0000000..528a047 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/anthropic_43_stop_sequences.json @@ -0,0 +1,14 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "tell a joke", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "stop_sequences": [ + "END", + "STOP" + ] +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_01_plain_user_text.json b/sidecars/cc_convert/tests/fixtures/requests/openai_01_plain_user_text.json new file mode 100644 index 0000000..67e735c --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_01_plain_user_text.json @@ -0,0 +1,10 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "hello", + "role": "user" + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_02_system_string.json b/sidecars/cc_convert/tests/fixtures/requests/openai_02_system_string.json new file mode 100644 index 0000000..8d45e0e --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_02_system_string.json @@ -0,0 +1,14 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "Be concise.", + "role": "system" + }, + { + "content": "hi", + "role": "user" + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_03_system_blocks_with_cache_control.json b/sidecars/cc_convert/tests/fixtures/requests/openai_03_system_blocks_with_cache_control.json new file mode 100644 index 0000000..21f553f --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_03_system_blocks_with_cache_control.json @@ -0,0 +1,19 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": [ + { + "text": "rule 1", + "type": "text" + } + ], + "role": "system" + }, + { + "content": "hi", + "role": "user" + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_04_multi_turn.json b/sidecars/cc_convert/tests/fixtures/requests/openai_04_multi_turn.json new file mode 100644 index 0000000..35ffcc0 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_04_multi_turn.json @@ -0,0 +1,19 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "hi", + "role": "user" + }, + { + "content": "hello!", + "role": "assistant", + "thinking_blocks": null + }, + { + "content": "ok", + "role": "user" + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_05_user_image_base64.json b/sidecars/cc_convert/tests/fixtures/requests/openai_05_user_image_base64.json new file mode 100644 index 0000000..f6bf7b2 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_05_user_image_base64.json @@ -0,0 +1,21 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": [ + { + "image_url": { + "url": "data:image/png;base64,AAAA" + }, + "type": "image_url" + }, + { + "text": "what is this?", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_06_user_image_url.json b/sidecars/cc_convert/tests/fixtures/requests/openai_06_user_image_url.json new file mode 100644 index 0000000..12aebdf --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_06_user_image_url.json @@ -0,0 +1,17 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": [ + { + "image_url": { + "url": "https://example.com/x.png" + }, + "type": "image_url" + } + ], + "role": "user" + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_07_assistant_single_tool_use.json b/sidecars/cc_convert/tests/fixtures/requests/openai_07_assistant_single_tool_use.json new file mode 100644 index 0000000..64feb7a --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_07_assistant_single_tool_use.json @@ -0,0 +1,21 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": null, + "role": "assistant", + "thinking_blocks": null, + "tool_calls": [ + { + "function": { + "arguments": "{\"city\": \"Paris\"}", + "name": "get_weather" + }, + "id": "toolu_1", + "type": "function" + } + ] + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_08_assistant_two_parallel_tool_uses.json b/sidecars/cc_convert/tests/fixtures/requests/openai_08_assistant_two_parallel_tool_uses.json new file mode 100644 index 0000000..5655ab5 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_08_assistant_two_parallel_tool_uses.json @@ -0,0 +1,29 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": null, + "role": "assistant", + "thinking_blocks": null, + "tool_calls": [ + { + "function": { + "arguments": "{\"a\": 1}", + "name": "f1" + }, + "id": "tu_a", + "type": "function" + }, + { + "function": { + "arguments": "{\"b\": 2}", + "name": "f2" + }, + "id": "tu_b", + "type": "function" + } + ] + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_09_user_single_tool_result.json b/sidecars/cc_convert/tests/fixtures/requests/openai_09_user_single_tool_result.json new file mode 100644 index 0000000..c694e6e --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_09_user_single_tool_result.json @@ -0,0 +1,11 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "21C", + "role": "tool", + "tool_call_id": "toolu_1" + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_10_user_three_tool_results.json b/sidecars/cc_convert/tests/fixtures/requests/openai_10_user_three_tool_results.json new file mode 100644 index 0000000..25a36ea --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_10_user_three_tool_results.json @@ -0,0 +1,21 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "a", + "role": "tool", + "tool_call_id": "t1" + }, + { + "content": "b", + "role": "tool", + "tool_call_id": "t2" + }, + { + "content": "c", + "role": "tool", + "tool_call_id": "t3" + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_11_user_tool_result_multipart.json b/sidecars/cc_convert/tests/fixtures/requests/openai_11_user_tool_result_multipart.json new file mode 100644 index 0000000..b1a1d5d --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_11_user_tool_result_multipart.json @@ -0,0 +1,22 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": [ + { + "text": "see image:", + "type": "text" + }, + { + "image_url": { + "url": "https://x/y.png" + }, + "type": "image_url" + } + ], + "role": "tool", + "tool_call_id": "t1" + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_12_tools_input_schema.json b/sidecars/cc_convert/tests/fixtures/requests/openai_12_tools_input_schema.json new file mode 100644 index 0000000..52f6fa2 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_12_tools_input_schema.json @@ -0,0 +1,30 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "tools": [ + { + "function": { + "description": "weather lookup", + "name": "get_weather", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + "type": "function" + } + ] +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_13_long_tool_name.json b/sidecars/cc_convert/tests/fixtures/requests/openai_13_long_tool_name.json new file mode 100644 index 0000000..1cc5ea6 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_13_long_tool_name.json @@ -0,0 +1,21 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "tools": [ + { + "function": { + "name": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx_d929cdee", + "parameters": { + "type": "object" + } + }, + "type": "function" + } + ] +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_14_tool_choice_any.json b/sidecars/cc_convert/tests/fixtures/requests/openai_14_tool_choice_any.json new file mode 100644 index 0000000..219c784 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_14_tool_choice_any.json @@ -0,0 +1,11 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "tool_choice": "required" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_15_tool_choice_named.json b/sidecars/cc_convert/tests/fixtures/requests/openai_15_tool_choice_named.json new file mode 100644 index 0000000..ded0a20 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_15_tool_choice_named.json @@ -0,0 +1,27 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "tool_choice": { + "function": { + "name": "f" + }, + "type": "function" + }, + "tools": [ + { + "function": { + "name": "f", + "parameters": { + "type": "object" + } + }, + "type": "function" + } + ] +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_16_metadata_user_id.json b/sidecars/cc_convert/tests/fixtures/requests/openai_16_metadata_user_id.json new file mode 100644 index 0000000..575d027 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_16_metadata_user_id.json @@ -0,0 +1,11 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "user": "u-123" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_17_thinking_medium.json b/sidecars/cc_convert/tests/fixtures/requests/openai_17_thinking_medium.json new file mode 100644 index 0000000..a7bd87f --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_17_thinking_medium.json @@ -0,0 +1,11 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "reasoning_effort": "medium" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_18_top_k_dropped.json b/sidecars/cc_convert/tests/fixtures/requests/openai_18_top_k_dropped.json new file mode 100644 index 0000000..6d021f1 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_18_top_k_dropped.json @@ -0,0 +1,11 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "top_k": 20 +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_19_stream_include_usage.json b/sidecars/cc_convert/tests/fixtures/requests/openai_19_stream_include_usage.json new file mode 100644 index 0000000..4fa1dc6 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_19_stream_include_usage.json @@ -0,0 +1,11 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "stream": true +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_20_o3_mini_uses_max_completion_tokens.json b/sidecars/cc_convert/tests/fixtures/requests/openai_20_o3_mini_uses_max_completion_tokens.json new file mode 100644 index 0000000..725c70a --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_20_o3_mini_uses_max_completion_tokens.json @@ -0,0 +1,10 @@ +{ + "max_tokens": 200, + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "model": "o3-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_32_agent_tool_loop.json b/sidecars/cc_convert/tests/fixtures/requests/openai_32_agent_tool_loop.json new file mode 100644 index 0000000..262bdb2 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_32_agent_tool_loop.json @@ -0,0 +1,35 @@ +{ + "max_tokens": 200, + "messages": [ + { + "content": "What's the weather in Tokyo?", + "role": "user" + }, + { + "content": "Let me check.", + "role": "assistant", + "thinking_blocks": null, + "tool_calls": [ + { + "function": { + "arguments": "{\"city\": \"Tokyo\"}", + "name": "get_weather" + }, + "id": "toolu_w1", + "type": "function" + } + ] + }, + { + "content": "Sunny, 25C", + "role": "tool", + "tool_call_id": "toolu_w1" + }, + { + "content": "It's sunny and 25\u00b0C in Tokyo.", + "role": "assistant", + "thinking_blocks": null + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_33_user_content_cache_control.json b/sidecars/cc_convert/tests/fixtures/requests/openai_33_user_content_cache_control.json new file mode 100644 index 0000000..534e0ba --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_33_user_content_cache_control.json @@ -0,0 +1,19 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": [ + { + "text": "Very long cached context", + "type": "text" + }, + { + "text": "Question: summarize.", + "type": "text" + } + ], + "role": "user" + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_34_assistant_content_cache_control.json b/sidecars/cc_convert/tests/fixtures/requests/openai_34_assistant_content_cache_control.json new file mode 100644 index 0000000..0a9de8e --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_34_assistant_content_cache_control.json @@ -0,0 +1,19 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "ok", + "role": "user" + }, + { + "content": "remembered answer", + "role": "assistant", + "thinking_blocks": null + }, + { + "content": "again", + "role": "user" + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_35_assistant_thinking_history.json b/sidecars/cc_convert/tests/fixtures/requests/openai_35_assistant_thinking_history.json new file mode 100644 index 0000000..d5c0c99 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_35_assistant_thinking_history.json @@ -0,0 +1,26 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "Hard math problem", + "role": "user" + }, + { + "content": "The answer is 42.", + "role": "assistant", + "thinking_blocks": [ + { + "cache_control": {}, + "signature": "sig_xyz", + "thinking": "Let me work this out step by step...", + "type": "thinking" + } + ] + }, + { + "content": "Why?", + "role": "user" + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_36_user_mixed_content.json b/sidecars/cc_convert/tests/fixtures/requests/openai_36_user_mixed_content.json new file mode 100644 index 0000000..4f89ebf --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_36_user_mixed_content.json @@ -0,0 +1,26 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "previous tool output", + "role": "tool", + "tool_call_id": "tu_x" + }, + { + "content": [ + { + "text": "Now look at this image:", + "type": "text" + }, + { + "image_url": { + "url": "https://example.com/a.png" + }, + "type": "image_url" + } + ], + "role": "user" + } + ], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_37_empty_string_content.json b/sidecars/cc_convert/tests/fixtures/requests/openai_37_empty_string_content.json new file mode 100644 index 0000000..7c9cc54 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_37_empty_string_content.json @@ -0,0 +1,5 @@ +{ + "max_tokens": 100, + "messages": [], + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_38_complex_tool_schema.json b/sidecars/cc_convert/tests/fixtures/requests/openai_38_complex_tool_schema.json new file mode 100644 index 0000000..e589698 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_38_complex_tool_schema.json @@ -0,0 +1,63 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "ok", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "tools": [ + { + "function": { + "description": "Find flights", + "name": "search_flights", + "parameters": { + "properties": { + "departure_date": { + "format": "date", + "type": "string" + }, + "destination": { + "type": "string" + }, + "origin": { + "type": "string" + }, + "passengers": { + "items": { + "properties": { + "age": { + "minimum": 0, + "type": "integer" + }, + "class": { + "enum": [ + "economy", + "business", + "first" + ], + "type": "string" + } + }, + "required": [ + "age", + "class" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "origin", + "destination", + "departure_date" + ], + "type": "object" + } + }, + "type": "function" + } + ] +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_39_tool_choice_auto_no_parallel.json b/sidecars/cc_convert/tests/fixtures/requests/openai_39_tool_choice_auto_no_parallel.json new file mode 100644 index 0000000..4928a6c --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_39_tool_choice_auto_no_parallel.json @@ -0,0 +1,22 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "ok", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "tool_choice": "auto", + "tools": [ + { + "function": { + "name": "f", + "parameters": { + "type": "object" + } + }, + "type": "function" + } + ] +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_40_tool_choice_none.json b/sidecars/cc_convert/tests/fixtures/requests/openai_40_tool_choice_none.json new file mode 100644 index 0000000..3e013b1 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_40_tool_choice_none.json @@ -0,0 +1,22 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "ok", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "tool_choice": "none", + "tools": [ + { + "function": { + "name": "f", + "parameters": { + "type": "object" + } + }, + "type": "function" + } + ] +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_41_thinking_high.json b/sidecars/cc_convert/tests/fixtures/requests/openai_41_thinking_high.json new file mode 100644 index 0000000..428c6ae --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_41_thinking_high.json @@ -0,0 +1,11 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "reasoning_effort": "high" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_42_thinking_low.json b/sidecars/cc_convert/tests/fixtures/requests/openai_42_thinking_low.json new file mode 100644 index 0000000..c8ad4fb --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_42_thinking_low.json @@ -0,0 +1,11 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "hi", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "reasoning_effort": "low" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/openai_43_stop_sequences.json b/sidecars/cc_convert/tests/fixtures/requests/openai_43_stop_sequences.json new file mode 100644 index 0000000..528a047 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/openai_43_stop_sequences.json @@ -0,0 +1,14 @@ +{ + "max_tokens": 100, + "messages": [ + { + "content": "tell a joke", + "role": "user" + } + ], + "model": "gpt-4o-mini", + "stop_sequences": [ + "END", + "STOP" + ] +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_01_plain_user_text.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_01_plain_user_text.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_01_plain_user_text.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_02_system_string.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_02_system_string.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_02_system_string.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_03_system_blocks_with_cache_control.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_03_system_blocks_with_cache_control.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_03_system_blocks_with_cache_control.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_04_multi_turn.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_04_multi_turn.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_04_multi_turn.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_05_user_image_base64.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_05_user_image_base64.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_05_user_image_base64.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_06_user_image_url.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_06_user_image_url.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_06_user_image_url.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_07_assistant_single_tool_use.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_07_assistant_single_tool_use.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_07_assistant_single_tool_use.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_08_assistant_two_parallel_tool_uses.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_08_assistant_two_parallel_tool_uses.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_08_assistant_two_parallel_tool_uses.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_09_user_single_tool_result.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_09_user_single_tool_result.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_09_user_single_tool_result.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_10_user_three_tool_results.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_10_user_three_tool_results.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_10_user_three_tool_results.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_11_user_tool_result_multipart.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_11_user_tool_result_multipart.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_11_user_tool_result_multipart.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_12_tools_input_schema.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_12_tools_input_schema.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_12_tools_input_schema.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_13_long_tool_name.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_13_long_tool_name.json new file mode 100644 index 0000000..8633d4a --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_13_long_tool_name.json @@ -0,0 +1,3 @@ +{ + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx_d929cdee": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" +} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_14_tool_choice_any.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_14_tool_choice_any.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_14_tool_choice_any.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_15_tool_choice_named.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_15_tool_choice_named.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_15_tool_choice_named.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_16_metadata_user_id.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_16_metadata_user_id.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_16_metadata_user_id.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_17_thinking_medium.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_17_thinking_medium.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_17_thinking_medium.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_18_top_k_dropped.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_18_top_k_dropped.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_18_top_k_dropped.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_19_stream_include_usage.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_19_stream_include_usage.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_19_stream_include_usage.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_20_o3_mini_uses_max_completion_tokens.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_20_o3_mini_uses_max_completion_tokens.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_20_o3_mini_uses_max_completion_tokens.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_32_agent_tool_loop.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_32_agent_tool_loop.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_32_agent_tool_loop.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_33_user_content_cache_control.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_33_user_content_cache_control.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_33_user_content_cache_control.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_34_assistant_content_cache_control.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_34_assistant_content_cache_control.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_34_assistant_content_cache_control.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_35_assistant_thinking_history.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_35_assistant_thinking_history.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_35_assistant_thinking_history.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_36_user_mixed_content.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_36_user_mixed_content.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_36_user_mixed_content.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_37_empty_string_content.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_37_empty_string_content.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_37_empty_string_content.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_38_complex_tool_schema.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_38_complex_tool_schema.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_38_complex_tool_schema.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_39_tool_choice_auto_no_parallel.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_39_tool_choice_auto_no_parallel.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_39_tool_choice_auto_no_parallel.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_40_tool_choice_none.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_40_tool_choice_none.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_40_tool_choice_none.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_41_thinking_high.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_41_thinking_high.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_41_thinking_high.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_42_thinking_low.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_42_thinking_low.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_42_thinking_low.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/requests/tool_map_43_stop_sequences.json b/sidecars/cc_convert/tests/fixtures/requests/tool_map_43_stop_sequences.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/requests/tool_map_43_stop_sequences.json @@ -0,0 +1 @@ +{} diff --git a/sidecars/cc_convert/tests/fixtures/responses/anthropic_21_plain_text.json b/sidecars/cc_convert/tests/fixtures/responses/anthropic_21_plain_text.json new file mode 100644 index 0000000..8b6cf3f --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/responses/anthropic_21_plain_text.json @@ -0,0 +1,18 @@ +{ + "content": [ + { + "text": "hello world", + "type": "text" + } + ], + "id": "chatcmpl-abc", + "model": "gpt-4o-mini", + "role": "assistant", + "stop_reason": "end_turn", + "stop_sequence": null, + "type": "message", + "usage": { + "input_tokens": 10, + "output_tokens": 4 + } +} diff --git a/sidecars/cc_convert/tests/fixtures/responses/anthropic_22_empty_content.json b/sidecars/cc_convert/tests/fixtures/responses/anthropic_22_empty_content.json new file mode 100644 index 0000000..d716967 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/responses/anthropic_22_empty_content.json @@ -0,0 +1,13 @@ +{ + "content": [], + "id": "chatcmpl-1", + "model": "gpt-4o-mini", + "role": "assistant", + "stop_reason": "end_turn", + "stop_sequence": null, + "type": "message", + "usage": { + "input_tokens": 0, + "output_tokens": 0 + } +} diff --git a/sidecars/cc_convert/tests/fixtures/responses/anthropic_23_single_tool_call_no_text.json b/sidecars/cc_convert/tests/fixtures/responses/anthropic_23_single_tool_call_no_text.json new file mode 100644 index 0000000..988ffb3 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/responses/anthropic_23_single_tool_call_no_text.json @@ -0,0 +1,23 @@ +{ + "content": [ + { + "id": "call_1", + "input": { + "city": "Paris" + }, + "name": "get_weather", + "provider_specific_fields": null, + "type": "tool_use" + } + ], + "id": "chatcmpl-x", + "model": "gpt-4o-mini", + "role": "assistant", + "stop_reason": "tool_use", + "stop_sequence": null, + "type": "message", + "usage": { + "input_tokens": 0, + "output_tokens": 0 + } +} diff --git a/sidecars/cc_convert/tests/fixtures/responses/anthropic_24_multiple_tool_calls.json b/sidecars/cc_convert/tests/fixtures/responses/anthropic_24_multiple_tool_calls.json new file mode 100644 index 0000000..5ae947d --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/responses/anthropic_24_multiple_tool_calls.json @@ -0,0 +1,28 @@ +{ + "content": [ + { + "id": "c1", + "input": {}, + "name": "f", + "provider_specific_fields": null, + "type": "tool_use" + }, + { + "id": "c2", + "input": {}, + "name": "g", + "provider_specific_fields": null, + "type": "tool_use" + } + ], + "id": "chatcmpl-y", + "model": "gpt-4o-mini", + "role": "assistant", + "stop_reason": "tool_use", + "stop_sequence": null, + "type": "message", + "usage": { + "input_tokens": 0, + "output_tokens": 0 + } +} diff --git a/sidecars/cc_convert/tests/fixtures/responses/anthropic_25_length_max_tokens.json b/sidecars/cc_convert/tests/fixtures/responses/anthropic_25_length_max_tokens.json new file mode 100644 index 0000000..6eca1f2 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/responses/anthropic_25_length_max_tokens.json @@ -0,0 +1,18 @@ +{ + "content": [ + { + "text": "trunc", + "type": "text" + } + ], + "id": "chatcmpl-1", + "model": "gpt-4o-mini", + "role": "assistant", + "stop_reason": "max_tokens", + "stop_sequence": null, + "type": "message", + "usage": { + "input_tokens": 0, + "output_tokens": 0 + } +} diff --git a/sidecars/cc_convert/tests/fixtures/responses/anthropic_26_cached_tokens.json b/sidecars/cc_convert/tests/fixtures/responses/anthropic_26_cached_tokens.json new file mode 100644 index 0000000..bd3ec3f --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/responses/anthropic_26_cached_tokens.json @@ -0,0 +1,19 @@ +{ + "content": [ + { + "text": "hi", + "type": "text" + } + ], + "id": "chatcmpl-1", + "model": "gpt-4o-mini", + "role": "assistant", + "stop_reason": "end_turn", + "stop_sequence": null, + "type": "message", + "usage": { + "cache_read_input_tokens": 32, + "input_tokens": 68, + "output_tokens": 2 + } +} diff --git a/sidecars/cc_convert/tests/fixtures/responses/meta_21_plain_text.json b/sidecars/cc_convert/tests/fixtures/responses/meta_21_plain_text.json new file mode 100644 index 0000000..d29db7e --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/responses/meta_21_plain_text.json @@ -0,0 +1,4 @@ +{ + "original_model": "claude-opus-4-7", + "tool_map": {} +} diff --git a/sidecars/cc_convert/tests/fixtures/responses/meta_22_empty_content.json b/sidecars/cc_convert/tests/fixtures/responses/meta_22_empty_content.json new file mode 100644 index 0000000..d29db7e --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/responses/meta_22_empty_content.json @@ -0,0 +1,4 @@ +{ + "original_model": "claude-opus-4-7", + "tool_map": {} +} diff --git a/sidecars/cc_convert/tests/fixtures/responses/meta_23_single_tool_call_no_text.json b/sidecars/cc_convert/tests/fixtures/responses/meta_23_single_tool_call_no_text.json new file mode 100644 index 0000000..d29db7e --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/responses/meta_23_single_tool_call_no_text.json @@ -0,0 +1,4 @@ +{ + "original_model": "claude-opus-4-7", + "tool_map": {} +} diff --git a/sidecars/cc_convert/tests/fixtures/responses/meta_24_multiple_tool_calls.json b/sidecars/cc_convert/tests/fixtures/responses/meta_24_multiple_tool_calls.json new file mode 100644 index 0000000..d29db7e --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/responses/meta_24_multiple_tool_calls.json @@ -0,0 +1,4 @@ +{ + "original_model": "claude-opus-4-7", + "tool_map": {} +} diff --git a/sidecars/cc_convert/tests/fixtures/responses/meta_25_length_max_tokens.json b/sidecars/cc_convert/tests/fixtures/responses/meta_25_length_max_tokens.json new file mode 100644 index 0000000..d29db7e --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/responses/meta_25_length_max_tokens.json @@ -0,0 +1,4 @@ +{ + "original_model": "claude-opus-4-7", + "tool_map": {} +} diff --git a/sidecars/cc_convert/tests/fixtures/responses/meta_26_cached_tokens.json b/sidecars/cc_convert/tests/fixtures/responses/meta_26_cached_tokens.json new file mode 100644 index 0000000..d29db7e --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/responses/meta_26_cached_tokens.json @@ -0,0 +1,4 @@ +{ + "original_model": "claude-opus-4-7", + "tool_map": {} +} diff --git a/sidecars/cc_convert/tests/fixtures/responses/openai_21_plain_text.json b/sidecars/cc_convert/tests/fixtures/responses/openai_21_plain_text.json new file mode 100644 index 0000000..4957179 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/responses/openai_21_plain_text.json @@ -0,0 +1,18 @@ +{ + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "hello world", + "role": "assistant" + } + } + ], + "id": "chatcmpl-abc", + "model": "gpt-4o-mini", + "usage": { + "completion_tokens": 4, + "prompt_tokens": 10 + } +} diff --git a/sidecars/cc_convert/tests/fixtures/responses/openai_22_empty_content.json b/sidecars/cc_convert/tests/fixtures/responses/openai_22_empty_content.json new file mode 100644 index 0000000..2284095 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/responses/openai_22_empty_content.json @@ -0,0 +1,14 @@ +{ + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": null, + "role": "assistant" + } + } + ], + "id": "chatcmpl-1", + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/responses/openai_23_single_tool_call_no_text.json b/sidecars/cc_convert/tests/fixtures/responses/openai_23_single_tool_call_no_text.json new file mode 100644 index 0000000..7e58d7e --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/responses/openai_23_single_tool_call_no_text.json @@ -0,0 +1,24 @@ +{ + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{\"city\":\"Paris\"}", + "name": "get_weather" + }, + "id": "call_1", + "type": "function" + } + ] + } + } + ], + "id": "chatcmpl-x", + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/responses/openai_24_multiple_tool_calls.json b/sidecars/cc_convert/tests/fixtures/responses/openai_24_multiple_tool_calls.json new file mode 100644 index 0000000..67c8575 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/responses/openai_24_multiple_tool_calls.json @@ -0,0 +1,32 @@ +{ + "choices": [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "content": null, + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "{}", + "name": "f" + }, + "id": "c1", + "type": "function" + }, + { + "function": { + "arguments": "{}", + "name": "g" + }, + "id": "c2", + "type": "function" + } + ] + } + } + ], + "id": "chatcmpl-y", + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/responses/openai_25_length_max_tokens.json b/sidecars/cc_convert/tests/fixtures/responses/openai_25_length_max_tokens.json new file mode 100644 index 0000000..e777bb1 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/responses/openai_25_length_max_tokens.json @@ -0,0 +1,14 @@ +{ + "choices": [ + { + "finish_reason": "length", + "index": 0, + "message": { + "content": "trunc", + "role": "assistant" + } + } + ], + "id": "chatcmpl-1", + "model": "gpt-4o-mini" +} diff --git a/sidecars/cc_convert/tests/fixtures/responses/openai_26_cached_tokens.json b/sidecars/cc_convert/tests/fixtures/responses/openai_26_cached_tokens.json new file mode 100644 index 0000000..e1c710a --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/responses/openai_26_cached_tokens.json @@ -0,0 +1,21 @@ +{ + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "hi", + "role": "assistant" + } + } + ], + "id": "chatcmpl-1", + "model": "gpt-4o-mini", + "usage": { + "completion_tokens": 2, + "prompt_tokens": 100, + "prompt_tokens_details": { + "cached_tokens": 32 + } + } +} diff --git a/sidecars/cc_convert/tests/fixtures/streams/anthropic_27_text_only.jsonl b/sidecars/cc_convert/tests/fixtures/streams/anthropic_27_text_only.jsonl new file mode 100644 index 0000000..fc69873 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/streams/anthropic_27_text_only.jsonl @@ -0,0 +1,7 @@ +{"message": {"content": [], "id": "msg_168c6410-02b2-495c-ab0a-d9e294f19e40", "model": "claude-opus-4-7", "role": "assistant", "stop_reason": null, "stop_sequence": null, "type": "message", "usage": {"cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, "input_tokens": 0, "output_tokens": 0}}, "type": "message_start"} +{"content_block": {"text": "", "type": "text"}, "index": 0, "type": "content_block_start"} +{"delta": {"text": "hel", "type": "text_delta"}, "index": 0, "type": "content_block_delta"} +{"delta": {"text": "lo", "type": "text_delta"}, "index": 0, "type": "content_block_delta"} +{"index": 0, "type": "content_block_stop"} +{"delta": {"stop_reason": "end_turn"}, "type": "message_delta", "usage": {"input_tokens": 0, "output_tokens": 0}} +{"type": "message_stop"} diff --git a/sidecars/cc_convert/tests/fixtures/streams/anthropic_28_single_tool_call_fragments.jsonl b/sidecars/cc_convert/tests/fixtures/streams/anthropic_28_single_tool_call_fragments.jsonl new file mode 100644 index 0000000..389c1af --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/streams/anthropic_28_single_tool_call_fragments.jsonl @@ -0,0 +1,9 @@ +{"message": {"content": [], "id": "msg_444b0173-71c5-4703-b1e2-970510339fe2", "model": "claude-opus-4-7", "role": "assistant", "stop_reason": null, "stop_sequence": null, "type": "message", "usage": {"cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, "input_tokens": 0, "output_tokens": 0}}, "type": "message_start"} +{"content_block": {"text": "", "type": "text"}, "index": 0, "type": "content_block_start"} +{"index": 0, "type": "content_block_stop"} +{"content_block": {"id": "call_1", "input": {}, "name": "get_weather", "type": "tool_use"}, "index": 1, "type": "content_block_start"} +{"delta": {"partial_json": "{\"city\":", "type": "input_json_delta"}, "index": 1, "type": "content_block_delta"} +{"delta": {"partial_json": "\"Paris\"}", "type": "input_json_delta"}, "index": 1, "type": "content_block_delta"} +{"index": 1, "type": "content_block_stop"} +{"delta": {"stop_reason": "tool_use"}, "type": "message_delta", "usage": {"input_tokens": 0, "output_tokens": 0}} +{"type": "message_stop"} diff --git a/sidecars/cc_convert/tests/fixtures/streams/anthropic_29_two_parallel_tool_calls.jsonl b/sidecars/cc_convert/tests/fixtures/streams/anthropic_29_two_parallel_tool_calls.jsonl new file mode 100644 index 0000000..d820c14 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/streams/anthropic_29_two_parallel_tool_calls.jsonl @@ -0,0 +1,8 @@ +{"message": {"content": [], "id": "msg_03550e46-97fc-4419-a073-81b6caf142fa", "model": "claude-opus-4-7", "role": "assistant", "stop_reason": null, "stop_sequence": null, "type": "message", "usage": {"cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, "input_tokens": 0, "output_tokens": 0}}, "type": "message_start"} +{"content_block": {"text": "", "type": "text"}, "index": 0, "type": "content_block_start"} +{"index": 0, "type": "content_block_stop"} +{"content_block": {"id": "a", "input": {}, "name": "f", "type": "tool_use"}, "index": 1, "type": "content_block_start"} +{"delta": {"partial_json": "{}{}", "type": "input_json_delta"}, "index": 1, "type": "content_block_delta"} +{"index": 1, "type": "content_block_stop"} +{"delta": {"stop_reason": "tool_use"}, "type": "message_delta", "usage": {"input_tokens": 0, "output_tokens": 0}} +{"type": "message_stop"} diff --git a/sidecars/cc_convert/tests/fixtures/streams/anthropic_30_stream_ends_without_finish_reason.jsonl b/sidecars/cc_convert/tests/fixtures/streams/anthropic_30_stream_ends_without_finish_reason.jsonl new file mode 100644 index 0000000..c154931 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/streams/anthropic_30_stream_ends_without_finish_reason.jsonl @@ -0,0 +1,4 @@ +{"message": {"content": [], "id": "msg_73c3bbe2-d1f6-49f0-a0dc-43f0a829acec", "model": "claude-opus-4-7", "role": "assistant", "stop_reason": null, "stop_sequence": null, "type": "message", "usage": {"cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, "input_tokens": 0, "output_tokens": 0}}, "type": "message_start"} +{"content_block": {"text": "", "type": "text"}, "index": 0, "type": "content_block_start"} +{"delta": {"text": "partial", "type": "text_delta"}, "index": 0, "type": "content_block_delta"} +{"type": "message_stop"} diff --git a/sidecars/cc_convert/tests/fixtures/streams/anthropic_31_reasoning_then_text.jsonl b/sidecars/cc_convert/tests/fixtures/streams/anthropic_31_reasoning_then_text.jsonl new file mode 100644 index 0000000..7ff620c --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/streams/anthropic_31_reasoning_then_text.jsonl @@ -0,0 +1,7 @@ +{"message": {"content": [], "id": "msg_ea5061d1-e15f-4204-864f-1d0cddc75b9c", "model": "claude-opus-4-7", "role": "assistant", "stop_reason": null, "stop_sequence": null, "type": "message", "usage": {"cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, "input_tokens": 0, "output_tokens": 0}}, "type": "message_start"} +{"content_block": {"text": "", "type": "text"}, "index": 0, "type": "content_block_start"} +{"delta": {"thinking": "let me think...", "type": "thinking_delta"}, "index": 0, "type": "content_block_delta"} +{"delta": {"text": "Done.", "type": "text_delta"}, "index": 0, "type": "content_block_delta"} +{"index": 0, "type": "content_block_stop"} +{"delta": {"stop_reason": "end_turn"}, "type": "message_delta", "usage": {"input_tokens": 0, "output_tokens": 0}} +{"type": "message_stop"} diff --git a/sidecars/cc_convert/tests/fixtures/streams/openai_27_text_only.sse b/sidecars/cc_convert/tests/fixtures/streams/openai_27_text_only.sse new file mode 100644 index 0000000..091c1f6 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/streams/openai_27_text_only.sse @@ -0,0 +1,8 @@ +data: {"id":"chatcmpl-1","choices":[{"index":0,"delta":{"role":"assistant","content":"hel"}}]} + +data: {"id":"chatcmpl-1","choices":[{"index":0,"delta":{"content":"lo"}}]} + +data: {"id":"chatcmpl-1","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]} + +data: [DONE] + diff --git a/sidecars/cc_convert/tests/fixtures/streams/openai_28_single_tool_call_fragments.sse b/sidecars/cc_convert/tests/fixtures/streams/openai_28_single_tool_call_fragments.sse new file mode 100644 index 0000000..9d9d1e6 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/streams/openai_28_single_tool_call_fragments.sse @@ -0,0 +1,8 @@ +data: {"id":"chatcmpl-x","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"get_weather","arguments":""}}]}}]} + +data: {"id":"chatcmpl-x","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"arguments":"{\"city\":"}}]}}]} + +data: {"id":"chatcmpl-x","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"arguments":"\"Paris\"}"}}]}}]} + +data: {"id":"chatcmpl-x","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]} + diff --git a/sidecars/cc_convert/tests/fixtures/streams/openai_29_two_parallel_tool_calls.sse b/sidecars/cc_convert/tests/fixtures/streams/openai_29_two_parallel_tool_calls.sse new file mode 100644 index 0000000..9a8e867 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/streams/openai_29_two_parallel_tool_calls.sse @@ -0,0 +1,4 @@ +data: {"id":"chatcmpl-y","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"a","type":"function","function":{"name":"f","arguments":"{}"}},{"index":1,"id":"b","type":"function","function":{"name":"g","arguments":"{}"}}]}}]} + +data: {"id":"chatcmpl-y","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]} + diff --git a/sidecars/cc_convert/tests/fixtures/streams/openai_30_stream_ends_without_finish_reason.sse b/sidecars/cc_convert/tests/fixtures/streams/openai_30_stream_ends_without_finish_reason.sse new file mode 100644 index 0000000..5bb3cf5 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/streams/openai_30_stream_ends_without_finish_reason.sse @@ -0,0 +1,2 @@ +data: {"id":"chatcmpl-z","choices":[{"index":0,"delta":{"role":"assistant","content":"partial"}}]} + diff --git a/sidecars/cc_convert/tests/fixtures/streams/openai_31_reasoning_then_text.sse b/sidecars/cc_convert/tests/fixtures/streams/openai_31_reasoning_then_text.sse new file mode 100644 index 0000000..5581fe5 --- /dev/null +++ b/sidecars/cc_convert/tests/fixtures/streams/openai_31_reasoning_then_text.sse @@ -0,0 +1,6 @@ +data: {"id":"chatcmpl-r","choices":[{"index":0,"delta":{"reasoning_content":"let me think..."}}]} + +data: {"id":"chatcmpl-r","choices":[{"index":0,"delta":{"content":"Done."}}]} + +data: {"id":"chatcmpl-r","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]} + diff --git a/sidecars/tito/.github/workflows/python-package.yml b/sidecars/tito/.github/workflows/python-package.yml new file mode 100644 index 0000000..662e7f9 --- /dev/null +++ b/sidecars/tito/.github/workflows/python-package.yml @@ -0,0 +1,61 @@ +name: Python package + +on: + push: + branches: [master] + tags: ["v*"] + pull_request: + branches: [master] + +jobs: + test: + name: Test Python ${{ matrix.python-version }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install package + run: python -m pip install -e '.[test]' + - name: Run package tests + run: pytest tests/package -q + + build: + name: Build distributions + runs-on: ubuntu-latest + needs: test + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install build frontend + run: python -m pip install build + - name: Build wheel and sdist + run: python -m build + - uses: actions/upload-artifact@v4 + with: + name: python-distributions + path: dist/* + + publish: + name: Publish to PyPI + runs-on: ubuntu-latest + needs: build + if: startsWith(github.ref, 'refs/tags/v') + permissions: + id-token: write + environment: + name: pypi + url: https://pypi.org/p/tito-gateway + steps: + - uses: actions/download-artifact@v4 + with: + name: python-distributions + path: dist + - uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/sidecars/tito/.gitignore b/sidecars/tito/.gitignore new file mode 100644 index 0000000..9d78d95 --- /dev/null +++ b/sidecars/tito/.gitignore @@ -0,0 +1,9 @@ +__pycache__/ +*.py[cod] +.humanize/ +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ +dist/ +build/ +*.egg-info/ diff --git a/sidecars/tito/LICENSE b/sidecars/tito/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/sidecars/tito/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/sidecars/tito/README.md b/sidecars/tito/README.md new file mode 100644 index 0000000..14449ff --- /dev/null +++ b/sidecars/tito/README.md @@ -0,0 +1,83 @@ +# TITO Gateway + +[English](README.md) | [中文](README.zh-CN.md) + +TITO Gateway is a standalone Python package and CLI wrapper around the Miles +Agentic Chat Template / TITO session-server work. It is a public-facing +packaging and usage layer for Miles' TITO implementation, not a rewrite of the +underlying algorithms. + +The TITO tokenizer, fixed chat-template approach, session trajectory model, +proxy behavior, and verifier flow are credited to Miles and its contributors. +Vendored source is tracked in `tito_gateway/VENDORED_MILES_AUDIT.md`. + +## Documentation + +- [Docs Home](docs/index.md) +- [Quickstart](docs/quickstart.md) +- [Concepts](docs/concepts.md) +- [Python API](docs/api.md) +- [CLI Reference](docs/cli.md) +- [Verification And Tests](docs/verification.md) +- [Development Notes](docs/development.md) +- [中文文档入口](docs/index.zh-CN.md) + +## Quickstart + +Install from PyPI: + +```bash +pip install tito-gateway +``` + +The copied Miles TITO logic is bundled in this package. Install optional +verifier dependencies only when you want to run the heavy verifier path locally: + +```bash +pip install 'tito-gateway[verify]' +``` + +Start the gateway beside an OpenAI-compatible backend: + +```bash +tito-gateway --hf-checkpoint Qwen/Qwen3-0.6B \ + --tito-model qwen3 \ + --tito-allowed-append-roles tool user \ + --backend-url http://127.0.0.1:8000 \ + --session-server-port 30000 +``` + +Or embed it in Python: + +```python +from tito_gateway import TITOGateway, TITOGatewayConfig + +gateway = TITOGateway( + TITOGatewayConfig( + hf_checkpoint="Qwen/Qwen3-0.6B", + backend_url="http://127.0.0.1:8000", + tito_model="qwen3", + tito_allowed_append_roles=("tool", "user"), + ) +) + +app = gateway.app +``` + +Run the CPU-fast test suite: + +```bash +pip install -e '.[test]' +pytest tests/upstream tests/package -q +``` + +## Acknowledgement + +This package explicitly acknowledges Miles as the source of the underlying TITO +design and implementation: + +- Miles repository: https://github.com/radixark/miles +- Miles documentation: https://www.radixark.com/miles/docs/user-guide/agentic-chat-template + +Future vendoring should preserve upstream notices, keep the source audit +current, and keep copied upstream tests as the compatibility contract. diff --git a/sidecars/tito/README.zh-CN.md b/sidecars/tito/README.zh-CN.md new file mode 100644 index 0000000..f81ce0c --- /dev/null +++ b/sidecars/tito/README.zh-CN.md @@ -0,0 +1,81 @@ +# TITO Gateway + +[English](README.md) | [中文](README.zh-CN.md) + +TITO Gateway 是围绕 Miles Agentic Chat Template / TITO session-server 工作做的 +独立 Python package 和 CLI 封装。它是对 Miles TITO 实现的公开封装和使用层，不是 +底层算法的重新实现。 + +TITO tokenizer、fixed chat-template 方案、session trajectory 模型、proxy 行为和验证 +流程都来自 Miles 及其贡献者。vendored source 的审计记录在 +`tito_gateway/VENDORED_MILES_AUDIT.md`。 + +## 文档入口 + +- [中文文档首页](docs/index.zh-CN.md) +- [快速开始](docs/quickstart.zh-CN.md) +- [核心概念](docs/concepts.zh-CN.md) +- [Python API](docs/api.zh-CN.md) +- [CLI 参考](docs/cli.zh-CN.md) +- [验证与测试](docs/verification.zh-CN.md) +- [开发说明](docs/development.zh-CN.md) +- [English Docs Home](docs/index.md) + +## 快速开始 + +从 PyPI 安装： + +```bash +pip install tito-gateway +``` + +Miles TITO 逻辑已经 copy/vendor 在这个 package 里。只有需要在本地跑 heavy verifier +路径时，才安装 verifier 可选依赖： + +```bash +pip install 'tito-gateway[verify]' +``` + +在 OpenAI-compatible backend 旁边启动 gateway： + +```bash +tito-gateway --hf-checkpoint Qwen/Qwen3-0.6B \ + --tito-model qwen3 \ + --tito-allowed-append-roles tool user \ + --backend-url http://127.0.0.1:8000 \ + --session-server-port 30000 +``` + +或在 Python 中嵌入： + +```python +from tito_gateway import TITOGateway, TITOGatewayConfig + +gateway = TITOGateway( + TITOGatewayConfig( + hf_checkpoint="Qwen/Qwen3-0.6B", + backend_url="http://127.0.0.1:8000", + tito_model="qwen3", + tito_allowed_append_roles=("tool", "user"), + ) +) + +app = gateway.app +``` + +运行 CPU-fast 测试： + +```bash +pip install -e '.[test]' +pytest tests/upstream tests/package -q +``` + +## 致谢 + +本 package 明确致谢 Miles：底层 TITO 设计和实现来自 Miles 项目。 + +- Miles repository: https://github.com/radixark/miles +- Miles documentation: https://www.radixark.com/miles/docs/user-guide/agentic-chat-template + +后续 vendor Miles 源码时，需要保留 upstream notices，更新 source audit，并保持 copied +upstream tests 作为兼容性合同。 diff --git a/sidecars/tito/docs/api.md b/sidecars/tito/docs/api.md new file mode 100644 index 0000000..7051b4f --- /dev/null +++ b/sidecars/tito/docs/api.md @@ -0,0 +1,50 @@ +# Python API + +[Docs Home](index.md) | [中文](api.zh-CN.md) + +## Import Surface + +```python +from tito_gateway import TITOGateway, TITOGatewayConfig, SessionServer, get_tito_tokenizer +``` + +## Gateway Configuration + +```python +from tito_gateway import TITOGateway, TITOGatewayConfig + +config = TITOGatewayConfig( + hf_checkpoint="Qwen/Qwen3-0.6B", + backend_url="http://127.0.0.1:8000", + chat_template_path=None, + apply_chat_template_kwargs={"enable_thinking": False}, + tito_model="qwen3", + tito_allowed_append_roles=("tool", "user"), + session_server_ip="127.0.0.1", + session_server_port=30000, + miles_router_timeout=600.0, +) + +gateway = TITOGateway(config) +app = gateway.app +``` + +## Important Fields + +- `hf_checkpoint`: required checkpoint or model ID for tokenizer loading. +- `backend_url`: explicit backend URL. If omitted, discovery is used. +- `chat_template_path`: optional fixed chat template. +- `apply_chat_template_kwargs`: JSON-like dict passed into template rendering. +- `tito_model`: Miles TITO tokenizer family. +- `tito_allowed_append_roles`: roles allowed after an assistant turn. +- `backend_probe_candidates`: optional tuple of URLs to probe. +- `backend_probe_timeout`: per-endpoint probe timeout. + +## Running Directly + +```python +gateway.run() +``` + +For ASGI composition, prefer `gateway.app` and mount or serve it with your own +server process. diff --git a/sidecars/tito/docs/api.zh-CN.md b/sidecars/tito/docs/api.zh-CN.md new file mode 100644 index 0000000..b3fe19d --- /dev/null +++ b/sidecars/tito/docs/api.zh-CN.md @@ -0,0 +1,49 @@ +# Python API + +[文档首页](index.zh-CN.md) | [English](api.md) + +## Import Surface + +```python +from tito_gateway import TITOGateway, TITOGatewayConfig, SessionServer, get_tito_tokenizer +``` + +## Gateway 配置 + +```python +from tito_gateway import TITOGateway, TITOGatewayConfig + +config = TITOGatewayConfig( + hf_checkpoint="Qwen/Qwen3-0.6B", + backend_url="http://127.0.0.1:8000", + chat_template_path=None, + apply_chat_template_kwargs={"enable_thinking": False}, + tito_model="qwen3", + tito_allowed_append_roles=("tool", "user"), + session_server_ip="127.0.0.1", + session_server_port=30000, + miles_router_timeout=600.0, +) + +gateway = TITOGateway(config) +app = gateway.app +``` + +## 重要字段 + +- `hf_checkpoint`：必填 checkpoint 或 model ID，用于 tokenizer loading。 +- `backend_url`：显式 backend URL；不填时走 discovery。 +- `chat_template_path`：可选 fixed chat template。 +- `apply_chat_template_kwargs`：传给 template rendering 的 dict。 +- `tito_model`：Miles TITO tokenizer family。 +- `tito_allowed_append_roles`：assistant turn 后允许追加的 roles。 +- `backend_probe_candidates`：可选 probe URL 列表。 +- `backend_probe_timeout`：每个 probe endpoint 的 timeout。 + +## 直接运行 + +```python +gateway.run() +``` + +如果要做 ASGI 组合，建议使用 `gateway.app`，交给自己的 server process mount 或 serve。 diff --git a/sidecars/tito/docs/cli.md b/sidecars/tito/docs/cli.md new file mode 100644 index 0000000..85f270f --- /dev/null +++ b/sidecars/tito/docs/cli.md @@ -0,0 +1,50 @@ +# CLI Reference + +[Docs Home](index.md) | [中文](cli.zh-CN.md) + +## Serve + +```bash +tito-gateway serve --hf-checkpoint Qwen/Qwen3-0.6B \ + --tito-model qwen3 \ + --tito-allowed-append-roles tool user \ + --backend-url http://127.0.0.1:8000 \ + --session-server-port 30000 +``` + +The top-level command aliases to `serve`, so the `serve` word can be omitted. + +## Template Kwargs + +```bash +tito-gateway serve --hf-checkpoint Qwen/Qwen3-0.6B \ + --backend-url http://127.0.0.1:8000 \ + --apply-chat-template-kwargs '{"enable_thinking": false}' +``` + +## Backend Probe Flags + +```bash +tito-gateway serve --hf-checkpoint Qwen/Qwen3-0.6B \ + --backend-probe-candidate http://127.0.0.1:8000 \ + --backend-probe-candidate http://127.0.0.1:30000 \ + --backend-probe-timeout 0.5 +``` + +## Verifiers + +```bash +tito-gateway verify-chat-template --template path/to/template.jinja --thinking off +``` + +```bash +tito-gateway verify-session-tito-tokenizer \ + --hf-checkpoint Qwen/Qwen3-4B \ + --tito-model qwen3 \ + --tito-allowed-append-roles tool user \ + --sglang-reasoning-parser qwen3 \ + --sglang-tool-call-parser qwen25 \ + --rollout-num-gpus-per-engine 1 +``` + +Use `--help` on any subcommand for the full parser surface. diff --git a/sidecars/tito/docs/cli.zh-CN.md b/sidecars/tito/docs/cli.zh-CN.md new file mode 100644 index 0000000..4d0ca49 --- /dev/null +++ b/sidecars/tito/docs/cli.zh-CN.md @@ -0,0 +1,50 @@ +# CLI 参考 + +[文档首页](index.zh-CN.md) | [English](cli.md) + +## Serve + +```bash +tito-gateway serve --hf-checkpoint Qwen/Qwen3-0.6B \ + --tito-model qwen3 \ + --tito-allowed-append-roles tool user \ + --backend-url http://127.0.0.1:8000 \ + --session-server-port 30000 +``` + +顶层命令会 alias 到 `serve`，所以可以省略 `serve`。 + +## Template Kwargs + +```bash +tito-gateway serve --hf-checkpoint Qwen/Qwen3-0.6B \ + --backend-url http://127.0.0.1:8000 \ + --apply-chat-template-kwargs '{"enable_thinking": false}' +``` + +## Backend Probe Flags + +```bash +tito-gateway serve --hf-checkpoint Qwen/Qwen3-0.6B \ + --backend-probe-candidate http://127.0.0.1:8000 \ + --backend-probe-candidate http://127.0.0.1:30000 \ + --backend-probe-timeout 0.5 +``` + +## Verifiers + +```bash +tito-gateway verify-chat-template --template path/to/template.jinja --thinking off +``` + +```bash +tito-gateway verify-session-tito-tokenizer \ + --hf-checkpoint Qwen/Qwen3-4B \ + --tito-model qwen3 \ + --tito-allowed-append-roles tool user \ + --sglang-reasoning-parser qwen3 \ + --sglang-tool-call-parser qwen25 \ + --rollout-num-gpus-per-engine 1 +``` + +每个 subcommand 都可以用 `--help` 查看完整 parser surface。 diff --git a/sidecars/tito/docs/concepts.md b/sidecars/tito/docs/concepts.md new file mode 100644 index 0000000..a53f549 --- /dev/null +++ b/sidecars/tito/docs/concepts.md @@ -0,0 +1,55 @@ +# Concepts + +[Docs Home](index.md) | [中文](concepts.zh-CN.md) + +## Runtime Model + +The gateway has three parts: + +1. An OpenAI-compatible backend. +2. A FastAPI gateway that wraps the Miles session server. +3. Clients that create sessions and send chat completions through session routes. + +Typical flow: + +1. Start or discover a backend. +2. Start the gateway with checkpoint and TITO options. +3. Create a session with `POST /sessions`. +4. Send chat requests through `/sessions/{session_id}/v1/chat/completions`. +5. The gateway injects session/TITO metadata and updates token state from backend response metadata. + +## Session Invariants + +Miles' TITO path relies on append-only message history. The exception is the +Miles-supported rollback around the latest assistant checkpoint. + +`tito_allowed_append_roles` declares which roles may be appended after an +assistant turn. `tool` is the default role surface. + +## Backend Discovery + +Backend selection is deterministic: + +1. Explicit config or `--backend-url`. +2. Environment variables: + - `TITO_BACKEND_URL` + - `OPENAI_BASE_URL` + - `SGLANG_BASE_URL` +3. Configured probe candidates, in order. + +Each probe candidate is checked at `/health` first and `/v1/models` second. The +first live candidate wins. If no backend is found, startup fails before binding +the gateway. + +## Session Routes + +- `GET /health` +- `POST /sessions` +- `GET /sessions/{session_id}` +- `DELETE /sessions/{session_id}` +- `POST /sessions/{session_id}/v1/chat/completions` + +Chat completion requests are proxied to the backend. The proxy path injects the +fields expected by the Miles implementation, including token IDs and metadata +requests. Backend responses must include output token logprob metadata so the +session trajectory can update. diff --git a/sidecars/tito/docs/concepts.zh-CN.md b/sidecars/tito/docs/concepts.zh-CN.md new file mode 100644 index 0000000..c508363 --- /dev/null +++ b/sidecars/tito/docs/concepts.zh-CN.md @@ -0,0 +1,53 @@ +# 核心概念 + +[文档首页](index.zh-CN.md) | [English](concepts.md) + +## 运行模型 + +Gateway 有三部分： + +1. 一个 OpenAI-compatible backend。 +2. 一个封装 Miles session server 的 FastAPI gateway。 +3. 客户端先创建 session，再通过 session routes 发送 chat completions。 + +典型流程： + +1. 启动或发现 backend。 +2. 用 checkpoint 和 TITO 配置启动 gateway。 +3. 通过 `POST /sessions` 创建 session。 +4. 通过 `/sessions/{session_id}/v1/chat/completions` 发送 chat 请求。 +5. Gateway 注入 session/TITO metadata，并从 backend response metadata 更新 token 状态。 + +## Session 不变量 + +Miles 的 TITO 路径依赖 append-only message history。例外是 Miles 支持的围绕最新 +assistant checkpoint 的 rollback。 + +`tito_allowed_append_roles` 声明 assistant turn 后允许追加的角色。默认 role surface 是 +`tool`。 + +## Backend 自动发现 + +Backend 选择是 deterministic 的： + +1. 显式配置或 `--backend-url`。 +2. 环境变量： + - `TITO_BACKEND_URL` + - `OPENAI_BASE_URL` + - `SGLANG_BASE_URL` +3. 配置的 probe candidates，按顺序。 + +每个 probe candidate 先检查 `/health`，再检查 `/v1/models`。第一个 live candidate +会被选中。如果找不到 backend，gateway 会在绑定前失败。 + +## Session Routes + +- `GET /health` +- `POST /sessions` +- `GET /sessions/{session_id}` +- `DELETE /sessions/{session_id}` +- `POST /sessions/{session_id}/v1/chat/completions` + +Chat completion 请求会被 proxy 到 backend。proxy 路径会注入 Miles 实现所需字段，包括 +token IDs 和 metadata requests。Backend response 必须包含 output token logprob +metadata，session trajectory 才能更新。 diff --git a/sidecars/tito/docs/development.md b/sidecars/tito/docs/development.md new file mode 100644 index 0000000..680cb52 --- /dev/null +++ b/sidecars/tito/docs/development.md @@ -0,0 +1,66 @@ +# Development Notes + +[Docs Home](index.md) | [中文](development.zh-CN.md) + +## Source Policy + +This package is a public wrapper around Miles' work. The implementation should +reuse the upstream logic wherever possible. + +Maintenance rules: + +- Preserve Miles attribution and upstream notices. +- Keep the vendored source audit current. +- Prefer wrapper code over changing TITO algorithms. +- Keep copied upstream tests as the compatibility contract. +- Do not weaken negative tests. + +## Public Documentation Standard + +Public docs should make the relationship clear: + +- Miles owns the underlying TITO design and implementation. +- This package makes that work importable and runnable as a standalone gateway. +- Optional heavy verification depends on the Miles/SGLang training stack. + +## Local Setup + +Editable installs are for repository development: + +```bash +pip install -e '.[test]' +pytest tests/package -q +``` + +Public users should install the package distribution instead: + +```bash +pip install tito-gateway +pip install 'tito-gateway[verify]' +``` + +## Build A Distribution + +```bash +python -m pip install build +python -m build +python -m pip install dist/tito_gateway-0.1.0-py3-none-any.whl +tito-gateway --help +``` + +Releases should be tagged as `vX.Y.Z`. The GitHub Actions workflow builds on +every push and publishes to PyPI only from version tags, using PyPI trusted +publishing for the `pypi` environment. + +## Pre-Push Checklist + +1. Run CPU-fast tests. +2. Run CLI help smoke checks. +3. Confirm staged files do not include cache, model weights, credentials, or local env files. +4. Run a secret scan over staged and outgoing changes. +5. Check `git diff --check`. + +## Current Secret Guard + +The `guard-secret` skill can be used before pushing. If installed, run it before +`git push` and only push when it reports `SAFE_TO_PUSH`. diff --git a/sidecars/tito/docs/development.zh-CN.md b/sidecars/tito/docs/development.zh-CN.md new file mode 100644 index 0000000..cd573a6 --- /dev/null +++ b/sidecars/tito/docs/development.zh-CN.md @@ -0,0 +1,64 @@ +# 开发说明 + +[文档首页](index.zh-CN.md) | [English](development.md) + +## 源码策略 + +这个 package 是对 Miles 工作的公开封装。实现上应尽可能复用 upstream 逻辑。 + +维护规则： + +- 保留 Miles attribution 和 upstream notices。 +- 持续更新 vendored source audit。 +- 优先写 wrapper，不改 TITO 算法。 +- copied upstream tests 是兼容性合同。 +- 不弱化 negative tests。 + +## 公开文档标准 + +公开文档需要讲清楚关系： + +- 底层 TITO 设计和实现来自 Miles。 +- 这个 package 让这套工作可以作为 standalone gateway 被 import 和运行。 +- optional heavy verification 依赖 Miles/SGLang training stack。 + +## 本地设置 + +Editable install 是给仓库开发用的： + +```bash +pip install -e '.[test]' +pytest tests/package -q +``` + +公开用户应该安装 package distribution： + +```bash +pip install tito-gateway +pip install 'tito-gateway[verify]' +``` + +## 构建 Distribution + +```bash +python -m pip install build +python -m build +python -m pip install dist/tito_gateway-0.1.0-py3-none-any.whl +tito-gateway --help +``` + +Release tag 使用 `vX.Y.Z`。GitHub Actions workflow 会在每次 push 时构建，在 version +tag 上通过 PyPI trusted publishing 发布到 `pypi` environment。 + +## Push 前检查 + +1. 跑 CPU-fast tests。 +2. 跑 CLI help smoke checks。 +3. 确认 staged files 不包含 cache、模型权重、credentials 或本地 env 文件。 +4. 对 staged 和 outgoing changes 做 secret scan。 +5. 检查 `git diff --check`。 + +## 当前 Secret Guard + +push 前可以使用 `guard-secret` skill。安装后，在 `git push` 前运行它；只有报告 +`SAFE_TO_PUSH` 时才继续 push。 diff --git a/sidecars/tito/docs/guide.md b/sidecars/tito/docs/guide.md new file mode 100644 index 0000000..9431391 --- /dev/null +++ b/sidecars/tito/docs/guide.md @@ -0,0 +1,13 @@ +# User Guide + +The user guide has been split into layered public documentation: + +- [Docs Home](index.md) +- [Quickstart](quickstart.md) +- [Concepts](concepts.md) +- [Python API](api.md) +- [CLI Reference](cli.md) +- [Verification And Tests](verification.md) +- [Development Notes](development.md) + +For Chinese documentation, see [中文文档入口](index.zh-CN.md). diff --git a/sidecars/tito/docs/guide.zh-CN.md b/sidecars/tito/docs/guide.zh-CN.md new file mode 100644 index 0000000..2530fb3 --- /dev/null +++ b/sidecars/tito/docs/guide.zh-CN.md @@ -0,0 +1,13 @@ +# 用户指南 + +用户指南已经拆成分层公开文档： + +- [中文文档首页](index.zh-CN.md) +- [快速开始](quickstart.zh-CN.md) +- [核心概念](concepts.zh-CN.md) +- [Python API](api.zh-CN.md) +- [CLI 参考](cli.zh-CN.md) +- [验证与测试](verification.zh-CN.md) +- [开发说明](development.zh-CN.md) + +英文文档见 [Docs Home](index.md)。 diff --git a/sidecars/tito/docs/index.md b/sidecars/tito/docs/index.md new file mode 100644 index 0000000..9ce76eb --- /dev/null +++ b/sidecars/tito/docs/index.md @@ -0,0 +1,22 @@ +# TITO Gateway Documentation + +[English](index.md) | [中文](index.zh-CN.md) + +TITO Gateway is a public package and CLI wrapper around Miles' Agentic Chat +Template / TITO session-server work. It reuses Miles logic and exposes it in a +standalone package. + +## Start Here + +- [Quickstart](quickstart.md): install, run, and test the package. +- [Concepts](concepts.md): runtime model, session flow, and invariants. +- [Python API](api.md): embedding the gateway beside a backend. +- [CLI Reference](cli.md): serve command and backend discovery. +- [Verification And Tests](verification.md): verifier commands and CPU-fast test setup. +- [Development Notes](development.md): source policy, attribution, and release checks. + +## Public Attribution + +This package explicitly credits Miles as the source of the underlying TITO +implementation. It is a wrapper and usage layer, not a reimplementation. The +source audit lives in `tito_gateway/VENDORED_MILES_AUDIT.md`. diff --git a/sidecars/tito/docs/index.zh-CN.md b/sidecars/tito/docs/index.zh-CN.md new file mode 100644 index 0000000..56d76aa --- /dev/null +++ b/sidecars/tito/docs/index.zh-CN.md @@ -0,0 +1,20 @@ +# TITO Gateway 文档 + +[English](index.md) | [中文](index.zh-CN.md) + +TITO Gateway 是围绕 Miles Agentic Chat Template / TITO session-server 工作做的 +公开 package 和 CLI 封装。它复用 Miles 逻辑，并把这条路径整理成独立 package。 + +## 从这里开始 + +- [快速开始](quickstart.zh-CN.md)：安装、运行和测试。 +- [核心概念](concepts.zh-CN.md)：运行模型、session flow 和不变量。 +- [Python API](api.zh-CN.md)：把 gateway 嵌入到 backend 旁边。 +- [CLI 参考](cli.zh-CN.md)：serve 命令和 backend discovery。 +- [验证与测试](verification.zh-CN.md)：verifier 命令和 CPU-fast 测试准备。 +- [开发说明](development.zh-CN.md)：源码策略、致谢和发布检查。 + +## 公开致谢 + +本 package 明确致谢 Miles：底层 TITO 实现来自 Miles。这个项目是封装和使用层，不是 +重新实现。源码审计记录在 `tito_gateway/VENDORED_MILES_AUDIT.md`。 diff --git a/sidecars/tito/docs/quickstart.md b/sidecars/tito/docs/quickstart.md new file mode 100644 index 0000000..2817cde --- /dev/null +++ b/sidecars/tito/docs/quickstart.md @@ -0,0 +1,70 @@ +# Quickstart + +[Docs Home](index.md) | [中文](quickstart.zh-CN.md) + +## Install + +```bash +pip install tito-gateway +``` + +The copied Miles TITO logic is bundled in this package. Install optional +verifier dependencies only when you want to run the heavy verifier path locally: + +```bash +pip install 'tito-gateway[verify]' +``` + +If the console script is not on `PATH`, use: + +```bash +python -m tito_gateway.cli --help +``` + +## Start With An Explicit Backend + +```bash +tito-gateway --hf-checkpoint Qwen/Qwen3-0.6B \ + --tito-model qwen3 \ + --tito-allowed-append-roles tool user \ + --backend-url http://127.0.0.1:8000 \ + --session-server-port 30000 +``` + +The default command is `serve`, so `tito-gateway ...` and `tito-gateway serve ...` +use the same startup path. + +## Start With Backend Probing + +```bash +tito-gateway serve --hf-checkpoint Qwen/Qwen3-0.6B \ + --backend-probe-candidate http://127.0.0.1:8000 \ + --backend-probe-candidate http://127.0.0.1:30000 \ + --backend-probe-timeout 0.5 +``` + +## Embed In Python + +```python +from tito_gateway import TITOGateway, TITOGatewayConfig + +gateway = TITOGateway( + TITOGatewayConfig( + hf_checkpoint="Qwen/Qwen3-0.6B", + backend_url="http://127.0.0.1:8000", + apply_chat_template_kwargs={"enable_thinking": False}, + tito_model="qwen3", + tito_allowed_append_roles=("tool", "user"), + ) +) + +app = gateway.app +``` + +## Run CPU-Fast Tests + +```bash +pip install -e '.[test]' +python scripts/prepare_test_tokenizer_cache.py --endpoint https://huggingface.co +pytest tests/upstream tests/package -q +``` diff --git a/sidecars/tito/docs/quickstart.zh-CN.md b/sidecars/tito/docs/quickstart.zh-CN.md new file mode 100644 index 0000000..391a1dd --- /dev/null +++ b/sidecars/tito/docs/quickstart.zh-CN.md @@ -0,0 +1,70 @@ +# 快速开始 + +[文档首页](index.zh-CN.md) | [English](quickstart.md) + +## 安装 + +```bash +pip install tito-gateway +``` + +Miles TITO 逻辑已经 copy/vendor 在这个 package 里。只有需要在本地跑 heavy verifier +路径时，才安装 verifier 可选依赖： + +```bash +pip install 'tito-gateway[verify]' +``` + +如果 console script 不在 `PATH` 中，可以用： + +```bash +python -m tito_gateway.cli --help +``` + +## 使用显式 backend 启动 + +```bash +tito-gateway --hf-checkpoint Qwen/Qwen3-0.6B \ + --tito-model qwen3 \ + --tito-allowed-append-roles tool user \ + --backend-url http://127.0.0.1:8000 \ + --session-server-port 30000 +``` + +默认命令就是 `serve`，所以 `tito-gateway ...` 和 `tito-gateway serve ...` 走同一套 +启动逻辑。 + +## 使用 backend probing 启动 + +```bash +tito-gateway serve --hf-checkpoint Qwen/Qwen3-0.6B \ + --backend-probe-candidate http://127.0.0.1:8000 \ + --backend-probe-candidate http://127.0.0.1:30000 \ + --backend-probe-timeout 0.5 +``` + +## 在 Python 中嵌入 + +```python +from tito_gateway import TITOGateway, TITOGatewayConfig + +gateway = TITOGateway( + TITOGatewayConfig( + hf_checkpoint="Qwen/Qwen3-0.6B", + backend_url="http://127.0.0.1:8000", + apply_chat_template_kwargs={"enable_thinking": False}, + tito_model="qwen3", + tito_allowed_append_roles=("tool", "user"), + ) +) + +app = gateway.app +``` + +## 运行 CPU-fast 测试 + +```bash +pip install -e '.[test]' +python scripts/prepare_test_tokenizer_cache.py --endpoint https://huggingface.co +pytest tests/upstream tests/package -q +``` diff --git a/sidecars/tito/docs/verification.md b/sidecars/tito/docs/verification.md new file mode 100644 index 0000000..a8f6aff --- /dev/null +++ b/sidecars/tito/docs/verification.md @@ -0,0 +1,46 @@ +# Verification And Tests + +[Docs Home](index.md) | [中文](verification.zh-CN.md) + +## Tokenizer Cache + +Copied upstream fast tests need tokenizer assets. Prepare the local cache +without downloading model weights: + +```bash +python scripts/prepare_test_tokenizer_cache.py --endpoint https://huggingface.co +``` + +## CPU-Fast Suite + +```bash +pytest tests/upstream tests/package -q +``` + +Targeted checks: + +```bash +pytest tests/upstream/fast/utils/chat_template_utils/test_tito_tokenizer.py +pytest tests/upstream/fast/utils/chat_template_utils/test_pretokenized_via_tito.py +pytest tests/upstream/fast/router/test_sessions.py +pytest tests/upstream/fast/router/test_session_race_conditions.py +pytest tests/upstream/fast/router/test_session_pretokenized_e2e.py +pytest tests/upstream/fast/utils/test_utils/test_session_verify_runner.py +pytest tests/package +``` + +## CLI Smoke Checks + +```bash +tito-gateway --help +tito-gateway serve --help +tito-gateway verify-chat-template --help +tito-gateway verify-session-tito-tokenizer --help +``` + +## Optional Heavy Verifier + +`verify-session-tito-tokenizer` runs the migrated Miles/SGLang session verifier +when the optional training stack is installed. Without that stack, it exits +with a clear dependency or runtime error. That dependency-gated exit is not a +GPU/e2e pass. diff --git a/sidecars/tito/docs/verification.zh-CN.md b/sidecars/tito/docs/verification.zh-CN.md new file mode 100644 index 0000000..313faed --- /dev/null +++ b/sidecars/tito/docs/verification.zh-CN.md @@ -0,0 +1,45 @@ +# 验证与测试 + +[文档首页](index.zh-CN.md) | [English](verification.md) + +## Tokenizer Cache + +复制过来的 upstream fast tests 需要 tokenizer assets。可以只准备本地 tokenizer cache， +不下载模型权重： + +```bash +python scripts/prepare_test_tokenizer_cache.py --endpoint https://huggingface.co +``` + +## CPU-Fast Suite + +```bash +pytest tests/upstream tests/package -q +``` + +Targeted checks： + +```bash +pytest tests/upstream/fast/utils/chat_template_utils/test_tito_tokenizer.py +pytest tests/upstream/fast/utils/chat_template_utils/test_pretokenized_via_tito.py +pytest tests/upstream/fast/router/test_sessions.py +pytest tests/upstream/fast/router/test_session_race_conditions.py +pytest tests/upstream/fast/router/test_session_pretokenized_e2e.py +pytest tests/upstream/fast/utils/test_utils/test_session_verify_runner.py +pytest tests/package +``` + +## CLI Smoke Checks + +```bash +tito-gateway --help +tito-gateway serve --help +tito-gateway verify-chat-template --help +tito-gateway verify-session-tito-tokenizer --help +``` + +## Optional Heavy Verifier + +`verify-session-tito-tokenizer` 会在安装可选 Miles/SGLang training stack 后运行迁移后的 +session verifier。没有这套依赖时，它会以清晰的 dependency 或 runtime error 退出。 +这个 dependency-gated exit 不能算作 GPU/e2e pass。 diff --git a/sidecars/tito/miles/__init__.py b/sidecars/tito/miles/__init__.py new file mode 100644 index 0000000..9c61bb1 --- /dev/null +++ b/sidecars/tito/miles/__init__.py @@ -0,0 +1,5 @@ +"""Compatibility namespace for vendored Miles TITO modules.""" + +from pkgutil import extend_path + +__path__ = extend_path(__path__, __name__) diff --git a/sidecars/tito/miles/_upstream_loader.py b/sidecars/tito/miles/_upstream_loader.py new file mode 100644 index 0000000..49b3e21 --- /dev/null +++ b/sidecars/tito/miles/_upstream_loader.py @@ -0,0 +1,75 @@ +"""Helpers for delegating compatibility wrappers to an installed Miles tree.""" + +from __future__ import annotations + +import hashlib +import importlib.util +import sys +from pathlib import Path +from types import ModuleType + + +class UpstreamModuleLoadError(ImportError): + """Raised when a present upstream module cannot be imported.""" + + +def _candidate_files(module_name: str, search_root: Path) -> tuple[Path, Path]: + module_parts = module_name.split(".") + module_path = search_root.joinpath(*module_parts) + return module_path.with_suffix(".py"), module_path / "__init__.py" + + +def load_upstream_module(module_name: str, local_file: str) -> ModuleType | None: + """Load an upstream Miles module with the same public name, if available. + + The local compatibility package intentionally occupies `miles.*` import + paths. Exact-name wrappers use this function to look past the current repo + and delegate to a real upstream Miles installation when one is present. + """ + local_path = Path(local_file).resolve() + for entry in sys.path: + search_root = Path(entry or ".").resolve() + for candidate in _candidate_files(module_name, search_root): + try: + candidate = candidate.resolve() + except OSError: + continue + if not candidate.exists() or candidate == local_path: + continue + + digest = hashlib.sha1(str(candidate).encode("utf-8")).hexdigest()[:12] + alias = f"_tito_gateway_upstream_{module_name.replace('.', '_')}_{digest}" + if alias in sys.modules: + return sys.modules[alias] + + is_package = candidate.name == "__init__.py" + spec = importlib.util.spec_from_file_location( + alias, + candidate, + submodule_search_locations=[str(candidate.parent)] if is_package else None, + ) + if spec is None or spec.loader is None: + continue + module = importlib.util.module_from_spec(spec) + sys.modules[alias] = module + try: + spec.loader.exec_module(module) + except Exception as exc: + sys.modules.pop(alias, None) + raise UpstreamModuleLoadError( + f"Found upstream candidate for {module_name} at {candidate}, " + "but importing it failed. Fix the upstream Miles installation " + "or remove it from sys.path." + ) from exc + return module + return None + + +def export_public(module: ModuleType, namespace: dict[str, object]) -> list[str]: + """Copy public symbols from `module` into `namespace`.""" + names = getattr(module, "__all__", None) + if names is None: + names = [name for name in vars(module) if not name.startswith("_")] + for name in names: + namespace[name] = getattr(module, name) + return list(names) diff --git a/sidecars/tito/miles/rollout/__init__.py b/sidecars/tito/miles/rollout/__init__.py new file mode 100644 index 0000000..09befd7 --- /dev/null +++ b/sidecars/tito/miles/rollout/__init__.py @@ -0,0 +1,5 @@ +"""Compatibility namespace for vendored Miles rollout modules.""" + +from pkgutil import extend_path + +__path__ = extend_path(__path__, __name__) diff --git a/sidecars/tito/miles/rollout/base_types.py b/sidecars/tito/miles/rollout/base_types.py new file mode 100644 index 0000000..61c9fd7 --- /dev/null +++ b/sidecars/tito/miles/rollout/base_types.py @@ -0,0 +1,9 @@ +"""Compatibility wrapper for Miles rollout base types.""" + +from miles._upstream_loader import export_public, load_upstream_module + +_upstream = load_upstream_module(__name__, __file__) +if _upstream is not None: + __all__ = export_public(_upstream, globals()) +else: + from tito_gateway.vendor.miles_compat.rollout.base_types import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/rollout/generate_hub/__init__.py b/sidecars/tito/miles/rollout/generate_hub/__init__.py new file mode 100644 index 0000000..cdc6153 --- /dev/null +++ b/sidecars/tito/miles/rollout/generate_hub/__init__.py @@ -0,0 +1,5 @@ +"""Compatibility namespace for Miles generate helpers.""" + +from pkgutil import extend_path + +__path__ = extend_path(__path__, __name__) diff --git a/sidecars/tito/miles/rollout/generate_hub/agentic_tool_call.py b/sidecars/tito/miles/rollout/generate_hub/agentic_tool_call.py new file mode 100644 index 0000000..aab875b --- /dev/null +++ b/sidecars/tito/miles/rollout/generate_hub/agentic_tool_call.py @@ -0,0 +1,9 @@ +"""Compatibility wrapper for Miles agentic tool-call generate bridge.""" + +from miles._upstream_loader import export_public, load_upstream_module + +_upstream = load_upstream_module(__name__, __file__) +if _upstream is not None: + __all__ = export_public(_upstream, globals()) +else: + from tito_gateway.vendor.miles_compat.rollout.generate_hub.agentic_tool_call import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/rollout/session/__init__.py b/sidecars/tito/miles/rollout/session/__init__.py new file mode 100644 index 0000000..ebc5c93 --- /dev/null +++ b/sidecars/tito/miles/rollout/session/__init__.py @@ -0,0 +1 @@ +"""Compatibility wrapper for Miles session modules.""" diff --git a/sidecars/tito/miles/rollout/session/linear_trajectory.py b/sidecars/tito/miles/rollout/session/linear_trajectory.py new file mode 100644 index 0000000..f45f2a8 --- /dev/null +++ b/sidecars/tito/miles/rollout/session/linear_trajectory.py @@ -0,0 +1,3 @@ +"""Compatibility wrapper for Miles linear trajectory implementation.""" + +from tito_gateway.vendor.miles_compat.rollout.session.linear_trajectory import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/rollout/session/session_errors.py b/sidecars/tito/miles/rollout/session/session_errors.py new file mode 100644 index 0000000..0dbaae6 --- /dev/null +++ b/sidecars/tito/miles/rollout/session/session_errors.py @@ -0,0 +1,3 @@ +"""Compatibility wrapper for Miles session errors.""" + +from tito_gateway.vendor.miles_compat.rollout.session.session_errors import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/rollout/session/session_server.py b/sidecars/tito/miles/rollout/session/session_server.py new file mode 100644 index 0000000..2d8496f --- /dev/null +++ b/sidecars/tito/miles/rollout/session/session_server.py @@ -0,0 +1,3 @@ +"""Compatibility wrapper for Miles session server.""" + +from tito_gateway.vendor.miles_compat.rollout.session.session_server import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/rollout/session/session_types.py b/sidecars/tito/miles/rollout/session/session_types.py new file mode 100644 index 0000000..8327c0c --- /dev/null +++ b/sidecars/tito/miles/rollout/session/session_types.py @@ -0,0 +1,3 @@ +"""Compatibility wrapper for Miles session types.""" + +from tito_gateway.vendor.miles_compat.rollout.session.session_types import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/rollout/session/sessions.py b/sidecars/tito/miles/rollout/session/sessions.py new file mode 100644 index 0000000..714e037 --- /dev/null +++ b/sidecars/tito/miles/rollout/session/sessions.py @@ -0,0 +1,3 @@ +"""Compatibility wrapper for Miles session route setup.""" + +from tito_gateway.vendor.miles_compat.rollout.session.sessions import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/__init__.py b/sidecars/tito/miles/utils/__init__.py new file mode 100644 index 0000000..d8cb848 --- /dev/null +++ b/sidecars/tito/miles/utils/__init__.py @@ -0,0 +1,5 @@ +"""Compatibility wrappers for Miles utility modules vendored by TITO Gateway.""" + +from pkgutil import extend_path + +__path__ = extend_path(__path__, __name__) diff --git a/sidecars/tito/miles/utils/chat_template_utils/__init__.py b/sidecars/tito/miles/utils/chat_template_utils/__init__.py new file mode 100644 index 0000000..14a35f8 --- /dev/null +++ b/sidecars/tito/miles/utils/chat_template_utils/__init__.py @@ -0,0 +1,3 @@ +"""Compatibility wrapper for Miles chat-template utilities.""" + +from tito_gateway.vendor.miles_compat.utils.chat_template_utils import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/chat_template_utils/deepseek_v32.py b/sidecars/tito/miles/utils/chat_template_utils/deepseek_v32.py new file mode 100644 index 0000000..1c3fe1f --- /dev/null +++ b/sidecars/tito/miles/utils/chat_template_utils/deepseek_v32.py @@ -0,0 +1,3 @@ +"""Compatibility wrapper for Miles DeepSeek V3.2 chat-template bridge.""" + +from tito_gateway.vendor.miles_compat.utils.chat_template_utils.deepseek_v32 import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/chat_template_utils/deepseek_v4.py b/sidecars/tito/miles/utils/chat_template_utils/deepseek_v4.py new file mode 100644 index 0000000..22fa69c --- /dev/null +++ b/sidecars/tito/miles/utils/chat_template_utils/deepseek_v4.py @@ -0,0 +1,3 @@ +"""Compatibility wrapper for Miles DeepSeek V4 chat-template bridge.""" + +from tito_gateway.vendor.miles_compat.utils.chat_template_utils.deepseek_v4 import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/chat_template_utils/template.py b/sidecars/tito/miles/utils/chat_template_utils/template.py new file mode 100644 index 0000000..3d99596 --- /dev/null +++ b/sidecars/tito/miles/utils/chat_template_utils/template.py @@ -0,0 +1,3 @@ +"""Compatibility wrapper for Miles chat-template rendering helpers.""" + +from tito_gateway.vendor.miles_compat.utils.chat_template_utils.template import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/chat_template_utils/tito_tokenizer.py b/sidecars/tito/miles/utils/chat_template_utils/tito_tokenizer.py new file mode 100644 index 0000000..5fc2216 --- /dev/null +++ b/sidecars/tito/miles/utils/chat_template_utils/tito_tokenizer.py @@ -0,0 +1,4 @@ +"""Compatibility wrapper for Miles TITO tokenizer implementations.""" + +from tito_gateway.vendor.miles_compat.utils.chat_template_utils.tito_tokenizer import * # noqa: F401,F403 +from tito_gateway.vendor.miles_compat.utils.chat_template_utils.tito_tokenizer import _build_dummy_assistant diff --git a/sidecars/tito/miles/utils/chat_template_utils/token_seq_comparator.py b/sidecars/tito/miles/utils/chat_template_utils/token_seq_comparator.py new file mode 100644 index 0000000..432b0e9 --- /dev/null +++ b/sidecars/tito/miles/utils/chat_template_utils/token_seq_comparator.py @@ -0,0 +1,3 @@ +"""Compatibility wrapper for Miles token sequence comparator.""" + +from tito_gateway.vendor.miles_compat.utils.chat_template_utils.token_seq_comparator import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/external_utils/__init__.py b/sidecars/tito/miles/utils/external_utils/__init__.py new file mode 100644 index 0000000..736a6d1 --- /dev/null +++ b/sidecars/tito/miles/utils/external_utils/__init__.py @@ -0,0 +1,5 @@ +"""Compatibility namespace for optional Miles external utilities.""" + +from pkgutil import extend_path + +__path__ = extend_path(__path__, __name__) diff --git a/sidecars/tito/miles/utils/external_utils/command_utils.py b/sidecars/tito/miles/utils/external_utils/command_utils.py new file mode 100644 index 0000000..29f2708 --- /dev/null +++ b/sidecars/tito/miles/utils/external_utils/command_utils.py @@ -0,0 +1,9 @@ +"""Compatibility wrapper for optional Miles command helpers.""" + +from miles._upstream_loader import export_public, load_upstream_module + +_upstream = load_upstream_module(__name__, __file__) +if _upstream is not None: + __all__ = export_public(_upstream, globals()) +else: + from tito_gateway.vendor.miles_compat.utils.external_utils.command_utils import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/hf_config.py b/sidecars/tito/miles/utils/hf_config.py new file mode 100644 index 0000000..3270150 --- /dev/null +++ b/sidecars/tito/miles/utils/hf_config.py @@ -0,0 +1,3 @@ +"""Compatibility wrapper for Miles HuggingFace config helpers.""" + +from tito_gateway.vendor.miles_compat.utils.hf_config import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/http_utils.py b/sidecars/tito/miles/utils/http_utils.py new file mode 100644 index 0000000..0dcdb86 --- /dev/null +++ b/sidecars/tito/miles/utils/http_utils.py @@ -0,0 +1,3 @@ +"""Compatibility wrapper for Miles HTTP utilities.""" + +from tito_gateway.vendor.miles_compat.utils.http_utils import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/processing_utils.py b/sidecars/tito/miles/utils/processing_utils.py new file mode 100644 index 0000000..eef08c4 --- /dev/null +++ b/sidecars/tito/miles/utils/processing_utils.py @@ -0,0 +1,3 @@ +"""Compatibility wrapper for Miles tokenizer loading helpers.""" + +from tito_gateway.vendor.miles_compat.utils.processing_utils import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/test_utils/__init__.py b/sidecars/tito/miles/utils/test_utils/__init__.py new file mode 100644 index 0000000..92f08be --- /dev/null +++ b/sidecars/tito/miles/utils/test_utils/__init__.py @@ -0,0 +1 @@ +"""Compatibility wrappers for vendored Miles test utilities.""" diff --git a/sidecars/tito/miles/utils/test_utils/chat_template_verify.py b/sidecars/tito/miles/utils/test_utils/chat_template_verify.py new file mode 100644 index 0000000..c2fc82a --- /dev/null +++ b/sidecars/tito/miles/utils/test_utils/chat_template_verify.py @@ -0,0 +1,3 @@ +"""Compatibility wrapper for Miles chat-template verifier utilities.""" + +from tito_gateway.vendor.miles_compat.utils.test_utils.chat_template_verify import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/test_utils/mock_sglang_server.py b/sidecars/tito/miles/utils/test_utils/mock_sglang_server.py new file mode 100644 index 0000000..623a6e4 --- /dev/null +++ b/sidecars/tito/miles/utils/test_utils/mock_sglang_server.py @@ -0,0 +1,3 @@ +"""Compatibility wrapper for Miles mock SGLang process result types.""" + +from tito_gateway.vendor.miles_compat.utils.test_utils.mock_sglang_server import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/test_utils/mock_trajectories.py b/sidecars/tito/miles/utils/test_utils/mock_trajectories.py new file mode 100644 index 0000000..81aa765 --- /dev/null +++ b/sidecars/tito/miles/utils/test_utils/mock_trajectories.py @@ -0,0 +1,3 @@ +"""Compatibility wrapper for Miles mock trajectories.""" + +from tito_gateway.vendor.miles_compat.utils.test_utils.mock_trajectories import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/test_utils/session_verify_agent.py b/sidecars/tito/miles/utils/test_utils/session_verify_agent.py new file mode 100644 index 0000000..98d0127 --- /dev/null +++ b/sidecars/tito/miles/utils/test_utils/session_verify_agent.py @@ -0,0 +1,3 @@ +"""Compatibility wrapper for Miles session verifier agent.""" + +from tito_gateway.vendor.miles_compat.utils.test_utils.session_verify_agent import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/test_utils/session_verify_runner.py b/sidecars/tito/miles/utils/test_utils/session_verify_runner.py new file mode 100644 index 0000000..4b2e4a0 --- /dev/null +++ b/sidecars/tito/miles/utils/test_utils/session_verify_runner.py @@ -0,0 +1,3 @@ +"""Compatibility wrapper for Miles session verifier runner.""" + +from tito_gateway.vendor.miles_compat.utils.test_utils.session_verify_runner import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/test_utils/uvicorn_thread_server.py b/sidecars/tito/miles/utils/test_utils/uvicorn_thread_server.py new file mode 100644 index 0000000..f5b75a3 --- /dev/null +++ b/sidecars/tito/miles/utils/test_utils/uvicorn_thread_server.py @@ -0,0 +1,3 @@ +"""Compatibility wrapper for Miles uvicorn thread test server.""" + +from tito_gateway.vendor.miles_compat.utils.test_utils.uvicorn_thread_server import * # noqa: F401,F403 diff --git a/sidecars/tito/plan.md b/sidecars/tito/plan.md new file mode 100644 index 0000000..74f6d39 --- /dev/null +++ b/sidecars/tito/plan.md @@ -0,0 +1,228 @@ +# TITO Gateway Package Extraction Plan + +## Goal Description + +把 Miles 文档中 Agentic Chat Template / TITO session-server 路径抽成一个独立 Python package。这个项目必须明确 ack Miles 同学/团队的原创工作，定位为“对 Miles TITO 工作的封装、复用和使用层”，不是重新发明或改写 Miles 的算法。核心算法尽可能原样复用 Miles upstream 代码，不重写、不改测试逻辑。新 package 需要同时支持两种使用方式： + +1. Python import 方式：应用在已有 server 旁边包裹一层 gateway，自动发现或接收后端 OpenAI-compatible server 地址，捕捉 `/v1/chat/completions` 调用并维护 TITO session/token 轨迹。 +2. CLI 方式：一行命令启动 gateway，参数与 Miles 对应 TITO/session-server 参数保持兼容，尤其是 `--hf-checkpoint`、`--chat-template-path`、`--apply-chat-template-kwargs`、`--tito-model`、`--tito-allowed-append-roles`、`--session-server-ip`、`--session-server-port`、router/backend URL 相关参数。 + +## Source Exploration Summary + +- 本 package 的技术来源与核心能力来自 Miles 项目；计划、文档和代码注释中需要明确 acknowledgement：TITO tokenizer、fixed chat templates、session trajectory、session-server proxy 和验证体系均基于 Miles 同学/团队已有工作。 +- 文档页 `Agentic Chat Templates (TITO)` 明确了运行不变量：messages 必须 append-only；只允许最新 assistant checkpoint 的单步 rollback；`--tito-allowed-append-roles` 必须准确声明追加角色；`tool` 总是隐含允许。 +- 文档页指向的核心验证脚本是 `scripts/tools/verify_chat_template.py` 和 `scripts/tools/verify_session_tito_tokenizer.py`，它们分别验证固定模板 append-only 与真实 session-server TITO e2e。 +- 代码核心集中在 Miles: + - `miles/utils/chat_template_utils/tito_tokenizer.py` + - `miles/utils/chat_template_utils/template.py` + - `miles/utils/chat_template_utils/token_seq_comparator.py` + - `miles/utils/chat_template_utils/templates/*.jinja` + - `miles/rollout/session/session_server.py` + - `miles/rollout/session/sessions.py` + - `miles/rollout/session/linear_trajectory.py` + - `miles/rollout/session/session_errors.py` + - `miles/rollout/session/session_types.py` +- 现有 session server 是 FastAPI + httpx proxy：创建 `/sessions`，然后通过 `/sessions/{session_id}/v1/chat/completions` 将请求代理到后端，并注入 `logprobs=True`、`return_meta_info=True`、`input_ids`，再从 SGLang/OpenAI-compatible 响应中取 `meta_info.output_token_logprobs` 更新 token checkpoint。 +- 现有测试应作为迁移合同原样保留，优先复制并运行以下测试簇： + - `tests/fast/utils/chat_template_utils/test_tito_tokenizer.py` + - `tests/fast/utils/chat_template_utils/test_pretokenized_via_tito.py` + - `tests/fast/router/test_sessions.py` + - `tests/fast/router/test_session_race_conditions.py` + - `tests/fast/router/test_session_pretokenized_e2e.py` + - `tests/fast/utils/test_utils/test_session_verify_runner.py` + - 可选 GPU/e2e: `tests/e2e/sglang/test_session_server_multi_role/*` + +## Acceptance Criteria + +- AC-1: Package import surface works. + - Positive Tests (expected to PASS): + - `python -c "import tito_gateway; from tito_gateway import TITOGateway, SessionServer, get_tito_tokenizer"` succeeds. + - A test constructs `TITOGateway(...)` with an explicit backend URL and starts the same FastAPI route behavior as Miles `SessionServer`. + - Negative Tests (expected to FAIL): + - Constructing gateway without `hf_checkpoint` raises the same skip/error behavior defined by the wrapper contract, without silently enabling broken TITO tracking. + +- AC-2: Miles TITO core logic is reused with minimal source edits. + - Positive Tests (expected to PASS): + - Upstream copied tests for `TITOTokenizer`, fixed-template resolution, decode-roundtrip verifier, and session routes pass without test body edits. + - A source audit shows `tito_tokenizer.py`, fixed templates, `template.py`, `token_seq_comparator.py`, `linear_trajectory.py`, `sessions.py`, and session error/type models are copied verbatim except import path rewrites required by package namespace. + - Negative Tests (expected to FAIL): + - Any implementation that rewrites TITO merge/tokenize algorithms instead of vendoring upstream code fails review. + +- AC-3: Existing Miles tests are preserved. + - Positive Tests (expected to PASS): + - Migrated tests keep assertions, parametrization, expected failures, and mock trajectory behavior identical to upstream. + - Compatibility shims make original import paths usable where practical, e.g. `miles.utils.chat_template_utils...` can resolve to vendored modules during tests. + - Negative Tests (expected to FAIL): + - Changing upstream test assertions, deleting negative tests like buggy Qwen3 boundary tests, or weakening expected mismatch checks is not allowed. + +- AC-4: CLI starts gateway with Miles-compatible arguments. + - Positive Tests (expected to PASS): + - `tito-gateway --hf-checkpoint Qwen/Qwen3-0.6B --tito-model qwen3 --tito-allowed-append-roles tool user --backend-url http://127.0.0.1:8000 --session-server-port 30000` starts the FastAPI gateway. + - CLI supports JSON parsing for `--apply-chat-template-kwargs` in the same convention as Miles. + - `tito-gateway verify-chat-template ...` delegates to the migrated `verify_chat_template` logic. + - Negative Tests (expected to FAIL): + - Invalid `--tito-model` exits non-zero with argparse/typer validation. + - Unsupported append roles fail before server startup. + +- AC-5: Backend server address can be auto-detected for wrapper usage. + - Positive Tests (expected to PASS): + - Explicit `backend_url` always wins. + - Environment variables are checked in deterministic order, e.g. `TITO_BACKEND_URL`, `OPENAI_BASE_URL`, `SGLANG_BASE_URL`. + - If a common local backend port is configured for probing, `/health` or `/v1/models` detection selects a live backend and logs the selected URL. + - Negative Tests (expected to FAIL): + - If no backend can be found, startup fails with a clear error instead of binding a gateway that cannot proxy calls. + +- AC-6: Session proxy behavior matches Miles. + - Positive Tests (expected to PASS): + - `/health`, `/sessions`, `/sessions/{session_id}`, `DELETE /sessions/{session_id}`, and `/sessions/{session_id}/v1/chat/completions` behave like upstream tests. + - Proxied chat requests inject `input_ids`, `logprobs=True`, `return_meta_info=True`, and `no_stop_trim=False`. + - Concurrent same-session, different-session, and delete-while-inflight race tests pass unchanged. + - Negative Tests (expected to FAIL): + - Missing upstream `meta_info.output_token_logprobs` returns the same upstream-response error behavior. + - Non-append-only messages or forbidden appended roles return 400. + +- AC-7: Verification commands remain available. + - Positive Tests (expected to PASS): + - `tito-gateway verify-chat-template` prints the same PASS/FAIL verdicts as Miles `scripts/tools/verify_chat_template.py`. + - `tito-gateway verify-session-tito-tokenizer` exists as an optional command and either runs the migrated runner when Miles/SGLang training dependencies are installed, or exits with a clear dependency error. + - Negative Tests (expected to FAIL): + - The package must not pretend GPU/e2e verification passed when optional heavy dependencies are unavailable. + +## Path Boundaries + +### Upper Bound (Maximum Scope) + +- New package scaffold with `pyproject.toml`, importable `tito_gateway` package, CLI entrypoints, vendored Miles TITO/session code, compatibility imports, copied tests, and CI commands for CPU-fast tests. +- Optional command namespace for e2e verification that preserves Miles arguments but documents dependency requirements. +- Minimal docs: import usage, CLI usage, backend discovery order, and test commands. + +### Lower Bound (Minimum Scope) + +- Importable package exposing the TITO tokenizer factory and session gateway. +- CLI that starts FastAPI gateway with explicit `--backend-url`. +- Upstream fast tests copied and passing with only import-path compatibility changes outside the test bodies. + +### Allowed Choices + +- Can use FastAPI, httpx, uvicorn, transformers, huggingface_hub, jinja2, pydantic, pytest, requests, typer or argparse. +- Can add a thin namespace compatibility layer so upstream test imports keep working. +- Can add wrapper-only modules such as `tito_gateway.cli`, `tito_gateway.gateway`, `tito_gateway.config`, and `tito_gateway.discovery`. +- Can vendor upstream Miles source with attribution and an upstream commit marker. + +### Disallowed Choices + +- Cannot rewrite TITO tokenization/merge behavior when upstream code can be copied. +- Cannot weaken or edit upstream test assertions. +- Cannot remove negative tests that prove broken templates/subclasses fail. +- Cannot require full Miles training stack for basic package import or gateway startup. +- Cannot silently auto-detect a backend when multiple candidates are alive without deterministic precedence. + +## Proposed Package Layout + +```text +tito_gateway/ + __init__.py + cli.py + config.py + discovery.py + gateway.py + server.py + vendor/ + miles_compat/ + utils/ + chat_template_utils/ + __init__.py + template.py + token_seq_comparator.py + tito_tokenizer.py + deepseek_v32.py + deepseek_v4.py + templates/ + rollout/ + session/ + session_server.py + sessions.py + linear_trajectory.py + session_errors.py + session_types.py + utils/ + processing_utils.py + http_utils.py + test_utils/ + miles/ + __init__.py + ... optional compatibility re-export modules for unchanged tests ... +scripts/ + verify_chat_template.py + verify_session_tito_tokenizer.py +tests/ + upstream/ + ... copied Miles tests, unchanged ... + package/ + test_import_surface.py + test_cli_args.py + test_backend_discovery.py +``` + +## Dependencies and Sequence + +### Milestone 1: Baseline Scaffold + +- Create `pyproject.toml` with package metadata, runtime dependencies, optional test/e2e extras, and console script `tito-gateway`. +- Add `tito_gateway.__init__` export surface. +- Add an upstream metadata file recording Miles repository URL and commit SHA used for extraction. + +### Milestone 2: Vendor Core Miles Logic + +- Copy TITO tokenizer, chat template helpers, fixed jinja templates, token comparator, session types/errors, linear trajectory, sessions route setup, and session server. +- Rewrite only import paths or provide compatibility modules so upstream code remains functionally unchanged. +- Copy required lightweight utility helpers used by tests, especially tokenizer loading, port discovery, mock SGLang server, uvicorn thread server, and mock trajectories. + +### Milestone 3: Python Wrapper API + +- Implement `TITOGatewayConfig` with Miles-compatible names. +- Implement `TITOGateway.from_server(...)` / `TITOGateway(...)` that accepts explicit backend URL or uses discovery. +- Expose `app`, `run()`, and helper methods so users can mount/run beside an existing server. + +### Milestone 4: CLI + +- Implement `tito-gateway serve` and default command alias for one-line startup. +- Preserve Miles-compatible argument names. +- Implement `tito-gateway verify-chat-template` by delegating to copied verifier. +- Implement `tito-gateway verify-session-tito-tokenizer` as optional heavy command with explicit dependency checks. + +### Milestone 5: Test Migration Without Test Edits + +- Copy selected upstream tests into `tests/upstream`. +- Prefer compatibility shims so copied tests import `miles.*` unchanged. +- If import path edits are absolutely unavoidable, perform mechanical path rewrites only and document each changed line in a migration ledger; do not alter assertions, cases, expected exceptions, or parametrization. + +### Milestone 6: New Wrapper Tests + +- Add package-specific tests for import surface, CLI parsing, backend discovery precedence, and explicit backend startup. +- Use Miles mock server utilities to avoid requiring a real SGLang server for CPU-fast tests. + +### Milestone 7: Verification + +- Run CPU-fast subset: + - `pytest tests/upstream/fast/utils/chat_template_utils/test_tito_tokenizer.py` + - `pytest tests/upstream/fast/utils/chat_template_utils/test_pretokenized_via_tito.py` + - `pytest tests/upstream/fast/router/test_sessions.py` + - `pytest tests/upstream/fast/router/test_session_race_conditions.py` + - `pytest tests/upstream/fast/router/test_session_pretokenized_e2e.py` + - `pytest tests/upstream/fast/utils/test_utils/test_session_verify_runner.py` + - `pytest tests/package` +- Run CLI smoke tests: + - `tito-gateway --help` + - `tito-gateway serve --help` + - `tito-gateway verify-chat-template --help` +- Document optional e2e command separately because it requires model/GPU/SGLang dependencies. + +## Implementation Notes + +- The package should treat Miles as the source of truth. Add wrapper code around it; do not “simplify” the TITO algorithm. +- Public docs, README, package metadata, and copied source headers should clearly state that this is a standalone packaging/wrapper effort around Miles TITO work, with attribution to Miles and its contributors. +- Keep upstream test files as a contract. The test migration should be boring and traceable. +- Backend auto-discovery must be deterministic and observable in logs. +- The default import path should be `tito_gateway`, but a `miles` compatibility namespace is acceptable for tests and copied code. +- Preserve Apache-2.0 license notices and upstream attribution when copying Miles code. diff --git a/sidecars/tito/pyproject.toml b/sidecars/tito/pyproject.toml new file mode 100644 index 0000000..2169583 --- /dev/null +++ b/sidecars/tito/pyproject.toml @@ -0,0 +1,91 @@ +[build-system] +requires = ["hatchling>=1.25"] +build-backend = "hatchling.build" + +[project] +name = "tito-gateway" +version = "0.1.0" +description = "Standalone wrapper package for Miles TITO session gateway work." +readme = "README.md" +requires-python = ">=3.10" +license = "Apache-2.0" +authors = [ + { name = "TITO Gateway maintainers" }, +] +keywords = ["tito", "agentic", "chat-template", "miles", "gateway"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Environment :: Console", + "Framework :: FastAPI", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Internet :: WWW/HTTP :: HTTP Servers", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] +dependencies = [ + "fastapi>=0.110", + "httpx>=0.27", + "pydantic>=2", + "setproctitle>=1.3", + "uvicorn>=0.29", +] + +[project.optional-dependencies] +verify = [ + "huggingface-hub>=0.23", + "jinja2>=3.1", + "sglang>=0.4", + "tokenizers>=0.19", + "transformers>=4.44", +] +test = [ + "huggingface-hub>=0.23", + "jinja2>=3.1", + "pytest>=8", + "requests>=2.31", + "sglang>=0.4", + "tokenizers>=0.19", + "transformers>=4.44", +] + +[project.urls] +Homepage = "https://github.com/yitianlian/tito_gateway" +Documentation = "https://github.com/yitianlian/tito_gateway/tree/master/docs" +Repository = "https://github.com/yitianlian/tito_gateway" +"Miles upstream" = "https://github.com/radixark/miles" +"Miles documentation" = "https://www.radixark.com/miles/docs/user-guide/agentic-chat-template" + +[project.scripts] +tito-gateway = "tito_gateway.cli:main" + +[tool.hatch.build.targets.wheel] +packages = ["tito_gateway", "miles"] + +[tool.hatch.build.targets.wheel.force-include] +"LICENSE" = "LICENSE" + +[tool.hatch.build.targets.sdist] +include = [ + "LICENSE", + "README.md", + "README.zh-CN.md", + "docs/**/*.md", + "miles/**/*.py", + "pyproject.toml", + "scripts/**/*.py", + "tests/**/*.py", + "tests/**/*.jinja", + "tito_gateway/**/*.json", + "tito_gateway/**/*.md", + "tito_gateway/**/*.py", + "tito_gateway/**/*.jinja", +] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["."] +addopts = "-q" diff --git a/sidecars/tito/scripts/prepare_test_tokenizer_cache.py b/sidecars/tito/scripts/prepare_test_tokenizer_cache.py new file mode 100644 index 0000000..884068f --- /dev/null +++ b/sidecars/tito/scripts/prepare_test_tokenizer_cache.py @@ -0,0 +1,83 @@ +"""Prepare tokenizer-only HF cache assets for copied Miles upstream tests.""" + +from __future__ import annotations + +import argparse +import os +from collections.abc import Sequence + +from huggingface_hub import snapshot_download + + +TOKENIZER_REPOS: tuple[str, ...] = ( + "Qwen/Qwen3-0.6B", + "Qwen/Qwen3-4B", + "Qwen/Qwen3-4B-Instruct-2507", + "Qwen/Qwen3-4B-Thinking-2507", + "Qwen/Qwen3-Next-80B-A3B-Thinking", + "Qwen/Qwen3.5-0.8B", + "zai-org/GLM-4.7-Flash", +) + +ALLOW_PATTERNS: tuple[str, ...] = ( + "added_tokens.json", + "chat_template*.jinja", + "config.json", + "configuration*.py", + "generation_config.json", + "merges.txt", + "modeling*.py", + "special_tokens_map.json", + "tokenization*.py", + "tokenizer.json", + "tokenizer.model", + "tokenizer_config.json", + "vocab*.json", +) + +IGNORE_PATTERNS: tuple[str, ...] = ( + "*.bin", + "*.gguf", + "*.h5", + "*.msgpack", + "*.onnx", + "*.pt", + "*.safetensors", + "*.tflite", + "*.th", + "*.weights", +) + + +def prepare_tokenizer_cache(repos: Sequence[str], *, endpoint: str | None = None) -> None: + for repo_id in repos: + print(f"Preparing tokenizer cache for {repo_id}") + snapshot_download( + repo_id=repo_id, + endpoint=endpoint, + allow_patterns=ALLOW_PATTERNS, + ignore_patterns=IGNORE_PATTERNS, + ) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--endpoint", + default=os.environ.get("HF_ENDPOINT"), + help="HF endpoint to use. Defaults to HF_ENDPOINT when set.", + ) + parser.add_argument( + "--repo", + action="append", + dest="repos", + help="Override repo list; may be passed multiple times.", + ) + args = parser.parse_args(argv) + + prepare_tokenizer_cache(tuple(args.repos or TOKENIZER_REPOS), endpoint=args.endpoint) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/sidecars/tito/tests/ci/__init__.py b/sidecars/tito/tests/ci/__init__.py new file mode 100644 index 0000000..14e7098 --- /dev/null +++ b/sidecars/tito/tests/ci/__init__.py @@ -0,0 +1 @@ +"""Compatibility package for copied Miles CI marker helpers.""" diff --git a/sidecars/tito/tests/ci/ci_register.py b/sidecars/tito/tests/ci/ci_register.py new file mode 100644 index 0000000..5c72d0a --- /dev/null +++ b/sidecars/tito/tests/ci/ci_register.py @@ -0,0 +1,12 @@ +"""Runtime no-op compatibility for copied Miles CPU CI markers.""" + + +def register_cpu_ci( + est_time: float, + suite: str, + *, + labels: list[str] | None = None, + nightly: bool = False, + disabled: str | None = None, +): + return None diff --git a/sidecars/tito/tests/fast/__init__.py b/sidecars/tito/tests/fast/__init__.py new file mode 100644 index 0000000..5e0f4b4 --- /dev/null +++ b/sidecars/tito/tests/fast/__init__.py @@ -0,0 +1 @@ +"""Compatibility namespace for copied Miles fast-test helpers.""" diff --git a/sidecars/tito/tests/fast/router/__init__.py b/sidecars/tito/tests/fast/router/__init__.py new file mode 100644 index 0000000..136f6f4 --- /dev/null +++ b/sidecars/tito/tests/fast/router/__init__.py @@ -0,0 +1 @@ +"""Compatibility namespace for copied Miles router test helpers.""" diff --git a/sidecars/tito/tests/fast/router/session_pretokenized_test_utils.py b/sidecars/tito/tests/fast/router/session_pretokenized_test_utils.py new file mode 100644 index 0000000..ca8f7bb --- /dev/null +++ b/sidecars/tito/tests/fast/router/session_pretokenized_test_utils.py @@ -0,0 +1,202 @@ +from __future__ import annotations + +from dataclasses import dataclass +from types import SimpleNamespace +from typing import Any + +import requests +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse + +from miles.rollout.session.session_server import SessionServer +from miles.utils.chat_template_utils import MismatchType, apply_chat_template, get_tito_tokenizer +from miles.utils.http_utils import find_available_port +from miles.utils.processing_utils import load_tokenizer +from miles.utils.test_utils.uvicorn_thread_server import UvicornThreadServer + +FORBIDDEN_MISMATCH_TYPES: frozenset[str] = frozenset( + { + MismatchType.SPECIAL_TOKEN_COUNT.value, + MismatchType.SPECIAL_TOKEN_TYPE.value, + MismatchType.NON_ASSISTANT_TEXT.value, + } +) + + +@dataclass(frozen=True) +class ScriptedBackendTurn: + response_message: dict[str, Any] + render_message: dict[str, Any] + + +def load_test_tokenizer(hf_checkpoint: str, chat_template_path: str | None): + return load_tokenizer( + hf_checkpoint, + chat_template_path=chat_template_path, + trust_remote_code=True, + ) + + +def make_router_env( + backend, + *, + hf_checkpoint: str, + chat_template_path: str | None, + tito_model: str, + allowed_append_roles: list[str], +): + args = SimpleNamespace( + miles_router_timeout=30, + hf_checkpoint=hf_checkpoint, + chat_template_path=chat_template_path, + tito_model=tito_model, + tito_allowed_append_roles=allowed_append_roles, + use_rollout_routing_replay=False, + ) + session_server = SessionServer(args, backend_url=backend.url) + + port = find_available_port(31000) + server = UvicornThreadServer(session_server.app, host="127.0.0.1", port=port) + server.start() + + return SimpleNamespace( + url=f"http://127.0.0.1:{port}", + backend=backend, + server=server, + ) + + +def teardown_router_env(env) -> None: + env.server.stop() + env.backend.stop() + + +def fetch_session_payload(base_url: str, session_id: str) -> dict[str, Any]: + response = requests.get(f"{base_url}/sessions/{session_id}", timeout=5.0) + response.raise_for_status() + return response.json() + + +def compute_local_session_mismatch( + tokenizer, + *, + tito_model: str, + allowed_append_roles: list[str], + messages: list[dict[str, Any]], + accumulated_token_ids: list[int], + tools: list[dict[str, Any]] | None, +) -> list[dict[str, Any]]: + comparator = get_tito_tokenizer( + tokenizer, + tokenizer_type=tito_model, + allowed_append_roles=allowed_append_roles, + ).create_comparator() + expected_ids = apply_chat_template( + messages, + tokenizer=tokenizer, + tools=tools, + add_generation_prompt=False, + tokenize=True, + ) + return [m.to_dict() for m in comparator.compare_sequences(expected_ids, accumulated_token_ids)] + + +def forbidden_mismatches(mismatch: list[dict[str, Any]]) -> list[dict[str, Any]]: + return [m for m in mismatch if m.get("type") in FORBIDDEN_MISMATCH_TYPES] + + +class ScriptedChatBackend: + def __init__(self, tokenizer, scripted_turns: list[ScriptedBackendTurn]): + self.tokenizer = tokenizer + self._scripted_turns = scripted_turns + self._call_count = 0 + self.request_log: list[dict[str, Any]] = [] + self.host = "127.0.0.1" + self.port = find_available_port(32000) + self.app = FastAPI() + self._server = UvicornThreadServer(self.app, host=self.host, port=self.port) + self._setup_routes() + + @property + def url(self) -> str: + return f"http://{self.host}:{self.port}" + + def start(self): + self._server.start() + + def stop(self): + self._server.stop() + + def reset_stats(self): + self.request_log.clear() + self._call_count = 0 + + def _setup_routes(self): + @self.app.get("/health") + async def health(): + return JSONResponse(content={"status": "ok"}) + + @self.app.post("/v1/chat/completions") + async def chat_completions(request: Request): + payload = await request.json() + self.request_log.append(payload) + + idx = self._call_count + assert idx < len(self._scripted_turns), f"Unexpected extra request #{idx + 1}" + self._call_count += 1 + + turn = self._scripted_turns[idx] + messages = payload["messages"] + tools = payload.get("tools") + + prompt_text = apply_chat_template( + messages, + tokenizer=self.tokenizer, + tools=tools, + add_generation_prompt=True, + tokenize=False, + ) + with_assistant_text = apply_chat_template( + messages + [turn.render_message], + tokenizer=self.tokenizer, + tools=tools, + add_generation_prompt=False, + tokenize=False, + ) + assert with_assistant_text.startswith(prompt_text), "Scripted assistant must extend prompt text" + + response_text = with_assistant_text[len(prompt_text) :] + output_ids = self.tokenizer.encode(response_text, add_special_tokens=False) + input_ids = payload.get("input_ids") + prompt_ids = ( + list(input_ids) + if input_ids is not None + else apply_chat_template( + messages, + tokenizer=self.tokenizer, + tools=tools, + add_generation_prompt=True, + tokenize=True, + ) + ) + + return JSONResponse( + content={ + "id": f"scripted-{idx}", + "object": "chat.completion", + "created": 0, + "model": "scripted-model", + "choices": [ + { + "index": 0, + "message": turn.response_message, + "prompt_token_ids": prompt_ids, + "finish_reason": "tool_calls" if turn.response_message.get("tool_calls") else "stop", + "meta_info": { + "completion_tokens": len(output_ids), + "output_token_logprobs": [[-i / 128, tid] for i, tid in enumerate(output_ids)], + }, + } + ], + } + ) diff --git a/sidecars/tito/tests/miles/utils/chat_template_utils/templates/qwen3_thinking_2507_and_next_fixed.jinja b/sidecars/tito/tests/miles/utils/chat_template_utils/templates/qwen3_thinking_2507_and_next_fixed.jinja new file mode 100644 index 0000000..1588dfb --- /dev/null +++ b/sidecars/tito/tests/miles/utils/chat_template_utils/templates/qwen3_thinking_2507_and_next_fixed.jinja @@ -0,0 +1,82 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if (not (clear_thinking | default(true))) or loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n\n' }} +{%- endif %} diff --git a/sidecars/tito/tests/package/test_cli.py b/sidecars/tito/tests/package/test_cli.py new file mode 100644 index 0000000..cd8147c --- /dev/null +++ b/sidecars/tito/tests/package/test_cli.py @@ -0,0 +1,212 @@ +import os + +from miles.utils.test_utils import session_verify_runner as runner +from tito_gateway.cli import build_parser, main + + +def test_cli_top_level_help(capsys): + try: + main(["--help"]) + except SystemExit as exc: + assert exc.code == 0 + + assert "Standalone wrapper around Miles TITO" in capsys.readouterr().out + + +def test_cli_serve_help(capsys): + try: + main(["serve", "--help"]) + except SystemExit as exc: + assert exc.code == 0 + + assert "--hf-checkpoint" in capsys.readouterr().out + + +def test_cli_verify_session_returns_clear_dependency_error(capsys, monkeypatch): + monkeypatch.setenv("http_proxy", "http://proxy.example:8888") + monkeypatch.setenv("HTTPS_PROXY", "http://secure-proxy.example:8888") + + code = main( + [ + "verify-session-tito-tokenizer", + "--hf-checkpoint", + "Qwen/Qwen3-4B", + "--tito-model", + "qwen3", + "--tito-allowed-append-roles", + "tool", + "user", + "--sglang-reasoning-parser", + "qwen3", + "--sglang-tool-call-parser", + "qwen25", + "--rollout-num-gpus-per-engine", + "1", + ] + ) + + assert code == 2 + assert "requires the optional Miles/SGLang training stack" in capsys.readouterr().out + assert os.environ.get("http_proxy") == "http://proxy.example:8888" + assert os.environ.get("HTTPS_PROXY") == "http://secure-proxy.example:8888" + + +def test_cli_verify_session_help(capsys): + try: + main(["verify-session-tito-tokenizer", "--help"]) + except SystemExit as exc: + assert exc.code == 0 + + out = capsys.readouterr().out + assert "--hf-checkpoint" in out + assert "--rollout-num-gpus-per-engine" in out + assert "--assistant-text-threshold" in out + assert f"Default {runner.ASSISTANT_TEXT_MISMATCH_RATIO_THRESHOLD}" in out + + +def test_cli_verify_session_default_threshold_matches_runner_constant(): + args = build_parser().parse_args( + [ + "verify-session-tito-tokenizer", + "--hf-checkpoint", + "Qwen/Qwen3-4B", + "--tito-model", + "qwen3", + ] + ) + + assert args.assistant_text_threshold == runner.ASSISTANT_TEXT_MISMATCH_RATIO_THRESHOLD + + +def test_cli_verify_session_parses_representative_miles_style_invocation(tmp_path): + prompt_data = tmp_path / "session.jsonl" + template_path = tmp_path / "template.jinja" + args = build_parser().parse_args( + [ + "verify-session-tito-tokenizer", + "--hf-checkpoint", + "Qwen/Qwen3-4B", + "--chat-template-path", + str(template_path), + "--apply-chat-template", + "--apply-chat-template-kwargs", + '{"enable_thinking": false}', + "--tito-model", + "qwen3", + "--tito-allowed-append-roles", + "tool", + "user", + "--prompt-data", + str(prompt_data), + "--input-key", + "messages", + "--backend-url", + "http://127.0.0.1:8000", + "--session-server-ip", + "127.0.0.1", + "--session-server-port", + "31000", + "--miles-router-timeout", + "12.5", + "--sglang-reasoning-parser", + "qwen3", + "--sglang-tool-call-parser", + "qwen25", + "--rollout-num-gpus-per-engine", + "2", + "--sglang-expert-parallel-size", + "4", + "--num-rollout", + "2", + "--rollout-batch-size", + "8", + "--rollout-max-response-len", + "4096", + "--rollout-temperature", + "0.2", + "--global-batch-size", + "32", + "--actor-num-nodes", + "2", + "--actor-num-gpus-per-node", + "4", + "--n-samples-per-prompt", + "6", + "--session-verify-cycles", + "5", + "--tool-call-failure-mode", + "skip", + "--assistant-text-threshold", + "0.4", + "--train-backend", + "fsdp", + "--custom-generate-function-path", + "pkg.verify.generate", + "--custom-agent-function-path", + "pkg.verify.run_agent", + "--rm-type", + "random", + "--use-session-server", + "--debug-rollout-only", + "--ci-test", + "--colocate", + ] + ) + + assert args.verify_command == "session-tito-tokenizer" + assert args.chat_template_path == str(template_path) + assert args.apply_chat_template is True + assert args.apply_chat_template_kwargs == {"enable_thinking": False} + assert args.backend_url == "http://127.0.0.1:8000" + assert args.session_server_port == 31000 + assert args.miles_router_timeout == 12.5 + assert args.assistant_text_threshold == 0.4 + + train_args = runner.namespace_to_train_args(args) + assert f"--prompt-data {prompt_data}" in train_args + assert "--input-key messages" in train_args + assert "--rollout-batch-size 8" in train_args + assert "--n-samples-per-prompt 6" in train_args + assert "--rollout-max-response-len 4096" in train_args + assert "--rollout-temperature 0.2" in train_args + assert "--global-batch-size 32" in train_args + assert "--custom-generate-function-path pkg.verify.generate" in train_args + assert "--custom-agent-function-path pkg.verify.run_agent" in train_args + assert "--session-verify-cycles 5" in train_args + assert "--tool-call-failure-mode skip" in train_args + assert "--rollout-num-gpus-per-engine 2" in train_args + assert "--sglang-expert-parallel-size 4" in train_args + assert "--actor-num-nodes 2" in train_args + assert "--actor-num-gpus-per-node 4" in train_args + assert "--train-backend fsdp" in train_args + assert "--use-session-server" in train_args + assert "--debug-rollout-only" in train_args + assert "--ci-test" in train_args + assert "--colocate" in train_args + + +def test_cli_verify_chat_template_help(capsys): + try: + main(["verify-chat-template", "--help"]) + except SystemExit as exc: + assert exc.code == 0 + + out = capsys.readouterr().out + assert "--template" in out + assert "--tito-allowed-append-roles" in out + + +def test_cli_verify_chat_template_runs_real_verifier(tmp_path, capsys): + template = tmp_path / "simple.jinja" + template.write_text( + "{%- for message in messages -%}" + "{{ '<|' + message['role'] + '|>' + (message.get('content') or '') }}" + "{%- endfor -%}" + "{%- if add_generation_prompt -%}{{ '<|assistant|>' }}{%- endif -%}" + ) + + code = main(["verify-chat-template", "--template", str(template), "--thinking", "off"]) + + captured = capsys.readouterr() + assert code == 0 + assert "Verdict: PASS - template IS append-only" in captured.out diff --git a/sidecars/tito/tests/package/test_config_discovery.py b/sidecars/tito/tests/package/test_config_discovery.py new file mode 100644 index 0000000..6b6752b --- /dev/null +++ b/sidecars/tito/tests/package/test_config_discovery.py @@ -0,0 +1,139 @@ +import pytest + +from tito_gateway.config import TITOGatewayConfig +from tito_gateway import discovery + + +def test_explicit_backend_url_wins_over_environment_and_probe(monkeypatch): + env = {"TITO_BACKEND_URL": "http://env.example:3000"} + + def fail_if_probed(*args, **kwargs): + raise AssertionError("explicit backend URL must not probe candidates") + + monkeypatch.setattr(discovery, "probe_backend_url", fail_if_probed) + + assert ( + discovery.discover_backend_url( + "localhost:8000", + env=env, + probe_candidates=("http://probe.example:8000",), + ) + == "http://localhost:8000" + ) + + +def test_environment_precedence_is_deterministic_and_wins_over_probe(monkeypatch): + env = { + "OPENAI_BASE_URL": "http://openai.example:8000", + "SGLANG_BASE_URL": "http://sglang.example:8000", + } + + def fail_if_probed(*args, **kwargs): + raise AssertionError("environment backend URL must not probe candidates") + + monkeypatch.setattr(discovery, "probe_backend_url", fail_if_probed) + + assert ( + discovery.discover_backend_url(env=env, probe_candidates=("http://probe.example:8000",)) + == "http://openai.example:8000" + ) + + +def test_probe_selects_health_success(monkeypatch): + calls = [] + + def endpoint_probe(url, timeout): + calls.append((url, timeout)) + return url == "http://candidate.example:8000/health" + + monkeypatch.setattr(discovery, "_probe_endpoint", endpoint_probe) + + assert ( + discovery.discover_backend_url( + env={}, + probe_candidates=("candidate.example:8000",), + probe_timeout=1.5, + ) + == "http://candidate.example:8000" + ) + assert calls == [("http://candidate.example:8000/health", 1.5)] + + +def test_probe_falls_back_to_models_endpoint(monkeypatch): + calls = [] + + def endpoint_probe(url, timeout): + calls.append(url) + return url == "http://candidate.example:8000/v1/models" + + monkeypatch.setattr(discovery, "_probe_endpoint", endpoint_probe) + + assert ( + discovery.discover_backend_url(env={}, probe_candidates=("http://candidate.example:8000",)) + == "http://candidate.example:8000" + ) + assert calls == [ + "http://candidate.example:8000/health", + "http://candidate.example:8000/v1/models", + ] + + +def test_probe_selection_uses_first_live_candidate(monkeypatch): + calls = [] + + def endpoint_probe(url, timeout): + calls.append(url) + return url == "http://second.example:8000/health" + + monkeypatch.setattr(discovery, "_probe_endpoint", endpoint_probe) + + assert ( + discovery.discover_backend_url( + env={}, + probe_candidates=("http://first.example:8000", "http://second.example:8000"), + ) + == "http://second.example:8000" + ) + assert calls == [ + "http://first.example:8000/health", + "http://first.example:8000/v1/models", + "http://second.example:8000/health", + ] + + +def test_missing_backend_url_fails_clearly(): + with pytest.raises(RuntimeError, match="backend URL not found"): + discovery.discover_backend_url(env={}, probe_candidates=()) + + +def test_no_live_probe_candidate_fails_clearly(monkeypatch): + monkeypatch.setattr(discovery, "_probe_endpoint", lambda url, timeout: False) + + with pytest.raises(RuntimeError, match="start a live backend"): + discovery.discover_backend_url(env={}, probe_candidates=("http://dead.example:8000",)) + + +def test_cli_json_kwargs_parse_to_dict(): + config = TITOGatewayConfig.from_cli_values( + hf_checkpoint="model", + backend_url="http://backend", + chat_template_path=None, + apply_chat_template_kwargs='{"enable_thinking": false}', + tito_model="qwen3", + tito_allowed_append_roles=["tool", "user"], + session_server_ip="127.0.0.1", + session_server_port=30000, + miles_router_timeout=30, + backend_probe_candidates=["http://probe-a:8000", "probe-b:8001"], + backend_probe_timeout=2.0, + ) + + assert config.apply_chat_template_kwargs == {"enable_thinking": False} + assert config.tito_allowed_append_roles == ("tool", "user") + assert config.backend_probe_candidates == ("http://probe-a:8000", "probe-b:8001") + assert config.backend_probe_timeout == 2.0 + + +def test_invalid_append_role_fails(): + with pytest.raises(ValueError, match="unsupported tito append roles"): + TITOGatewayConfig(hf_checkpoint="model", tito_allowed_append_roles=("assistant",)) diff --git a/sidecars/tito/tests/package/test_gateway_integration.py b/sidecars/tito/tests/package/test_gateway_integration.py new file mode 100644 index 0000000..429d56f --- /dev/null +++ b/sidecars/tito/tests/package/test_gateway_integration.py @@ -0,0 +1,82 @@ +from unittest.mock import patch + +import requests + +from miles.utils.http_utils import find_available_port +from miles.utils.test_utils.mock_sglang_server import MockSGLangServer, ProcessResult, with_mock_server +from miles.utils.test_utils.uvicorn_thread_server import UvicornThreadServer +from tito_gateway import TITOGateway, TITOGatewayConfig + + +def test_tito_gateway_serves_real_session_routes_with_explicit_backend(): + def process_fn(prompt: str) -> ProcessResult: + return ProcessResult(text=f"echo: {prompt}", finish_reason="stop") + + original_chat_response = MockSGLangServer._compute_chat_completions_response + + def patched_chat_response(self, payload: dict) -> dict: + response = original_chat_response(self, payload) + choice = response["choices"][0] + logprobs_content = choice["logprobs"]["content"] + output_token_logprobs = [ + (item["logprob"], self.tokenizer.convert_tokens_to_ids(item["token"])) for item in logprobs_content + ] + choice["meta_info"] = { + "output_token_logprobs": output_token_logprobs, + "completion_tokens": len(output_token_logprobs), + } + return response + + with ( + patch.object(MockSGLangServer, "_compute_chat_completions_response", new=patched_chat_response), + with_mock_server(process_fn=process_fn) as backend, + ): + gateway = TITOGateway( + TITOGatewayConfig( + hf_checkpoint="Qwen/Qwen3-0.6B", + backend_url=backend.url, + apply_chat_template_kwargs={"enable_thinking": False}, + tito_model="default", + tito_allowed_append_roles=("tool",), + miles_router_timeout=30, + ) + ) + + port = find_available_port(33000) + server = UvicornThreadServer(gateway.app, host="127.0.0.1", port=port) + server.start() + url = f"http://127.0.0.1:{port}" + + try: + health = requests.get(f"{url}/health", timeout=5.0) + assert health.status_code == 200 + assert health.json()["status"] == "ok" + + session_id = requests.post(f"{url}/sessions", timeout=5.0).json()["session_id"] + payload = { + "messages": [{"role": "user", "content": "What is 1+2?"}], + "return_logprob": True, + } + response = requests.post( + f"{url}/sessions/{session_id}/v1/chat/completions", + json=payload, + timeout=10.0, + ) + + assert response.status_code == 200 + assert response.json()["choices"] + assert len(backend.request_log) == 1 + proxied_payload = backend.request_log[0] + assert proxied_payload["messages"] == payload["messages"] + assert proxied_payload["logprobs"] is True + assert proxied_payload["return_meta_info"] is True + assert proxied_payload["no_stop_trim"] is False + assert isinstance(proxied_payload["input_ids"], list) + assert proxied_payload["input_ids"] + + session = requests.get(f"{url}/sessions/{session_id}", timeout=5.0).json() + assert len(session["records"]) == 1 + assert session["records"][0]["path"] == "/v1/chat/completions" + assert session["records"][0]["status_code"] == 200 + finally: + server.stop() diff --git a/sidecars/tito/tests/package/test_import_surface.py b/sidecars/tito/tests/package/test_import_surface.py new file mode 100644 index 0000000..facb732 --- /dev/null +++ b/sidecars/tito/tests/package/test_import_surface.py @@ -0,0 +1,37 @@ +import pytest + + +def test_public_import_surface(): + import tito_gateway + from tito_gateway import TITOGateway, TITOGatewayConfig, SessionServer, get_tito_tokenizer + + assert tito_gateway.TITOGateway is TITOGateway + assert tito_gateway.TITOGatewayConfig is TITOGatewayConfig + assert tito_gateway.SessionServer is SessionServer + assert callable(get_tito_tokenizer) + + +def test_config_requires_hf_checkpoint(): + from tito_gateway import TITOGatewayConfig + + with pytest.raises(ValueError, match="hf_checkpoint is required"): + TITOGatewayConfig(hf_checkpoint="") + + +def test_gateway_constructs_with_explicit_backend(monkeypatch): + import tito_gateway.gateway as gateway_module + from tito_gateway import TITOGateway + + class FakeSessionServer: + def __init__(self, args, backend_url): + self.args = args + self.backend_url = backend_url + self.app = object() + + monkeypatch.setattr(gateway_module, "SessionServer", FakeSessionServer) + + gateway = TITOGateway.from_server(hf_checkpoint="Qwen/Qwen3-0.6B", backend_url="127.0.0.1:8000") + + assert gateway.config.backend_url == "http://127.0.0.1:8000" + assert gateway.app is gateway.server.app + assert gateway.server.args.hf_checkpoint == "Qwen/Qwen3-0.6B" diff --git a/sidecars/tito/tests/package/test_session_verifier_plumbing.py b/sidecars/tito/tests/package/test_session_verifier_plumbing.py new file mode 100644 index 0000000..69f7a65 --- /dev/null +++ b/sidecars/tito/tests/package/test_session_verifier_plumbing.py @@ -0,0 +1,74 @@ +import argparse +import importlib +import json + +from miles.utils.test_utils import session_verify_runner as runner + + +def _load_dotted(path): + module_name, attr_name = path.rsplit(".", 1) + return getattr(importlib.import_module(module_name), attr_name) + + +def _build_args(tmp_path): + values = { + **runner.SESSION_VERIFY_INVARIANT_ARGS, + "hf_checkpoint": str(tmp_path / "local-model"), + "tito_model": "qwen3", + "tito_allowed_append_roles": ["tool", "user"], + "rollout_num_gpus_per_engine": 1, + "actor_num_nodes": 1, + "actor_num_gpus_per_node": 1, + "n_samples_per_prompt": 4, + "session_verify_cycles": 3, + "tool_call_failure_mode": "rollback", + "sglang_reasoning_parser": "qwen3", + "sglang_tool_call_parser": "qwen25", + "assistant_text_threshold": 0.1, + "sglang_expert_parallel_size": 1, + } + model_dir = tmp_path / "local-model" + model_dir.mkdir() + return argparse.Namespace(**values) + + +def test_session_verify_agent_function_paths_are_importable(): + generate = _load_dotted(runner.SESSION_VERIFY_INVARIANT_ARGS["custom_generate_function_path"]) + run_agent = _load_dotted(runner.SESSION_VERIFY_INVARIANT_ARGS["custom_agent_function_path"]) + + assert callable(generate) + assert callable(run_agent) + + +def test_run_session_verify_cpu_fast_positive_path(tmp_path, monkeypatch): + import miles.utils.external_utils.command_utils as command_utils + + calls = [] + + def fake_execute_train(**kwargs): + calls.append(kwargs) + metrics_path = kwargs["extra_env_vars"]["MILES_SESSION_VERIFY_METRICS_PATH"] + with open(metrics_path, "w") as f: + f.write( + json.dumps( + { + "driver_events": ["initial", "append_tool", "rollback"], + "had_assistant_mismatch": False, + } + ) + + "\n" + ) + + monkeypatch.setattr(command_utils, "execute_train", fake_execute_train) + monkeypatch.setattr(runner, "PROMPT_DATA_PATH", str(tmp_path / "session_multi_role_verify.jsonl")) + + args = _build_args(tmp_path) + runner.run_session_verify(args) + + assert len(calls) == 1 + assert calls[0]["num_gpus_per_node"] == 1 + assert calls[0]["megatron_model_type"] is None + train_args = calls[0]["train_args"] + assert f"--hf-checkpoint {tmp_path / 'local-model'}" in train_args + assert "--custom-generate-function-path miles.utils.test_utils.session_verify_agent.generate" in train_args + assert "--custom-agent-function-path miles.utils.test_utils.session_verify_agent.run_agent" in train_args diff --git a/sidecars/tito/tests/package/test_upstream_delegation.py b/sidecars/tito/tests/package/test_upstream_delegation.py new file mode 100644 index 0000000..8e7cdd5 --- /dev/null +++ b/sidecars/tito/tests/package/test_upstream_delegation.py @@ -0,0 +1,121 @@ +import importlib +import sys + +import pytest + +from miles._upstream_loader import UpstreamModuleLoadError, load_upstream_module + + +TARGET_MODULES = { + "miles.utils.external_utils.command_utils", + "miles.rollout.generate_hub.agentic_tool_call", + "miles.rollout.base_types", +} + + +def _clear_target_modules(): + for name in list(sys.modules): + if name in TARGET_MODULES or name.startswith("_tito_gateway_upstream_"): + sys.modules.pop(name, None) + + +@pytest.fixture(autouse=True) +def clear_target_modules(): + _clear_target_modules() + yield + _clear_target_modules() + + +def _write_fake_upstream(root): + command_utils = root / "miles" / "utils" / "external_utils" + command_utils.mkdir(parents=True) + (command_utils / "command_utils.py").write_text( + "SOURCE = 'fake-upstream-command-utils'\n" + "def exec_command(*args, **kwargs):\n" + " return ('upstream-exec', args, kwargs)\n" + "def execute_train(*args, **kwargs):\n" + " return ('upstream-train', args, kwargs)\n" + ) + + generate_hub = root / "miles" / "rollout" / "generate_hub" + generate_hub.mkdir(parents=True) + (generate_hub / "agentic_tool_call.py").write_text( + "SOURCE = 'fake-upstream-agentic-tool-call'\n" + "async def generate(input):\n" + " return ('upstream-generate', input)\n" + "def _add_arguments(parser):\n" + " parser.add_argument('--fake-upstream-agentic-flag')\n" + "generate.add_arguments = _add_arguments\n" + ) + + rollout = root / "miles" / "rollout" + (rollout / "base_types.py").write_text( + "SOURCE = 'fake-upstream-base-types'\n" + "class GenerateFnInput:\n" + " ORIGIN = 'upstream'\n" + "class GenerateFnOutput:\n" + " ORIGIN = 'upstream'\n" + ) + + +def test_exact_name_wrappers_delegate_to_later_upstream_sys_path(tmp_path, monkeypatch): + fake_root = tmp_path / "fake_upstream" + _write_fake_upstream(fake_root) + monkeypatch.setattr(sys, "path", [*sys.path, str(fake_root)]) + importlib.invalidate_caches() + + command_utils = importlib.import_module("miles.utils.external_utils.command_utils") + agentic_tool_call = importlib.import_module("miles.rollout.generate_hub.agentic_tool_call") + base_types = importlib.import_module("miles.rollout.base_types") + + assert command_utils.SOURCE == "fake-upstream-command-utils" + assert command_utils.exec_command("x")[0] == "upstream-exec" + assert command_utils.execute_train(train_args="--debug")[0] == "upstream-train" + + assert agentic_tool_call.SOURCE == "fake-upstream-agentic-tool-call" + assert callable(agentic_tool_call.generate) + assert callable(agentic_tool_call.generate.add_arguments) + + assert base_types.SOURCE == "fake-upstream-base-types" + assert base_types.GenerateFnInput.ORIGIN == "upstream" + assert base_types.GenerateFnOutput.ORIGIN == "upstream" + + +def test_loader_considers_upstream_candidate_under_shared_install_root(tmp_path, monkeypatch): + shared_root = tmp_path / "site-packages" + upstream = shared_root / "miles" / "utils" / "external_utils" + upstream.mkdir(parents=True) + (upstream / "command_utils.py").write_text("SOURCE = 'shared-root-upstream'\n") + + local_file = shared_root / "tito_gateway_wrapper" / "miles" / "utils" / "external_utils" / "command_utils.py" + local_file.parent.mkdir(parents=True) + local_file.write_text("SOURCE = 'local-wrapper'\n") + + monkeypatch.setattr(sys, "path", [str(shared_root)]) + + module = load_upstream_module("miles.utils.external_utils.command_utils", str(local_file)) + + assert module is not None + assert module.SOURCE == "shared-root-upstream" + + +def test_present_upstream_import_failure_is_not_masked(tmp_path, monkeypatch): + fake_root = tmp_path / "broken_upstream" + command_utils = fake_root / "miles" / "utils" / "external_utils" + command_utils.mkdir(parents=True) + (command_utils / "command_utils.py").write_text("raise RuntimeError('upstream exploded')\n") + monkeypatch.setattr(sys, "path", [*sys.path, str(fake_root)]) + importlib.invalidate_caches() + + with pytest.raises(UpstreamModuleLoadError, match="Found upstream candidate") as exc_info: + importlib.import_module("miles.utils.external_utils.command_utils") + + assert isinstance(exc_info.value.__cause__, RuntimeError) + assert "upstream exploded" in str(exc_info.value.__cause__) + + +def test_command_utils_fallback_remains_clear_without_upstream(): + command_utils = importlib.import_module("miles.utils.external_utils.command_utils") + + with pytest.raises(command_utils.MissingMilesTrainingStackError, match="not bundled with tito-gateway"): + command_utils.execute_train(train_args="--debug", num_gpus_per_node=1, megatron_model_type=None) diff --git a/sidecars/tito/tests/package/test_vendored_miles.py b/sidecars/tito/tests/package/test_vendored_miles.py new file mode 100644 index 0000000..175590a --- /dev/null +++ b/sidecars/tito/tests/package/test_vendored_miles.py @@ -0,0 +1,31 @@ +from pathlib import Path + + +def test_miles_chat_template_compat_import_resolves_fixed_template(): + from miles.utils.chat_template_utils import TITOTokenizerType, resolve_fixed_chat_template + + template_path, kwargs = resolve_fixed_chat_template(TITOTokenizerType.QWEN3, ["tool"]) + + assert template_path is not None + assert Path(template_path).name == "qwen3_fixed.jinja" + assert Path(template_path).is_file() + assert kwargs == {} + + +def test_miles_session_compat_import_resolves_errors(): + from miles.rollout.session.session_errors import SessionError, SessionNotFoundError + + assert SessionError.status_code == 500 + assert SessionNotFoundError.status_code == 404 + + +def test_public_get_tito_tokenizer_delegates_to_vendored_default(): + from tito_gateway import get_tito_tokenizer + from tito_gateway.vendor.miles_compat.utils.chat_template_utils.tito_tokenizer import TITOTokenizer + + fake_tokenizer = object() + + result = get_tito_tokenizer(fake_tokenizer, tokenizer_type="default") + + assert isinstance(result, TITOTokenizer) + assert result.tokenizer is fake_tokenizer diff --git a/sidecars/tito/tests/upstream/fast/router/test_session_pretokenized_e2e.py b/sidecars/tito/tests/upstream/fast/router/test_session_pretokenized_e2e.py new file mode 100644 index 0000000..d7a9dfb --- /dev/null +++ b/sidecars/tito/tests/upstream/fast/router/test_session_pretokenized_e2e.py @@ -0,0 +1,174 @@ +"""Representative session-layer smoke tests for miles-maintained fixed templates. + +These tests intentionally stay narrow: + +- only bundled fixed templates maintained by miles +- only tool-only multi-turn session flow +- only session/TITO plumbing + mismatch taxonomy checks + +Detailed template correctness remains covered by the lower-level chat-template +tests in ``tests/fast/utils/chat_template_utils/``. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import pytest +import requests +from tests.fast.router.session_pretokenized_test_utils import ( + ScriptedBackendTurn, + ScriptedChatBackend, + compute_local_session_mismatch, + fetch_session_payload, + forbidden_mismatches, + load_test_tokenizer, + make_router_env, + teardown_router_env, +) + +from miles.utils.chat_template_utils import TITOTokenizerType, resolve_fixed_chat_template +from miles.utils.test_utils.mock_trajectories import LongChainTrajectory, build_trajectory + + +@dataclass(frozen=True) +class FixedTemplateSmokeConfig: + name: str + hf_checkpoint: str + chat_template_path: str + tito_model: str + + +FIXED_TEMPLATE_SMOKE_CONFIGS: tuple[FixedTemplateSmokeConfig, ...] = ( + FixedTemplateSmokeConfig( + name="qwen3-fixed", + hf_checkpoint="Qwen/Qwen3-0.6B", + chat_template_path=resolve_fixed_chat_template(TITOTokenizerType.QWEN3, ["tool"])[0], + tito_model=TITOTokenizerType.QWEN3.value, + ), + FixedTemplateSmokeConfig( + name="qwen3.5-fixed", + hf_checkpoint="Qwen/Qwen3.5-0.8B", + chat_template_path=resolve_fixed_chat_template(TITOTokenizerType.QWEN35, ["tool"])[0], + tito_model=TITOTokenizerType.QWEN35.value, + ), + FixedTemplateSmokeConfig( + name="qwen3-thinking2507-fixed", + hf_checkpoint="Qwen/Qwen3-4B-Thinking-2507", + chat_template_path=resolve_fixed_chat_template(TITOTokenizerType.QWENNEXT, ["tool"])[0], + tito_model=TITOTokenizerType.QWENNEXT.value, + ), + FixedTemplateSmokeConfig( + name="qwen3-next-thinking-fixed", + hf_checkpoint="Qwen/Qwen3-Next-80B-A3B-Thinking", + chat_template_path=resolve_fixed_chat_template(TITOTokenizerType.QWENNEXT, ["tool"])[0], + tito_model=TITOTokenizerType.QWENNEXT.value, + ), +) + + +def _get_followup_messages_after_assistant(full_messages: list[dict], assistant_idx: int) -> list[dict]: + followup = [] + i = assistant_idx + 1 + while i < len(full_messages) and full_messages[i]["role"] != "assistant": + followup.append(full_messages[i]) + i += 1 + return followup + + +def _remap_followup_messages(followup_msgs: list[dict], response_tool_calls: list[dict]) -> list[dict]: + remapped = [] + tool_idx = 0 + for msg in followup_msgs: + new_msg = dict(msg) + if msg["role"] == "tool": + if tool_idx < len(response_tool_calls): + new_msg["tool_call_id"] = response_tool_calls[tool_idx]["id"] + tool_idx += 1 + remapped.append(new_msg) + return remapped + + +@pytest.mark.parametrize("config", FIXED_TEMPLATE_SMOKE_CONFIGS, ids=[c.name for c in FIXED_TEMPLATE_SMOKE_CONFIGS]) +def test_bundled_fixed_template_session_smoke(config: FixedTemplateSmokeConfig): + assert config.chat_template_path is not None, f"{config.name} should resolve to a bundled fixed template" + + try: + tokenizer = load_test_tokenizer(config.hf_checkpoint, config.chat_template_path) + except (ValueError, OSError) as exc: + pytest.skip(f"Cannot load tokenizer for {config.hf_checkpoint}: {exc}") + + trajectory = build_trajectory(tokenizer, LongChainTrajectory) + scripted_turns = [ + ScriptedBackendTurn( + response_message={**turn.assistant_message, "content": turn.assistant_message.get("content") or ""}, + render_message=turn.assistant_message, + ) + for turn in trajectory.turns + ] + backend = ScriptedChatBackend(tokenizer, scripted_turns) + backend.start() + env = make_router_env( + backend, + hf_checkpoint=config.hf_checkpoint, + chat_template_path=config.chat_template_path, + tito_model=config.tito_model, + allowed_append_roles=["tool"], + ) + + try: + backend.reset_stats() + session_id = requests.post(f"{env.url}/sessions", timeout=5.0).json()["session_id"] + assistant_indices = [i for i, m in enumerate(trajectory.full_messages) if m["role"] == "assistant"] + + accumulated_messages: list[dict] = [] + for turn_idx, turn in enumerate(trajectory.turns): + messages = list(accumulated_messages) if turn_idx > 0 else list(turn.request_messages) + if turn_idx == 0: + accumulated_messages = list(messages) + + payload = {"messages": messages, "tools": trajectory.tools} + response = requests.post( + f"{env.url}/sessions/{session_id}/v1/chat/completions", + json=payload, + timeout=10.0, + ) + assert response.status_code == 200, f"{config.name} turn {turn_idx} failed: {response.text}" + + body = response.json() + assert len(body["choices"]) == 1 + if turn_idx > 0: + assert "input_ids" in backend.request_log[turn_idx], f"{config.name} turn {turn_idx} missing input_ids" + + assistant_msg = body["choices"][0]["message"] + session_messages = list(messages) + [assistant_msg] + + session_payload = fetch_session_payload(env.url, session_id) + metadata = session_payload["metadata"] + remote_mismatch = metadata.get("tito_session_mismatch", []) + local_mismatch = compute_local_session_mismatch( + tokenizer, + tito_model=config.tito_model, + allowed_append_roles=["tool"], + messages=session_messages, + accumulated_token_ids=metadata["accumulated_token_ids"], + tools=trajectory.tools, + ) + assert remote_mismatch == local_mismatch + assert ( + forbidden_mismatches(remote_mismatch) == [] + ), f"{config.name} turn {turn_idx} has forbidden mismatch types: {remote_mismatch}" + + accumulated_messages.append(assistant_msg) + ass_idx = assistant_indices[turn_idx] + followup_msgs = _get_followup_messages_after_assistant(trajectory.full_messages, ass_idx) + response_tool_calls = assistant_msg.get("tool_calls") or [] + accumulated_messages.extend(_remap_followup_messages(followup_msgs, response_tool_calls)) + + final_session_payload = fetch_session_payload(env.url, session_id) + records = final_session_payload["records"] + assert len(records) == len(trajectory.turns) + assert all(r["status_code"] == 200 for r in records) + assert all(r["path"] == "/v1/chat/completions" for r in records) + finally: + teardown_router_env(env) diff --git a/sidecars/tito/tests/upstream/fast/router/test_session_race_conditions.py b/sidecars/tito/tests/upstream/fast/router/test_session_race_conditions.py new file mode 100644 index 0000000..95c6a69 --- /dev/null +++ b/sidecars/tito/tests/upstream/fast/router/test_session_race_conditions.py @@ -0,0 +1,422 @@ +"""E2E session stress tests. + +Contract under test (with split-lock / session.closing): +- Phase 1 (prepare) and Phase 3 (state update) hold session.lock briefly; + Phase 2 (proxy to SGLang) does NOT hold the lock. +- Concurrent same-session requests can overlap at the backend (Phase 2), + but state updates (Phase 3) are serialized; stale-update guard + (expected_num_assistant check) ensures only one concurrent writer wins. +- Different sessions can run in parallel (no global lock). +- Per-session clients can run turn-by-turn without idle gaps while global load stays parallel. +- Delete marks session.closing=True, acquires session.lock, then removes. + Because the lock is not held during Phase 2, delete can proceed while a + chat request is mid-proxy; the chat's Phase 3 will see closing=True and + skip the state update gracefully. +- Chat requests to a closing session get 404 immediately (pre-lock check). +- Chat requests arriving while delete waits for lock get 404 (double-check after lock). +- Concurrent deletes on the same session: second delete gets 404. +""" + +from __future__ import annotations + +import time +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from types import SimpleNamespace +from unittest.mock import patch + +import requests + +from miles.rollout.session.session_server import SessionServer +from miles.utils.http_utils import find_available_port +from miles.utils.test_utils.mock_sglang_server import MockSGLangServer, ProcessResult, with_mock_server +from miles.utils.test_utils.uvicorn_thread_server import UvicornThreadServer + +HF_CHECKPOINT = "Qwen/Qwen3-0.6B" + + +def _patch_mock_chat_response(): + original_chat_response = MockSGLangServer._compute_chat_completions_response + + def patched_chat_response(self, payload: dict) -> dict: + response = original_chat_response(self, payload) + # Session server expects output_token_logprobs as (logprob, token_id). + choice = response["choices"][0] + logprobs_content = choice["logprobs"]["content"] + output_token_logprobs = [ + (item["logprob"], self.tokenizer.convert_tokens_to_ids(item["token"])) for item in logprobs_content + ] + choice["meta_info"] = { + "output_token_logprobs": output_token_logprobs, + "completion_tokens": len(output_token_logprobs), + } + return response + + return patch.object(MockSGLangServer, "_compute_chat_completions_response", new=patched_chat_response) + + +@contextmanager +def _router_env(process_fn, *, latency: float = 0.0): + with _patch_mock_chat_response(): + with with_mock_server(model_name=HF_CHECKPOINT, process_fn=process_fn, latency=latency) as backend: + args = SimpleNamespace( + miles_router_timeout=30, + hf_checkpoint=HF_CHECKPOINT, + chat_template_path=None, + trajectory_manager="linear_trajectory", + tito_allowed_append_roles=["tool", "system"], + ) + server_obj = SessionServer(args, backend_url=backend.url) + + port = find_available_port(31000) + server = UvicornThreadServer(server_obj.app, host="127.0.0.1", port=port) + server.start() + url = f"http://127.0.0.1:{port}" + + try: + yield SimpleNamespace(url=url, backend=backend, server=server) + finally: + server.stop() + + +def _create_session(url: str) -> str: + response = requests.post(f"{url}/sessions", timeout=5.0) + assert response.status_code == 200 + return response.json()["session_id"] + + +def _chat(url: str, session_id: str, payload: dict, timeout: float = 20.0) -> requests.Response: + return requests.post( + f"{url}/sessions/{session_id}/v1/chat/completions", + json=payload, + timeout=timeout, + ) + + +class TestSessionConcurrencyContracts: + def test_same_session_concurrent_requests_reach_backend(self): + """With the split-lock, same-session requests CAN overlap at the backend. + + Phase 2 (proxy) runs without the lock, so concurrent requests are not + serialized at the backend level. Phase 3 state updates are still + serialized; the stale-update guard ensures only one writer wins per + generation, so no state corruption occurs. + """ + + def process_fn(prompt: str) -> ProcessResult: + return ProcessResult(text="concurrent-ok", finish_reason="stop") + + with _router_env(process_fn, latency=0.2) as env: + session_id = _create_session(env.url) + + # Warm up one assistant checkpoint so repeated identical retry payloads are valid. + warmup_payload = {"messages": [{"role": "user", "content": "warmup"}]} + warmup_resp = _chat(env.url, session_id, warmup_payload) + assert warmup_resp.status_code == 200 + assistant = warmup_resp.json()["choices"][0]["message"] + + retry_payload = { + "messages": [ + {"role": "user", "content": "warmup"}, + assistant, + {"role": "system", "content": "retry-from-assistant-checkpoint"}, + ] + } + + env.backend.reset_stats() + with ThreadPoolExecutor(max_workers=4) as pool: + futures = [pool.submit(_chat, env.url, session_id, retry_payload) for _ in range(4)] + responses = [f.result(timeout=30.0) for f in futures] + + # All requests should succeed (200) — no 500s. + assert all(resp.status_code == 200 for resp in responses) + assert len(env.backend.request_log) == 4 + # With split-lock, concurrent backend access is expected (not == 1). + assert env.backend.max_concurrent >= 1 + + def test_different_sessions_can_run_in_parallel(self): + def process_fn(prompt: str) -> ProcessResult: + return ProcessResult(text="parallel-ok", finish_reason="stop") + + with _router_env(process_fn, latency=0.2) as env: + session_ids = [_create_session(env.url) for _ in range(6)] + + env.backend.reset_stats() + with ThreadPoolExecutor(max_workers=6) as pool: + futures = [ + pool.submit( + _chat, + env.url, + sid, + {"messages": [{"role": "user", "content": f"parallel-{i}"}]}, + ) + for i, sid in enumerate(session_ids) + ] + responses = [f.result(timeout=30.0) for f in futures] + + assert all(resp.status_code == 200 for resp in responses) + assert len(env.backend.request_log) == 6 + assert env.backend.max_concurrent >= 3 + + def test_e2e_pressure_serial_per_session_parallel_globally(self): + num_sessions = 8 + turns_per_session = 3 + + def process_fn(prompt: str) -> ProcessResult: + return ProcessResult(text="turn-ok", finish_reason="stop") + + with _router_env(process_fn, latency=0.08) as env: + session_ids = [_create_session(env.url) for _ in range(num_sessions)] + + def run_session_worker(session_id: str, idx: int) -> bool: + messages: list[dict] = [{"role": "user", "content": f"session-{idx}-turn-0"}] + for turn in range(turns_per_session): + resp = _chat(env.url, session_id, {"messages": messages}, timeout=30.0) + assert resp.status_code == 200 + assistant = resp.json()["choices"][0]["message"] + if turn < turns_per_session - 1: + messages = [ + *messages, + assistant, + {"role": "system", "content": f"session-{idx}-continue-{turn}"}, + ] + return True + + env.backend.reset_stats() + with ThreadPoolExecutor(max_workers=num_sessions) as pool: + futures = [pool.submit(run_session_worker, sid, idx) for idx, sid in enumerate(session_ids)] + results = [f.result(timeout=120.0) for f in futures] + + assert all(results) + assert len(env.backend.request_log) == num_sessions * turns_per_session + assert env.backend.max_concurrent >= 4 + + def test_delete_can_proceed_while_chat_is_mid_proxy(self): + """With split-lock, delete can acquire the lock while chat is in Phase 2. + + The inflight chat's Phase 3 sees session.closing=True and skips + state update gracefully. Both chat and delete complete without error. + """ + + def process_fn(prompt: str) -> ProcessResult: + return ProcessResult(text="slow-turn", finish_reason="stop") + + with _router_env(process_fn, latency=0.35) as env: + session_id = _create_session(env.url) + payload = {"messages": [{"role": "user", "content": "slow-turn-0"}]} + + with ThreadPoolExecutor(max_workers=2) as pool: + inflight = pool.submit(_chat, env.url, session_id, payload, 30.0) + + # Wait until the first request has reached backend before deleting. + deadline = time.time() + 5.0 + while time.time() < deadline: + if env.backend.request_log: + break + time.sleep(0.01) + else: + raise AssertionError("in-flight request did not reach backend in time") + + delete_resp = requests.delete(f"{env.url}/sessions/{session_id}", timeout=30.0) + inflight_resp = inflight.result(timeout=30.0) + + # Chat returns 200 (backend responded); delete returns 204. + assert inflight_resp.status_code == 200 + assert delete_resp.status_code == 204 + # Session is gone after delete. + post_delete = _chat(env.url, session_id, payload, timeout=10.0) + assert post_delete.status_code == 404 + + +class TestClosingRaceConditions: + """Tests for race conditions around session.closing flag.""" + + def test_chat_during_delete_returns_404(self): + """Chat requests arriving after delete sets closing=True get 404. + + Timeline: + 1. Chat A starts, acquires lock (Phase 1), releases it, proxying (Phase 2) + 2. Delete arrives, sets session.closing=True, acquires lock, removes session + 3. Chat B arrives, sees session.closing=True, returns 404 immediately + 4. Chat A's Phase 3 sees closing=True, skips state update, returns 200 + """ + + def process_fn(prompt: str) -> ProcessResult: + return ProcessResult(text="slow", finish_reason="stop") + + with _router_env(process_fn, latency=0.5) as env: + session_id = _create_session(env.url) + payload = {"messages": [{"role": "user", "content": "slow-chat"}]} + + with ThreadPoolExecutor(max_workers=3) as pool: + # 1. Start slow chat A + chat_a = pool.submit(_chat, env.url, session_id, payload, 30.0) + + # Wait for chat A to reach backend + deadline = time.time() + 5.0 + while time.time() < deadline: + if env.backend.request_log: + break + time.sleep(0.01) + + # 2. Start delete (will block waiting for lock) + delete_future = pool.submit( + requests.delete, + f"{env.url}/sessions/{session_id}", + timeout=30.0, + ) + # Small delay to ensure delete has set closing=True + time.sleep(0.05) + + # 3. Chat B should get 404 because session.closing=True + chat_b = _chat(env.url, session_id, payload, timeout=10.0) + assert chat_b.status_code == 404, f"Chat during closing should return 404, got {chat_b.status_code}" + + # Wait for remaining futures + chat_a_resp = chat_a.result(timeout=30.0) + delete_resp = delete_future.result(timeout=30.0) + + assert chat_a_resp.status_code == 200 + assert delete_resp.status_code == 204 + + def test_double_delete_second_returns_404(self): + """Concurrent delete on the same session: second delete gets 404. + + With session.closing flag, the first delete sets closing=True. + The second delete sees closing=True and returns 404. + """ + + def process_fn(prompt: str) -> ProcessResult: + return ProcessResult(text="ok", finish_reason="stop") + + with _router_env(process_fn, latency=0.3) as env: + session_id = _create_session(env.url) + + # Start a slow chat to hold the lock + payload = {"messages": [{"role": "user", "content": "hold-lock"}]} + with ThreadPoolExecutor(max_workers=3) as pool: + chat_future = pool.submit(_chat, env.url, session_id, payload, 30.0) + + # Wait for chat to reach backend + deadline = time.time() + 5.0 + while time.time() < deadline: + if env.backend.request_log: + break + time.sleep(0.01) + + # Fire two deletes concurrently + delete_1 = pool.submit( + requests.delete, + f"{env.url}/sessions/{session_id}", + timeout=30.0, + ) + time.sleep(0.02) # tiny delay to let first delete set closing + delete_2 = pool.submit( + requests.delete, + f"{env.url}/sessions/{session_id}", + timeout=30.0, + ) + + chat_resp = chat_future.result(timeout=30.0) + d1 = delete_1.result(timeout=30.0) + d2 = delete_2.result(timeout=30.0) + + assert chat_resp.status_code == 200 + # One delete succeeds, the other gets 404 + codes = sorted([d1.status_code, d2.status_code]) + assert codes == [204, 404], f"Expected [204, 404], got {codes}" + + def test_chat_after_delete_returns_404(self): + """Chat request after session is fully deleted returns 404.""" + + def process_fn(prompt: str) -> ProcessResult: + return ProcessResult(text="ok", finish_reason="stop") + + with _router_env(process_fn) as env: + session_id = _create_session(env.url) + + # Delete the session + delete_resp = requests.delete(f"{env.url}/sessions/{session_id}", timeout=5.0) + assert delete_resp.status_code == 204 + + # Chat should get 404 + payload = {"messages": [{"role": "user", "content": "hello"}]} + chat_resp = _chat(env.url, session_id, payload, timeout=5.0) + assert chat_resp.status_code == 404 + + # GET should also get 404 + get_resp = requests.get(f"{env.url}/sessions/{session_id}", timeout=5.0) + assert get_resp.status_code == 404 + + def test_multiple_chats_queued_then_delete(self): + """Multiple chat requests queued behind session.lock, then delete. + + After delete marks closing=True, queued chats that acquire the lock + should check closing and return 404. + """ + + def process_fn(prompt: str) -> ProcessResult: + return ProcessResult(text="queued-ok", finish_reason="stop") + + with _router_env(process_fn, latency=0.3) as env: + session_id = _create_session(env.url) + payload = {"messages": [{"role": "user", "content": "queued"}]} + + with ThreadPoolExecutor(max_workers=6) as pool: + # Fire 3 chats (first holds lock, others queue) + chat_futures = [pool.submit(_chat, env.url, session_id, payload, 30.0) for _ in range(3)] + + # Wait for first to reach backend + deadline = time.time() + 5.0 + while time.time() < deadline: + if env.backend.request_log: + break + time.sleep(0.01) + + # Now delete - sets closing, waits for first chat to finish + delete_future = pool.submit( + requests.delete, + f"{env.url}/sessions/{session_id}", + timeout=30.0, + ) + + results = [f.result(timeout=30.0) for f in chat_futures] + delete_resp = delete_future.result(timeout=30.0) + + assert delete_resp.status_code == 204 + + # At least one chat must succeed (the one holding the lock when + # delete arrived). Others may get 200 (acquired lock before + # closing) or 404 (saw closing=True). No 500s allowed. + status_codes = [r.status_code for r in results] + assert all(c in (200, 404) for c in status_codes), f"Unexpected status codes: {status_codes}" + assert 200 in status_codes, f"Expected at least one 200, got {status_codes}" + + def test_rapid_create_chat_delete_cycles(self): + """Rapidly create, chat, and delete sessions to stress the lifecycle. + + Ensures no deadlocks or crashes from rapid session lifecycle operations. + """ + + def process_fn(prompt: str) -> ProcessResult: + return ProcessResult(text="cycle-ok", finish_reason="stop") + + with _router_env(process_fn) as env: + + def lifecycle_cycle(idx: int) -> bool: + session_id = _create_session(env.url) + payload = {"messages": [{"role": "user", "content": f"cycle-{idx}"}]} + chat_resp = _chat(env.url, session_id, payload, timeout=10.0) + assert chat_resp.status_code == 200 + delete_resp = requests.delete(f"{env.url}/sessions/{session_id}", timeout=5.0) + assert delete_resp.status_code == 204 + # Verify gone + get_resp = requests.get(f"{env.url}/sessions/{session_id}", timeout=5.0) + assert get_resp.status_code == 404 + return True + + with ThreadPoolExecutor(max_workers=8) as pool: + futures = [pool.submit(lifecycle_cycle, i) for i in range(20)] + results = [f.result(timeout=60.0) for f in futures] + + assert all(results) diff --git a/sidecars/tito/tests/upstream/fast/router/test_sessions.py b/sidecars/tito/tests/upstream/fast/router/test_sessions.py new file mode 100644 index 0000000..86f2e2d --- /dev/null +++ b/sidecars/tito/tests/upstream/fast/router/test_sessions.py @@ -0,0 +1,140 @@ +"""Integration tests for session HTTP routes (create / get / delete / proxy).""" + +import re +import uuid +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +import requests + +from miles.rollout.session.session_server import SessionServer +from miles.utils.http_utils import find_available_port +from miles.utils.test_utils.mock_sglang_server import MockSGLangServer, ProcessResult, with_mock_server +from miles.utils.test_utils.uvicorn_thread_server import UvicornThreadServer + + +@pytest.fixture(scope="class") +def router_env(): + """Create a standalone SessionServer with session routes and a mock backend.""" + + def process_fn(prompt: str) -> ProcessResult: + return ProcessResult(text=f"echo: {prompt}", finish_reason="stop") + + original_chat_response = MockSGLangServer._compute_chat_completions_response + + def patched_chat_response(self, payload: dict) -> dict: + response = original_chat_response(self, payload) + choice = response["choices"][0] + logprobs_content = choice["logprobs"]["content"] + output_token_logprobs = [ + (item["logprob"], self.tokenizer.convert_tokens_to_ids(item["token"])) for item in logprobs_content + ] + choice["meta_info"] = { + "output_token_logprobs": output_token_logprobs, + "completion_tokens": len(output_token_logprobs), + } + return response + + with patch.object(MockSGLangServer, "_compute_chat_completions_response", new=patched_chat_response): + with with_mock_server(process_fn=process_fn) as backend: + args = SimpleNamespace( + miles_router_timeout=30, + hf_checkpoint="Qwen/Qwen3-0.6B", + chat_template_path=None, + apply_chat_template_kwargs={"enable_thinking": False}, + tito_model="default", + tito_allowed_append_roles=["tool"], + trajectory_manager="linear_trajectory", + session_server_instance_id=uuid.uuid4().hex, + ) + server_obj = SessionServer(args, backend_url=backend.url) + + port = find_available_port(31000) + server = UvicornThreadServer(server_obj.app, host="127.0.0.1", port=port) + server.start() + + url = f"http://127.0.0.1:{port}" + + try: + yield SimpleNamespace(url=url, backend=backend) + finally: + server.stop() + + +class TestSessionRoutes: + def test_health_reports_stable_instance_id(self, router_env): + first = requests.get(f"{router_env.url}/health", timeout=5.0) + second = requests.get(f"{router_env.url}/health", timeout=5.0) + + assert first.status_code == 200 + assert second.status_code == 200 + first_body = first.json() + second_body = second.json() + assert first_body["status"] == "ok" + assert second_body["status"] == "ok" + assert re.fullmatch(r"[0-9a-f]{32}", first_body["session_server_instance_id"]) + assert second_body["session_server_instance_id"] == first_body["session_server_instance_id"] + + def test_create_session(self, router_env): + response = requests.post(f"{router_env.url}/sessions", timeout=5.0) + assert response.status_code == 200 + data = response.json() + assert "session_id" in data + assert len(data["session_id"]) == 32 + + def test_get_session_initial_state(self, router_env): + session_id = requests.post(f"{router_env.url}/sessions", timeout=5.0).json()["session_id"] + + get_resp = requests.get(f"{router_env.url}/sessions/{session_id}", timeout=5.0) + assert get_resp.status_code == 200 + data = get_resp.json() + assert data["session_id"] == session_id + assert data["records"] == [] + + def test_get_session_not_found(self, router_env): + response = requests.get(f"{router_env.url}/sessions/nonexistent", timeout=5.0) + assert response.status_code == 404 + assert response.json()["error"] == "session not found: session_id=nonexistent" + + def test_delete_session(self, router_env): + session_id = requests.post(f"{router_env.url}/sessions", timeout=5.0).json()["session_id"] + + delete_resp = requests.delete(f"{router_env.url}/sessions/{session_id}", timeout=5.0) + assert delete_resp.status_code == 204 + assert delete_resp.text == "" + + assert requests.delete(f"{router_env.url}/sessions/{session_id}", timeout=5.0).status_code == 404 + + def test_delete_session_not_found(self, router_env): + response = requests.delete(f"{router_env.url}/sessions/nonexistent", timeout=5.0) + assert response.status_code == 404 + assert response.json()["error"] == "session not found: session_id=nonexistent" + + +class TestSessionProxy: + def test_proxy_chat_appends_record(self, router_env): + session_id = requests.post(f"{router_env.url}/sessions", timeout=5.0).json()["session_id"] + + payload = { + "messages": [{"role": "user", "content": "What is 1+2?"}], + "return_logprob": True, + } + resp = requests.post( + f"{router_env.url}/sessions/{session_id}/v1/chat/completions", + json=payload, + timeout=10.0, + ) + assert resp.status_code == 200 + body = resp.json() + assert "choices" in body + assert body["choices"] + + get_resp = requests.get(f"{router_env.url}/sessions/{session_id}", timeout=5.0) + records = get_resp.json()["records"] + + assert isinstance(records, list) + assert len(records) == 1 + record = records[0] + assert record["path"] == "/v1/chat/completions" + assert record["status_code"] == 200 diff --git a/sidecars/tito/tests/upstream/fast/utils/chat_template_utils/test_pretokenized_via_tito.py b/sidecars/tito/tests/upstream/fast/utils/chat_template_utils/test_pretokenized_via_tito.py new file mode 100644 index 0000000..68b392c --- /dev/null +++ b/sidecars/tito/tests/upstream/fast/utils/chat_template_utils/test_pretokenized_via_tito.py @@ -0,0 +1,161 @@ +"""Unit tests for ``verify_append_only_via_tito_instance`` / +``run_all_checks_via_tito``: PASS on registered TITO families, FAIL on the +unfixed Qwen3 chat template, FAIL on a test-local ``_BuggyQwen3TITOTokenizer`` +that omits the ``\\n`` insertion at the ``<|im_end|>`` boundary. +""" + +from copy import deepcopy + +import pytest +from tests.ci.ci_register import register_cpu_ci +from transformers import AutoTokenizer + +register_cpu_ci(est_time=120, suite="stage-b-cpu", labels=[]) + + +from miles.utils.chat_template_utils import TITOTokenizerType, resolve_fixed_chat_template +from miles.utils.chat_template_utils.tito_tokenizer import Qwen3TITOTokenizer +from miles.utils.test_utils.chat_template_verify import run_all_checks_via_tito, verify_append_only_via_tito_instance +from miles.utils.test_utils.mock_trajectories import SingleToolTrajectory + +# --------------------------------------------------------------------------- +# Test helpers +# --------------------------------------------------------------------------- + + +def _setup_tokenizer_with_registered_template( + model_id: str, + family: TITOTokenizerType, + roles: list[str], +): + """Mirror what production wiring does at startup. + + Loads tokenizer, looks up the registered ``SUPPORTED_TEMPLATES`` row for + ``(family, roles)``, and applies the resolved fixed template (if any) onto + ``tokenizer.chat_template``. Returns ``(tokenizer, extra_kwargs)``. + + A fresh tokenizer instance per call avoids state-mutation hazards from + overwriting ``chat_template``. + """ + tokenizer = AutoTokenizer.from_pretrained(model_id) + fixed_path, extra_kwargs = resolve_fixed_chat_template(family, roles) + if fixed_path is not None: + with open(fixed_path) as f: + tokenizer.chat_template = f.read() + return tokenizer, dict(extra_kwargs) + + +# --------------------------------------------------------------------------- +# (1) PASS on registered families × role surfaces +# --------------------------------------------------------------------------- + + +_PASS_PARAMS = [ + pytest.param(TITOTokenizerType.QWEN3, "Qwen/Qwen3-0.6B", frozenset({"tool"}), id="qwen3-tool"), + pytest.param(TITOTokenizerType.QWEN3, "Qwen/Qwen3-0.6B", frozenset({"tool", "user"}), id="qwen3-tool_user"), + pytest.param(TITOTokenizerType.QWEN35, "Qwen/Qwen3.5-0.8B", frozenset({"tool"}), id="qwen35-tool"), + pytest.param(TITOTokenizerType.QWEN35, "Qwen/Qwen3.5-0.8B", frozenset({"tool", "user"}), id="qwen35-tool_user"), + pytest.param(TITOTokenizerType.QWENNEXT, "Qwen/Qwen3-4B-Thinking-2507", frozenset({"tool"}), id="qwennext-tool"), + pytest.param( + TITOTokenizerType.QWENNEXT, + "Qwen/Qwen3-4B-Thinking-2507", + frozenset({"tool", "user"}), + id="qwennext-tool_user", + ), + pytest.param(TITOTokenizerType.GLM47, "zai-org/GLM-4.7-Flash", frozenset({"tool"}), id="glm47-tool"), + pytest.param(TITOTokenizerType.GLM47, "zai-org/GLM-4.7-Flash", frozenset({"tool", "user"}), id="glm47-tool_user"), + pytest.param( + TITOTokenizerType.GLM47, + "zai-org/GLM-4.7-Flash", + frozenset({"tool", "user", "system"}), + id="glm47-tool_user_system", + ), +] + + +@pytest.mark.parametrize("family,model_id,roles", _PASS_PARAMS) +def test_via_tito_pass_on_registered_families(family, model_id, roles): + """All 4 registered TITO families round-trip cleanly via decode-roundtrip.""" + tokenizer, extra_kwargs = _setup_tokenizer_with_registered_template(model_id, family, sorted(roles)) + results = run_all_checks_via_tito( + tokenizer, + family, + allowed_append_roles=set(roles), + thinking="both", + extra_template_kwargs=extra_kwargs, + ) + failures = [r for r in results if not r.passed] + assert not failures, ( + f"Expected all PASS for {family.value} × {sorted(roles)} via TITO primitive; " + f"got {len(failures)} FAIL(s) out of {len(results)}:\n" + + "\n".join(f" [{r.case_name}] {r.error}" for r in failures[:5]) + ) + + +# --------------------------------------------------------------------------- +# (2) FAIL on the original unfixed Qwen3 chat template +# --------------------------------------------------------------------------- + + +def test_via_tito_fail_on_original_qwen3_template(): + """The original Qwen3 chat template uses ``loop.last`` and breaks append-only. + + Bypass ``resolve_fixed_chat_template`` entirely — keep the HF default + ``tokenizer.chat_template`` and assert the primitive surfaces a FAIL. + """ + tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B") + # Do NOT overwrite tokenizer.chat_template — keep the broken HF default. + + # Cast wide: thinking=both + multi-user-turn surface so trajectories that + # actually advance ``last_query_index`` between prefix and full are exercised. + # Those are the ones where ``loop.index0 > ns.last_query_index`` truncation + # in the original Qwen3 template renders the same assistant turn differently + # depending on the boundary position. + results = run_all_checks_via_tito( + tokenizer, + TITOTokenizerType.QWEN3, + allowed_append_roles={"tool", "user"}, + thinking="both", + ) + failures = [r for r in results if not r.passed] + assert failures, "Expected ≥1 FAIL on the original (unfixed) Qwen3 chat template; got all PASS." + + +# --------------------------------------------------------------------------- +# (3) FAIL on a test-local buggy subclass +# --------------------------------------------------------------------------- + + +class _BuggyQwen3TITOTokenizer(Qwen3TITOTokenizer): + """Test-only Qwen3 variant with the ``\\n`` boundary insertion deleted. + + Real ``Qwen3TITOTokenizer.merge_tokens`` appends ``self._newline_id`` after + a trailing ``<|im_end|>`` because the model stops without emitting the + newline the chat template would otherwise produce. This variant skips + that fixup; the decode-roundtrip primitive is expected to surface it as a + single-character diff at the prefix-suffix junction. + """ + + def merge_tokens(self, old_messages, new_messages, pretokenized_token_ids, tools=None): + incremental = self.tokenize_additional_non_assistant(old_messages, new_messages, tools) + # Intentionally omit the `+\n` insertion — that's the bug we're catching. + return list(pretokenized_token_ids) + incremental + + +def test_via_tito_fail_on_buggy_qwen3_subclass(): + """A buggy ``merge_tokens`` produces a junction-level diff that the verifier surfaces.""" + tokenizer, _ = _setup_tokenizer_with_registered_template("Qwen/Qwen3-0.6B", TITOTokenizerType.QWEN3, ["tool"]) + buggy = _BuggyQwen3TITOTokenizer(tokenizer, allowed_append_roles=["tool"]) + + result = verify_append_only_via_tito_instance( + buggy, + tokenizer, + deepcopy(SingleToolTrajectory.MESSAGES), + pretokenized_num_message=3, + tools=SingleToolTrajectory.TOOLS, + case_name="buggy_qwen3-single_tool-N3", + ) + assert not result.passed, "Expected FAIL on _BuggyQwen3TITOTokenizer (omits the `+\\n` boundary patch); got PASS." + assert "Decode-roundtrip mismatch" in ( + result.error or "" + ), f"Expected decode-roundtrip diff in error message; got: {result.error}" diff --git a/sidecars/tito/tests/upstream/fast/utils/chat_template_utils/test_tito_tokenizer.py b/sidecars/tito/tests/upstream/fast/utils/chat_template_utils/test_tito_tokenizer.py new file mode 100644 index 0000000..31b2aff --- /dev/null +++ b/sidecars/tito/tests/upstream/fast/utils/chat_template_utils/test_tito_tokenizer.py @@ -0,0 +1,582 @@ +"""Tests for TITOTokenizer: merge_tokens boundary logic, incremental tokenization, and factory. + +## Test structure + +TestConfig + Smoke-checks that each subclass stores the correct model-specific config + (assistant_start_str, trailing_token_ids, max_trim_tokens) and propagates + them to the comparator. These are NOT behavioral tests — they guard + against accidental config regressions when modifying __init__. + +TestMergeTokensBoundary + Unit tests for the core merge_tokens boundary logic, using *synthetic* + prefix IDs ([100, 200, ...]) so the assertions are purely about prefix + manipulation — not about template rendering. + + Why synthetic IDs? merge_tokens is: ``prefix + [boundary fix] + incremental``. + The incremental part comes from tokenize_additional_non_assistant (tested + separately); boundary logic depends only on the last token of the prefix. + Synthetic IDs isolate this and make failures trivially diagnosable. + + Covers three subclass behaviors: + - Qwen3: inserts ``\\n`` when prefix ends with ``<|im_end|>`` (model stops + at im_end without the trailing newline the template expects). + - GLM47: strips trailing ``<|observation|>`` or ``<|user|>`` (model emits + the stop token, but the template also emits it as the next turn's opener). + - Default: plain concatenation (no boundary handling). + +TestTokenizeAdditional + Behavioral tests for tokenize_additional_non_assistant — the role-segmented + synthetic-prefix diff that computes incremental token IDs for appended + non-assistant messages. + + ``test_produces_nonempty_incremental`` is parametrized over: + _TOOL_TRAJECTORIES (trajectory classes) × _TITO_MODELS (qwen3, glm47) + Split points are auto-detected by _find_tito_splits from message structure, + so adding a trajectory to _TOOL_TRAJECTORIES automatically extends coverage. + + Remaining tests cover segmentation logic, generation-prompt timing, + reasoning-content shape, merge structure preservation, and append-only + validation (reject prefix mutation, fewer messages, or forbidden roles). + +TestFactory + get_tito_tokenizer factory: string/enum dispatch, invalid input handling. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from transformers import AutoTokenizer + +from miles.utils.chat_template_utils import MismatchType, apply_chat_template, resolve_fixed_chat_template +from miles.utils.chat_template_utils.tito_tokenizer import ( + GLM47TITOTokenizer, + Qwen3TITOTokenizer, + Qwen35TITOTokenizer, + QwenNextTITOTokenizer, + TITOTokenizer, + TITOTokenizerType, + _build_dummy_assistant, + get_tito_tokenizer, +) +from miles.utils.processing_utils import load_tokenizer +from miles.utils.test_utils.mock_trajectories import ( + IntermediateSystemTrajectory, + LongChainTrajectory, + MultiToolSingleTurnTrajectory, + MultiTurnTrajectory, + ParallelToolsTrajectory, + RetrySystemTrajectory, + SingleToolThinkingTrajectory, + SingleToolTrajectory, +) + +# --------------------------------------------------------------------------- +# Tokenizer cache +# --------------------------------------------------------------------------- + +_TOK_CACHE: dict[tuple[str, str | None], AutoTokenizer] = {} + + +def _get_tokenizer(model_id: str, tito_type: TITOTokenizerType | None = None) -> AutoTokenizer: + chat_template_path = resolve_fixed_chat_template(tito_type, ["tool"])[0] if tito_type is not None else None + cache_key = (model_id, chat_template_path) + if cache_key not in _TOK_CACHE: + _TOK_CACHE[cache_key] = load_tokenizer( + model_id, + chat_template_path=chat_template_path, + trust_remote_code=True, + ) + return _TOK_CACHE[cache_key] + + +# --------------------------------------------------------------------------- +# Fixtures — model-specific TITO tokenizers +# +# `tito` is parametrized over all supported models; use it for tests that +# should run against every model. Named fixtures (qwen3_tito, etc.) are +# for tests specific to one subclass's boundary logic. +# --------------------------------------------------------------------------- + +_TITO_MODELS: dict[str, tuple[str, type[TITOTokenizer], TITOTokenizerType]] = { + "qwen3": ("Qwen/Qwen3-4B", Qwen3TITOTokenizer, TITOTokenizerType.QWEN3), + "glm47": ("zai-org/GLM-4.7-Flash", GLM47TITOTokenizer, TITOTokenizerType.GLM47), +} + +_ALLOWED_APPEND_ROLES = ["tool", "user", "system"] + + +@pytest.fixture(params=list(_TITO_MODELS.keys())) +def tito(request) -> TITOTokenizer: + model_id, cls, tito_type = _TITO_MODELS[request.param] + return cls(_get_tokenizer(model_id, tito_type), allowed_append_roles=_ALLOWED_APPEND_ROLES) + + +@pytest.fixture +def qwen3_tito() -> Qwen3TITOTokenizer: + return Qwen3TITOTokenizer( + _get_tokenizer("Qwen/Qwen3-4B", TITOTokenizerType.QWEN3), + allowed_append_roles=_ALLOWED_APPEND_ROLES, + ) + + +@pytest.fixture +def glm47_tito() -> GLM47TITOTokenizer: + return GLM47TITOTokenizer( + _get_tokenizer("zai-org/GLM-4.7-Flash", TITOTokenizerType.GLM47), + allowed_append_roles=_ALLOWED_APPEND_ROLES, + ) + + +@pytest.fixture +def default_tito() -> TITOTokenizer: + return TITOTokenizer(_get_tokenizer("Qwen/Qwen3-4B"), allowed_append_roles=_ALLOWED_APPEND_ROLES) + + +# --------------------------------------------------------------------------- +# Trajectory parametrization +# +# Instead of relying on PRETOKENIZE_POSITIONS (which serves the pretokenized +# *chat* tests), we derive TITO split points directly from message structure: +# every assistant(tool_calls) followed by a tool/system message is a valid +# split. This way new trajectories get coverage automatically. +# +# To extend: add a trajectory class to _TOOL_TRAJECTORIES. +# To add a model: add an entry to _TITO_MODELS above. +# --------------------------------------------------------------------------- + + +def _find_tito_splits(traj_cls) -> list[int]: + """Find TITO split positions from message structure. + + A valid split is at index ``i+1`` whenever ``messages[i]`` is an assistant + message with tool_calls and ``messages[i+1]`` is a tool or system message. + Returns a list of such positions (the index of the first appended message). + """ + msgs = traj_cls.MESSAGES + splits = [] + for i, msg in enumerate(msgs): + if ( + msg.get("role") == "assistant" + and msg.get("tool_calls") + and i + 1 < len(msgs) + and msgs[i + 1].get("role") in ("tool", "system") + ): + splits.append(i + 1) + return splits + + +def _split_at(traj_cls, pos: int): + """Split trajectory at *pos* into ``(old_msgs, new_msgs, tools)``. + + ``old_msgs = messages[:pos]`` — the pretokenized prefix (ends with assistant turn). + ``new_msgs`` extends through all subsequent non-assistant messages + (tool/user/system), stopping before the next assistant turn. + """ + msgs = traj_cls.MESSAGES + end = pos + while end < len(msgs) and msgs[end].get("role") != "assistant": + end += 1 + return msgs[:pos], msgs[:end], traj_cls.TOOLS + + +_TOOL_TRAJECTORIES = [ + SingleToolTrajectory, # 1 tool call, 1 response + MultiTurnTrajectory, # 2 sequential tool turns + MultiToolSingleTurnTrajectory, # 2 parallel tool calls (weather + date) + ParallelToolsTrajectory, # 3 parallel tool calls + LongChainTrajectory, # 3 sequential turns (weather → date → weather) + RetrySystemTrajectory, # tool + system retry injection mid-conversation + IntermediateSystemTrajectory, # system messages interleaved with tool turns + SingleToolThinkingTrajectory, # tool call with reasoning_content +] + +_TRAJ_CASES = [ + pytest.param(traj_cls, pos, id=f"{traj_cls.__name__}-N{pos}") + for traj_cls in _TOOL_TRAJECTORIES + for pos in _find_tito_splits(traj_cls) +] + +# --------------------------------------------------------------------------- +# TestConfig — subclass configuration smoke-checks +# --------------------------------------------------------------------------- + + +class TestConfig: + """Each subclass stores the correct model-specific configuration at init.""" + + def test_qwen3(self, qwen3_tito: Qwen3TITOTokenizer): + assert qwen3_tito._assistant_start_str == "<|im_start|>assistant" + assert qwen3_tito._newline_id in qwen3_tito.trailing_token_ids + + def test_glm47(self, glm47_tito: GLM47TITOTokenizer): + assert glm47_tito._assistant_start_str == "<|assistant|>" + assert glm47_tito._observation_id in glm47_tito.trailing_token_ids + assert glm47_tito._user_id in glm47_tito.trailing_token_ids + assert glm47_tito.max_trim_tokens == 1 + + def test_default(self, default_tito: TITOTokenizer): + assert default_tito._assistant_start_str is None + assert default_tito.trailing_token_ids == frozenset() + + def test_comparator_inherits_trailing_ids(self, qwen3_tito: Qwen3TITOTokenizer): + """create_comparator propagates trailing_token_ids to the comparator's trim set.""" + comp = qwen3_tito.create_comparator() + assert comp._trim_trailing_ids == set(qwen3_tito.trailing_token_ids) + + +# --------------------------------------------------------------------------- +# TestMergeTokensBoundary — prefix manipulation with synthetic IDs +# +# All tests use the same trajectory (SingleToolTrajectory split at pos=3) +# to compute incremental tokens, then verify prefix manipulation with +# synthetic IDs like [100, 200, ]. +# --------------------------------------------------------------------------- + +_BND_OLD, _BND_NEW, _BND_TOOLS = _split_at(SingleToolTrajectory, 3) + + +class TestMergeTokensBoundary: + """merge_tokens correctly manipulates the prefix before concatenating incremental tokens.""" + + # -- Qwen3: insert \n after <|im_end|> -- + + def test_qwen3_inserts_newline_after_im_end(self, qwen3_tito: Qwen3TITOTokenizer): + """Model stops at <|im_end|> without trailing \\n; merge_tokens inserts it.""" + incremental = qwen3_tito.tokenize_additional_non_assistant(_BND_OLD, _BND_NEW, _BND_TOOLS) + im_end = qwen3_tito._im_end_id + nl = qwen3_tito._newline_id + + result = qwen3_tito.merge_tokens(_BND_OLD, _BND_NEW, [100, 200, im_end], _BND_TOOLS) + assert result == [100, 200, im_end, nl] + incremental + + def test_qwen3_no_newline_otherwise(self, qwen3_tito: Qwen3TITOTokenizer): + """No insertion when prefix does not end with <|im_end|>.""" + incremental = qwen3_tito.tokenize_additional_non_assistant(_BND_OLD, _BND_NEW, _BND_TOOLS) + result = qwen3_tito.merge_tokens(_BND_OLD, _BND_NEW, [100, 200, 300], _BND_TOOLS) + assert result == [100, 200, 300] + incremental + + # -- GLM47: strip ambiguous boundary tokens -- + + def test_glm47_strips_observation(self, glm47_tito: GLM47TITOTokenizer): + """Model emits <|observation|> as stop token; merge_tokens strips the duplicate.""" + incremental = glm47_tito.tokenize_additional_non_assistant(_BND_OLD, _BND_NEW, _BND_TOOLS) + result = glm47_tito.merge_tokens(_BND_OLD, _BND_NEW, [100, 200, glm47_tito._observation_id], _BND_TOOLS) + assert result == [100, 200] + incremental + + def test_glm47_strips_user(self, glm47_tito: GLM47TITOTokenizer): + """<|user|> is also an ambiguous boundary — stripped the same way.""" + incremental = glm47_tito.tokenize_additional_non_assistant(_BND_OLD, _BND_NEW, _BND_TOOLS) + result = glm47_tito.merge_tokens(_BND_OLD, _BND_NEW, [100, 200, glm47_tito._user_id], _BND_TOOLS) + assert result == [100, 200] + incremental + + def test_glm47_no_strip_otherwise(self, glm47_tito: GLM47TITOTokenizer): + """Non-boundary trailing token is preserved.""" + incremental = glm47_tito.tokenize_additional_non_assistant(_BND_OLD, _BND_NEW, _BND_TOOLS) + result = glm47_tito.merge_tokens(_BND_OLD, _BND_NEW, [100, 200, 300], _BND_TOOLS) + assert result == [100, 200, 300] + incremental + + # -- Default: no boundary handling -- + + def test_default_concatenates(self, default_tito: TITOTokenizer): + """Base class does plain concatenation without any prefix modification.""" + incremental = default_tito.tokenize_additional_non_assistant(_BND_OLD, _BND_NEW, _BND_TOOLS) + result = default_tito.merge_tokens(_BND_OLD, _BND_NEW, [100, 200, 300], _BND_TOOLS) + assert result == [100, 200, 300] + incremental + + # -- Edge case -- + + def test_empty_prefix(self, qwen3_tito: Qwen3TITOTokenizer): + """Empty prefix → no boundary handling, result is just incremental.""" + incremental = qwen3_tito.tokenize_additional_non_assistant(_BND_OLD, _BND_NEW, _BND_TOOLS) + result = qwen3_tito.merge_tokens(_BND_OLD, _BND_NEW, [], _BND_TOOLS) + assert result == incremental + + +# --------------------------------------------------------------------------- +# TestTokenizeAdditional — incremental tokenization via role-segmented synthetic diff +# +# test_produces_nonempty_incremental is the scalable core: parametrized over +# _TRAJ_CASES (trajectories × split points) × tito fixture (models). +# 8 trajectories × ~14 splits × 2 models = 28 test cases currently. +# +# Validation tests use a single trajectory since the validation logic +# (assert_messages_append_only_with_allowed_role) is model/trajectory-independent. +# --------------------------------------------------------------------------- + + +class TestTokenizeAdditional: + """tokenize_additional_non_assistant produces valid incremental tokens.""" + + @pytest.mark.parametrize("traj_cls, pos", _TRAJ_CASES) + def test_produces_nonempty_incremental(self, tito: TITOTokenizer, traj_cls, pos): + """Every valid TITO split yields non-empty incremental tokens. + + This is the primary scalability test — it runs every trajectory's + TITO splits against every model tokenizer. + """ + old_msgs, new_msgs, tools = _split_at(traj_cls, pos) + incremental = tito.tokenize_additional_non_assistant(old_msgs, new_msgs, tools) + assert len(incremental) > 0 + + def test_contiguous_tool_segment_is_tokenized_together(self, qwen3_tito: Qwen3TITOTokenizer): + old_msgs, new_msgs, tools = _split_at(MultiToolSingleTurnTrajectory, 3) + appended = new_msgs[len(old_msgs) :] + + segments = qwen3_tito._split_appended_segments(appended) + assert len(segments) == 1 + assert [msg["role"] for msg in segments[0]] == ["tool", "tool"] + + incremental = qwen3_tito.tokenize_additional_non_assistant(old_msgs, new_msgs, tools) + decoded = qwen3_tito.tokenizer.decode(incremental) + assert MultiToolSingleTurnTrajectory.MESSAGES[3]["content"] in decoded + assert MultiToolSingleTurnTrajectory.MESSAGES[4]["content"] in decoded + + def test_user_and_system_segments_are_singletons(self, default_tito: TITOTokenizer): + appended = [ + {"role": "system", "content": "Use JSON."}, + {"role": "user", "content": "Hello"}, + {"role": "tool", "tool_call_id": "call_1", "content": '{"ok": true}'}, + {"role": "tool", "tool_call_id": "call_2", "content": '{"ok": false}'}, + {"role": "user", "content": "Try again"}, + ] + + segments = default_tito._split_appended_segments(appended) + assert [[msg["role"] for msg in segment] for segment in segments] == [ + ["system"], + ["user"], + ["tool", "tool"], + ["user"], + ] + + def test_generation_prompt_is_appended_once_for_full_suffix(self, qwen3_tito: Qwen3TITOTokenizer): + old_msgs = list(SingleToolThinkingTrajectory.MESSAGES[:3]) + new_msgs = old_msgs + [ + SingleToolThinkingTrajectory.MESSAGES[3], + {"role": "user", "content": "Now check Shanghai too."}, + ] + tools = SingleToolThinkingTrajectory.TOOLS + + incremental = qwen3_tito.tokenize_additional_non_assistant(old_msgs, new_msgs, tools) + decoded = qwen3_tito.tokenizer.decode(incremental) + assert decoded.count(qwen3_tito._assistant_start_str) == 1 + assert decoded.endswith( + qwen3_tito.tokenizer.decode( + qwen3_tito._tokenize_rendered_suffix(new_msgs, [], tools=tools, add_generation_prompt=True) + ) + ) + + def test_qwen3_tool_dummy_assistant_preserves_reasoning_shape(self): + thinking_template_path = ( + Path(__file__).resolve().parents[4] + / "miles/utils/chat_template_utils/templates/qwen3_thinking_2507_and_next_fixed.jinja" + ) + thinking_tito = Qwen3TITOTokenizer( + load_tokenizer( + "Qwen/Qwen3-4B-Instruct-2507", + chat_template_path=str(thinking_template_path), + trust_remote_code=True, + ), + allowed_append_roles=_ALLOWED_APPEND_ROLES, + ) + tool_messages = [SingleToolThinkingTrajectory.MESSAGES[3]] + dummy_assistant = _build_dummy_assistant(tool_messages) + rendered = thinking_tito.render_messages( + [{"role": "system", "content": "dummy system"}, dummy_assistant], + add_generation_prompt=False, + tools=SingleToolThinkingTrajectory.TOOLS, + ) + + assert dummy_assistant["reasoning_content"] == " " + assert rendered.endswith( + '<|im_start|>assistant\n\n{"name": "dummy_func", "arguments": {}}\n<|im_end|>\n' + ) + + @pytest.mark.parametrize( + "traj_cls, pos", + [ + pytest.param(SingleToolTrajectory, 3, id="single-tool"), + pytest.param(RetrySystemTrajectory, 3, id="tool-plus-system"), + pytest.param(IntermediateSystemTrajectory, 3, id="intermediate-system"), + ], + ) + def test_qwen3_merge_preserves_non_assistant_structure(self, qwen3_tito: Qwen3TITOTokenizer, traj_cls, pos): + """Merged tokens may differ in assistant text, but not in tool/system structure.""" + old_msgs, new_msgs, tools = _split_at(traj_cls, pos) + pretokenized = apply_chat_template( + old_msgs, + tokenizer=qwen3_tito.tokenizer, + tokenize=True, + add_generation_prompt=False, + tools=tools, + ) + merged = qwen3_tito.merge_tokens(old_msgs, new_msgs, pretokenized, tools) + expected = apply_chat_template( + new_msgs, + tokenizer=qwen3_tito.tokenizer, + tokenize=True, + add_generation_prompt=True, + tools=tools, + ) + mismatches = qwen3_tito.create_comparator().compare_sequences(expected, merged) + assert all(m.type == MismatchType.ASSISTANT_TEXT for m in mismatches) + + # -- Append-only validation (assert_messages_append_only_with_allowed_role is called internally) -- + + def test_rejects_prefix_mutation(self, qwen3_tito: Qwen3TITOTokenizer): + """Modifying an existing message in new_messages raises ValueError.""" + old_msgs, new_msgs, _ = _split_at(SingleToolTrajectory, 3) + mutated_old = [{"role": "user", "content": "CHANGED"}] + list(old_msgs[1:]) + mutated_new = mutated_old + list(new_msgs[len(old_msgs) :]) + with pytest.raises(ValueError, match="mismatch"): + qwen3_tito.tokenize_additional_non_assistant(old_msgs, mutated_new) + + def test_rejects_fewer_messages(self, qwen3_tito: Qwen3TITOTokenizer): + """new_messages shorter than old_messages raises ValueError.""" + old_msgs = SingleToolTrajectory.MESSAGES[:3] + with pytest.raises(ValueError, match="fewer"): + qwen3_tito.tokenize_additional_non_assistant(old_msgs, old_msgs[:1]) + + def test_rejects_assistant_append(self, qwen3_tito: Qwen3TITOTokenizer): + """Appending an assistant message (not tool/system) raises ValueError.""" + old_msgs = SingleToolTrajectory.MESSAGES[:3] + bad_new = list(old_msgs) + [{"role": "assistant", "content": "hi"}] + with pytest.raises(ValueError, match="role"): + qwen3_tito.tokenize_additional_non_assistant(old_msgs, bad_new) + + +# --------------------------------------------------------------------------- +# TestFactory — get_tito_tokenizer dispatch +# --------------------------------------------------------------------------- + + +class TestFactory: + """get_tito_tokenizer creates the correct subclass from string or enum type.""" + + @pytest.mark.parametrize( + "type_str, model_id, cls", + [ + ("qwen3", "Qwen/Qwen3-4B", Qwen3TITOTokenizer), + ("qwen35", "Qwen/Qwen3-4B", Qwen35TITOTokenizer), + ("qwennext", "Qwen/Qwen3-4B", QwenNextTITOTokenizer), + ("glm47", "zai-org/GLM-4.7-Flash", GLM47TITOTokenizer), + ("default", "Qwen/Qwen3-4B", TITOTokenizer), + ], + ) + def test_creates_correct_type(self, type_str, model_id, cls): + tito = get_tito_tokenizer(_get_tokenizer(model_id), tokenizer_type=type_str) + assert isinstance(tito, cls) + + def test_enum_input(self): + """Enum values work the same as string values.""" + tito = get_tito_tokenizer(_get_tokenizer("Qwen/Qwen3-4B"), tokenizer_type=TITOTokenizerType.QWEN3) + assert isinstance(tito, Qwen3TITOTokenizer) + + @pytest.mark.parametrize( + "type_str, cls", + [("qwen35", Qwen35TITOTokenizer), ("qwennext", QwenNextTITOTokenizer)], + ) + def test_qwen_variant_inherits_qwen3_boundary_logic(self, type_str, cls): + """Qwen3.5 / Qwen3-Next reuse Qwen3's boundary handling via inheritance. + The named subclass exists so fixed_templates can key on (tito_model, + surface) — but token-level merge behavior is identical to Qwen3.""" + tito = get_tito_tokenizer(_get_tokenizer("Qwen/Qwen3-4B"), tokenizer_type=type_str) + assert isinstance(tito, cls) + assert isinstance(tito, Qwen3TITOTokenizer) + + def test_invalid_type_raises(self): + with pytest.raises(ValueError): + get_tito_tokenizer(_get_tokenizer("Qwen/Qwen3-4B"), tokenizer_type="nonexistent") + + def test_none_tokenizer_raises(self): + with pytest.raises(ValueError, match="must not be None"): + get_tito_tokenizer(None) + + +class TestParserBinding: + """Each TITO subclass binds sglang ``--reasoning-parser`` and + ``--tool-call-parser`` values; ``resolve_reasoning_and_tool_call_parser`` + enforces user-supplied values agree with the bindings (or returns the + bound values when the user didn't pass one). The two parsers are + resolved independently — a missing binding on one doesn't suppress the + assert on the other.""" + + @pytest.mark.parametrize( + "tito_model, expected_reasoning, expected_tool_call", + [ + (TITOTokenizerType.QWEN3, "qwen3", "qwen25"), + (TITOTokenizerType.QWEN35, "qwen3", "qwen3_coder"), + (TITOTokenizerType.QWENNEXT, "qwen3", "qwen25"), + (TITOTokenizerType.GLM47, "glm45", "glm47"), + (TITOTokenizerType.NEMOTRON3, "nemotron_3", "qwen3_coder"), + (TITOTokenizerType.KIMI25, None, None), + (TITOTokenizerType.KIMI26, "kimi_k2", "kimi_k2_raw_id"), + (TITOTokenizerType.MINIMAX_M25, "minimax-append-think", "minimax-m2"), + (TITOTokenizerType.MINIMAX_M27, "minimax-append-think", "minimax-m2"), + (TITOTokenizerType.DEEPSEEKV32, "deepseek-v3", "deepseekv32"), + (TITOTokenizerType.DEEPSEEKV4, "deepseek-v4", "deepseekv4"), + (TITOTokenizerType.DEFAULT, None, None), + ], + ) + def test_subclass_binding(self, tito_model, expected_reasoning, expected_tool_call): + cls = TITOTokenizerType.get_tokenizer_class(tito_model) + assert cls.reasoning_parser == expected_reasoning + assert cls.tool_call_parser == expected_tool_call + + def test_resolve_returns_binding_when_user_omits(self): + from miles.utils.chat_template_utils import resolve_reasoning_and_tool_call_parser + + assert resolve_reasoning_and_tool_call_parser(TITOTokenizerType.QWEN3) == ("qwen3", "qwen25") + assert resolve_reasoning_and_tool_call_parser(TITOTokenizerType.QWEN35) == ("qwen3", "qwen3_coder") + assert resolve_reasoning_and_tool_call_parser(TITOTokenizerType.GLM47) == ("glm45", "glm47") + assert resolve_reasoning_and_tool_call_parser(TITOTokenizerType.DEEPSEEKV4) == ("deepseek-v4", "deepseekv4") + # DEFAULT family has no binding for either parser; both come back None. + assert resolve_reasoning_and_tool_call_parser(TITOTokenizerType.DEFAULT) == (None, None) + + def test_resolve_accepts_matching_user_value(self): + from miles.utils.chat_template_utils import resolve_reasoning_and_tool_call_parser + + assert resolve_reasoning_and_tool_call_parser("qwen3", "qwen3", "qwen25") == ("qwen3", "qwen25") + assert resolve_reasoning_and_tool_call_parser(TITOTokenizerType.QWEN35, "qwen3", "qwen3_coder") == ( + "qwen3", + "qwen3_coder", + ) + + def test_resolve_raises_on_reasoning_mismatch(self): + from miles.utils.chat_template_utils import resolve_reasoning_and_tool_call_parser + + with pytest.raises(ValueError, match="--reasoning-parser='glm45' disagrees"): + resolve_reasoning_and_tool_call_parser(TITOTokenizerType.QWEN3, user_reasoning_parser="glm45") + + def test_resolve_raises_on_tool_call_mismatch(self): + from miles.utils.chat_template_utils import resolve_reasoning_and_tool_call_parser + + with pytest.raises(ValueError, match="--tool-call-parser='glm47' disagrees"): + resolve_reasoning_and_tool_call_parser(TITOTokenizerType.QWEN3, user_tool_call_parser="glm47") + + def test_resolve_accepts_user_value_when_family_unbound(self): + # DEFAULT family has no binding for either parser; user-provided wins + # (for families that haven't been wired up to a sglang parser yet). + from miles.utils.chat_template_utils import resolve_reasoning_and_tool_call_parser + + assert resolve_reasoning_and_tool_call_parser( + TITOTokenizerType.DEFAULT, "custom_reasoning", "custom_tool_call" + ) == ("custom_reasoning", "custom_tool_call") + + def test_resolve_partial_user_input(self): + # User can pass only one of the two; the other auto-resolves from + # the family binding independently. + from miles.utils.chat_template_utils import resolve_reasoning_and_tool_call_parser + + # User passes reasoning only — tool_call comes from binding. + assert resolve_reasoning_and_tool_call_parser(TITOTokenizerType.QWEN3, user_reasoning_parser="qwen3") == ( + "qwen3", + "qwen25", + ) + # User passes tool_call only — reasoning comes from binding. + assert resolve_reasoning_and_tool_call_parser(TITOTokenizerType.GLM47, user_tool_call_parser="glm47") == ( + "glm45", + "glm47", + ) diff --git a/sidecars/tito/tests/upstream/fast/utils/test_utils/test_session_verify_runner.py b/sidecars/tito/tests/upstream/fast/utils/test_utils/test_session_verify_runner.py new file mode 100644 index 0000000..770f7f5 --- /dev/null +++ b/sidecars/tito/tests/upstream/fast/utils/test_utils/test_session_verify_runner.py @@ -0,0 +1,85 @@ +import argparse +import json + +import pytest + +from miles.utils.test_utils.session_verify_runner import ( + SESSION_VERIFY_INVARIANT_ARGS, + assert_session_verify_metrics, + namespace_to_train_args, +) + + +def _build_args(**overrides) -> str: + values = { + **SESSION_VERIFY_INVARIANT_ARGS, + "hf_checkpoint": "/root/models/test-model", + "tito_model": "qwen3", + "tito_allowed_append_roles": ["tool", "user"], + "rollout_num_gpus_per_engine": 2, + "actor_num_nodes": 1, + "actor_num_gpus_per_node": 8, + "n_samples_per_prompt": 4, + "session_verify_cycles": 3, + "tool_call_failure_mode": "rollback", + "sglang_reasoning_parser": "qwen3", + "sglang_tool_call_parser": "qwen25", + } + values.update(overrides) + return namespace_to_train_args(argparse.Namespace(**values)) + + +def test_namespace_to_train_args_uses_default_rollout_max_response_len(): + train_args = _build_args() + + assert "--rollout-max-response-len 8192" in train_args + + +def test_namespace_to_train_args_allows_model_specific_rollout_max_response_len(): + train_args = _build_args(rollout_max_response_len=16384) + + assert "--rollout-max-response-len 16384" in train_args + + +def test_namespace_to_train_args_keeps_ci_test_enabled_for_fsdp_debug_rollout(): + train_args = _build_args() + + assert "--train-backend fsdp" in train_args + assert "--ci-test" in train_args + + +def test_namespace_to_train_args_omits_expert_parallel_for_single_expert(): + train_args = _build_args() + + assert "--sglang-expert-parallel-size" not in train_args + + +def test_namespace_to_train_args_emits_expert_parallel_for_moe(): + train_args = _build_args(sglang_expert_parallel_size=8) + + assert "--sglang-expert-parallel-size 8" in train_args + + +def _write_metrics(path, entries: list[dict]) -> None: + path.write_text("\n".join(json.dumps(entry) for entry in entries) + "\n") + + +def test_session_verify_metrics_accepts_cross_sample_append_tool(tmp_path): + metrics_path = tmp_path / "metrics.jsonl" + _write_metrics( + metrics_path, + [ + {"driver_events": ["initial", "append_user"], "had_assistant_mismatch": False}, + {"driver_events": ["initial", "append_tool"], "had_assistant_mismatch": False}, + ], + ) + + assert_session_verify_metrics(str(metrics_path), assistant_text_threshold=0.1) + + +def test_session_verify_metrics_requires_at_least_one_append_tool(tmp_path): + metrics_path = tmp_path / "metrics.jsonl" + _write_metrics(metrics_path, [{"driver_events": ["initial", "append_user"], "had_assistant_mismatch": False}]) + + with pytest.raises(AssertionError, match="no sample produced an append_tool action"): + assert_session_verify_metrics(str(metrics_path), assistant_text_threshold=0.1) diff --git a/sidecars/tito/tito_gateway/VENDORED_MILES_AUDIT.md b/sidecars/tito/tito_gateway/VENDORED_MILES_AUDIT.md new file mode 100644 index 0000000..6f053f4 --- /dev/null +++ b/sidecars/tito/tito_gateway/VENDORED_MILES_AUDIT.md @@ -0,0 +1,29 @@ +# Vendored Miles Source Audit + +This package is a standalone wrapper/packaging layer around Miles TITO work. It is not a rewrite of Miles TITO algorithms. Core Miles files below were audited against: + +- Repository: `https://github.com/radixark/miles` +- Commit: `9437366e0aa3a25294720f70d18b081067595f85` +- Local upstream checkout used for audit: `/tmp/miles-explore` + +## Audit Result + +| Upstream file | Vendored file | Status | Allowed differences | +|---|---|---|---| +| `miles/utils/chat_template_utils/tito_tokenizer.py` | `tito_gateway/vendor/miles_compat/utils/chat_template_utils/tito_tokenizer.py` | Import-path rewrite plus Python 3.10 enum compatibility | `miles.*` imports rewritten to `tito_gateway.vendor.miles_compat.*`; upstream `StrEnum` replaced by `str, Enum` because this package supports Python 3.10. TITO tokenization, merge, fixed-template resolution, and factory logic are otherwise preserved. | +| `miles/utils/chat_template_utils/template.py` | `tito_gateway/vendor/miles_compat/utils/chat_template_utils/template.py` | Import-path rewrite only | `miles.utils.chat_template_utils` import rewritten to `tito_gateway.vendor.miles_compat.utils.chat_template_utils`. | +| `miles/utils/chat_template_utils/token_seq_comparator.py` | `tito_gateway/vendor/miles_compat/utils/chat_template_utils/token_seq_comparator.py` | Byte-identical | None. | +| `miles/utils/chat_template_utils/templates/*.jinja` | `tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/*.jinja` | Byte-identical | None. Audited templates: `kimi_k25_fixed.jinja`, `minimax_m25_fixed.jinja`, `minimax_m27_fixed.jinja`, `qwen3.5_fixed.jinja`, `qwen3_fixed.jinja`, `qwen3_thinking_2507_and_next_fixed.jinja`. | +| `miles/rollout/session/linear_trajectory.py` | `tito_gateway/vendor/miles_compat/rollout/session/linear_trajectory.py` | Import-path rewrite only | `miles.*` imports rewritten to `tito_gateway.vendor.miles_compat.*`. | +| `miles/rollout/session/sessions.py` | `tito_gateway/vendor/miles_compat/rollout/session/sessions.py` | Import-path rewrite only | `miles.*` imports rewritten to `tito_gateway.vendor.miles_compat.*`. | +| `miles/rollout/session/session_errors.py` | `tito_gateway/vendor/miles_compat/rollout/session/session_errors.py` | Byte-identical | None. | +| `miles/rollout/session/session_types.py` | `tito_gateway/vendor/miles_compat/rollout/session/session_types.py` | Byte-identical | None. | +| `miles/rollout/session/session_server.py` | `tito_gateway/vendor/miles_compat/rollout/session/session_server.py` | Import-path rewrite only | `miles.rollout.session.sessions` import rewritten to `tito_gateway.vendor.miles_compat.rollout.session.sessions`. | + +## Compatibility Test Path + +The copied upstream tokenizer test computes a repository-root-relative template path from its own `tests/upstream/...` location. To preserve the unchanged test body, this package provides: + +- `tests/miles/utils/chat_template_utils/templates/qwen3_thinking_2507_and_next_fixed.jinja` + +That file is byte-identical to the vendored/upstream template. diff --git a/sidecars/tito/tito_gateway/__init__.py b/sidecars/tito/tito_gateway/__init__.py new file mode 100644 index 0000000..0b531b2 --- /dev/null +++ b/sidecars/tito/tito_gateway/__init__.py @@ -0,0 +1,16 @@ +"""Standalone wrapper package around Miles TITO session gateway work.""" + +from tito_gateway.config import TITOGatewayConfig +from tito_gateway.discovery import discover_backend_url +from tito_gateway.gateway import TITOGateway +from tito_gateway.server import SessionServer +from tito_gateway.tokenizer import TITOTokenizerType, get_tito_tokenizer + +__all__ = [ + "TITOGateway", + "TITOGatewayConfig", + "SessionServer", + "TITOTokenizerType", + "discover_backend_url", + "get_tito_tokenizer", +] diff --git a/sidecars/tito/tito_gateway/cli.py b/sidecars/tito/tito_gateway/cli.py new file mode 100644 index 0000000..f348e8f --- /dev/null +++ b/sidecars/tito/tito_gateway/cli.py @@ -0,0 +1,241 @@ +"""Command-line entrypoint for TITO Gateway.""" + +from __future__ import annotations + +import argparse +import json +import sys + +from tito_gateway.config import TITOGatewayConfig +from tito_gateway.gateway import TITOGateway +from tito_gateway.tokenizer import TITOTokenizerType + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="tito-gateway", + description="Standalone wrapper around Miles TITO session gateway work.", + ) + subparsers = parser.add_subparsers(dest="command") + + _add_serve_parser(subparsers) + _add_verify_chat_template_parser(subparsers) + _add_verify_session_parser(subparsers) + return parser + + +def _add_serve_parser(subparsers: argparse._SubParsersAction) -> argparse.ArgumentParser: + serve = subparsers.add_parser("serve", help="Start the TITO gateway server.") + _add_serve_arguments(serve) + return serve + + +def _add_serve_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--hf-checkpoint", required=True, help="HuggingFace model ID or local checkpoint path.") + parser.add_argument("--backend-url", default=None, help="OpenAI-compatible backend URL to proxy to.") + parser.add_argument("--chat-template-path", default=None, help="Optional fixed chat template path.") + parser.add_argument( + "--apply-chat-template-kwargs", + default=None, + help="JSON object forwarded as chat-template kwargs, matching Miles convention.", + ) + parser.add_argument( + "--tito-model", + choices=[item.value for item in TITOTokenizerType], + default=TITOTokenizerType.DEFAULT.value, + help="Miles TITO tokenizer family.", + ) + parser.add_argument( + "--tito-allowed-append-roles", + nargs="+", + choices=["tool", "user", "system"], + default=["tool"], + help="Roles allowed after an assistant turn; tool is the default.", + ) + parser.add_argument("--session-server-ip", default="127.0.0.1", help="Gateway bind host.") + parser.add_argument("--session-server-port", type=int, default=30000, help="Gateway bind port.") + parser.add_argument("--miles-router-timeout", type=float, default=600.0, help="Proxy timeout in seconds.") + parser.add_argument( + "--backend-probe-candidate", + action="append", + default=None, + metavar="URL", + help="Local backend URL candidate to probe after explicit and environment URLs; repeatable.", + ) + parser.add_argument( + "--backend-probe-timeout", + type=float, + default=0.25, + help="Per-endpoint backend probe timeout in seconds.", + ) + + +def _add_verify_chat_template_parser(subparsers: argparse._SubParsersAction) -> None: + parser = subparsers.add_parser( + "verify-chat-template", + help="Verify that a chat template is append-only after last user message.", + ) + parser.add_argument("--template", metavar="PATH") + parser.add_argument("--model", metavar="MODEL_ID") + parser.add_argument( + "--tito-model", + choices=[item.value for item in TITOTokenizerType], + default=None, + ) + parser.add_argument( + "--tito-allowed-append-roles", + nargs="+", + default=["tool"], + choices=["tool", "user", "system"], + metavar="ROLE", + ) + parser.add_argument("--thinking", choices=["off", "on", "both"], default="on") + parser.add_argument("--chat-template-kwargs", type=json.loads, default=None, metavar="JSON") + parser.set_defaults(verify_command="chat-template") + + +def _add_verify_session_parser(subparsers: argparse._SubParsersAction) -> None: + from miles.utils.test_utils.session_verify_runner import ( + ASSISTANT_TEXT_MISMATCH_RATIO_THRESHOLD, + SESSION_VERIFY_INVARIANT_ARGS, + ) + + parser = subparsers.add_parser( + "verify-session-tito-tokenizer", + help="Run the optional Miles/SGLang session-server TITO verifier.", + ) + parser.add_argument("--hf-checkpoint", required=True, help="HuggingFace model ID or local checkpoint path.") + parser.add_argument("--chat-template-path", default=None, help="Optional fixed chat template path.") + parser.add_argument( + "--apply-chat-template", + action="store_true", + default=False, + help="Miles-compatible flag for applying the chat template in data preprocessing.", + ) + parser.add_argument( + "--apply-chat-template-kwargs", + type=json.loads, + default={}, + metavar="JSON", + help="JSON object forwarded as Miles chat-template kwargs.", + ) + parser.add_argument( + "--tito-model", + choices=[item.value for item in TITOTokenizerType], + required=True, + help="Miles TITO tokenizer family.", + ) + parser.add_argument( + "--tito-allowed-append-roles", + nargs="+", + default=["tool"], + choices=["tool", "user", "system"], + metavar="ROLE", + ) + parser.add_argument("--prompt-data", default=SESSION_VERIFY_INVARIANT_ARGS["prompt_data"]) + parser.add_argument("--input-key", default=SESSION_VERIFY_INVARIANT_ARGS["input_key"]) + parser.add_argument( + "--custom-generate-function-path", + default=SESSION_VERIFY_INVARIANT_ARGS["custom_generate_function_path"], + ) + parser.add_argument( + "--custom-agent-function-path", + default=SESSION_VERIFY_INVARIANT_ARGS["custom_agent_function_path"], + ) + parser.add_argument("--backend-url", default=None, help="Optional OpenAI-compatible backend URL for session tests.") + parser.add_argument("--session-server-ip", default="127.0.0.1") + parser.add_argument("--session-server-port", type=int, default=30000) + parser.add_argument("--miles-router-timeout", type=float, default=600.0) + parser.add_argument("--sglang-reasoning-parser", default=None) + parser.add_argument("--sglang-tool-call-parser", default=None) + parser.add_argument("--rollout-num-gpus-per-engine", type=int, default=1) + parser.add_argument("--sglang-expert-parallel-size", type=int, default=1) + parser.add_argument("--num-rollout", type=int, default=SESSION_VERIFY_INVARIANT_ARGS["num_rollout"]) + parser.add_argument("--rollout-batch-size", type=int, default=SESSION_VERIFY_INVARIANT_ARGS["rollout_batch_size"]) + parser.add_argument( + "--rollout-max-response-len", + type=int, + default=SESSION_VERIFY_INVARIANT_ARGS["rollout_max_response_len"], + ) + parser.add_argument( + "--rollout-temperature", + type=float, + default=SESSION_VERIFY_INVARIANT_ARGS["rollout_temperature"], + ) + parser.add_argument("--global-batch-size", type=int, default=SESSION_VERIFY_INVARIANT_ARGS["global_batch_size"]) + parser.add_argument("--rm-type", default=SESSION_VERIFY_INVARIANT_ARGS["rm_type"]) + parser.add_argument("--actor-num-nodes", type=int, default=1) + parser.add_argument("--actor-num-gpus-per-node", type=int, default=1) + parser.add_argument("--n-samples-per-prompt", type=int, default=4) + parser.add_argument("--session-verify-cycles", type=int, default=3) + parser.add_argument("--tool-call-failure-mode", default="rollback") + parser.add_argument( + "--assistant-text-threshold", + type=float, + default=ASSISTANT_TEXT_MISMATCH_RATIO_THRESHOLD, + help=( + "Soft threshold for assistant_text mismatch ratio. " + f"Default {ASSISTANT_TEXT_MISMATCH_RATIO_THRESHOLD}." + ), + ) + parser.add_argument( + "--train-backend", + choices=["megatron", "fsdp"], + default=SESSION_VERIFY_INVARIANT_ARGS["train_backend"], + ) + parser.add_argument("--use-session-server", action=argparse.BooleanOptionalAction, default=None) + parser.add_argument("--debug-rollout-only", action=argparse.BooleanOptionalAction, default=None) + parser.add_argument("--ci-test", action=argparse.BooleanOptionalAction, default=None) + parser.add_argument("--colocate", action=argparse.BooleanOptionalAction, default=None) + parser.set_defaults(**SESSION_VERIFY_INVARIANT_ARGS) + parser.set_defaults(verify_command="session-tito-tokenizer") + + +def _serve(args: argparse.Namespace) -> int: + config = TITOGatewayConfig.from_cli_values( + hf_checkpoint=args.hf_checkpoint, + backend_url=args.backend_url, + chat_template_path=args.chat_template_path, + apply_chat_template_kwargs=args.apply_chat_template_kwargs, + tito_model=args.tito_model, + tito_allowed_append_roles=args.tito_allowed_append_roles, + session_server_ip=args.session_server_ip, + session_server_port=args.session_server_port, + miles_router_timeout=args.miles_router_timeout, + backend_probe_candidates=args.backend_probe_candidate, + backend_probe_timeout=args.backend_probe_timeout, + ) + TITOGateway(config).run() + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = build_parser() + raw_args = list(sys.argv[1:] if argv is None else argv) + commands = {"serve", "verify-chat-template", "verify-session-tito-tokenizer", "-h", "--help"} + if not raw_args or raw_args[0] not in commands: + raw_args.insert(0, "serve") + args = parser.parse_args(raw_args) + + if getattr(args, "verify_command", None): + if args.verify_command == "chat-template": + from tito_gateway.verify_chat_template import run_from_args + + try: + return run_from_args(args) + except Exception as exc: + print(f"tito-gateway verify-chat-template: error: {exc}", file=sys.stderr) + return 1 + from tito_gateway.verify_session_tito_tokenizer import run_from_args + + return run_from_args(args) + + try: + return _serve(args) + except Exception as exc: + print(f"tito-gateway: error: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/sidecars/tito/tito_gateway/config.py b/sidecars/tito/tito_gateway/config.py new file mode 100644 index 0000000..801ecd0 --- /dev/null +++ b/sidecars/tito/tito_gateway/config.py @@ -0,0 +1,91 @@ +"""Configuration objects for the TITO Gateway wrapper.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from typing import Any + +from tito_gateway.discovery import DEFAULT_BACKEND_PROBE_CANDIDATES + + +_VALID_APPEND_ROLES = frozenset({"tool", "user", "system"}) + + +@dataclass(frozen=True) +class TITOGatewayConfig: + """Miles-compatible configuration for the standalone gateway wrapper.""" + + hf_checkpoint: str + backend_url: str | None = None + chat_template_path: str | None = None + apply_chat_template_kwargs: dict[str, Any] = field(default_factory=dict) + tito_model: str = "default" + tito_allowed_append_roles: tuple[str, ...] = ("tool",) + session_server_ip: str = "127.0.0.1" + session_server_port: int = 30000 + miles_router_timeout: float = 600.0 + backend_probe_candidates: tuple[str, ...] = field(default_factory=lambda: DEFAULT_BACKEND_PROBE_CANDIDATES) + backend_probe_timeout: float = 0.25 + + def __post_init__(self) -> None: + if not self.hf_checkpoint: + raise ValueError("hf_checkpoint is required for TITO token tracking") + + normalized_roles = tuple(dict.fromkeys(role.lower() for role in self.tito_allowed_append_roles)) + invalid = sorted(set(normalized_roles) - _VALID_APPEND_ROLES) + if invalid: + raise ValueError(f"unsupported tito append roles: {invalid}") + object.__setattr__(self, "tito_allowed_append_roles", normalized_roles or ("tool",)) + + @classmethod + def from_cli_values( + cls, + *, + hf_checkpoint: str, + backend_url: str | None, + chat_template_path: str | None, + apply_chat_template_kwargs: str | None, + tito_model: str, + tito_allowed_append_roles: list[str], + session_server_ip: str, + session_server_port: int, + miles_router_timeout: float, + backend_probe_candidates: list[str] | None = None, + backend_probe_timeout: float = 0.25, + ) -> "TITOGatewayConfig": + kwargs: dict[str, Any] = {} + if apply_chat_template_kwargs: + parsed = json.loads(apply_chat_template_kwargs) + if not isinstance(parsed, dict): + raise ValueError("--apply-chat-template-kwargs must decode to a JSON object") + kwargs = parsed + + return cls( + hf_checkpoint=hf_checkpoint, + backend_url=backend_url, + chat_template_path=chat_template_path, + apply_chat_template_kwargs=kwargs, + tito_model=tito_model, + tito_allowed_append_roles=tuple(tito_allowed_append_roles), + session_server_ip=session_server_ip, + session_server_port=session_server_port, + miles_router_timeout=miles_router_timeout, + backend_probe_candidates=tuple(backend_probe_candidates or DEFAULT_BACKEND_PROBE_CANDIDATES), + backend_probe_timeout=backend_probe_timeout, + ) + + def as_miles_namespace(self): + """Return an argparse-like namespace for vendored Miles session code.""" + from types import SimpleNamespace + + return SimpleNamespace( + hf_checkpoint=self.hf_checkpoint, + chat_template_path=self.chat_template_path, + apply_chat_template_kwargs=self.apply_chat_template_kwargs, + tito_model=self.tito_model, + tito_allowed_append_roles=list(self.tito_allowed_append_roles), + session_server_ip=self.session_server_ip, + session_server_port=self.session_server_port, + miles_router_timeout=self.miles_router_timeout, + ) diff --git a/sidecars/tito/tito_gateway/discovery.py b/sidecars/tito/tito_gateway/discovery.py new file mode 100644 index 0000000..9741a28 --- /dev/null +++ b/sidecars/tito/tito_gateway/discovery.py @@ -0,0 +1,99 @@ +"""Backend URL discovery for wrapping an OpenAI-compatible server.""" + +from __future__ import annotations + +import logging +import os +from collections.abc import Callable, Iterable, Mapping +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + + +DEFAULT_BACKEND_ENV_VARS = ("TITO_BACKEND_URL", "OPENAI_BASE_URL", "SGLANG_BASE_URL") +DEFAULT_BACKEND_PROBE_CANDIDATES = ( + "http://127.0.0.1:8000", + "http://localhost:8000", + "http://127.0.0.1:30000", + "http://localhost:30000", +) +BACKEND_PROBE_PATHS = ("/health", "/v1/models") + +logger = logging.getLogger(__name__) + + +def normalize_backend_url(url: str) -> str: + normalized = url.strip().rstrip("/") + if not normalized: + raise ValueError("backend URL is empty") + if "://" not in normalized: + normalized = f"http://{normalized}" + return normalized + + +def _probe_endpoint(url: str, timeout: float) -> bool: + request = Request(url, method="GET") + try: + with urlopen(request, timeout=timeout) as response: + return 200 <= response.status < 300 + except HTTPError as exc: + return 200 <= exc.code < 300 + except (OSError, URLError, TimeoutError, ValueError): + return False + + +def probe_backend_url( + candidate_url: str, + *, + timeout: float = 0.25, + endpoint_probe: Callable[[str, float], bool] | None = None, +) -> str | None: + """Return the successful probe path for a backend candidate, if live.""" + backend_url = normalize_backend_url(candidate_url) + probe = _probe_endpoint if endpoint_probe is None else endpoint_probe + for path in BACKEND_PROBE_PATHS: + if probe(f"{backend_url}{path}", timeout): + return path + return None + + +def discover_backend_url( + explicit_url: str | None = None, + *, + env: Mapping[str, str] | None = None, + env_vars: Iterable[str] = DEFAULT_BACKEND_ENV_VARS, + probe_candidates: Iterable[str] | None = DEFAULT_BACKEND_PROBE_CANDIDATES, + probe_timeout: float = 0.25, +) -> str: + """Resolve the backend URL with deterministic precedence. + + Explicit config wins, followed by environment variables in + ``DEFAULT_BACKEND_ENV_VARS`` order, followed by configured local probe + candidates in the supplied order. + """ + if explicit_url: + backend_url = normalize_backend_url(explicit_url) + logger.info("Selected backend URL from explicit config: %s", backend_url) + return backend_url + + source = os.environ if env is None else env + for key in env_vars: + value = source.get(key) + if value: + backend_url = normalize_backend_url(value) + logger.info("Selected backend URL from %s: %s", key, backend_url) + return backend_url + + candidates = tuple(probe_candidates or ()) + for candidate in candidates: + backend_url = normalize_backend_url(candidate) + live_path = probe_backend_url(backend_url, timeout=probe_timeout) + if live_path: + logger.info("Selected backend URL from probe %s via %s", backend_url, live_path) + return backend_url + + names = ", ".join(env_vars) + candidate_text = ", ".join(candidates) if candidates else "none configured" + raise RuntimeError( + "backend URL not found; pass --backend-url, " + f"set one of: {names}, or start a live backend on one of: {candidate_text}" + ) diff --git a/sidecars/tito/tito_gateway/gateway.py b/sidecars/tito/tito_gateway/gateway.py new file mode 100644 index 0000000..f573504 --- /dev/null +++ b/sidecars/tito/tito_gateway/gateway.py @@ -0,0 +1,40 @@ +"""Python wrapper API for launching TITO Gateway beside a backend server.""" + +from __future__ import annotations + +from dataclasses import replace + +from tito_gateway.config import TITOGatewayConfig +from tito_gateway.discovery import discover_backend_url +from tito_gateway.server import SessionServer + + +class TITOGateway: + """Small wrapper that resolves a backend and owns a session server app.""" + + def __init__(self, config: TITOGatewayConfig): + backend_url = discover_backend_url( + config.backend_url, + probe_candidates=config.backend_probe_candidates, + probe_timeout=config.backend_probe_timeout, + ) + self.config = replace(config, backend_url=backend_url) + self.server = SessionServer(self.config.as_miles_namespace(), backend_url) + + @classmethod + def from_server(cls, *, hf_checkpoint: str, backend_url: str | None = None, **kwargs) -> "TITOGateway": + return cls(TITOGatewayConfig(hf_checkpoint=hf_checkpoint, backend_url=backend_url, **kwargs)) + + @property + def app(self): + return self.server.app + + def run(self) -> None: + import uvicorn + + uvicorn.run( + self.app, + host=self.config.session_server_ip, + port=self.config.session_server_port, + log_level="info", + ) diff --git a/sidecars/tito/tito_gateway/server.py b/sidecars/tito/tito_gateway/server.py new file mode 100644 index 0000000..10fc79c --- /dev/null +++ b/sidecars/tito/tito_gateway/server.py @@ -0,0 +1,19 @@ +"""Session server wrapper around the vendored Miles implementation.""" + +from __future__ import annotations + +from typing import Any + + +class SessionServer: + """Lazy wrapper for Miles' standalone FastAPI session server.""" + + def __init__(self, args: Any, backend_url: str): + from tito_gateway.vendor.miles_compat.rollout.session.session_server import ( + SessionServer as MilesSessionServer, + ) + + self._impl = MilesSessionServer(args, backend_url) + self.args = args + self.backend_url = backend_url + self.app = self._impl.app diff --git a/sidecars/tito/tito_gateway/tokenizer.py b/sidecars/tito/tito_gateway/tokenizer.py new file mode 100644 index 0000000..bab7c9e --- /dev/null +++ b/sidecars/tito/tito_gateway/tokenizer.py @@ -0,0 +1,30 @@ +"""Public tokenizer entrypoints for the vendored Miles TITO implementation.""" + +from __future__ import annotations + +from enum import Enum +from typing import Any + + +class TITOTokenizerType(str, Enum): + DEFAULT = "default" + QWEN3 = "qwen3" + QWEN35 = "qwen35" + QWENNEXT = "qwennext" + GLM47 = "glm47" + NEMOTRON3 = "nemotron3" + KIMI25 = "kimi25" + KIMI26 = "kimi26" + MINIMAX_M25 = "minimax_m25" + MINIMAX_M27 = "minimax_m27" + DEEPSEEKV32 = "deepseekv32" + DEEPSEEKV4 = "deepseekv4" + + +def get_tito_tokenizer(*args: Any, **kwargs: Any) -> Any: + """Return a vendored Miles TITO tokenizer instance.""" + from tito_gateway.vendor.miles_compat.utils.chat_template_utils import ( + get_tito_tokenizer as _get_tito_tokenizer, + ) + + return _get_tito_tokenizer(*args, **kwargs) diff --git a/sidecars/tito/tito_gateway/upstream.json b/sidecars/tito/tito_gateway/upstream.json new file mode 100644 index 0000000..ef8ca13 --- /dev/null +++ b/sidecars/tito/tito_gateway/upstream.json @@ -0,0 +1,7 @@ +{ + "project": "Miles", + "repository": "https://github.com/radixark/miles", + "documentation": "https://www.radixark.com/miles/docs/user-guide/agentic-chat-template", + "source_commit": "9437366e0aa3a25294720f70d18b081067595f85", + "acknowledgement": "TITO Gateway is a standalone wrapper and packaging layer around Miles TITO work, not a rewrite of the Miles TITO algorithms." +} diff --git a/sidecars/tito/tito_gateway/vendor/__init__.py b/sidecars/tito/tito_gateway/vendor/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/__init__.py b/sidecars/tito/tito_gateway/vendor/miles_compat/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/__init__.py b/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/base_types.py b/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/base_types.py new file mode 100644 index 0000000..1041a6f --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/base_types.py @@ -0,0 +1,29 @@ +"""Lightweight Miles rollout type shapes used by verifier imports. + +The full Miles training stack owns the production rollout implementation. This +module preserves the small dataclass surface that `session_verify_agent` needs +for import and CPU-fast wrapper tests. +""" + +from __future__ import annotations + +from argparse import Namespace +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class GenerateFnInput: + state: Any + sample: Any + sampling_params: dict[str, Any] + evaluation: bool + + @property + def args(self) -> Namespace: + return self.state.args + + +@dataclass(frozen=True) +class GenerateFnOutput: + samples: Any diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/generate_hub/__init__.py b/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/generate_hub/__init__.py new file mode 100644 index 0000000..d5a9064 --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/generate_hub/__init__.py @@ -0,0 +1 @@ +"""Lightweight compatibility namespace for Miles generate helpers.""" diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/generate_hub/agentic_tool_call.py b/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/generate_hub/agentic_tool_call.py new file mode 100644 index 0000000..1bfdd8b --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/generate_hub/agentic_tool_call.py @@ -0,0 +1,29 @@ +"""Optional Miles agentic tool-call generate bridge. + +TITO Gateway vendors the session verifier wrapper, but the full Miles rollout +engine remains an optional dependency. The callable exists so verifier helper +paths are importable and testable; real e2e execution must install/provide the +Miles rollout stack or monkeypatch this bridge in CPU-fast tests. +""" + +from __future__ import annotations + +import argparse +from typing import Any + + +async def generate(input: Any) -> Any: + raise RuntimeError( + "Miles agentic tool-call rollout generation is not bundled with " + "tito-gateway. Install the optional Miles/SGLang training stack before " + "running full session verifier e2e jobs." + ) + + +def _add_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--custom-agent-function-path", type=str) + parser.add_argument("--generate-multi-samples", action="store_true", default=False) + parser.add_argument("--max-seq-len", type=int, default=None, dest="max_seq_len") + + +generate.add_arguments = _add_arguments diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/__init__.py b/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/linear_trajectory.py b/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/linear_trajectory.py new file mode 100644 index 0000000..491bd64 --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/linear_trajectory.py @@ -0,0 +1,285 @@ +import asyncio +import logging +import uuid +from dataclasses import dataclass, field +from typing import Any + +from tito_gateway.vendor.miles_compat.rollout.session.session_errors import MessageValidationError, SessionNotFoundError, TokenizationError +from tito_gateway.vendor.miles_compat.rollout.session.session_types import SessionRecord +from tito_gateway.vendor.miles_compat.utils.chat_template_utils import assert_messages_append_only_with_allowed_role, message_matches +from tito_gateway.vendor.miles_compat.utils.chat_template_utils.tito_tokenizer import TITOTokenizer + +logger = logging.getLogger(__name__) + + +# TODO: hardcoded to 1 for now; if multi-step rollback is actually needed, +# raise this limit or make it configurable and remove the restriction. +MAX_ASSISTANT_ROLLBACK_STEPS = 1 + + +@dataclass +class LinearTrajectory: + """State for a linear trajectory. + + Tracks the full message history and accumulated token IDs for one session. + The typical message sequence is: [system?, user, assistant, tool, assistant, tool, …], + but the agent may retry from an earlier point (e.g. re-running a tool call), + in which case the session is rolled back at most one assistant step. + + Concurrency contract: all mutating methods must be called under ``self.lock``. + """ + + lock: asyncio.Lock = field(default_factory=asyncio.Lock, repr=False, compare=False) + closing: bool = field(default=False, repr=False, compare=False) + messages: list[dict[str, Any]] = field(default_factory=list) + records: list[SessionRecord] = field(default_factory=list) + trajectory_token_ids: list[list[int]] = field(default_factory=list) + num_assistant: int = 0 + + @property + def token_ids(self) -> list[int]: + """Current token IDs — the latest assistant checkpoint.""" + return self.trajectory_token_ids[-1] if self.trajectory_token_ids else [] + + def append_record(self, record: SessionRecord) -> None: + self.records.append(record) + + def prepare_pretokenized( + self, + request_messages: list[dict[str, Any]], + tools: list[dict[str, Any]] | None = None, + *, + tito_tokenizer: TITOTokenizer, + ) -> list[int]: + """Build the full prompt input_ids for *request_messages*. + + On the first turn (no stored token_ids), renders *request_messages* + from scratch via the chat template. On subsequent turns, validates + that *request_messages* extends the stored history (rolling back at + most one assistant step on agent retries) and reuses the stored + token_ids as the pretokenized prefix. + + Must be called under ``self.lock``. + """ + if not self.token_ids: + return tito_tokenizer.render_messages( + request_messages, + tools=tools, + add_generation_prompt=True, + tokenize=True, + ) + + # 1. Detect agent retries and roll back (at most one assistant step). + self._try_detect_and_rollback_to_assistant_checkpoint(request_messages) + # 2. Confirm the (possibly rolled-back) stored messages are a prefix of request, + # and that each appended message role is in tito_tokenizer.allowed_append_roles. + try: + assert_messages_append_only_with_allowed_role( + self.messages, request_messages, tito_tokenizer.allowed_append_roles + ) + except ValueError as e: + raise MessageValidationError(f"{e}; to allow more roles use --tito-allowed-append-roles") from e + + return tito_tokenizer.merge_tokens( + old_messages=self.messages, + new_messages=request_messages, + pretokenized_token_ids=self.token_ids, + tools=tools, + ) + + def update_pretokenized_state( + self, + request_messages: list[dict[str, Any]], + assistant_message: dict[str, Any], + prompt_token_ids: list[int], + completion_token_ids: list[int], + max_trim_tokens: int, + ) -> None: + """Store raw token IDs after a successful response. + + Appends ``prompt_token_ids + completion_token_ids`` as a new checkpoint. + Validates that the previously stored token_ids are a prefix of the new + checkpoint (tolerating up to ``max_trim_tokens`` trailing differences). + Must be called under ``self.lock``. + """ + all_token_ids = prompt_token_ids + completion_token_ids + + prev = self.token_ids + if prev: + check_len = len(prev) - max_trim_tokens + if check_len > 0 and all_token_ids[:check_len] != prev[:check_len]: + first_mismatch = next( + ( + i + for i, (a, b) in enumerate(zip(all_token_ids[:check_len], prev[:check_len], strict=True)) + if a != b + ), + min(len(all_token_ids), check_len), + ) + raise TokenizationError( + f"pretokenized prefix mismatch: " + f"stored {len(prev)} tokens (checking first {check_len}, " + f"allowing {max_trim_tokens} trailing) are not a prefix of " + f"prompt_token_ids + completion_token_ids " + f"({len(all_token_ids)} tokens), " + f"first mismatch at index {first_mismatch}, " + f"matched {first_mismatch}/{check_len} prefix tokens\n" + f"request_messages={request_messages}\n" + f"assistant_message={assistant_message}" + ) + + self.messages = list(request_messages) + [assistant_message] + self.trajectory_token_ids.append(all_token_ids) + self.num_assistant += 1 + + def _try_detect_and_rollback_to_assistant_checkpoint( + self, + request_messages: list[dict[str, Any]], + ) -> None: + """Detect if *request_messages* diverges from stored history and roll back. + + In agentic workflows the agent may retry from an earlier point — for + example, re-running a tool call with different arguments. When that + happens the new request shares a common prefix with the stored messages + but diverges before the end. This method truncates session state back + to the last assistant checkpoint within the matching prefix. + + Only a single-step rollback is allowed (controlled by + ``MAX_ASSISTANT_ROLLBACK_STEPS``). Discarding exactly one assistant + message means the agent is retrying from the preceding checkpoint — + the request shares the stored prefix up to that assistant and then + continues with whatever the agent chooses (same or different tool + result, additional messages, etc.). Any request that would need to + discard more than one assistant (i.e. jump back across multiple + turns) is rejected with ``MessageValidationError`` and no state is + modified. + + Example — agent retries after the first tool call:: + + stored: [sys, user, assistant₁, tool₁, assistant₂] + ───────────────────── ▲ + checkpoint 0 (assistant₁) checkpoint 1 (assistant₂) + + request: [sys, user, assistant₁, tool₁_different, ...] + ↑ diverges here (index 3) + + match_len = 3 (sys, user, assistant₁ all match) + Last assistant in matched prefix → assistant₁ (checkpoint 0) + discard_count = 2 - 1 = 1 (≤ MAX_ASSISTANT_ROLLBACK_STEPS) + + After rollback: + messages = [sys, user, assistant₁] + trajectory_token_ids = [checkpoint_0_ids] + records = [record_0] + num_assistant = 1 + + No rollback occurs when: + - The stored history is empty. + - *request_messages* is a strict extension of stored messages + (``match_len >= len(stored)``). + """ + stored = self.messages + if not stored or not self.trajectory_token_ids: + return + + match_len = 0 + for i in range(min(len(request_messages), len(stored))): + if message_matches(stored[i], request_messages[i]): + match_len = i + 1 + else: + break + + if match_len >= len(stored): + return + + # Find the last assistant message within the matched prefix. + rollback_msg_end = None + checkpoint_index = -1 + assistant_count = 0 + for i in range(match_len): + if stored[i].get("role") == "assistant": + rollback_msg_end = i + 1 + checkpoint_index = assistant_count + assistant_count += 1 + + if checkpoint_index < 0: + raise MessageValidationError( + f"rollback failed: no assistant message found in the first " + f"{match_len} matched messages (stored has {len(stored)} messages, " + f"request has {len(request_messages)} messages)" + ) + + discard_count = self.num_assistant - (checkpoint_index + 1) + if discard_count > MAX_ASSISTANT_ROLLBACK_STEPS: + raise MessageValidationError( + f"rollback failed: discard_count={discard_count} exceeds " + f"max_assistant_rollback_steps={MAX_ASSISTANT_ROLLBACK_STEPS} " + f"(stored has {len(stored)} messages, " + f"request has {len(request_messages)} messages)" + ) + + logger.info( + "Rolling back session: stored %d messages / %d checkpoints -> " + "checkpoint %d (messages[:%d]), discarding %d assistant(s)", + len(stored), + self.num_assistant, + checkpoint_index, + rollback_msg_end, + discard_count, + ) + + self.messages = stored[:rollback_msg_end] + self.trajectory_token_ids = self.trajectory_token_ids[: checkpoint_index + 1] + self.records = self.records[: checkpoint_index + 1] + self.num_assistant = checkpoint_index + 1 + + +class SessionRegistry: + """Session ID -> trajectory mapping with shared tokenizer resources. + + Pure CRUD plus read-only computation (compute_session_mismatch). + Does NOT mutate session state - all mutations are methods on + LinearTrajectory; called by the route handler under session.lock. + """ + + def __init__(self, args, tokenizer: Any, *, tito_tokenizer: TITOTokenizer): + self.sessions: dict[str, LinearTrajectory] = {} + self.args = args + self.tokenizer = tokenizer + self.tito_tokenizer = tito_tokenizer + self.comparator = tito_tokenizer.create_comparator() + + def create_session(self) -> str: + session_id = uuid.uuid4().hex + self.sessions[session_id] = LinearTrajectory() + return session_id + + def get_session(self, session_id: str) -> LinearTrajectory: + session = self.sessions.get(session_id) + if session is None: + raise SessionNotFoundError(f"session not found: session_id={session_id}") + return session + + def remove_session(self, session_id: str) -> None: + if self.sessions.pop(session_id, None) is None: + raise SessionNotFoundError(f"session not found: session_id={session_id}") + + def compute_session_mismatch(self, session: LinearTrajectory) -> list[dict] | None: + """Compare accumulated token IDs against canonical chat template output. + + Read-only: does not mutate session state. + """ + if not session.token_ids: + return None + try: + tools = session.records[-1].request.get("tools") if session.records else None + expected_ids = self.tito_tokenizer.render_messages( + session.messages, + tools=tools, + add_generation_prompt=False, + tokenize=True, + ) + mismatches = self.comparator.compare_sequences(expected_ids, session.token_ids) + return [m.to_dict() for m in mismatches] + except Exception as e: + raise TokenizationError(f"failed to compute tito_session_mismatch: {e}") from e diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/session_errors.py b/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/session_errors.py new file mode 100644 index 0000000..30e6784 --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/session_errors.py @@ -0,0 +1,51 @@ +"""Error types for the session module. + +Hierarchy +--------- +SessionError (base) +├── SessionNotFoundError → 404 session does not exist +├── MessageValidationError → 400 messages structure/content invalid +├── TokenizationError → 500 TITO tokenizer / prefix mismatch +└── UpstreamResponseError → 502 SGLang response invalid or unexpected +""" + + +class SessionError(Exception): + """Base class for all session-related errors.""" + + status_code: int = 500 + + +class SessionNotFoundError(SessionError): + """Raised when the requested session ID does not exist.""" + + status_code: int = 404 + + +class MessageValidationError(SessionError): + """Raised when request messages fail structural validation. + + Examples: user message after assistant, messages not append-only, + rollback failed (no assistant checkpoint in matched prefix). + """ + + status_code: int = 400 + + +class TokenizationError(SessionError): + """Raised when TITO tokenization invariants are violated. + + Examples: pretokenized prefix mismatch between stored and new token IDs. + """ + + status_code: int = 500 + + +class UpstreamResponseError(SessionError): + """Raised when the upstream SGLang response is invalid or unexpected. + + Examples: missing meta_info, assistant content is None, + output_token_logprobs length mismatch. + """ + + status_code: int = 502 diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/session_server.py b/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/session_server.py new file mode 100644 index 0000000..b787e16 --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/session_server.py @@ -0,0 +1,111 @@ +"""Standalone Session Server that proxies through the inference router. + +This decouples session/TITO logic from the Miles Router, allowing sessions +to work with the SGLang Rust Router or any other backend. Inference +requests are proxied through the router (sglang or miles), which handles +load balancing and forwarding to worker engines. +""" + +import json +import logging + +import httpx +import setproctitle +import uvicorn +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse +from starlette.responses import Response + +from tito_gateway.vendor.miles_compat.rollout.session.sessions import setup_session_routes + +logger = logging.getLogger(__name__) + + +class SessionServer: + """Lightweight FastAPI server that manages sessions and proxies inference + requests through the inference router (sglang or miles).""" + + def __init__(self, args, backend_url: str): + self.backend_url = backend_url + self.app = FastAPI() + + timeout = getattr(args, "miles_router_timeout", 600.0) + self.client = httpx.AsyncClient( + limits=httpx.Limits(max_connections=1024), + timeout=httpx.Timeout(timeout), + ) + + # Close the httpx connection pool when uvicorn shuts down to avoid FD leaks. + self.app.router.on_shutdown.append(self.client.aclose) + + setup_session_routes(self.app, self, args) + + async def do_proxy( + self, + request: Request, + path: str, + body: bytes | None = None, + headers: dict | None = None, + ) -> dict: + url = f"{self.backend_url}/{path}" + if request.url.query: + url = f"{url}?{request.url.query}" + + if body is None: + body = await request.body() + if headers is None: + headers = dict(request.headers) + headers = { + k: v for k, v in headers.items() if k.lower() not in ("content-length", "transfer-encoding", "host") + } + + try: + response = await self.client.request(request.method, url, content=body, headers=headers) + except httpx.TransportError as exc: + logger.warning("Proxy transport error for %s %s: %s", request.method, path, exc) + error_body = json.dumps({"error": f"backend transport error: {type(exc).__name__}: {exc}"}).encode() + return { + "request_body": body, + "response_body": error_body, + "status_code": 502, + "headers": {"content-type": "application/json"}, + } + content = await response.aread() + return { + "request_body": body, + "response_body": content, + "status_code": response.status_code, + "headers": dict(response.headers), + } + + def build_proxy_response(self, result: dict) -> Response: + content = result["response_body"] + status_code = result["status_code"] + # Drop wire-level framing headers from upstream so Starlette rebuilds them + # from the body we actually send: transfer-encoding is hop-by-hop + headers = { + k: v + for k, v in result["headers"].items() + if k.lower() not in ("content-length", "transfer-encoding", "content-encoding") + } + content_type = headers.get("content-type", "") + try: + data = json.loads(content) + return JSONResponse(content=data, status_code=status_code, headers=headers) + except (json.JSONDecodeError, UnicodeDecodeError): + return Response(content=content, status_code=status_code, headers=headers, media_type=content_type) + + +def run_session_server(args, backend_url: str): + """Entry point to start the standalone session server as a subprocess.""" + # Visible to `pkill -9 miles`; without this the daemon inherits "python". + setproctitle.setproctitle("miles-session-server") + + server = SessionServer(args, backend_url) + logger.info( + "[session-server] Starting on %s:%s, proxying to %s", + args.session_server_ip, + args.session_server_port, + backend_url, + ) + uvicorn.run(server.app, host=args.session_server_ip, port=args.session_server_port, log_level="info") diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/session_types.py b/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/session_types.py new file mode 100644 index 0000000..6548902 --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/session_types.py @@ -0,0 +1,16 @@ +from pydantic import BaseModel, Field + + +class SessionRecord(BaseModel): + timestamp: float + method: str + path: str + request: dict + response: dict + status_code: int + + +class GetSessionResponse(BaseModel): + session_id: str + records: list[SessionRecord] + metadata: dict = Field(default_factory=dict) diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/sessions.py b/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/sessions.py new file mode 100644 index 0000000..17e722b --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/sessions.py @@ -0,0 +1,251 @@ +import json +import logging +import time + +from fastapi import Request +from fastapi.responses import JSONResponse +from starlette.responses import Response + +from tito_gateway.vendor.miles_compat.rollout.session.linear_trajectory import SessionRegistry +from tito_gateway.vendor.miles_compat.rollout.session.session_errors import ( + SessionError, + SessionNotFoundError, + TokenizationError, + UpstreamResponseError, +) +from tito_gateway.vendor.miles_compat.rollout.session.session_types import GetSessionResponse, SessionRecord +from tito_gateway.vendor.miles_compat.utils.chat_template_utils import get_tito_tokenizer +from tito_gateway.vendor.miles_compat.utils.processing_utils import load_tokenizer + +logger = logging.getLogger(__name__) + + +def setup_session_routes(app, backend, args): + hf_checkpoint = getattr(args, "hf_checkpoint", None) + if not hf_checkpoint: + logger.info("[session] Skipping session routes (hf_checkpoint not set).") + return + + session_server_instance_id = getattr(args, "session_server_instance_id", None) + + tokenizer = load_tokenizer( + hf_checkpoint, chat_template_path=getattr(args, "chat_template_path", None), trust_remote_code=True + ) + + tito_tokenizer = get_tito_tokenizer( + tokenizer, + tokenizer_type=getattr(args, "tito_model", "default"), + chat_template_kwargs=getattr(args, "apply_chat_template_kwargs", None), + allowed_append_roles=getattr(args, "tito_allowed_append_roles", None), + ) + + registry = SessionRegistry(args, tokenizer, tito_tokenizer=tito_tokenizer) + + @app.get("/health") + async def health(): + body = {"status": "ok"} + if session_server_instance_id is not None: + body["session_server_instance_id"] = session_server_instance_id + return body + + # --- DEBUG: track in-flight chat_completions --- + _inflight_chat = {"count": 0} + + @app.middleware("http") + async def debug_request_logger(request: Request, call_next): + client = request.client + client_info = f"{client.host}:{client.port}" if client else "unknown" + logger.info( + f"[session-server] REQUEST ARRIVED: {request.method} {request.url.path} from={client_info} inflight_chat={_inflight_chat['count']}" + ) + t0 = time.time() + response = await call_next(request) + elapsed = time.time() - t0 + logger.info( + f"[session-server] REQUEST DONE: {request.method} {request.url.path} status={response.status_code} elapsed={elapsed:.3f}s from={client_info}" + ) + return response + + @app.exception_handler(SessionError) + async def session_error_handler(request: Request, exc: SessionError): + return JSONResponse(status_code=exc.status_code, content={"error": str(exc)}) + + @app.post("/sessions") + async def create_session(): + session_id = registry.create_session() + return {"session_id": session_id} + + @app.get("/sessions/{session_id}") + async def get_session(session_id: str): + session = registry.get_session(session_id) + metadata = {} + try: + mismatch = registry.compute_session_mismatch(session) + except TokenizationError: + logger.exception("Failed to compute tito_session_mismatch for session %s", session_id) + mismatch = None + if mismatch is not None: + metadata["tito_session_mismatch"] = mismatch + metadata["accumulated_token_ids"] = session.token_ids + metadata["max_trim_tokens"] = registry.tito_tokenizer.max_trim_tokens + return GetSessionResponse( + session_id=session_id, + records=session.records, + metadata=metadata, + ) + + @app.delete("/sessions/{session_id}") + async def delete_session(session_id: str): + session = registry.get_session(session_id) + if session.closing: + raise SessionNotFoundError(f"session not found: session_id={session_id}") + session.closing = True + logger.debug( + f"[session-server] DELETE waiting for lock: session={session_id} lock_locked={session.lock.locked()}" + ) + await session.lock.acquire() + logger.debug(f"[session-server] DELETE acquired lock: session={session_id}") + try: + registry.remove_session(session_id) + finally: + session.lock.release() + return Response(status_code=204) + + @app.post("/sessions/{session_id}/v1/chat/completions") + async def chat_completions(request: Request, session_id: str): + """Proxy a chat completion through SGLang with TITO token tracking. + + Flow: prepare pretokenized input_ids (lock held briefly) → inject + SGLang flags → proxy to backend (NO lock) → validate response → + update trajectory checkpoint (lock held briefly) → append session record. + + The lock is NOT held during the slow proxy call to avoid blocking + DELETE/other operations when the agent disconnects mid-request. + """ + _inflight_chat["count"] += 1 + try: + session = registry.get_session(session_id) + if session.closing: + raise SessionNotFoundError(f"session not found: session_id={session_id}") + + # --- Phase 1: prepare request (lock held briefly) --- + async with session.lock: + # Double-check: session may have been marked closing while waiting for lock. + if session.closing: + raise SessionNotFoundError(f"session not found: session_id={session_id}") + + body = await request.body() + request_body = json.loads(body) if body else {} + + # TITO token tracking requires Miles-owned input_ids plus SGLang + # output-token metadata: + # logprobs=True → populates meta_info.output_token_logprobs + # return_meta_info → wraps the above in choice.meta_info + # Both flags are hardcoded (not set default) to prevent agent-side + # overrides from breaking the token accumulation invariants. + request_body["logprobs"] = True + request_body["return_meta_info"] = True + if getattr(args, "use_rollout_routing_replay", False): + request_body["return_routed_experts"] = True + if getattr(args, "use_rollout_indexer_replay", False): + request_body["return_indexer_topk"] = True + # Must be False so stop-token text is trimmed from assistant + # message content; token IDs are still taken from logprobs below. + request_body["no_stop_trim"] = False + + request_messages = request_body.get("messages", []) + prompt_token_ids = session.prepare_pretokenized( + request_messages, + tools=request_body.get("tools"), + tito_tokenizer=registry.tito_tokenizer, + ) + request_body["input_ids"] = prompt_token_ids + logger.debug( + "Using TITO input_ids: %d tokens", + len(prompt_token_ids), + ) + + body = json.dumps(request_body).encode() + expected_num_assistant = session.num_assistant + # --- lock released here --- + + # --- Phase 2: proxy to SGLang (NO lock held) --- + result = await backend.do_proxy(request, "v1/chat/completions", body=body) + + # If SGLang returned a non-200 error (e.g. 400 for context too long), + # pass it through to the agent without recording — the agent can retry + # or handle the error. + if result["status_code"] != 200: + return backend.build_proxy_response(result) + + response = json.loads(result["response_body"]) + + choice = response.get("choices", [{}])[0] + + meta_info = choice.get("meta_info") + if not isinstance(meta_info, dict) or "output_token_logprobs" not in meta_info: + raise UpstreamResponseError( + "meta_info and output_token_logprobs must be in choice (requires logprobs=True)" + ) + assistant_message = choice.get("message", {}) + if assistant_message.get("content") is None: + raise UpstreamResponseError( + "assistant message content is None, when tool call parser failed SGLang should still return " + "an empty content rather than None. Please check your modified SGLang version." + ) + + output_token_logprobs = meta_info["output_token_logprobs"] + completion_tokens = meta_info["completion_tokens"] + + actual_output_logprobs_len = len(output_token_logprobs) + if actual_output_logprobs_len != completion_tokens: + raise UpstreamResponseError( + "invalid chat completion response: " + f"len(output_token_logprobs)={actual_output_logprobs_len} " + f"!= completion_tokens={completion_tokens}. " + f"Please check whether you use the correct SGLang branch which has fix the tokenizer batch decode issue." + ) + + completion_token_ids = [t[1] for t in output_token_logprobs] + + # --- Phase 3: update state (lock held briefly) --- + async with session.lock: + if session.closing: + logger.warning(f"Session {session_id} closed during proxy, skipping state update") + return backend.build_proxy_response(result) + + if session.num_assistant != expected_num_assistant: + logger.warning( + f"Session {session_id} state changed during proxy " + f"(expected num_assistant={expected_num_assistant}, " + f"got {session.num_assistant}), skipping state update" + ) + return backend.build_proxy_response(result) + + session.update_pretokenized_state( + request_messages, + assistant_message, + prompt_token_ids=prompt_token_ids, + completion_token_ids=completion_token_ids, + max_trim_tokens=registry.tito_tokenizer.max_trim_tokens, + ) + + record = SessionRecord( + timestamp=time.time(), + method=request.method, + path="/v1/chat/completions", + status_code=result["status_code"], + request=request_body, + response=response, + ) + session.append_record(record) + # --- lock released here --- + + return backend.build_proxy_response(result) + finally: + _inflight_chat["count"] -= 1 + + @app.api_route("/sessions/{session_id}/{path:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"]) + async def session_proxy(request: Request, session_id: str, path: str): + result = await backend.do_proxy(request, path) + return backend.build_proxy_response(result) diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/__init__.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/__init__.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/__init__.py new file mode 100644 index 0000000..24976a4 --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/__init__.py @@ -0,0 +1,39 @@ +"""Chat template utilities for agentic-workflow token consistency.""" + +from tito_gateway.vendor.miles_compat.utils.chat_template_utils.template import ( + apply_chat_template, + apply_chat_template_from_str, + assert_messages_append_only_with_allowed_role, + extract_tool_dicts, + load_hf_chat_template, + message_matches, + normalize_tool_arguments, +) +from tito_gateway.vendor.miles_compat.utils.chat_template_utils.tito_tokenizer import ( + TEMPLATE_DIR, + TITOTokenizer, + TITOTokenizerType, + get_tito_tokenizer, + resolve_fixed_chat_template, + resolve_reasoning_and_tool_call_parser, +) +from tito_gateway.vendor.miles_compat.utils.chat_template_utils.token_seq_comparator import Mismatch, MismatchType, TokenSeqComparator + +__all__ = [ + "TITOTokenizer", + "TITOTokenizerType", + "get_tito_tokenizer", + "TEMPLATE_DIR", + "resolve_fixed_chat_template", + "resolve_reasoning_and_tool_call_parser", + "load_hf_chat_template", + "apply_chat_template", + "apply_chat_template_from_str", + "assert_messages_append_only_with_allowed_role", + "message_matches", + "extract_tool_dicts", + "normalize_tool_arguments", + "Mismatch", + "TokenSeqComparator", + "MismatchType", +] diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/deepseek_v32.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/deepseek_v32.py new file mode 100644 index 0000000..bf29b22 --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/deepseek_v32.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import copy +import functools +import json +import logging +import os +from typing import Any + +from sglang.srt.entrypoints.openai.protocol import Tool + +try: + from sglang.srt.entrypoints.openai import encoding_dsv32 +except ImportError: # pragma: no cover - depends on the installed sglang build. + encoding_dsv32 = None + +logger = logging.getLogger(__name__) + +_MODEL_TYPE = "deepseek_v32" + +_KNOWN_KWARGS = frozenset( + { + "thinking_mode", + "drop_thinking", + "add_default_bos_token", + "context", + } +) + + +@functools.cache +def _read_model_type(name_or_path: str) -> str: + """Read ``model_type`` from a checkpoint's ``config.json`` (cached per path).""" + if not name_or_path: + return "" + config_path = os.path.join(name_or_path, "config.json") + if not os.path.isfile(config_path): + return "" + try: + with open(config_path, encoding="utf-8") as f: + config = json.load(f) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + return "" + if not isinstance(config, dict): + return "" + return config.get("model_type", "") or "" + + +def is_deepseek_v32(tokenizer: Any) -> bool: + """Return True when *tokenizer* is a DeepSeek V3.2 checkpoint.""" + return _read_model_type(tokenizer.name_or_path) == _MODEL_TYPE + + +def _build_deepseek_encode_config(kwargs: dict) -> dict: + kwargs = dict(kwargs) + if (enable_thinking := kwargs.pop("enable_thinking", None)) is not None: + kwargs.setdefault("thinking_mode", "thinking" if enable_thinking else "chat") + # reject unknown kwargs to avoid silent config drop + unknown = set(kwargs) - _KNOWN_KWARGS + if unknown: + raise ValueError( + f"apply_chat_template_kwargs has unsupported kwargs {sorted(unknown)} " + f"for the DeepSeek encoder. Known keys: {sorted(_KNOWN_KWARGS)}" + ) + cfg = {"thinking_mode": "thinking", "drop_thinking": True, "add_default_bos_token": True} + for key in _KNOWN_KWARGS: + if key in kwargs: + cfg[key] = kwargs[key] + return cfg + + +def _inject_tools_into_system(messages: list[dict[str, Any]], tools: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Put *tools* in the system message, where ``encode_messages`` reads them. + + The encoder serializes each tool dict verbatim into ````, so they + must round-trip through ``Tool.model_dump()`` (fills defaults / orders fields) + or the token ids drift from what sglang serves. + """ + out = copy.deepcopy(messages) + if not out or out[0].get("role") != "system": + out.insert(0, {"role": "system", "content": ""}) + out[0]["tools"] = [Tool.model_validate(t).model_dump() for t in tools] + return out + + +def render_messages(messages: list[dict[str, Any]], *, tools: list[dict] | None = None, **kwargs: Any) -> str: + """Render *messages* into a DeepSeek V3.2 prompt via sglang ``encode_messages``. + + Tool_call ``arguments`` must already be JSON strings; *tools*, if given, are + injected into the system message (see ``_inject_tools_into_system``). + """ + encode_config = _build_deepseek_encode_config(kwargs) + if tools: + messages = _inject_tools_into_system(messages, tools) + if encoding_dsv32 is None: + raise ImportError("sglang encoding_dsv32 is required for DeepSeek V3.2 chat-template rendering") + return encoding_dsv32.encode_messages(messages, **encode_config) diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/deepseek_v4.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/deepseek_v4.py new file mode 100644 index 0000000..e8bf2a0 --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/deepseek_v4.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +import copy +import functools +import json +import logging +import os +from typing import Any + +from sglang.srt.entrypoints.openai.protocol import Tool + +try: + from sglang.srt.entrypoints.openai import encoding_dsv4 +except ImportError: # pragma: no cover - depends on the installed sglang build. + encoding_dsv4 = None + +logger = logging.getLogger(__name__) + +_MODEL_TYPE = "deepseek_v4" + +_KNOWN_KWARGS = frozenset( + { + "thinking_mode", + "drop_thinking", + "add_default_bos_token", + "context", + "reasoning_effort", + } +) + + +@functools.cache +def _read_model_type(name_or_path: str) -> str: + """Read ``model_type`` from a checkpoint's ``config.json`` (cached per path).""" + if not name_or_path: + return "" + config_path = os.path.join(name_or_path, "config.json") + if not os.path.isfile(config_path): + return "" + try: + with open(config_path, encoding="utf-8") as f: + config = json.load(f) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + return "" + if not isinstance(config, dict): + return "" + return config.get("model_type", "") or "" + + +def is_deepseek_v4(tokenizer: Any) -> bool: + """Return True when *tokenizer* is a DeepSeek V4 checkpoint.""" + return _read_model_type(tokenizer.name_or_path) == _MODEL_TYPE + + +def _build_deepseek_encode_config(kwargs: dict) -> dict: + kwargs = dict(kwargs) + if (enable_thinking := kwargs.pop("enable_thinking", None)) is not None: + kwargs.setdefault("thinking_mode", "thinking" if enable_thinking else "chat") + # reject unknown kwargs to avoid silent config drop + unknown = set(kwargs) - _KNOWN_KWARGS + if unknown: + raise ValueError( + f"apply_chat_template_kwargs has unsupported kwargs {sorted(unknown)} " + f"for the DeepSeek encoder. Known keys: {sorted(_KNOWN_KWARGS)}" + ) + # reasoning_effort has no default: like context, it is only forwarded when the + # caller supplies it, and its value is validated by encoding_dsv4 (not here). + cfg = {"thinking_mode": "thinking", "drop_thinking": True, "add_default_bos_token": True} + for key in _KNOWN_KWARGS: + if key in kwargs: + cfg[key] = kwargs[key] + return cfg + + +def _inject_tools_into_system(messages: list[dict[str, Any]], tools: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Put *tools* in the system message, where ``encode_messages`` reads them. + + The encoder serializes each tool dict verbatim into ````, so they + must round-trip through ``Tool.model_dump()`` (fills defaults / orders fields) + or the token ids drift from what sglang serves. + """ + out = copy.deepcopy(messages) + if not out or out[0].get("role") != "system": + out.insert(0, {"role": "system", "content": ""}) + out[0]["tools"] = [Tool.model_validate(t).model_dump() for t in tools] + return out + + +def render_messages(messages: list[dict[str, Any]], *, tools: list[dict] | None = None, **kwargs: Any) -> str: + """Render *messages* into a DeepSeek V4 prompt via sglang ``encode_messages``. + + Tool_call ``arguments`` must already be JSON strings; *tools*, if given, are + injected into the system message (see ``_inject_tools_into_system``). + """ + encode_config = _build_deepseek_encode_config(kwargs) + if tools: + messages = _inject_tools_into_system(messages, tools) + if encoding_dsv4 is None: + raise ImportError("sglang encoding_dsv4 is required for DeepSeek V4 chat-template rendering") + return encoding_dsv4.encode_messages(messages, **encode_config) diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/template.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/template.py new file mode 100644 index 0000000..45464c4 --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/template.py @@ -0,0 +1,256 @@ +"""Core chat template operations: load from HuggingFace and render from string. + +``load_hf_chat_template`` fetches original (unmodified) chat templates via +``hf_hub_download``. Files are cached locally after the first download — +subsequent calls read from disk without network access. + +``apply_chat_template_from_str`` renders a Jinja2 chat template string +without depending on a HuggingFace tokenizer, equivalent to +``tokenizer.apply_chat_template(..., tokenize=False)``. + +``apply_chat_template`` applies via an HF tokenizer object (returns +``str`` or ``list[int]``). Both functions normalize tool arguments, +canonicalize tool definitions, and fall back between tool dict formats. +""" + +from __future__ import annotations + +import copy +import json +from typing import Any, Literal + +from huggingface_hub import hf_hub_download +from jinja2 import TemplateError +from pydantic import TypeAdapter +from sglang.srt.entrypoints.openai.protocol import Tool +from transformers.utils.chat_template_utils import render_jinja_template + +from tito_gateway.vendor.miles_compat.utils.chat_template_utils import deepseek_v4, deepseek_v32 + + +def load_hf_chat_template(model_id: str) -> str: + """Load an original chat template from HuggingFace (cached locally). + + Handles two layouts: + - ``chat_template`` field in ``tokenizer_config.json`` (most models) + - Separate ``chat_template.jinja`` file (e.g. GLM-5) + """ + config_path = hf_hub_download(model_id, "tokenizer_config.json") + with open(config_path) as f: + config = json.load(f) + template = config.get("chat_template", "") + if template: + if isinstance(template, list): + for t in template: + if t.get("name") == "default" or not t.get("name"): + return t["template"] + return template[0]["template"] + return template + + jinja_path = hf_hub_download(model_id, "chat_template.jinja") + with open(jinja_path) as f: + return f.read() + + +def normalize_tool_arguments(messages: list[dict], format: Literal["dict", "json"]) -> list[dict]: + """Deep-copy *messages*, normalize assistant ``content: None`` -> "", and coerce + tool_call ``arguments`` to the form the downstream renderer needs (``format`` picks + the direction; never mutates the input): + - ``"dict"``: JSON string -> dict, for HF-Jinja templates (they index args as objects). + - ``"json"``: dict -> JSON string, for the DeepSeek DSML encoders (they ``json.loads`` them). + """ + normalized = copy.deepcopy(messages) + for msg in normalized: + if msg.get("role") == "assistant": + if msg.get("content") is None: + msg["content"] = "" + if isinstance(msg.get("tool_calls"), list): + for item in msg["tool_calls"]: + func = item.get("function") + if not func: + continue + args = func.get("arguments") + if format == "dict" and isinstance(args, str): + func["arguments"] = json.loads(args) + elif format == "json" and isinstance(args, dict): + func["arguments"] = json.dumps(args, ensure_ascii=False) + return normalized + + +def extract_tool_dicts(tools: list[dict] | None) -> list[dict] | None: + """Canonicalize tools via Pydantic, returning full Tool model dumps. + + Matches SGLang's ``_process_messages`` (``serving_chat.py`` lines 343-344): + ``tools = [item.model_dump() for item in request.tools]`` — each tool is + a full ``Tool`` model dump (``{"type": "function", "function": {...}}``). + """ + if not tools: + return None + + wrapped = [t if isinstance(t, dict) and "function" in t else {"type": "function", "function": t} for t in tools] + validated = TypeAdapter(list[Tool]).validate_python(wrapped) + return [tool.model_dump() for tool in validated] + + +def apply_chat_template_from_str( + chat_template: str, + messages: list[dict], + add_generation_prompt: bool = True, + tools: list[dict] | None = None, + **kwargs, +) -> str: + """Render a Jinja2 chat template string (tokenize=False, no tokenizer needed). + + Calls HF transformers' ``render_jinja_template`` directly — the same + function that ``tokenizer.apply_chat_template`` uses internally. Both + SGLang and our ``apply_chat_template`` go through that same HF code path. + + Applies SGLang-style normalizations (tool argument parsing, tool dict + canonicalization, tool format fallback). + """ + + def _render(tool_defs): + rendered, _ = render_jinja_template( + conversations=[messages], + chat_template=chat_template, + add_generation_prompt=add_generation_prompt, + tools=tool_defs, + **kwargs, + ) + return rendered[0] + + messages = normalize_tool_arguments(messages, "dict") + tool_defs = extract_tool_dicts(tools) + try: + return _render(tool_defs) + except TemplateError as e: + if tool_defs is not None: + try: + return _render([t["function"] if "function" in t else t for t in tool_defs]) + except TemplateError as te: + raise ValueError(f"Chat template rendering failed (tool format fallback): {te}") from te + raise ValueError(f"Chat template rendering failed: {e}") from e + + +_TEMPLATE_RELEVANT_KEYS = ("role", "content", "reasoning_content", "tool_calls") + + +def _normalize_value(value: Any) -> Any: + """Normalize falsy sentinels that produce identical Jinja2 output. + + None, "" and [] are all falsy in Jinja2 and render the same way, + but client libraries may interchange them (e.g. content: null vs "" + for tool-call-only responses, or tool_calls: null vs []). + + Only collapses falsy values — non-falsy content (including whitespace + like trailing newlines) is returned as-is. Message boundary characters + must be preserved exactly so they tokenize identically across turns. + """ + if value is None or value == "" or value == []: + return None + return value + + +def message_matches(stored: dict[str, Any], new: dict[str, Any]) -> bool: + """Compare only the fields that affect chat-template tokenization. + + External client libraries (e.g. litellm) may inject extra keys like + ``provider_specific_fields`` into messages. These have no effect on + the Jinja2 chat template output, so we only compare the keys that + templates actually read: role, content, reasoning_content, tool_calls. + """ + for key in _TEMPLATE_RELEVANT_KEYS: + if _normalize_value(stored.get(key)) != _normalize_value(new.get(key)): + return False + return True + + +_DEFAULT_APPEND_ROLES: list[str] = ["tool"] + + +def assert_messages_append_only_with_allowed_role( + stored_messages: list[dict[str, Any]], + new_messages: list[dict[str, Any]], + allowed_append_roles: list[str] = _DEFAULT_APPEND_ROLES, +) -> None: + """Assert *new_messages* is an append-only extension of *stored_messages*. + + The stored prefix must match exactly (compared by template-relevant keys), + and any appended messages must have a role in *allowed_append_roles* + (default: ``{'tool'}``). + """ + if not stored_messages: + return + + if len(new_messages) < len(stored_messages): + raise ValueError( + f"new messages ({len(new_messages)}) are fewer than stored messages ({len(stored_messages)})", + new_messages, + stored_messages, + ) + + for i, stored_msg in enumerate(stored_messages): + if not message_matches(stored_msg, new_messages[i]): + diffs = { + key: {"stored": repr(stored_msg.get(key))[:200], "new": repr(new_messages[i].get(key))[:200]} + for key in _TEMPLATE_RELEVANT_KEYS + if stored_msg.get(key) != new_messages[i].get(key) + } + raise ValueError( + f"message mismatch at index {i} " + f"(role: stored={stored_msg.get('role')}, new={new_messages[i].get('role')}). " + f"Diffs: {diffs}" + ) + + for j, msg in enumerate(new_messages[len(stored_messages) :]): + if msg.get("role") not in allowed_append_roles: + raise ValueError( + f"appended message at index {len(stored_messages) + j} " + f"has role={msg.get('role')!r}, allowed={allowed_append_roles}" + ) + + +def apply_chat_template( + messages: list[dict], + *, + tokenizer, + tools: list[dict] | None = None, + add_generation_prompt: bool = True, + tokenize: bool = False, + **kwargs, +) -> str | list[int]: + """Apply chat template via HF tokenizer in SGLang style. + + Passes ``return_dict=False`` to match SGLang's ``serving_chat.py``, + ensuring the result is ``str`` (tokenize=False) or ``list[int]`` + (tokenize=True), not a ``BatchEncoding`` or ``dict``. + """ + if deepseek_v32.is_deepseek_v32(tokenizer): + rendered = deepseek_v32.render_messages(normalize_tool_arguments(messages, "json"), tools=tools, **kwargs) + return tokenizer.encode(rendered, add_special_tokens=False) if tokenize else rendered + + if deepseek_v4.is_deepseek_v4(tokenizer): + rendered = deepseek_v4.render_messages(normalize_tool_arguments(messages, "json"), tools=tools, **kwargs) + return tokenizer.encode(rendered, add_special_tokens=False) if tokenize else rendered + + messages = normalize_tool_arguments(messages, "dict") + tool_defs = extract_tool_dicts(tools) + render_kwargs = dict(add_generation_prompt=add_generation_prompt, **kwargs) + + try: + return tokenizer.apply_chat_template( + messages, tokenize=tokenize, tools=tool_defs, return_dict=False, **render_kwargs + ) + except TemplateError as e: + if tool_defs is not None: + try: + return tokenizer.apply_chat_template( + messages, + tokenize=tokenize, + tools=[t["function"] if "function" in t else t for t in tool_defs], + return_dict=False, + **render_kwargs, + ) + except TemplateError as te: + raise ValueError(f"Chat template rendering failed (tool format fallback): {te}") from te + raise ValueError(f"Chat template rendering failed: {e}") from e diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/kimi_k25_fixed.jinja b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/kimi_k25_fixed.jinja new file mode 100644 index 0000000..0f05753 --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/kimi_k25_fixed.jinja @@ -0,0 +1,111 @@ +{%- macro render_content(msg) -%} + {%- set c = msg.get('content') -%} + {%- if c is string -%} + {{ c }} + {%- elif c is not none -%} + {% for content in c -%} + {% if content['type'] == 'image' or content['type'] == 'image_url' -%} + <|media_begin|>image<|media_content|><|media_pad|><|media_end|> + {% elif content['type'] == 'video' or content['type']== 'video_url'-%} + <|kimi_k25_video_placeholder|> + {% else -%} + {{ content['text'] }} + {%- endif -%} + {%- endfor -%} + {%- endif -%} +{%- endmacro -%} + +{% macro set_roles(message) -%} + {%- set role_name = message.get('name') or message['role'] -%} + {%- if message['role'] == 'user' -%} + <|im_user|>{{role_name}}<|im_middle|> + {%- elif message['role'] == 'assistant' -%} + <|im_assistant|>{{role_name}}<|im_middle|> + {%- else -%} + <|im_system|>{{role_name}}<|im_middle|> + {%- endif -%} +{%- endmacro -%} + + +{%- macro render_toolcalls(message) -%} + <|tool_calls_section_begin|> + {%- for tool_call in message['tool_calls'] -%} + {%- set formatted_id = tool_call['id'] -%} + <|tool_call_begin|>{{ formatted_id }}<|tool_call_argument_begin|>{% if tool_call['function']['arguments'] is string %}{{ tool_call['function']['arguments'] }}{% else %}{{ tool_call['function']['arguments'] | tojson }}{% endif %}<|tool_call_end|> + {%- endfor -%} + <|tool_calls_section_end|> +{%- endmacro -%} + + +{%- set preserve_thinking = preserve_thinking | default(false) -%} +{# Find last non-tool-call assistant message. If preserve_thinking, keep -1 so hist is empty and all msgs use suffix (retain reasoning). #} +{%- set ns = namespace(last_non_tool_call_assistant_msg=-1) -%} +{%- if not preserve_thinking -%} +{%- for idx in range(messages|length-1, -1, -1) -%} + {%- if messages[idx]['role'] == 'assistant' and not messages[idx].get('tool_calls') -%} + {%- set ns.last_non_tool_call_assistant_msg = idx -%} + {%- break -%} + {%- endif -%} +{%- endfor -%} +{%- endif -%} + +{# split all messages into history & suffix, reasoning_content in suffix should be reserved.#} +{%- set hist_msgs = messages[:ns.last_non_tool_call_assistant_msg+1] -%} +{%- set suffix_msgs = messages[ns.last_non_tool_call_assistant_msg+1:] -%} + +{%- if tools -%} + {%- if tools_ts_str -%} + <|im_system|>tool_declare<|im_middle|>{{ tools_ts_str }}<|im_end|> + {%- else -%} + <|im_system|>tool_declare<|im_middle|>{{ tools | tojson(separators=(',', ':')) }}<|im_end|> + {%- endif -%} +{%- endif -%} + +{%- for message in hist_msgs -%} + {{set_roles(message)}} + {%- if message['role'] == 'assistant' -%} + {{render_content(message)}} + {%- if message.get('tool_calls') -%} + {{render_toolcalls(message)}} + {%- endif -%} + {%- elif message['role'] == 'tool' -%} + {%- set tool_call_id = message.tool_call_id -%} + ## Return of {{ tool_call_id }} +{{render_content(message)}} + {%- elif message['content'] is not none -%} + {{render_content(message)}} + {%- endif -%} + <|im_end|> +{%- endfor -%} + +{%- for message in suffix_msgs -%} + {{set_roles(message)}} + {%- if message['role'] == 'assistant' -%} + {%- if thinking is defined and thinking is false -%} + {{render_content(message)}} + {%- else -%} + {%- set rc = message.get('reasoning_content', '') -%} + {{rc}}{{render_content(message)}} + {%- endif -%} + {%- if message.get('tool_calls') -%} + {{render_toolcalls(message)}} + {%- endif -%} + {%- elif message['role'] == 'tool' -%} + {%- set tool_call_id = message.tool_call_id -%} + ## Return of {{ tool_call_id }} +{{render_content(message)}} + {%- elif message['content'] is not none -%} + {{render_content(message)}} + {%- endif -%} + <|im_end|> +{%- endfor -%} + + +{%- if add_generation_prompt -%} + <|im_assistant|>assistant<|im_middle|> + {%- if thinking is defined and thinking is false -%} + + {%- else -%} + + {%- endif -%} +{%- endif -%} diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/minimax_m25_fixed.jinja b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/minimax_m25_fixed.jinja new file mode 100644 index 0000000..c121f2d --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/minimax_m25_fixed.jinja @@ -0,0 +1,159 @@ +{# ----------‑‑‑ special token variables ‑‑‑---------- #} +{%- set toolcall_begin_token = '' -%} +{%- set toolcall_end_token = '' -%} +{#- Tool Rendering Functions ============================================== -#} +{%- macro render_tool_namespace(namespace_name, tool_list) -%} +{%- for tool in tool_list -%} +{{ tool.function | tojson(ensure_ascii=False) }} +{% endfor -%} +{%- endmacro -%} +{%- macro visible_text(content) -%} + {%- if content is string -%} + {{ content }} + {%- elif content is iterable and content is not mapping -%} + {%- for item in content -%} + {%- if item is mapping and item.type == 'text' -%} + {{- item.text }} + {%- elif item is string -%} + {{- item }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{- content }} + {%- endif -%} +{%- endmacro -%} +{#- System Message Construction ============================================ -#} +{%- macro build_system_message(system_message) -%} + {%- if system_message and system_message.content -%} + {{- visible_text(system_message.content) }} + {%- else -%} + {%- if model_identity is not defined -%} + {%- set model_identity = "You are a helpful assistant. Your name is MiniMax-M2.5 and is built by MiniMax." -%} + {%- endif -%} + {{- model_identity }} + {%- endif -%} + + {#- Handle current_date -#} + {%- if system_message and system_message.current_date -%} + {{- '\n' ~ 'Current date: ' + system_message.current_date }} + {%- endif -%} + {#- Handle current_location -#} + {%- if system_message and system_message.current_location -%} + {{- '\n' ~ 'Current location: ' + system_message.current_location }} + {%- endif -%} +{%- endmacro -%} +{#- Main Template Logic ================================================= -#} +{#- Extract system message (only first message if it's system) -#} +{%- set system_message = none -%} +{%- set conversation_messages = messages -%} +{%- if messages and messages[0].role == "system" -%} + {%- set system_message = messages[0] -%} + {%- set conversation_messages = messages[1:] -%} +{%- endif -%} +{#- Get the last user message turn, for interleved thinking -#} +{%- set ns = namespace(last_user_index=-1) %} +{% for m in conversation_messages %} + {%- if m.role == 'user' %} + {% set ns.last_user_index = loop.index0 -%} + {%- endif %} +{%- endfor %} +{#- Render system message -#} +{{- ']~!b[' ~ ']~b]system' ~ '\n' }} +{{- build_system_message(system_message) }} +{#- Render tools if available -#} +{%- if tools -%} + {{- '\n\n' ~ '# Tools' ~ '\n' ~ 'You may call one or more tools to assist with the user query.\nHere are the tools available in JSONSchema format:' ~ '\n' }} + {{- '\n' ~ '' ~ '\n' }} + {{- render_tool_namespace("functions", tools) }} + {{- '' ~ '\n\n' }} +{{- 'When making tool calls, use XML format to invoke tools and pass parameters:' ~ '\n' }} +{{- '\n' ~ toolcall_begin_token }} + +param-value-1 +param-value-2 +... + +{{- '\n' ~ toolcall_end_token }} +{%- endif -%} +{{- '[e~[\n' }} + +{#- Render messages -#} +{%- set last_tool_call = namespace(name=none) -%} +{%- for message in conversation_messages -%} + {%- if message.role == 'assistant' -%} + {#- Only render reasoning_content if no user message follows (or clear_thinking disabled) -#} + {{- ']~b]ai' ~ '\n' }} + + {%- set reasoning_content = '' %} + {%- set content = visible_text(message.content) %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].strip('\n').split('')[-1].strip('\n') %} + {%- set content = content.split('')[-1].strip('\n') %} + {%- endif %} + {%- endif %} + {%- if reasoning_content and (not (clear_thinking | default(true)) or loop.index0 > ns.last_user_index) -%} + {{- '' ~ '\n' ~ reasoning_content ~ '\n' ~ '' ~ '\n\n' }} + {%- endif -%} + {%- if content -%} + {{- content }} + {%- endif -%} + {%- if message.tool_calls -%} + {{- '\n' ~ toolcall_begin_token ~ '\n' }} + + {%- for tool_call in message.tool_calls -%} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '' }} + {% set _args = tool_call.arguments %} + {%- for k, v in _args.items() %} + {{- '' }} + {{- v | tojson(ensure_ascii=False) if v is not string else v }} + {{- '' }} + {% endfor %} + {{- '' ~ '\n' }} + {%- endfor -%} + + {{- toolcall_end_token}} + {%- set last_tool_call.name = message.tool_calls[-1].name -%} + {%- else -%} + {%- set last_tool_call.name = none -%} + {%- endif -%} + {{- '[e~[' ~ '\n' }} + + {%- elif message.role == 'tool' -%} + {%- if last_tool_call.name is none -%} + {{- raise_exception("Message has tool role, but there was no previous assistant message with a tool call!") }} + {%- endif -%} + {%- if loop.first or (conversation_messages[loop.index0 - 1].role != 'tool') -%} + {{- ']~b]tool' }} + {%- endif -%} + {%- if message.content is string -%} + {{- '\n' }} + {{- message.content }} + {{- '' }} + {%- else -%} + {%- for tr in message.content -%} + {{- '\n' }} + {{- tr.output if tr.output is defined else (tr.text if tr.type == 'text' and tr.text is defined else tr) }} + {{- '\n' }} + {%- endfor -%} + {%- endif -%} + {%- if loop.last or (conversation_messages[loop.index0 + 1].role != 'tool') -%} + {{- '[e~[\n' -}} + {%- endif -%} + + {%- elif message.role == 'user' -%} + {{- ']~b]user' ~ '\n' }} + {{- visible_text(message.content) }} + {{- '[e~[' ~ '\n' }} + {%- endif -%} +{%- endfor -%} + +{#- Generation prompt -#} +{%- if add_generation_prompt -%} +{{- ']~b]ai' ~ '\n' ~ '' ~ '\n' }} +{%- endif -%} diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/minimax_m27_fixed.jinja b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/minimax_m27_fixed.jinja new file mode 100644 index 0000000..b49b815 --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/minimax_m27_fixed.jinja @@ -0,0 +1,159 @@ +{# ----------‑‑‑ special token variables ‑‑‑---------- #} +{%- set toolcall_begin_token = '' -%} +{%- set toolcall_end_token = '' -%} +{#- Tool Rendering Functions ============================================== -#} +{%- macro render_tool_namespace(namespace_name, tool_list) -%} +{%- for tool in tool_list -%} +{{ tool.function | tojson(ensure_ascii=False) }} +{% endfor -%} +{%- endmacro -%} +{%- macro visible_text(content) -%} + {%- if content is string -%} + {{ content }} + {%- elif content is iterable and content is not mapping -%} + {%- for item in content -%} + {%- if item is mapping and item.type == 'text' -%} + {{- item.text }} + {%- elif item is string -%} + {{- item }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{- content }} + {%- endif -%} +{%- endmacro -%} +{#- System Message Construction ============================================ -#} +{%- macro build_system_message(system_message) -%} + {%- if system_message and system_message.content -%} + {{- visible_text(system_message.content) }} + {%- else -%} + {%- if model_identity is not defined -%} + {%- set model_identity = "You are a helpful assistant. Your name is MiniMax-M2.7 and is built by MiniMax." -%} + {%- endif -%} + {{- model_identity }} + {%- endif -%} + + {#- Handle current_date -#} + {%- if system_message and system_message.current_date -%} + {{- '\n' ~ 'Current date: ' + system_message.current_date }} + {%- endif -%} + {#- Handle current_location -#} + {%- if system_message and system_message.current_location -%} + {{- '\n' ~ 'Current location: ' + system_message.current_location }} + {%- endif -%} +{%- endmacro -%} +{#- Main Template Logic ================================================= -#} +{#- Extract system message (only first message if it's system) -#} +{%- set system_message = none -%} +{%- set conversation_messages = messages -%} +{%- if messages and messages[0].role == "system" -%} + {%- set system_message = messages[0] -%} + {%- set conversation_messages = messages[1:] -%} +{%- endif -%} +{#- Get the last user message turn, for interleved thinking -#} +{%- set ns = namespace(last_user_index=-1) %} +{% for m in conversation_messages %} + {%- if m.role == 'user' %} + {% set ns.last_user_index = loop.index0 -%} + {%- endif %} +{%- endfor %} +{#- Render system message -#} +{{- ']~!b[' ~ ']~b]system' ~ '\n' }} +{{- build_system_message(system_message) }} +{#- Render tools if available -#} +{%- if tools -%} + {{- '\n\n' ~ '# Tools' ~ '\n' ~ 'You may call one or more tools to assist with the user query.\nHere are the tools available in JSONSchema format:' ~ '\n' }} + {{- '\n' ~ '' ~ '\n' }} + {{- render_tool_namespace("functions", tools) }} + {{- '' ~ '\n\n' }} +{{- 'When making tool calls, use XML format to invoke tools and pass parameters:' ~ '\n' }} +{{- '\n' ~ toolcall_begin_token }} + +param-value-1 +param-value-2 +... + +{{- '\n' ~ toolcall_end_token }} +{%- endif -%} +{{- '[e~[\n' }} + +{#- Render messages -#} +{%- set last_tool_call = namespace(name=none) -%} +{%- for message in conversation_messages -%} + {%- if message.role == 'assistant' -%} + {#- Only render reasoning_content if no user message follows (or clear_thinking disabled) -#} + {{- ']~b]ai' ~ '\n' }} + + {%- set reasoning_content = '' %} + {%- set content = visible_text(message.content) %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].strip('\n').split('')[-1].strip('\n') %} + {%- set content = content.split('')[-1].strip('\n') %} + {%- endif %} + {%- endif %} + {%- if reasoning_content and (not (clear_thinking | default(true)) or loop.index0 > ns.last_user_index) -%} + {{- '' ~ '\n' ~ reasoning_content ~ '\n' ~ '' ~ '\n\n' }} + {%- endif -%} + {%- if content -%} + {{- content }} + {%- endif -%} + {%- if message.tool_calls -%} + {{- '\n' ~ toolcall_begin_token ~ '\n' }} + + {%- for tool_call in message.tool_calls -%} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '' }} + {% set _args = tool_call.arguments %} + {%- for k, v in _args.items() %} + {{- '' }} + {{- v | tojson(ensure_ascii=False) if v is not string else v }} + {{- '' }} + {% endfor %} + {{- '' ~ '\n' }} + {%- endfor -%} + + {{- toolcall_end_token}} + {%- set last_tool_call.name = message.tool_calls[-1].name -%} + {%- else -%} + {%- set last_tool_call.name = none -%} + {%- endif -%} + {{- '[e~[' ~ '\n' }} + + {%- elif message.role == 'tool' -%} + {%- if last_tool_call.name is none -%} + {{- raise_exception("Message has tool role, but there was no previous assistant message with a tool call!") }} + {%- endif -%} + {%- if loop.first or (conversation_messages[loop.index0 - 1].role != 'tool') -%} + {{- ']~b]tool' }} + {%- endif -%} + {%- if message.content is string -%} + {{- '\n' }} + {{- message.content }} + {{- '' }} + {%- else -%} + {%- for tr in message.content -%} + {{- '\n' }} + {{- tr.output if tr.output is defined else (tr.text if tr.type == 'text' and tr.text is defined else tr) }} + {{- '\n' }} + {%- endfor -%} + {%- endif -%} + {%- if loop.last or (conversation_messages[loop.index0 + 1].role != 'tool') -%} + {{- '[e~[\n' -}} + {%- endif -%} + + {%- elif message.role == 'user' -%} + {{- ']~b]user' ~ '\n' }} + {{- visible_text(message.content) }} + {{- '[e~[' ~ '\n' }} + {%- endif -%} +{%- endfor -%} + +{#- Generation prompt -#} +{%- if add_generation_prompt -%} +{{- ']~b]ai' ~ '\n' ~ '' ~ '\n' }} +{%- endif -%} diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/qwen3.5_fixed.jinja b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/qwen3.5_fixed.jinja new file mode 100644 index 0000000..b003bcb --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/qwen3.5_fixed.jinja @@ -0,0 +1,151 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if (not (clear_thinking | default(true))) or loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/qwen3_fixed.jinja b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/qwen3_fixed.jinja new file mode 100644 index 0000000..88ca535 --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/qwen3_fixed.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if (not (clear_thinking | default(true))) or loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/qwen3_thinking_2507_and_next_fixed.jinja b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/qwen3_thinking_2507_and_next_fixed.jinja new file mode 100644 index 0000000..1588dfb --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/qwen3_thinking_2507_and_next_fixed.jinja @@ -0,0 +1,82 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if (not (clear_thinking | default(true))) or loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n\n' }} +{%- endif %} diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/tito_tokenizer.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/tito_tokenizer.py new file mode 100644 index 0000000..df8df5a --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/tito_tokenizer.py @@ -0,0 +1,1014 @@ +"""TITO tokenizer — incremental tokenization for pretokenized prefix reuse. + +``TITOTokenizer`` computes incremental token IDs for non-assistant messages +(tool responses, user follow-ups, system injections) that follow the +assistant's generated token sequence, then merges them with the pretokenized +prefix — handling model-specific boundary tokens at the junction. + +The default implementation incrementally tokenizes appended non-assistant turns +with role-specific synthetic prefixes: + +- contiguous ``tool`` runs use ``[dummy_system, dummy_assistant]`` +- each ``user`` or ``system`` message uses ``[dummy_system]`` + +The appended suffix is processed left-to-right, then the generation prompt for +the next assistant turn is appended once at the end. Model-specific +subclasses only override ``merge_tokens`` for boundary quirks at the prefix +junction. +""" + +from __future__ import annotations + +import logging +from collections.abc import Iterable +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path +from typing import Any + +from tito_gateway.vendor.miles_compat.utils.chat_template_utils.template import apply_chat_template, assert_messages_append_only_with_allowed_role +from tito_gateway.vendor.miles_compat.utils.chat_template_utils.token_seq_comparator import TokenSeqComparator + +logger = logging.getLogger(__name__) + +# Bundled fixed-template files live under this directory; ``FixedTemplateRow.template`` +# values are filenames relative to it. +TEMPLATE_DIR = Path(__file__).parent / "templates" + +# Roles the TITO merge logic understands; passing anything else is a typo. +_VALID_ROLES = frozenset({"tool", "user", "system"}) + +_DUMMY_SYSTEM: dict[str, Any] = {"role": "system", "content": "dummy system"} + + +@dataclass(frozen=True) +class FixedTemplateRow: + """A ``(roles, template, extra_kwargs)`` row owned by a TITO tokenizer family. + + Each row says: when the session is configured for ``allowed_roles``, this + family expects the given chat template plus the given extra kwargs. + ``template`` is a path relative to ``TEMPLATE_DIR`` for a bundled fixed + template, or ``None`` to keep the HF-native template (kwargs-only fix). + """ + + allowed_roles: frozenset[str] + template: str | None = None + extra_kwargs: dict[str, Any] = field(default_factory=dict) + + +def _build_dummy_assistant(tool_responses: list[dict[str, Any]]) -> dict[str, Any]: + """Build a dummy assistant message with tool_calls matching *tool_responses*, + so the template correctly renders the subsequent tool-response turn boundaries.""" + return { + "role": "assistant", + "content": "", + "reasoning_content": " ", + "tool_calls": [ + { + "id": resp.get("tool_call_id") or f"call0000{i}", + "type": "function", + "function": { + "name": resp.get("name") or "dummy_func", + "arguments": {}, + }, + } + for i, resp in enumerate(tool_responses) + ], + } + + +# --------------------------------------------------------------------------- +# Base / default tokenizer +# --------------------------------------------------------------------------- +# TODO: split different model's TITO tokenizer into different files + + +class TITOTokenizer: + """Incremental tokenization and prefix merging for appended non-assistant turns.""" + + max_trim_tokens: int = 0 + trailing_token_ids: frozenset[int] = frozenset() + + # ``(roles, template, extra_kwargs)`` rows this family supports. Resolved + # by ``resolve_fixed_chat_template`` via smallest-superset match against + # the caller's ``allowed_append_roles``. + SUPPORTED_TEMPLATES: tuple[FixedTemplateRow, ...] = () + + # sglang ``--reasoning-parser`` and ``--tool-call-parser`` values bound to + # this family. + reasoning_parser: str | None = None + tool_call_parser: str | None = None + + def __init__( + self, + tokenizer: Any, + chat_template_kwargs: dict[str, Any] | None = None, + assistant_start_str: str | None = None, + special_token_ids: set[int] | None = None, + allowed_append_roles: list[str] | None = None, + ): + self.tokenizer = tokenizer + self.chat_template_kwargs = chat_template_kwargs or {} + self._assistant_start_str = assistant_start_str + self.allowed_append_roles: list[str] = allowed_append_roles if allowed_append_roles is not None else ["tool"] + self.special_token_ids: set[int] = special_token_ids + + def create_comparator(self) -> TokenSeqComparator: + """Create a :class:`TokenSeqComparator` configured with this + tokenizer's model-specific settings.""" + return TokenSeqComparator( + self.tokenizer, + assistant_start_str=self._assistant_start_str, + special_token_ids=self.special_token_ids, + trim_trailing_ids=self.trailing_token_ids or None, + ) + + def render_messages( + self, + messages: list[dict[str, Any]], + *, + add_generation_prompt: bool, + tools: list[dict[str, Any]] | None = None, + tokenize: bool = False, + ) -> str | list[int]: + return apply_chat_template( + messages, + tokenizer=self.tokenizer, + tokenize=tokenize, + add_generation_prompt=add_generation_prompt, + tools=tools, + **self.chat_template_kwargs, + ) + + def _encode_text(self, text: str) -> list[int]: + return self.tokenizer.encode(text, add_special_tokens=False) + + def _split_appended_segments(self, appended_messages: list[dict[str, Any]]) -> list[list[dict[str, Any]]]: + segments: list[list[dict[str, Any]]] = [] + i = 0 + while i < len(appended_messages): + role = appended_messages[i]["role"] + # Many templates wrap a contiguous tool-response run as one logical + # block, so tool messages are diffed together instead of one-by-one. + if role == "tool": + j = i + 1 + while j < len(appended_messages) and appended_messages[j]["role"] == "tool": + j += 1 + segments.append(appended_messages[i:j]) + i = j + continue + if role in {"user", "system"}: + segments.append([appended_messages[i]]) + i += 1 + continue + raise ValueError(f"unsupported appended role for TITO segmentation: {role}") + + return segments + + def _tokenize_rendered_suffix( + self, + base_messages: list[dict[str, Any]], + appended_messages: list[dict[str, Any]], + *, + tools: list[dict[str, Any]] | None = None, + add_generation_prompt: bool = False, + ) -> list[int]: + """Render *base_messages* and *base_messages + appended_messages*, return + tokens for the suffix. + + When *add_generation_prompt* is True and *appended_messages* is empty, + this computes the generation-prompt suffix (the assistant opener tokens). + """ + text_without = self.render_messages(base_messages, add_generation_prompt=False, tools=tools) + text_with = self.render_messages( + base_messages + appended_messages, + add_generation_prompt=add_generation_prompt, + tools=tools, + ) + if not text_with.startswith(text_without): + roles = [msg["role"] for msg in appended_messages] if appended_messages else ["generation_prompt"] + raise ValueError(f"rendered suffix diff failed for {roles}") + return self._encode_text(text_with[len(text_without) :]) + + def _tokenize_tool_segment( + self, + appended_messages: list[dict[str, Any]], + tools: list[dict[str, Any]] | None = None, + ) -> list[int]: + # No dummy user to avoid cut think issues. + return self._tokenize_rendered_suffix( + [_DUMMY_SYSTEM, _build_dummy_assistant(appended_messages)], + appended_messages, + tools=tools, + ) + + def _tokenize_user_and_system_segment( + self, + appended_message: dict[str, Any], + tools: list[dict[str, Any]] | None = None, + ) -> list[int]: + # User/system single-message appends share one synthetic context. + return self._tokenize_rendered_suffix( + [_DUMMY_SYSTEM], + [appended_message], + tools=tools, + ) + + def tokenize_additional_non_assistant( + self, + old_messages: list[dict[str, Any]], + new_messages: list[dict[str, Any]], + tools: list[dict[str, Any]] | None = None, + ) -> list[int]: + """Compute incremental token IDs for non-assistant messages appended + after the pretokenized prefix. + + Handles tool responses, user, and system messages — + never an assistant message. Validates that *new_messages* is an + append-only extension of *old_messages* via + ``assert_messages_append_only_with_allowed_role``. + + Args: + old_messages: Previously stored messages (prefix). + new_messages: Full new message list (must be a superset of + *old_messages* with only allowed-role messages appended). + tools: Tool definitions in OpenAI format (may vary per call). + + Returns: + Incremental token IDs (including the generation prompt) that, + when merged with pretokenized prefix via ``merge_tokens``, + form the full prompt token IDs. + """ + assert_messages_append_only_with_allowed_role(old_messages, new_messages, self.allowed_append_roles) + appended_messages = new_messages[len(old_messages) :] + incremental: list[int] = [] + + # Incremental non-assistant content is assembled segment-by-segment + # using the smallest synthetic context that preserves each role's + # boundary tokens. + for segment in self._split_appended_segments(appended_messages): + role = segment[0]["role"] + if role == "tool": + incremental.extend(self._tokenize_tool_segment(segment, tools)) + elif role == "user" or role == "system": + incremental.extend(self._tokenize_user_and_system_segment(segment[0], tools)) + else: + raise ValueError(f"unsupported appended role for TITO tokenization: {role}") + + # The next assistant opener depends on the full post-append history, so + # it is derived from the real messages once and appended only at the end. + return incremental + self._tokenize_rendered_suffix( + new_messages, + [], + tools=tools, + add_generation_prompt=True, + ) + + def merge_tokens( + self, + old_messages: list[dict[str, Any]], + new_messages: list[dict[str, Any]], + pretokenized_token_ids: list[int], + tools: list[dict[str, Any]] | None = None, + ) -> list[int]: + """Merge *pretokenized_token_ids* with incremental tokens to produce + the complete prompt token IDs (including generation prompt). + + The default implementation is simple concatenation. Subclasses + override this to handle model-specific boundary token logic. + """ + incremental = self.tokenize_additional_non_assistant(old_messages, new_messages, tools) + return list(pretokenized_token_ids) + incremental + + +# --------------------------------------------------------------------------- +# Qwen3 implementation +# --------------------------------------------------------------------------- + + +class Qwen3TITOTokenizer(TITOTokenizer): + """Qwen3 variant: handles missing newline at the boundary. + + The Qwen3 chat template emits ``<|im_end|>\\n`` after every message, but + the model stops at ``<|im_end|>`` without generating the trailing ``\\n``. + ``merge_tokens`` inserts the missing newline so that the pretokenized + prefix matches the canonical template output. + """ + + reasoning_parser = "qwen3" + tool_call_parser = "qwen25" + + SUPPORTED_TEMPLATES = ( + FixedTemplateRow( + allowed_roles=frozenset({"tool"}), + template="qwen3_fixed.jinja", + ), + FixedTemplateRow( + allowed_roles=frozenset({"tool", "user"}), + template="qwen3_fixed.jinja", + extra_kwargs={"clear_thinking": False}, + ), + ) + + _default_assistant_start_str: str = "<|im_start|>assistant" + + def __init__( + self, + tokenizer: Any, + chat_template_kwargs: dict[str, Any] | None = None, + assistant_start_str: str | None = None, + allowed_append_roles: list[str] | None = None, + ): + super().__init__( + tokenizer, + chat_template_kwargs, + assistant_start_str or self._default_assistant_start_str, + allowed_append_roles=allowed_append_roles, + ) + nl_ids = tokenizer.encode("\n", add_special_tokens=False) + assert len(nl_ids) == 1, f"Expected single newline token, got {nl_ids}" + self._newline_id: int = nl_ids[0] + self._im_end_id: int = tokenizer.convert_tokens_to_ids("<|im_end|>") + self.trailing_token_ids = frozenset({self._newline_id}) + + def merge_tokens( + self, + old_messages: list[dict[str, Any]], + new_messages: list[dict[str, Any]], + pretokenized_token_ids: list[int], + tools: list[dict[str, Any]] | None = None, + ) -> list[int]: + incremental = self.tokenize_additional_non_assistant(old_messages, new_messages, tools) + prefix = list(pretokenized_token_ids) + if prefix and prefix[-1] == self._im_end_id: + prefix.append(self._newline_id) + return prefix + incremental + + +# Qwen3.5 and Qwen3-Next-Thinking share the ``<|im_end|>`` boundary handling +# with Qwen3, so they reuse Qwen3TITOTokenizer's token-level logic via plain +# inheritance. They are still split into named subclasses because each owns +# its own ``SUPPORTED_TEMPLATES`` row pointing to a distinct fixed jinja, even +# though their boundary behavior is identical. + + +class Qwen35TITOTokenizer(Qwen3TITOTokenizer): + """Qwen3.5 — same boundary behavior as Qwen3, distinct fixed template.""" + + tool_call_parser = "qwen3_coder" + + SUPPORTED_TEMPLATES = ( + FixedTemplateRow( + allowed_roles=frozenset({"tool"}), + template="qwen3.5_fixed.jinja", + ), + FixedTemplateRow( + allowed_roles=frozenset({"tool", "user"}), + template="qwen3.5_fixed.jinja", + extra_kwargs={"clear_thinking": False}, + ), + ) + + +class QwenNextTITOTokenizer(Qwen3TITOTokenizer): + """Qwen3-Thinking-2507 / Qwen3-Next-Thinking — same boundary behavior as + Qwen3, distinct (shared) fixed template.""" + + SUPPORTED_TEMPLATES = ( + FixedTemplateRow( + allowed_roles=frozenset({"tool"}), + template="qwen3_thinking_2507_and_next_fixed.jinja", + ), + FixedTemplateRow( + allowed_roles=frozenset({"tool", "user"}), + template="qwen3_thinking_2507_and_next_fixed.jinja", + extra_kwargs={"clear_thinking": False}, + ), + ) + + +# --------------------------------------------------------------------------- +# GLM 4.7 implementation +# --------------------------------------------------------------------------- + + +class GLM47TITOTokenizer(TITOTokenizer): + """GLM 4.7 variant: handles ambiguous boundary tokens in ``merge_tokens``. + + ``<|user|>`` and ``<|observation|>`` are both assistant stop tokens *and* + next-message start tokens in the chat template. In ``merge_tokens``, + the last token of the pretokenized prefix is always stripped when it is + one of these boundary tokens — whether it matches the first incremental + token (overlap) or differs (e.g. model stopped with ``<|observation|>`` but + next turn is ``<|user|>`` because the tool call failed and a system message + is injected instead). + """ + + reasoning_parser = "glm45" + tool_call_parser = "glm47" + + # GLM's HF-native chat template already exposes a ``clear_thinking`` kwarg, + # so no fixed-jinja patch is needed for either append surface. + SUPPORTED_TEMPLATES = ( + FixedTemplateRow( + allowed_roles=frozenset({"tool"}), + template=None, + ), + FixedTemplateRow( + allowed_roles=frozenset({"tool", "user"}), + template=None, + extra_kwargs={"clear_thinking": False}, + ), + FixedTemplateRow( + allowed_roles=frozenset({"tool", "user", "system"}), + template=None, + extra_kwargs={"clear_thinking": False}, + ), + ) + + max_trim_tokens: int = 1 + _default_assistant_start_str: str = "<|assistant|>" + + def __init__( + self, + tokenizer: Any, + chat_template_kwargs: dict[str, Any] | None = None, + assistant_start_str: str | None = None, + allowed_append_roles: list[str] | None = None, + ): + super().__init__( + tokenizer, + chat_template_kwargs, + assistant_start_str or self._default_assistant_start_str, + allowed_append_roles=allowed_append_roles, + ) + self._observation_id: int = tokenizer.convert_tokens_to_ids("<|observation|>") + self._user_id: int = tokenizer.convert_tokens_to_ids("<|user|>") + self._ambiguous_boundary_ids: set[int] = {self._observation_id, self._user_id} + self.trailing_token_ids = frozenset(self._ambiguous_boundary_ids) + + def merge_tokens( + self, + old_messages: list[dict[str, Any]], + new_messages: list[dict[str, Any]], + pretokenized_token_ids: list[int], + tools: list[dict[str, Any]] | None = None, + ) -> list[int]: + incremental = self.tokenize_additional_non_assistant(old_messages, new_messages, tools) + prefix = list(pretokenized_token_ids) + if prefix and prefix[-1] in self._ambiguous_boundary_ids: + prefix = prefix[:-1] + return prefix + incremental + + +# --------------------------------------------------------------------------- +# Nemotron 3 implementation +# --------------------------------------------------------------------------- + + +class Nemotron3TITOTokenizer(Qwen3TITOTokenizer): + """NVIDIA Nemotron 3 family: ``<|im_end|>\\n`` message boundaries. + + Inherits Qwen3's boundary handling — Nemotron 3 emits the same + ``<|im_end|>\\n`` after every message and the model stops at + ``<|im_end|>`` without the trailing newline. + + No fixed jinja is shipped — HF native template is append-only when + ``truncate_history_thinking=False``. Multi-user-turn surfaces + auto-merge that kwarg via ``extra_kwargs`` below; ``{tool}``-only does + not need it (no user-turn boundary to truncate across). + + The plain-text assistant turn does not roundtrip cleanly under + sglang's upstream ``nemotron_3`` reasoning parser (it keeps a trailing + ``\\n`` in ``reasoning_content``), so step-4 ``assistant_text`` soft + assertion is expected to fail until the parser is patched upstream — + out of scope for this family registration. + """ + + reasoning_parser = "nemotron_3" + tool_call_parser = "qwen3_coder" + + SUPPORTED_TEMPLATES = ( + FixedTemplateRow( + allowed_roles=frozenset({"tool"}), + template=None, + ), + FixedTemplateRow( + allowed_roles=frozenset({"tool", "user"}), + template=None, + extra_kwargs={"truncate_history_thinking": False}, + ), + FixedTemplateRow( + allowed_roles=frozenset({"tool", "user", "system"}), + template=None, + extra_kwargs={"truncate_history_thinking": False}, + ), + ) + + _default_assistant_start_str: str = "<|im_start|>assistant\n" + + def __init__( + self, + tokenizer: Any, + chat_template_kwargs: dict[str, Any] | None = None, + assistant_start_str: str | None = None, + allowed_append_roles: list[str] | None = None, + ): + super().__init__( + tokenizer, + chat_template_kwargs, + assistant_start_str or self._default_assistant_start_str, + allowed_append_roles=allowed_append_roles, + ) + + +# --------------------------------------------------------------------------- +# Kimi K2 implementation +# --------------------------------------------------------------------------- + + +def _kimi_segment_special_token_ids(tokenizer: Any) -> set[int]: + """Kimi specials minus ``<|im_middle|>`` (intra-turn role-name/body + separator, not a role boundary; must not be a segment boundary).""" + return TokenSeqComparator.collect_special_ids(tokenizer) - {tokenizer.convert_tokens_to_ids("<|im_middle|>")} + + +class Kimi25TITOTokenizer(TITOTokenizer): + """Moonshot Kimi K2.5: ``<|im_end|>`` boundary (no trailing newline). + + K2.5 has no kwarg escape hatch for the "drop reasoning of prior assistants + once a new non-tool-call assistant arrives" behavior. Ships a + bundled fixed jinja that wraps the ``last_non_tool_call_assistant_msg`` + loop in ``{%- if not preserve_thinking -%}`` so multi-user-turn rollout + can pass ``preserve_thinking=True`` to keep history append-only. Only the + ``{tool, user}`` surface is registered (per current onboarding scope). + """ + + SUPPORTED_TEMPLATES = ( + FixedTemplateRow( + allowed_roles=frozenset({"tool", "user"}), + template="kimi_k25_fixed.jinja", + extra_kwargs={"preserve_thinking": True}, + ), + ) + + _default_assistant_start_str: str = "<|im_assistant|>" + + def __init__( + self, + tokenizer: Any, + chat_template_kwargs: dict[str, Any] | None = None, + assistant_start_str: str | None = None, + allowed_append_roles: list[str] | None = None, + ): + super().__init__( + tokenizer, + chat_template_kwargs, + assistant_start_str or self._default_assistant_start_str, + special_token_ids=_kimi_segment_special_token_ids(tokenizer), + allowed_append_roles=allowed_append_roles, + ) + + +class Kimi26TITOTokenizer(TITOTokenizer): + """Moonshot Kimi K2.6: same boundary as K2.5 + native ``preserve_thinking`` kwarg. + + K2.6's HF-native template already carries the ``preserve_thinking`` gate + that K2.5 needs patched in. No bundled fixed + template required; ``{tool, user}`` row registers ``template=None`` and + auto-merges ``preserve_thinking=True`` for multi-user-turn rollout. + + Tool-call parser is bound to ``kimi_k2_raw_id`` rather than ``kimi_k2``: + RL trajectories need the model-emitted ``tool_call_id`` to round-trip + verbatim across turns (no ``history_tool_calls_cnt`` renumbering), and + miles is the primary consumer of this TITO family. + """ + + reasoning_parser = "kimi_k2" + tool_call_parser = "kimi_k2_raw_id" + + SUPPORTED_TEMPLATES = ( + FixedTemplateRow( + allowed_roles=frozenset({"tool", "user"}), + template=None, + extra_kwargs={"preserve_thinking": True}, + ), + ) + + _default_assistant_start_str: str = "<|im_assistant|>" + + def __init__( + self, + tokenizer: Any, + chat_template_kwargs: dict[str, Any] | None = None, + assistant_start_str: str | None = None, + allowed_append_roles: list[str] | None = None, + ): + super().__init__( + tokenizer, + chat_template_kwargs, + assistant_start_str or self._default_assistant_start_str, + special_token_ids=_kimi_segment_special_token_ids(tokenizer), + allowed_append_roles=allowed_append_roles, + ) + + +# --------------------------------------------------------------------------- +# MiniMax M2 family implementation (M2.5 and M2.7 share tokenizer/arch and +# stop-token semantics; only their default system identity strings differ). +# --------------------------------------------------------------------------- + + +class MinimaxM25TITOTokenizer(TITOTokenizer): + """MiniMax-M2.5 family: bespoke ``]~!b[`` / ``[e~[`` / ``]~b]`` tag set. + + Shares tokenizer.json (sha256) and architecture (MiniMaxM2ForCausalLM) + with M2.7 — only the chat template's default system identity string + differs (``MiniMax-M2.5`` vs ``MiniMax-M2.7``). Stop-token handling + (``[e~[`` / trailing newline) is identical to M2.7. + + Reasoning is gated by a per-message ``last_user_index`` check: + ``reasoning_content`` is only rendered for assistant turns *after* the + last ``user`` — appending a new ``user`` therefore strips prior assistant + ```` blocks and breaks append-only. Only ``{tool}`` surface is + registered on HF-native template for that reason; multi-user-turn + requires the fixed jinja with ``clear_thinking=False`` to always + preserve history reasoning. + """ + + reasoning_parser = "minimax-append-think" + tool_call_parser = "minimax-m2" + + SUPPORTED_TEMPLATES = ( + FixedTemplateRow( + allowed_roles=frozenset({"tool"}), + template=None, + ), + FixedTemplateRow( + allowed_roles=frozenset({"tool", "user"}), + template="minimax_m25_fixed.jinja", + extra_kwargs={"clear_thinking": False}, + ), + ) + + _default_assistant_start_str: str = "]~b]ai" + + def __init__( + self, + tokenizer: Any, + chat_template_kwargs: dict[str, Any] | None = None, + assistant_start_str: str | None = None, + allowed_append_roles: list[str] | None = None, + ): + super().__init__( + tokenizer, + chat_template_kwargs, + assistant_start_str or self._default_assistant_start_str, + allowed_append_roles=allowed_append_roles, + ) + nl_ids = tokenizer.encode("\n", add_special_tokens=False) + assert len(nl_ids) == 1, f"Expected single newline token, got {nl_ids}" + self._newline_id: int = nl_ids[0] + self._eos_id: int = tokenizer.convert_tokens_to_ids("[e~[") + self.trailing_token_ids = frozenset({self._newline_id}) + + def merge_tokens( + self, + old_messages: list[dict[str, Any]], + new_messages: list[dict[str, Any]], + pretokenized_token_ids: list[int], + tools: list[dict[str, Any]] | None = None, + ) -> list[int]: + incremental = self.tokenize_additional_non_assistant(old_messages, new_messages, tools) + prefix = list(pretokenized_token_ids) + if prefix and prefix[-1] == self._eos_id: + prefix.append(self._newline_id) + return prefix + incremental + + +class MinimaxM27TITOTokenizer(MinimaxM25TITOTokenizer): + """MiniMax-M2.7 family: tokenizer / arch / stop-token semantics identical + to M2.5; the chat template only differs by default system identity string. + + Inherits parsers, ``__init__``, ``merge_tokens``, and + ``_default_assistant_start_str`` from M2.5; only ``SUPPORTED_TEMPLATES`` + is rebound to ``minimax_m27_fixed.jinja`` so the fixed-template lookup + points at the M2.7-derived jinja. + """ + + SUPPORTED_TEMPLATES = ( + FixedTemplateRow( + allowed_roles=frozenset({"tool"}), + template=None, + ), + FixedTemplateRow( + allowed_roles=frozenset({"tool", "user"}), + template="minimax_m27_fixed.jinja", + extra_kwargs={"clear_thinking": False}, + ), + ) + + +# --------------------------------------------------------------------------- +# DeepSeek V3.2 implementation +# --------------------------------------------------------------------------- + + +class DeepSeekV32TITOTokenizer(TITOTokenizer): + """DeepSeek V3.2 — official encoder via sglang's ``encoding_dsv32``. + + V3.2 ships no jinja chat_template; sglang renders prompts through + ``encoding_dsv32.encode_messages``, and miles' ``apply_chat_template`` routes + any V3.2 tokenizer to the thin ``chat_template_utils.deepseek_v32`` bridge. + TITO incremental tokenization rides that same bridge so it stays + byte-aligned with what the runtime serves. + + Only the ``{tool}`` surface is registered. DeepSeek's official + ``encoding_dsv32`` gates an assistant's thinking block on + ``index > last_user_idx``: appending a *user* turn re-classifies every prior + assistant as "before last user" and strips its thinking block, which is not + append-only. Tool-only append is safe because ``find_last_user_index`` + ignores tool roles, so the last-user position never moves. + """ + + reasoning_parser = "deepseek-v3" + tool_call_parser = "deepseekv32" + + SUPPORTED_TEMPLATES = ( + FixedTemplateRow( + allowed_roles=frozenset({"tool"}), + template=None, + ), + ) + + _DEFAULT_ASSISTANT_START = "<｜Assistant｜>" + + def __init__( + self, + tokenizer: Any, + chat_template_kwargs: dict[str, Any] | None = None, + assistant_start_str: str | None = None, + allowed_append_roles: list[str] | None = None, + ): + # V3.2 has no jinja template, so assistant_start_str can't be sniffed + # from one; pin it explicitly. The comparator keys off the User / + # Assistant sentinels to find assistant-content boundaries. + super().__init__( + tokenizer, + chat_template_kwargs=chat_template_kwargs, + assistant_start_str=assistant_start_str or self._DEFAULT_ASSISTANT_START, + special_token_ids={ + tokenizer.convert_tokens_to_ids("<｜User｜>"), + tokenizer.convert_tokens_to_ids("<｜Assistant｜>"), + }, + allowed_append_roles=allowed_append_roles, + ) + + +# --------------------------------------------------------------------------- +# DeepSeek V4 implementation +# --------------------------------------------------------------------------- + + +class DeepSeekV4TITOTokenizer(TITOTokenizer): + """DeepSeek V4 — official encoder via sglang's ``encoding_dsv4``. + + Like V3.2, V4 ships no jinja chat_template; miles' ``apply_chat_template`` + routes any V4 tokenizer to the ``chat_template_utils.deepseek_v4`` bridge, and + TITO incremental tokenization rides that same bridge to stay byte-aligned + with what the runtime serves. Only the ``{tool}`` surface is registered, so + the base ``_split_appended_segments`` (contiguous tool runs) covers it + without a custom override. + """ + + reasoning_parser = "deepseek-v4" + tool_call_parser = "deepseekv4" + + SUPPORTED_TEMPLATES = ( + FixedTemplateRow( + allowed_roles=frozenset({"tool"}), + template=None, + ), + ) + + _DEFAULT_ASSISTANT_START = "<｜Assistant｜>" + + def __init__( + self, + tokenizer: Any, + chat_template_kwargs: dict[str, Any] | None = None, + assistant_start_str: str | None = None, + allowed_append_roles: list[str] | None = None, + ): + super().__init__( + tokenizer, + chat_template_kwargs=chat_template_kwargs, + assistant_start_str=assistant_start_str or self._DEFAULT_ASSISTANT_START, + special_token_ids={ + tokenizer.convert_tokens_to_ids("<｜User｜>"), + tokenizer.convert_tokens_to_ids("<｜Assistant｜>"), + }, + allowed_append_roles=allowed_append_roles, + ) + + +# --------------------------------------------------------------------------- +# Enum + Factory +# --------------------------------------------------------------------------- + + +class TITOTokenizerType(str, Enum): + DEFAULT = "default" + QWEN3 = "qwen3" + QWEN35 = "qwen35" + QWENNEXT = "qwennext" + GLM47 = "glm47" + NEMOTRON3 = "nemotron3" + KIMI25 = "kimi25" + KIMI26 = "kimi26" + MINIMAX_M25 = "minimax_m25" + MINIMAX_M27 = "minimax_m27" + DEEPSEEKV32 = "deepseekv32" + DEEPSEEKV4 = "deepseekv4" + + @classmethod + def get_tokenizer_class(cls, t: TITOTokenizerType) -> type[TITOTokenizer]: + """Resolve the concrete ``TITOTokenizer`` subclass for *t*.""" + match t: + case cls.DEFAULT: + return TITOTokenizer + case cls.QWEN3: + return Qwen3TITOTokenizer + case cls.QWEN35: + return Qwen35TITOTokenizer + case cls.QWENNEXT: + return QwenNextTITOTokenizer + case cls.GLM47: + return GLM47TITOTokenizer + case cls.NEMOTRON3: + return Nemotron3TITOTokenizer + case cls.KIMI25: + return Kimi25TITOTokenizer + case cls.KIMI26: + return Kimi26TITOTokenizer + case cls.MINIMAX_M25: + return MinimaxM25TITOTokenizer + case cls.MINIMAX_M27: + return MinimaxM27TITOTokenizer + case cls.DEEPSEEKV32: + return DeepSeekV32TITOTokenizer + case cls.DEEPSEEKV4: + return DeepSeekV4TITOTokenizer + case _: + raise ValueError(f"Unknown TITOTokenizerType: {t!r}") + + +def get_tito_tokenizer( + tokenizer: Any, + tokenizer_type: TITOTokenizerType | str = TITOTokenizerType.DEFAULT, + chat_template_kwargs: dict[str, Any] | None = None, + assistant_start_str: str | None = None, + allowed_append_roles: list[str] | None = None, +) -> TITOTokenizer: + """Create a ``TITOTokenizer`` instance. + + Args: + tokenizer: HuggingFace tokenizer object. + tokenizer_type: Explicit type (string or enum). Corresponds to the + ``--tito-model`` CLI argument. + chat_template_kwargs: Extra kwargs forwarded to ``apply_chat_template``. + assistant_start_str: Decoded text prefix identifying assistant content + segments (e.g. ``"<|im_start|>assistant"``). Auto-detected from + the chat template by default; pass explicitly to override. + allowed_append_roles: Roles allowed in appended messages. Defaults to + ``["tool"]``. Passed to + ``assert_messages_append_only_with_allowed_role``. + """ + if tokenizer is None: + raise ValueError("tokenizer must not be None") + if isinstance(tokenizer_type, str): + tokenizer_type = TITOTokenizerType(tokenizer_type) + cls = TITOTokenizerType.get_tokenizer_class(tokenizer_type) + kwargs: dict[str, Any] = {"chat_template_kwargs": chat_template_kwargs} + if assistant_start_str is not None: + kwargs["assistant_start_str"] = assistant_start_str + if allowed_append_roles is not None: + kwargs["allowed_append_roles"] = allowed_append_roles + return cls(tokenizer, **kwargs) + + +# --------------------------------------------------------------------------- +# Fixed-template resolution (smallest-superset over SUPPORTED_TEMPLATES) +# --------------------------------------------------------------------------- + + +def resolve_fixed_chat_template( + tito_model: TITOTokenizerType | str, + allowed_append_roles: Iterable[str], +) -> tuple[str | None, dict[str, Any]]: + """Smallest-superset lookup over the requested family's ``SUPPORTED_TEMPLATES``. + + Returns ``(template_path, extra_kwargs)``: + + - ``template_path``: absolute path to a bundled ``.jinja`` file, or ``None`` + when the matched row registers HF-native (kwargs-only fix) or when no + row matches at all. + - ``extra_kwargs``: kwargs the caller should merge into + ``apply_chat_template`` (caller's explicit user kwargs win on conflict). + Empty when no row matches or the matched row needs none. + + Raises ``ValueError`` on equally-minimal supersets — register a stricter + row to disambiguate. + """ + if isinstance(tito_model, str): + tito_model = TITOTokenizerType(tito_model) + + requested = frozenset(allowed_append_roles) + invalid = requested - _VALID_ROLES + if invalid: + raise ValueError( + f"Unknown roles in allowed_append_roles: {sorted(invalid)}. " f"Supported: {sorted(_VALID_ROLES)}." + ) + + cls = TITOTokenizerType.get_tokenizer_class(tito_model) + candidates = [row for row in cls.SUPPORTED_TEMPLATES if requested.issubset(row.allowed_roles)] + if not candidates: + raise ValueError( + f"No SUPPORTED_TEMPLATES row registered for tito_model={tito_model.value} " + f"with allowed_append_roles={sorted(requested)}. Register a row in " + f"{cls.__name__}.SUPPORTED_TEMPLATES (template=None for HF-native models)." + ) + + # Pick the most specific superset. Ties surface registration mistakes + # immediately rather than depending on iteration order. + min_size = min(len(row.allowed_roles) for row in candidates) + minimal = [row for row in candidates if len(row.allowed_roles) == min_size] + if len(minimal) > 1: + raise ValueError( + f"Ambiguous fixed-template registration for tito_model={tito_model.value}, " + f"requested_roles={sorted(requested)}: multiple equally-minimal supersets " + f"{[sorted(row.allowed_roles) for row in minimal]}. Register a stricter row to disambiguate." + ) + row = minimal[0] + + path = str(TEMPLATE_DIR / row.template) if row.template else None + logger.info( + "tito_model=%s requested_roles=%s -> matched registered_roles=%s -> template=%s kwargs=%s", + tito_model.value, + sorted(requested), + sorted(row.allowed_roles), + path, + row.extra_kwargs, + ) + return path, dict(row.extra_kwargs) + + +# --------------------------------------------------------------------------- +# sglang parser resolution (per-family binding + assert-equal on user input) +# --------------------------------------------------------------------------- + + +def resolve_reasoning_and_tool_call_parser( + tito_model: TITOTokenizerType | str, + user_reasoning_parser: str | None = None, + user_tool_call_parser: str | None = None, +) -> tuple[str | None, str | None]: + """Resolve sglang ``--reasoning-parser`` and ``--tool-call-parser`` for the + given TITO family. + + Both parsers are bound on the TITO subclass as class attributes because + the model's reasoning / tool-call emission shapes are per-family facts. + For each parser independently: + + * If the user didn't pass a value, return the family's bound value + (which may itself be ``None`` for ``DEFAULT`` or unbound subclasses + — the caller is then responsible for supplying one downstream). + * If the user passed a value and the family is bound, assert equality; + a mismatch is a configuration bug and raises ``ValueError`` rather + than silently overriding. + * If the user passed a value and the family is unbound, accept it. + + Returns ``(reasoning_parser, tool_call_parser)``. + """ + if isinstance(tito_model, str): + tito_model = TITOTokenizerType(tito_model) + cls = TITOTokenizerType.get_tokenizer_class(tito_model) + + def _resolve_one(field: str, bound: str | None, user: str | None) -> str | None: + if user is None: + return bound + if bound is None: + return user + if user != bound: + raise ValueError( + f"--{field.replace('_', '-')}={user!r} disagrees with the parser " + f"registered for tito_model={tito_model.value!r}: {bound!r}. The " + f"parser is bound on the TITO subclass; either pass {bound!r} or " + f"omit the flag to auto-resolve." + ) + return user + + return ( + _resolve_one("reasoning_parser", cls.reasoning_parser, user_reasoning_parser), + _resolve_one("tool_call_parser", cls.tool_call_parser, user_tool_call_parser), + ) diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/token_seq_comparator.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/token_seq_comparator.py new file mode 100644 index 0000000..2f4b93d --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/token_seq_comparator.py @@ -0,0 +1,289 @@ +"""TokenSeqComparator: segment token IDs by special-token boundaries and compare sequences.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum + + +@dataclass +class Segment: + """A contiguous run of token IDs — either a special token or a content segment.""" + + token_ids: list[int] = field(default_factory=list) + is_special: bool = False + + +class MismatchType(Enum): + # Segment count or structure (special/content pattern) differs between + # expected and actual. When this happens, segments can't be aligned so + # no per-segment comparison is possible. + SPECIAL_TOKEN_COUNT = "special_token_count" + + # A special-token segment has the same position in both sequences but + # contains a different token ID. + SPECIAL_TOKEN_TYPE = "special_token_type" + + # Non-assistant content (user, system, tool, etc.) differs. This indicates + # a bug in the TITO algorithm — these regions should match exactly. + NON_ASSISTANT_TEXT = "non_assistant_text" + + # Assistant content differs. Expected and non-severe: assistant tokens + # are inherited directly from the pretokenized prefix across turns, + # so they may not match the chat template's canonical tokenization. + ASSISTANT_TEXT = "assistant_text" + + +@dataclass +class Mismatch: + """A single difference found between two token sequences.""" + + type: MismatchType + segment_index: int + expected_text: str + actual_text: str + detail: str = "" + + def to_dict(self) -> dict: + return { + "type": self.type.value, + "segment_index": self.segment_index, + "expected_text": self.expected_text, + "actual_text": self.actual_text, + "detail": self.detail, + } + + +class TokenSeqComparator: + """Segment token sequences by special tokens and compare them. + + Parameters + ---------- + tokenizer : PreTrainedTokenizerBase + special_token_ids : set[int] | None + Token IDs that mark segment boundaries. Default None auto-detects + them (see :meth:`collect_special_ids`). Pass an explicit set to + override — needed when the tokenizer flags a token ``special=True`` + even though it lives inside a role's turn (e.g. Kimi's + ``<|im_middle|>`` between role name and body) and so must not + split the segment. + assistant_start_str : str + Decoded text prefix identifying assistant content segments, e.g. + ``"<|im_start|>assistant"`` (Qwen3) or ``"<|assistant|>"`` (GLM). + Used to classify content mismatches as assistant vs non-assistant. + trim_trailing_ids : set[int] | None + Token IDs to strip from both sequence tails before comparison + (see :func:`_trim_trailing`). Stored as a default; callers of + :meth:`compare_sequences` may supply additional IDs that are + **unioned** with this set. + """ + + def __init__( + self, + tokenizer, + assistant_start_str: str, + special_token_ids: set[int] | None = None, + trim_trailing_ids: set[int] | None = None, + ): + self.tokenizer = tokenizer + if special_token_ids is not None: + self._special_ids = set(special_token_ids) + else: + self._special_ids = self.collect_special_ids(tokenizer) + self._assistant_start_str = assistant_start_str + self._trim_trailing_ids: set[int] | None = set(trim_trailing_ids) if trim_trailing_ids else None + + @staticmethod + def collect_special_ids(tokenizer) -> set[int]: + """Collect token IDs with ``special=True`` from the tokenizer. + + Special tokens are structural markers added by the chat template to + delimit messages, roles, and control flow — for example + ``<|im_start|>``, ``<|im_end|>``, ``<|endoftext|>``, ````, + ````, and ``<|assistant|>``. + + Tokens that encode *content* produced by a role are **not** special, + even if they look "special" to a human. For instance, ```` and + ```` in reasoning models are regular content tokens generated + by the assistant — the tokenizer does not flag them as + ``special=True``, so they are not collected here. + """ + ids = set(tokenizer.all_special_ids) + decoder = getattr(tokenizer, "added_tokens_decoder", None) + if decoder: + ids |= {k for k, v in decoder.items() if v.special} + return ids + + def segment_by_special_tokens(self, token_ids: list[int]) -> list[Segment]: + """Split *token_ids* into segments at special-token boundaries. + + Each special token becomes its own single-ID segment with + ``is_special=True``. Consecutive non-special tokens are grouped + into content segments. Example for Qwen3:: + + [<|im_start|>, "assistant", "\\n", "Hi", <|im_end|>, "\\n"] + → [special(<|im_start|>), content("assistant\\nHi"), special(<|im_end|>), content("\\n")] + """ + if not token_ids: + return [] + + segments: list[Segment] = [] + current: list[int] = [] + for tid in token_ids: + if tid in self._special_ids: + if current: + segments.append(Segment(token_ids=current)) + current = [] + segments.append(Segment(token_ids=[tid], is_special=True)) + else: + current.append(tid) + if current: + segments.append(Segment(token_ids=current)) + return segments + + def compare_sequences( + self, + expected_ids: list[int], + actual_ids: list[int], + trim_trailing_ids: set[int] | None = None, + ) -> list[Mismatch]: + """Compare two token-ID sequences and return mismatches. + + Parameters + ---------- + trim_trailing_ids : set[int] | None + Additional token IDs to strip from both sequence tails before + comparison. **Unioned** with the IDs passed at construction time. + """ + trim = self._trim_trailing_ids or set() + if trim_trailing_ids: + trim = trim | trim_trailing_ids + if trim: + expected_ids = _trim_trailing(expected_ids, trim) + actual_ids = _trim_trailing(actual_ids, trim) + + exp_segs = self.segment_by_special_tokens(expected_ids) + act_segs = self.segment_by_special_tokens(actual_ids) + + structural = self._check_segment_structure(exp_segs, act_segs) + if structural: + return [structural] + + mismatches: list[Mismatch] = [] + for idx, (exp, act) in enumerate(zip(exp_segs, act_segs, strict=True)): + is_assistant_content = self._is_assistant_content(exp_segs, idx) and self._is_assistant_content( + act_segs, idx + ) + m = self._compare_single_segment(idx, exp, act, is_assistant_content=is_assistant_content) + if m is not None: + mismatches.append(m) + return mismatches + + def _check_segment_structure( + self, + exp_segs: list[Segment], + act_segs: list[Segment], + ) -> Mismatch | None: + """Pre-check that expected and actual segment lists have the same count + and the same special/content pattern before per-segment comparison.""" + if len(exp_segs) != len(act_segs): + detail = f"segment count differs: expected {len(exp_segs)}, got {len(act_segs)}" + elif [s.is_special for s in exp_segs] != [s.is_special for s in act_segs]: + detail = "segment structure (special/content pattern) differs" + else: + return None + return Mismatch( + type=MismatchType.SPECIAL_TOKEN_COUNT, + segment_index=-1, + expected_text=self._describe_structure(exp_segs), + actual_text=self._describe_structure(act_segs), + detail=detail, + ) + + def _compare_single_segment( + self, + idx: int, + exp: Segment, + act: Segment, + *, + is_assistant_content: bool, + ) -> Mismatch | None: + """Compare a single aligned segment pair and return a mismatch if they differ. + + Special segments are compared by token ID. Content segments are decoded + and compared as stripped text — leading/trailing whitespace (``\\n``, + spaces) is ignored because chat templates may insert boundary newlines + that differ from the TITO prefix. This whitespace-only difference does + not cause meaningful misalignment with the chat template, so we strip + to avoid noisy false positives. + """ + if exp.is_special: + if exp.token_ids != act.token_ids: + return Mismatch( + type=MismatchType.SPECIAL_TOKEN_TYPE, + segment_index=idx, + expected_text=self._decode(exp.token_ids), + actual_text=self._decode(act.token_ids), + ) + return None + + # After ignoring assistant text diff, there is no need to keep the strip operator, + # as other text should be exact match. + exp_text = self._decode(exp.token_ids) + act_text = self._decode(act.token_ids) + if exp_text == act_text: + return None + + return Mismatch( + type=MismatchType.ASSISTANT_TEXT if is_assistant_content else MismatchType.NON_ASSISTANT_TEXT, + segment_index=idx, + expected_text=exp_text, + actual_text=act_text, + ) + + def _is_assistant_content(self, segments: list[Segment], idx: int) -> bool: + """Check if the content segment at *idx* belongs to an assistant message. + + Decodes the preceding special-token segment and the first few tokens of + the current segment *separately*, then concatenates the decoded strings. + If the result starts with ``assistant_start_str`` (e.g. + ``"<|im_start|>assistant"``), this segment is classified as assistant + content — mismatches there are expected and non-severe. + + """ + if self._assistant_start_str is None: + return False + if segments[idx].is_special: + return False + if idx == 0: + return False + prev = segments[idx - 1] + if not prev.is_special: + return False + special_text = self._decode(prev.token_ids) + # Decode enough prefix tokens to capture the role label (e.g. "assistant\n"). + content_prefix = self._decode(segments[idx].token_ids[:20]) + return (special_text + content_prefix).startswith(self._assistant_start_str) + + def _decode(self, token_ids: list[int]) -> str: + return self.tokenizer.decode(token_ids, skip_special_tokens=False) + + def _describe_structure(self, segments: list[Segment]) -> str: + return " ".join( + f"[{self._decode(s.token_ids)}]" if s.is_special else f"({len(s.token_ids)} tokens)" for s in segments + ) + + +def _trim_trailing(ids: list[int], to_remove: set[int]) -> list[int]: + """Strip trailing token IDs that belong to *to_remove*. + + The model's generated output typically ends with a stop token (e.g. + ``<|observation|>`` for GLM, ``<|im_end|>`` for Qwen) that won't appear + at the same position in the template-rendered expected sequence. Stripping + these trailing tokens from both sides before comparison avoids false + structural mismatches. + """ + end = len(ids) + while end > 0 and ids[end - 1] in to_remove: + end -= 1 + return ids[:end] diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/external_utils/__init__.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/external_utils/__init__.py new file mode 100644 index 0000000..101568c --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/external_utils/__init__.py @@ -0,0 +1 @@ +"""Optional Miles external utility compatibility namespace.""" diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/external_utils/command_utils.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/external_utils/command_utils.py new file mode 100644 index 0000000..db771f9 --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/external_utils/command_utils.py @@ -0,0 +1,33 @@ +"""Optional Miles command helpers used by session verifier e2e jobs.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +class MissingMilesTrainingStackError(RuntimeError): + """Raised when full Miles training helpers are required but unavailable.""" + + +@dataclass +class ExecuteTrainConfig: + cuda_core_dump: bool = False + num_nodes: int = 1 + extra_env_vars: str = "" + output_dir: str = "/root/shared_data" + + +def exec_command(*args, **kwargs): + raise MissingMilesTrainingStackError( + "Miles command execution helpers are not bundled with tito-gateway. " + "Install/provide the optional Miles training stack before running full " + "session verifier e2e jobs." + ) + + +def execute_train(*args, **kwargs): + raise MissingMilesTrainingStackError( + "Miles execute_train helper is not bundled with tito-gateway. " + "Install/provide the optional Miles training stack before running full " + "session verifier e2e jobs." + ) diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/hf_config.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/hf_config.py new file mode 100644 index 0000000..f949b02 --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/hf_config.py @@ -0,0 +1,108 @@ +"""HuggingFace config loader with model-type alias registration and overrides. + +`load_hf_config` is the single entry point miles uses to load an HF config from a +local checkpoint. It supports 2 customizations: + +- Registers model_type aliases before calling AutoConfig, in case the model is + not recognized in huggingface. +- Accepts an `overrides` dict applied via setattr after loading, so callers can + adjust fields without touching the checkpoint. + +The default behavior is exactly the same as `AutoConfig.from_pretrained`. +""" + +import importlib +from dataclasses import dataclass + +from transformers import AutoConfig, AutoModelForCausalLM +from transformers.models.auto.configuration_auto import CONFIG_MAPPING_NAMES + + +@dataclass(frozen=True) +class _HFConfigAlias: + model_type: str + base_module: str + base_class: str + compat_class_name: str + auto_model_classes: tuple = (AutoModelForCausalLM,) + # Set True to override transformers' native config. + override_hf_native: bool = False + + +_CONFIG_ALIASES: tuple[_HFConfigAlias, ...] = ( + _HFConfigAlias( + model_type="deepseek_v32", + base_module="transformers.models.deepseek_v3.configuration_deepseek_v3", + base_class="DeepseekV3Config", + compat_class_name="DeepseekV32Config", + ), + _HFConfigAlias( + model_type="deepseek_v4", + base_module="transformers.models.deepseek_v3.configuration_deepseek_v3", + base_class="DeepseekV3Config", + compat_class_name="DeepseekV4Config", + auto_model_classes=(), + override_hf_native=True, + ), +) + +_REGISTERED_ALIASES: set[str] = set() + + +def register_hf_config_aliases() -> None: + """Register miles model_type aliases with transformers. Idempotent. + + Already called inside `load_hf_config` and `load_tokenizer`. Only call + directly before a third-party entry point that won't go through either + (e.g. megatron's `_build_tokenizer`). + """ + for alias in _CONFIG_ALIASES: + if alias.model_type in _REGISTERED_ALIASES: + continue + if alias.model_type in CONFIG_MAPPING_NAMES and not alias.override_hf_native: + raise RuntimeError( + f"transformers now natively supports model_type={alias.model_type!r}; " + f"set override_hf_native=True to override." + ) + module = importlib.import_module(alias.base_module) + base_config = getattr(module, alias.base_class) + compat_config = type( + alias.compat_class_name, + (base_config,), + {"model_type": alias.model_type, "__module__": __name__}, + ) + AutoConfig.register(alias.model_type, compat_config, exist_ok=alias.override_hf_native) + for auto_cls in alias.auto_model_classes: + base_model_cls = auto_cls._model_mapping[base_config] + compat_model_cls = type( + base_model_cls.__name__, (base_model_cls,), {"config_class": compat_config, "__module__": __name__} + ) + auto_cls.register(compat_config, compat_model_cls, exist_ok=alias.override_hf_native) + _REGISTERED_ALIASES.add(alias.model_type) + + +def load_hf_config( + checkpoint_path: str, + *, + overrides: dict | None = None, + trust_remote_code: bool = True, + **autoconfig_kwargs, +): + """Load an HF config from a local checkpoint. + + Registers model aliases first for pre-set aliases. + + overrides: optional dict of attributes to setattr on the returned config + after loading. Lets callers patch fields without mutating the checkpoint. + """ + register_hf_config_aliases() + config = AutoConfig.from_pretrained(checkpoint_path, trust_remote_code=trust_remote_code, **autoconfig_kwargs) + + if overrides: + for key, value in overrides.items(): + setattr(config, key, value) + return config + + +def is_dsa(hf_config) -> bool: + return getattr(hf_config, "model_type", None) in ("deepseek_v32", "glm_moe_dsa") diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/http_utils.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/http_utils.py new file mode 100644 index 0000000..0aaf792 --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/http_utils.py @@ -0,0 +1,315 @@ +import asyncio +import ipaddress +import json +import logging +import multiprocessing +import os +import random +import socket +import time + +import httpx + +logger = logging.getLogger(__name__) + +MILES_HOST_IP_ENV = "MILES_HOST_IP" + + +def find_available_port(base_port: int): + port = base_port + random.randint(100, 1000) + while True: + if is_port_available(port): + return port + if port < 60000: + port += 42 + else: + port -= 43 + + +def is_port_available(port): + """Return whether a port is available.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind(("", port)) + s.listen(1) + return True + except OSError: + return False + except OverflowError: + return False + + +def wait_for_server_ready( + host: str, + port: int, + process: "multiprocessing.Process | None" = None, + timeout: float = 30, +) -> None: + """Poll until a TCP port is accepting connections. + + Raises ``RuntimeError`` if the process dies or the timeout is exceeded. + """ + deadline = time.time() + timeout + while time.time() < deadline: + if process is not None and not process.is_alive(): + raise RuntimeError(f"Server process died before port {port} became ready") + try: + with socket.create_connection((host, port), timeout=1): + return + except OSError: + time.sleep(0.5) + raise RuntimeError(f"Server at {host}:{port} not ready after {timeout}s") + + +def get_host_info(): + hostname = socket.gethostname() + + if env_overwrite_local_ip := os.getenv(MILES_HOST_IP_ENV, None): + return hostname, env_overwrite_local_ip + + def _is_loopback(ip): + return ip.startswith("127.") or ip == "::1" + + def _resolve_ip(family, test_target_ip): + """ + Attempt to get the local LAN IP for the specific family (IPv4/IPv6). + Strategy: UDP Probe (Preferred) -> Hostname Resolution (Fallback) -> None + """ + + # Strategy 1: UDP Connect Probe (Most accurate, relies on routing table) + # Useful when the machine has a default gateway or internet access. + try: + with socket.socket(family, socket.SOCK_DGRAM) as s: + # The IP doesn't need to be reachable, but the routing table must exist. + s.connect((test_target_ip, 80)) + ip = s.getsockname()[0] + if not _is_loopback(ip): + return ip + except Exception: + pass # Route unreachable or network error, move to next strategy. + + # Strategy 2: Hostname Resolution (Fallback for offline clusters) + # Useful for offline environments where UDP connect fails but /etc/hosts is configured. + try: + # getaddrinfo allows specifying the family (AF_INET or AF_INET6) + # Result format: [(family, type, proto, canonname, sockaddr), ...] + infos = socket.getaddrinfo(hostname, None, family=family, type=socket.SOCK_STREAM) + + for info in infos: + ip = info[4][0] # The first element of sockaddr is the IP + # Must filter out loopback addresses to avoid "127.0.0.1" issues + if not _is_loopback(ip): + return ip + except Exception: + pass + + return None + + prefer_ipv6 = os.getenv("MILES_PREFER_IPV6", "0").lower() in ("1", "true", "yes", "on") + local_ip = None + final_fallback = "127.0.0.1" + + if prefer_ipv6: + # [Strict Mode] IPv6 Only + # 1. Try UDP V6 Probe + # 2. Try Hostname Resolution (V6) + # If failed, fallback to V6 loopback. Never mix with V4. + local_ip = _resolve_ip(socket.AF_INET6, "2001:4860:4860::8888") + final_fallback = "::1" + else: + # [Strict Mode] IPv4 Only (Default) + # 1. Try UDP V4 Probe + # 2. Try Hostname Resolution (V4) + # If failed, fallback to V4 loopback. Never mix with V6. + local_ip = _resolve_ip(socket.AF_INET, "8.8.8.8") + final_fallback = "127.0.0.1" + + return hostname, local_ip or final_fallback + + +def _wrap_ipv6(host): + """Wrap IPv6 address in [] if needed.""" + try: + ipaddress.IPv6Address(host.strip("[]")) + return f"[{host.strip('[]')}]" + except ipaddress.AddressValueError: + return host + + +def run_router(args): + try: + from sglang_router.launch_router import launch_router + + router = launch_router(args) + if router is None: + return 1 + return 0 + except Exception as e: + logger.info(e) + return 1 + + +def terminate_process(process: multiprocessing.Process, timeout: float = 1.0) -> None: + """Terminate a process gracefully, with forced kill as fallback. + + Args: + process: The process to terminate + timeout: Seconds to wait for graceful termination before forcing kill + """ + if not process.is_alive(): + return + + process.terminate() + process.join(timeout=timeout) + if process.is_alive(): + process.kill() + process.join() + + +_http_client: httpx.AsyncClient | None = None +_client_concurrency: int = 0 + +# Optional Ray-based distributed POST dispatch +_distributed_post_enabled: bool = False +_post_actors: list[object] = [] +_post_actor_idx: int = 0 + + +def _next_actor(): + global _post_actor_idx + if not _post_actors: + return None + actor = _post_actors[_post_actor_idx % len(_post_actors)] + _post_actor_idx = (_post_actor_idx + 1) % len(_post_actors) + return actor + + +async def _post(client, url, payload, max_retries=60, action="post", headers=None): + retry_count = 0 + while retry_count < max_retries: + try: + if action in ("delete", "get"): + assert not payload + response = await getattr(client, action)(url, headers=headers) + else: + response = await getattr(client, action)(url, json=payload or {}, headers=headers) + response.raise_for_status() + try: + output = response.json() + except json.JSONDecodeError: + output = response.text + except Exception as e: + retry_count += 1 + + if isinstance(e, httpx.HTTPStatusError): + response_text = e.response.text + else: + response_text = None + + logger.info( + f"Error: {e}, retrying... (attempt {retry_count}/{max_retries}, url={url}, response={response_text})" + ) + if retry_count >= max_retries: + logger.info(f"Max retries ({max_retries}) reached, failing... (url={url})") + raise e + await asyncio.sleep(1) + continue + break + + return output + + +def init_http_client(args): + """Initialize HTTP client and optionally enable distributed POST via Ray.""" + global _http_client, _client_concurrency, _distributed_post_enabled + if not args.rollout_num_gpus: + return + + _client_concurrency = args.sglang_server_concurrency * args.rollout_num_gpus // args.rollout_num_gpus_per_engine + if _http_client is None: + _http_client = httpx.AsyncClient( + limits=httpx.Limits(max_connections=_client_concurrency), + timeout=httpx.Timeout(None), + ) + + # Optionally initialize distributed POST via Ray without changing interfaces + if args.use_distributed_post: + _init_ray_distributed_post(args) + _distributed_post_enabled = True + + +def _init_ray_distributed_post(args): + """Initialize one or more Ray async actors per node for HTTP POST. + + Uses NodeAffinitySchedulingStrategy to place actors on distinct nodes. + Controlled by MILES_HTTP_POST_ACTORS_PER_NODE. + """ + global _post_actors + if _post_actors: + return # Already initialized + + import ray + from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy + + # Discover alive nodes + nodes = [n for n in ray.nodes() if n.get("Alive")] + if not nodes: + raise RuntimeError("No alive Ray nodes to place HTTP POST actors.") + + # Define the async actor + @ray.remote + class _HttpPosterActor: + def __init__(self, concurrency: int): + # Lazy creation to this actor's event loop + self._client = httpx.AsyncClient( + limits=httpx.Limits(max_connections=max(1, concurrency)), + timeout=httpx.Timeout(None), + ) + + async def do_post(self, url, payload, max_retries=60, action="post", headers=None): + return await _post(self._client, url, payload, max_retries, action=action, headers=headers) + + # Create actors per node + created = [] + # Distribute client concurrency across actors (at least 1 per actor) + per_actor_conc = (_client_concurrency + len(nodes)) // len(nodes) + + for node in nodes: + node_id = node["NodeID"] + scheduling = NodeAffinitySchedulingStrategy(node_id=node_id, soft=False) + for _ in range(args.num_gpus_per_node): + actor = _HttpPosterActor.options( + name=None, + lifetime="detached", + scheduling_strategy=scheduling, + max_concurrency=per_actor_conc, + # Use tiny CPU to schedule + num_cpus=0.001, + ).remote(per_actor_conc) + created.append(actor) + + _post_actors = created + + +# TODO may generalize the name since it now contains http DELETE/GET etc (with retries and remote-execution) +async def post(url, payload, max_retries=60, action="post", headers=None): + # If distributed mode is enabled and actors exist, dispatch via Ray. + if _distributed_post_enabled and _post_actors: + try: + actor = _next_actor() + if actor is not None: + return await actor.do_post.remote(url, payload, max_retries, action=action, headers=headers) + except Exception as e: + logger.info(f"[http_utils] Distributed POST failed, falling back to local: {e} (url={url})") + # fall through to local + + return await _post(_http_client, url, payload, max_retries, action=action, headers=headers) + + +# TODO unify w/ `post` to add retries and remote-execution +async def get(url): + response = await _http_client.get(url) + response.raise_for_status() + output = response.json() + return output diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/processing_utils.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/processing_utils.py new file mode 100644 index 0000000..ac6e122 --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/processing_utils.py @@ -0,0 +1,175 @@ +import base64 +import inspect +import io +import logging +import os +from pathlib import Path + +from huggingface_hub import hf_hub_download +from tokenizers import Tokenizer as RawTokenizer +from transformers import AutoProcessor, AutoTokenizer, PreTrainedTokenizerBase, ProcessorMixin + +from tito_gateway.vendor.miles_compat.utils.hf_config import register_hf_config_aliases + +logger = logging.getLogger(__name__) + + +def _fix_v5_tokenizer_components(tokenizer: PreTrainedTokenizerBase, model_name_or_path: str) -> None: + # transformers v5's LlamaTokenizerFast rebuilds pre_tokenizer/decoder in + # __init__, discarding the originals from tokenizer.json. DeepSeek-V3.2 + # declares LlamaTokenizerFast but actually uses ByteLevel, so without this + # fix the loaded tokenizer decodes Metaspace ▁ instead of ByteLevel Ġ/Ċ + # and diverges from the sglang-served tokenizer. Mirrors sglang's + # _fix_v5_tokenizer_components (hf_transformers_utils.py). + backend = getattr(tokenizer, "_tokenizer", None) + if backend is None: + return + + try: + local_path = Path(model_name_or_path) / "tokenizer.json" + if local_path.is_file(): + tok_file = str(local_path) + else: + tok_file = hf_hub_download(model_name_or_path, "tokenizer.json", local_files_only=True) + raw = RawTokenizer.from_file(tok_file) + except Exception as e: + logger.warning("Could not load tokenizer.json for %s: %s", model_name_or_path, e) + return + + raw_pre = type(raw.pre_tokenizer).__name__ if raw.pre_tokenizer else None + loaded_pre = type(backend.pre_tokenizer).__name__ if backend.pre_tokenizer else None + + if raw_pre and loaded_pre and raw_pre != loaded_pre: + logger.info( + "Fixing v5 tokenizer component mismatch for %s: pre_tokenizer %s -> %s, decoder %s -> %s", + model_name_or_path, + loaded_pre, + raw_pre, + type(backend.decoder).__name__ if backend.decoder else None, + type(raw.decoder).__name__ if raw.decoder else None, + ) + backend.pre_tokenizer = raw.pre_tokenizer + backend.decoder = raw.decoder + + +# Default image patch size for vision-language models +# Note: Qwen3-VL uses 16, Qwen2.5-VL uses 14 +# Reference: https://github.com/QwenLM/Qwen3-VL/blob/main/qwen-vl-utils/README.md +DEFAULT_PATCH_SIZE = 14 + + +_TOKENIZER_CACHE: dict[tuple, PreTrainedTokenizerBase] = {} + + +def _make_cache_key(name_or_path: str, chat_template_path: str | None, kwargs: dict) -> tuple | None: + try: + kwargs_items = tuple(sorted(kwargs.items())) + hash(kwargs_items) + except TypeError: + return None + return (name_or_path, chat_template_path, kwargs_items) + + +def load_tokenizer(name_or_path: str, chat_template_path: str | None = None, **kwargs) -> PreTrainedTokenizerBase: + # Cache keyed by (name, chat_template_path, kwargs) — the fast suite creates + # hundreds of SessionServer / MockSGLangServer fixtures and each previously + # triggered a fresh AutoTokenizer.from_pretrained, tripping HF Hub rate limits. + cache_key = _make_cache_key(name_or_path, chat_template_path, kwargs) + if cache_key is not None and cache_key in _TOKENIZER_CACHE: + return _TOKENIZER_CACHE[cache_key] + + register_hf_config_aliases() + tokenizer = AutoTokenizer.from_pretrained(name_or_path, **kwargs) + _fix_v5_tokenizer_components(tokenizer, name_or_path) + if chat_template_path: + assert os.path.isfile(chat_template_path), ( + f"chat_template_path not found: {chat_template_path}. " + f"Ensure the path is accessible on this node (e.g. inside the miles repo or on a shared filesystem)." + ) + with open(chat_template_path) as f: + tokenizer.chat_template = f.read() + logger.info("Loaded custom chat template from %s", chat_template_path) + + if cache_key is not None: + _TOKENIZER_CACHE[cache_key] = tokenizer + return tokenizer + + +def build_processor_kwargs(multimodal_inputs: dict | None = None) -> dict: + + modality_forced = {"return_tensors": "pt"} + + result = dict(multimodal_inputs) if multimodal_inputs else {} + + # return_tensors=None for text (input_ids), "pt" for modality-specific outputs. + # Use per-modality dicts to avoid transformers >=5.0 duplicate kwarg error. + result["text_kwargs"] = {**result.get("text_kwargs", {}), "return_tensors": None} + for key in ("audio_kwargs", "images_kwargs", "videos_kwargs"): + if key in result: + result[key] = {**result[key], **modality_forced} + else: + result[key] = modality_forced.copy() + + return result + + +def processor_requires_medias(processor) -> bool: + try: + params = inspect.signature(processor).parameters + return "medias" in params and "text" in params + except (TypeError, ValueError): + return hasattr(processor, "media_processor") + + +def call_processor(processor, text, multimodal_inputs: dict | None = None): + multimodal_inputs = multimodal_inputs or {} + + # for kimi-vl & kimi-2.5 + if processor_requires_medias(processor): + medias = [] + if images := multimodal_inputs.get("images"): + medias.extend({"type": "image", "image": image} for image in images) + if videos := multimodal_inputs.get("videos"): + medias.extend({"type": "video", "video": video} for video in videos) + return processor(text=text, medias=medias) + + kwargs = build_processor_kwargs(multimodal_inputs) + return processor(text=text, **kwargs) + + +def load_processor(name_or_path: str, **kwargs): + try: + proc = AutoProcessor.from_pretrained(name_or_path, **kwargs) + except (OSError, ValueError) as e: + logger.warning(f"Failed to load processor from {name_or_path}: {e}") + proc = None + + # If HF returned a tokenizer, discard it. + if isinstance(proc, PreTrainedTokenizerBase) or not isinstance(proc, ProcessorMixin): + proc = None + + return proc + + +def process_vision_info(prompt, processor): + # TODO: temporary solution, will write image utils for miles later + from qwen_vl_utils import process_vision_info as qwen_process_vision_info + + if hasattr(processor.image_processor, "patch_size"): + image_patch_size = processor.image_processor.patch_size + else: + logger.info(f"Using default patch size: {DEFAULT_PATCH_SIZE}") + image_patch_size = DEFAULT_PATCH_SIZE + images, videos = qwen_process_vision_info(prompt, image_patch_size=image_patch_size) + multimodal_inputs = {"images": images, "videos": videos} + return multimodal_inputs + + +def encode_image_for_rollout_engine(image) -> str: + """Load an image from path, ensure RGB, encode as PNG base64 string.""" + buffer = io.BytesIO() + if image.mode != "RGB": + image = image.convert("RGB") + image.save(buffer, format="PNG") + image_base64 = base64.b64encode(buffer.getvalue()).decode("utf-8") + return f"data:image/png;base64,{image_base64}" diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/__init__.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/chat_template_verify.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/chat_template_verify.py new file mode 100644 index 0000000..79e2533 --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/chat_template_verify.py @@ -0,0 +1,602 @@ +"""Verify that a chat template satisfies the append-only invariant. + +The append-only invariant means: rendering the first N messages (without +generation prompt) produces a string that is an exact prefix of rendering +all messages (with generation prompt). This is required by sglang's +pretokenized prefix mechanism for agentic workflows. + +Core functions are used by both the CLI script +(``scripts/tools/verify_chat_template.py``) and the test suite +(``tests/fast/utils/chat_template_utils/test_pretokenized_chat.py``). +""" + +from __future__ import annotations + +from collections.abc import Iterable +from copy import deepcopy +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +from tito_gateway.vendor.miles_compat.utils.chat_template_utils.template import apply_chat_template_from_str + +if TYPE_CHECKING: + from tito_gateway.vendor.miles_compat.utils.chat_template_utils.tito_tokenizer import TITOTokenizer, TITOTokenizerType + + +def simulate_pretokenized_path( + chat_template: str, + messages: list[dict], + pretokenized_num_message: int, + tools: list[dict] | None = None, + **template_kwargs, +) -> str: + """Simulate the pretokenized incremental path at text level. + + 1. Render first N messages (no generation prompt) -> prefix_text + 2. Render ALL messages (with generation prompt) -> full_text + 3. Verify prefix_text is a prefix of full_text + + Raises ``ValueError`` on prefix mismatch. + """ + prefix_text = apply_chat_template_from_str( + chat_template, + messages[:pretokenized_num_message], + add_generation_prompt=False, + tools=tools, + **template_kwargs, + ) + + full_text = apply_chat_template_from_str( + chat_template, + messages, + add_generation_prompt=True, + tools=tools, + **template_kwargs, + ) + + if not full_text.startswith(prefix_text): + raise ValueError( + f"Prefix mismatch!\n" + f"prefix_text ({len(prefix_text)} chars):\n{repr(prefix_text[-200:])}\n\n" + f"full_text at same position:\n{repr(full_text[:len(prefix_text)][-200:])}" + ) + + return full_text + + +def get_standard_result( + chat_template: str, + messages: list[dict], + tools: list[dict] | None = None, + **template_kwargs, +) -> str: + """Standard path: render all messages with generation prompt.""" + return apply_chat_template_from_str( + chat_template, + messages, + add_generation_prompt=True, + tools=tools, + **template_kwargs, + ) + + +def assert_pretokenized_equals_standard(chat_template, messages, pretokenized_num_message, tools=None, **kwargs): + """Assert pretokenized incremental path produces same text as standard full render.""" + standard = get_standard_result(chat_template, messages, tools=tools, **kwargs) + pretokenized = simulate_pretokenized_path(chat_template, messages, pretokenized_num_message, tools=tools, **kwargs) + assert pretokenized == standard, f"Pretokenized (N={pretokenized_num_message}) != standard" + + +# --------------------------------------------------------------------------- +# Non-raising verification API for CLI / programmatic use +# --------------------------------------------------------------------------- + + +@dataclass +class VerifyResult: + """Result of a single append-only verification case.""" + + case_name: str + passed: bool + error: str | None = None + + +def verify_append_only( + chat_template: str, + messages: list[dict], + pretokenized_num_message: int, + tools: list[dict] | None = None, + case_name: str = "", + **template_kwargs, +) -> VerifyResult: + """Check that the template satisfies the append-only invariant. + + Returns a ``VerifyResult`` instead of raising, making it suitable for + batch verification in CLI scripts. + """ + try: + standard = get_standard_result(chat_template, deepcopy(messages), tools=tools, **template_kwargs) + pretokenized = simulate_pretokenized_path( + chat_template, deepcopy(messages), pretokenized_num_message, tools=tools, **template_kwargs + ) + if pretokenized != standard: + return VerifyResult( + case_name=case_name, passed=False, error=f"Pretokenized (N={pretokenized_num_message}) != standard" + ) + return VerifyResult(case_name=case_name, passed=True) + except ValueError as e: + return VerifyResult(case_name=case_name, passed=False, error=str(e)) + except Exception as e: + return VerifyResult(case_name=case_name, passed=False, error=f"{type(e).__name__}: {e}") + + +# --------------------------------------------------------------------------- +# Built-in test cases (shared between CLI and test suite) +# --------------------------------------------------------------------------- +# +# Trajectories expose two class attributes used for verify-layer filtering: +# +# * ``APPEND_ROLES: frozenset[str]`` — non-assistant roles that appear after +# the first assistant message. Drives ``--tito-allowed-append-roles``. +# * ``IS_THINKING: bool`` — any assistant carries ``reasoning_content``. +# Drives ``--thinking`` and whether ``enable_thinking`` kwarg is passed. +# +# Both are declared on the trajectory class (mock_trajectories.py), alongside +# ``TOOLS`` / ``PRETOKENIZE_POSITIONS`` / ``MESSAGES``. This file only lists +# which trajectories to exercise and expands them into concrete cases. + +import re # noqa: E402 + +from tito_gateway.vendor.miles_compat.utils.test_utils.mock_trajectories import ( # noqa: E402 + IntermediateSystemThinkingTrajectory, + IntermediateSystemTrajectory, + LongChainThinkingTrajectory, + LongChainTrajectory, + MultiRoleSequenceTrajectory, + MultiToolSingleTurnTrajectory, + MultiTurnNoToolThinkingTrajectory, + MultiTurnNoToolTrajectory, + MultiTurnThinkingTrajectory, + MultiTurnTrajectory, + MultiUserToolChainTrajectory, + MultiUserTurnThinkingTrajectory, + ParallelToolsTrajectory, + RetrySystemTrajectory, + SimpleNoToolTrajectory, + SingleToolThinkingTrajectory, + SingleToolTrajectory, +) + + +def _short_name(cls: type) -> str: + name = cls.__name__.replace("Trajectory", "") + return re.sub(r"(? list[CaseSpec]: + """Expand one trajectory into one CaseSpec per PRETOKENIZE_POSITIONS value.""" + short = _short_name(traj_cls) + return [ + CaseSpec( + case_name=f"{short}-N{n}", + traj_cls=traj_cls, + pretokenize_n=n, + tools=traj_cls.TOOLS, + append_roles=traj_cls.APPEND_ROLES, + is_thinking=traj_cls.IS_THINKING, + ) + for n in traj_cls.PRETOKENIZE_POSITIONS + ] + + +ALL_CASES: list[CaseSpec] = [c for t in _TRAJECTORIES for c in _expand(t)] + +THINKING_MODES: tuple[str, ...] = ("off", "on", "both") + + +def select_cases( + *, + allowed_append_roles: Iterable[str], + is_thinking: bool | None = None, +) -> list[CaseSpec]: + """Select trajectory cases by append-role surface and (optionally) thinking flag. + + A case is included iff ``case.append_roles`` is a subset of + *allowed_append_roles*, and (when *is_thinking* is not ``None``) + ``case.is_thinking`` matches. + + The caller is responsible for including ``"tool"`` in *allowed_append_roles* + when the session is tool-capable; this function does not silently union it. + """ + allowed = frozenset(allowed_append_roles) + out: list[CaseSpec] = [] + for c in ALL_CASES: + if not c.append_roles.issubset(allowed): + continue + if is_thinking is not None and c.is_thinking != is_thinking: + continue + out.append(c) + return out + + +def enable_thinking_variants(thinking: str) -> list[dict]: + """Return the list of ``enable_thinking`` kwarg variants to apply per case. + + * ``"off"`` → ``[{}]`` (no ``enable_thinking`` kwarg). + * ``"on"`` → ``[{"enable_thinking": True}]``. + * ``"both"`` → ``[{"enable_thinking": True}, {"enable_thinking": False}]``. + + Both CLI (:func:`run_all_checks`) and pytest parametrize callers use this + to avoid drifting in how the ``enable_thinking`` knob is exercised. + """ + if thinking == "off": + return [{}] + if thinking == "on": + return [{"enable_thinking": True}] + if thinking == "both": + return [{"enable_thinking": True}, {"enable_thinking": False}] + raise ValueError(f"thinking must be one of {THINKING_MODES}; got {thinking!r}") + + +def format_case_id(case: CaseSpec, kwargs: dict) -> str: + """Human-readable label for a ``(case, template_kwargs)`` tuple. + + Used for both CLI ``VerifyResult.case_name`` and pytest test ids so the + same tuple is identified the same way in both surfaces. Format: + + * empty kwargs → ``case.case_name``. + * otherwise → ``-_on/off-=val`` (keys sorted; + bool values emit ``key_on`` / ``key_off``; other values ``key=val``). + """ + if not kwargs: + return case.case_name + parts: list[str] = [] + for k, v in sorted(kwargs.items()): + if isinstance(v, bool): + parts.append(f"{k}_{'on' if v else 'off'}") + else: + parts.append(f"{k}={v}") + return f"{case.case_name}-{'-'.join(parts)}" + + +@dataclass +class CoverageReport: + """Coverage of cases across ``(is_thinking, append_roles \\ {tool})``. + + ``covered`` maps each combination to the case names that fall in it; + ``missing`` lists combinations with no case. ``tool`` is excluded from + the role axis because it is implicitly always allowed. + """ + + covered: dict[tuple[bool, tuple[str, ...]], list[str]] + missing: list[tuple[bool, tuple[str, ...]]] + + +def check_coverage( + cases: list[CaseSpec] | None = None, + *, + role_universe: set[str] | None = None, +) -> CoverageReport: + """Enumerate ``thinking × append-role-subset`` combinations and report gaps. + + Used as a sanity check that every meaningful combination of + ``--tito-allowed-append-roles`` and ``--thinking`` is backed by at least + one trajectory — otherwise certain CLI settings would be no-ops. + """ + if cases is None: + cases = ALL_CASES + if role_universe is None: + role_universe = {"user", "system"} + + from itertools import chain, combinations + + ordered_universe = sorted(role_universe) + all_subsets: list[tuple[str, ...]] = [ + tuple(sub) + for sub in chain.from_iterable(combinations(ordered_universe, r) for r in range(len(ordered_universe) + 1)) + ] + + covered: dict[tuple[bool, tuple[str, ...]], list[str]] = { + (is_thinking, sub): [] for is_thinking in (False, True) for sub in all_subsets + } + for c in cases: + roles_key = tuple(sorted(c.append_roles - {"tool"})) + key = (c.is_thinking, roles_key) + if key in covered: + covered[key].append(c.case_name) + + missing = [k for k, v in covered.items() if not v] + return CoverageReport(covered=covered, missing=missing) + + +def run_all_checks( + chat_template: str, + *, + allowed_append_roles: set[str] | frozenset[str] | None = None, + thinking: str = "off", + extra_template_kwargs: dict | None = None, +) -> list[VerifyResult]: + """Run verification cases filtered by *allowed_append_roles* and *thinking*. + + ``allowed_append_roles`` is the role surface the session may append after + an assistant turn; defaults to ``{"tool"}`` for the agentic baseline. + Trajectories whose ``append_roles`` are not a subset are skipped. Caller + must include ``"tool"`` explicitly when relevant — there is no implicit + union. + + ``thinking`` selects which ``enable_thinking`` variants are exercised — + see :func:`enable_thinking_variants`. When ``"both"``, **every** selected + trajectory (thinking or not) is rerun with ``enable_thinking=True`` and + ``enable_thinking=False``, so templates that branch on the flag are + validated against non-reasoning input too. + + ``extra_template_kwargs`` is merged into every invocation — use it to + thread template-specific kwargs (e.g. GLM's ``clear_thinking=False``) + through the CLI. + """ + if allowed_append_roles is None: + allowed_append_roles = {"tool"} + if thinking not in THINKING_MODES: + raise ValueError(f"thinking must be one of {THINKING_MODES}; got {thinking!r}") + extra = extra_template_kwargs or {} + + is_thinking_filter = {"off": False, "on": True, "both": None}[thinking] + selected = select_cases(allowed_append_roles=allowed_append_roles, is_thinking=is_thinking_filter) + variants = enable_thinking_variants(thinking) + + results: list[VerifyResult] = [] + for case in selected: + for variant in variants: + kwargs = {**variant, **extra} + results.append( + verify_append_only( + chat_template, + deepcopy(case.traj_cls.MESSAGES), + case.pretokenize_n, + tools=case.tools, + case_name=format_case_id(case, kwargs), + **kwargs, + ) + ) + + return results + + +# --------------------------------------------------------------------------- +# TITO-instance verification: decode-roundtrip equality +# --------------------------------------------------------------------------- +# +# The string-based primitive above asserts text-prefix at the chat-template +# layer. This is necessary but not sufficient for production correctness — +# production runs ``get_tito_tokenizer(...)`` and exercises ``merge_tokens`` +# (model-specific token-level boundary patches) plus +# ``tokenize_additional_non_assistant`` (renders appended segments under a +# synthetic ``[_DUMMY_SYSTEM, ...]`` context, not the real history). +# +# The primitive below mirrors the production path: it instantiates the actual +# TITO subclass + HF tokenizer, runs ``merge_tokens`` against the encoded +# prefix, decodes, and asserts text equality with the canonical full render. + + +def verify_append_only_via_tito_instance( + tito: TITOTokenizer, + tokenizer: Any, + messages: list[dict], + pretokenized_num_message: int, + tools: list[dict] | None = None, + case_name: str = "", + **template_kwargs, +) -> VerifyResult: + """Decode-roundtrip verify with a pre-built TITO instance. + + Asserts ``decode(tito.merge_tokens(prefix_msgs, full_msgs, encode(prefix_text))) + == full_text`` where ``prefix_text`` and ``full_text`` come from running the + chat template through ``tokenizer`` with the same kwargs ``tito`` was built + with. The test-only path (e.g. ``BuggyQwen3TITOTokenizer``) uses this + instance form directly; production-shape callers go through + :func:`verify_append_only_via_tito`. + """ + try: + # TITO's incremental path requires the appendix to be all non-assistant. + # From the pretokenized boundary N, greedily extend M forward over the + # maximal non-assistant run — that's the chunk production would call + # merge_tokens for (between two assistant generations, or up to end). + n = pretokenized_num_message + m = n + while m < len(messages) and messages[m].get("role") != "assistant": + m += 1 + if m == n: + return VerifyResult( + case_name=case_name, + passed=False, + error=( + f"Empty appendix at N={n}: messages[{n}] is assistant. " + "PRETOKENIZE_POSITIONS must land at a post-assistant boundary " + "where messages[N:] starts with a non-assistant turn." + ), + ) + + prefix_msgs = deepcopy(messages[:n]) + full_msgs = deepcopy(messages[:m]) + + prefix_text = tito.render_messages( + prefix_msgs, + tools=tools, + add_generation_prompt=False, + ) + full_text = tito.render_messages( + full_msgs, + tools=tools, + add_generation_prompt=True, + ) + + prefix_ids = tokenizer.encode(prefix_text, add_special_tokens=False) + # Simulate production's model-stop: in production, ``pretokenized_token_ids`` + # ends where the model actually stopped — typically before the trailing + # tokens the chat template would otherwise emit (Qwen's ``\n`` after + # ``<|im_end|>``, GLM's ambiguous ``<|user|>``/``<|observation|>`` boundary). + # The TITO subclass declares those as ``trailing_token_ids``. Trim them + # here so ``merge_tokens``'s boundary patches see the prefix in its + # production shape so the verifier sees the same prefix the + # subclass merge_tokens / trailing trim path operates on. + trailing = tito.trailing_token_ids + while prefix_ids and prefix_ids[-1] in trailing: + prefix_ids = prefix_ids[:-1] + merged_ids = tito.merge_tokens(prefix_msgs, full_msgs, prefix_ids, tools=tools) + merged_text = tokenizer.decode(merged_ids) + + if merged_text == full_text: + return VerifyResult(case_name=case_name, passed=True) + + # Find first divergence and quote ~60 chars of context on each side. + common_len = min(len(merged_text), len(full_text)) + diff_idx = next( + (i for i in range(common_len) if merged_text[i] != full_text[i]), + common_len, + ) + ctx_start = max(0, diff_idx - 60) + ctx_end = diff_idx + 60 + return VerifyResult( + case_name=case_name, + passed=False, + error=( + f"Decode-roundtrip mismatch (N={pretokenized_num_message}) at char {diff_idx}\n" + f" expected: ...{full_text[ctx_start:ctx_end]!r}...\n" + f" actual: ...{merged_text[ctx_start:ctx_end]!r}..." + ), + ) + except Exception as e: + return VerifyResult(case_name=case_name, passed=False, error=f"{type(e).__name__}: {e}") + + +def verify_append_only_via_tito( + tokenizer: Any, + tito_model: TITOTokenizerType | str, + allowed_append_roles: list[str], + messages: list[dict], + pretokenized_num_message: int, + tools: list[dict] | None = None, + case_name: str = "", + **template_kwargs, +) -> VerifyResult: + """Decode-roundtrip verify, building TITO from the registered family. + + Matches the production wiring at ``miles/rollout/session/sessions.py:35`` — + the same ``get_tito_tokenizer`` factory call, with ``chat_template_kwargs`` + threaded through so ``merge_tokens`` and the dummy-context segment renders + use the same kwargs as the reference full render. + """ + from tito_gateway.vendor.miles_compat.utils.chat_template_utils import get_tito_tokenizer + + tito = get_tito_tokenizer( + tokenizer, + tokenizer_type=tito_model, + chat_template_kwargs=dict(template_kwargs), + allowed_append_roles=list(allowed_append_roles), + ) + return verify_append_only_via_tito_instance( + tito, + tokenizer, + messages, + pretokenized_num_message, + tools=tools, + case_name=case_name, + **template_kwargs, + ) + + +def run_all_checks_via_tito( + tokenizer: Any, + tito_model: TITOTokenizerType | str, + *, + allowed_append_roles: set[str] | frozenset[str] | None = None, + thinking: str = "off", + extra_template_kwargs: dict | None = None, +) -> list[VerifyResult]: + """Same shape as :func:`run_all_checks` but routes through TITO + tokenizer. + + Per-case TITO rebuild: each (case, ``enable_thinking`` variant) gets a fresh + TITO instance constructed with the merged kwargs, so the dummy-context + segment renders inside ``tokenize_additional_non_assistant`` see the same + ``enable_thinking`` value as the reference render. Construction is + millisecond-level and runs ~50 times per CLI invocation; cheap. + + The caller is responsible for setting ``tokenizer.chat_template`` (e.g. via + ``resolve_fixed_chat_template`` lookup or ``--template`` override) before + calling this — this function does not consult ``SUPPORTED_TEMPLATES``. + """ + if allowed_append_roles is None: + allowed_append_roles = {"tool"} + if thinking not in THINKING_MODES: + raise ValueError(f"thinking must be one of {THINKING_MODES}; got {thinking!r}") + extra = extra_template_kwargs or {} + + is_thinking_filter = {"off": False, "on": True, "both": None}[thinking] + selected = select_cases(allowed_append_roles=allowed_append_roles, is_thinking=is_thinking_filter) + variants = enable_thinking_variants(thinking) + roles_list = sorted(allowed_append_roles) + + results: list[VerifyResult] = [] + for case in selected: + # TITO incremental requires a non-empty non-assistant appendix at the + # boundary. Trajectories that end at the assistant turn (e.g. plain + # ``[sys, user, assistant]``) have no appendix to verify and are + # silently skipped here — the string-based primitive still covers + # them at the text-prefix layer. + msgs = case.traj_cls.MESSAGES + n = case.pretokenize_n + if n >= len(msgs) or msgs[n].get("role") == "assistant": + continue + for variant in variants: + kwargs = {**variant, **extra} + results.append( + verify_append_only_via_tito( + tokenizer, + tito_model, + roles_list, + deepcopy(case.traj_cls.MESSAGES), + case.pretokenize_n, + tools=case.tools, + case_name=format_case_id(case, kwargs), + **kwargs, + ) + ) + + return results diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/mock_sglang_server.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/mock_sglang_server.py new file mode 100644 index 0000000..294aa41 --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/mock_sglang_server.py @@ -0,0 +1,270 @@ +import asyncio +import re +import time +import uuid +from collections.abc import Callable +from contextlib import contextmanager +from dataclasses import asdict, dataclass + +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse +from pydantic import TypeAdapter +from sglang.srt.entrypoints.openai.protocol import Tool +from sglang.srt.function_call.function_call_parser import FunctionCallParser + +from miles.utils.http_utils import find_available_port +from miles.utils.processing_utils import load_tokenizer +from miles.utils.test_utils.uvicorn_thread_server import UvicornThreadServer + + +@dataclass(frozen=True) +class ProcessResultMetaInfo: + weight_version: str | None = None + routed_experts: str | None = None + spec_accept_token_num: int | None = None + spec_draft_token_num: int | None = None + spec_verify_ct: int | None = None + + def to_dict(self) -> dict: + return {k: v for k, v in asdict(self).items() if v is not None} + + +@dataclass(frozen=True) +class ProcessResult: + text: str + finish_reason: str = "stop" + cached_tokens: int = 0 + meta_info: ProcessResultMetaInfo = ProcessResultMetaInfo() + + +ProcessFn = Callable[[str], ProcessResult] + + +class MockSGLangServer: + def __init__( + self, + model_name: str, + process_fn: ProcessFn, + host: str, + port: int, + latency: float = 0.0, + chat_template_path: str | None = None, + ): + self.tokenizer = load_tokenizer(model_name, chat_template_path=chat_template_path, trust_remote_code=True) + self.process_fn = process_fn + self.host = host + self.port = port or find_available_port(30000) + self.latency = latency + + self.app = FastAPI() + self._server: UvicornThreadServer | None = None + + self.request_log: list[dict] = [] + self._concurrency = Counter() + + self._setup_routes() + + @property + def max_concurrent(self) -> int: + return self._concurrency.max_value + + def reset_stats(self): + self.request_log.clear() + self._concurrency.reset() + + def start(self): + self._server = UvicornThreadServer(self.app, host=self.host, port=self.port) + self._server.start() + + def stop(self): + if self._server is not None: + self._server.stop() + + @property + def url(self) -> str: + return f"http://{self.host}:{self.port}" + + def _setup_routes(self): + @self.app.post("/generate") + async def generate(request: Request): + return await self._handle_generate_like_request(request, self._compute_generate_response) + + @self.app.post("/v1/chat/completions") + async def chat_completions(request: Request): + return await self._handle_generate_like_request(request, self._compute_chat_completions_response) + + @self.app.get("/health") + async def health(): + return JSONResponse(content={"status": "ok"}) + + @self.app.post("/abort_request") + async def abort_request(_request: Request): + return JSONResponse(content={"status": "ok"}) + + async def _handle_generate_like_request(self, request: Request, compute_fn: Callable[[dict], dict]): + payload = await request.json() + self.request_log.append(payload) + with self._concurrency.track(): + if self.latency > 0: + await asyncio.sleep(self.latency) + response = compute_fn(payload) + return JSONResponse(content=response) + + def _compute_generate_response(self, payload: dict) -> dict: + assert payload.get("return_logprob", True) is True, "MockSGLangServer requires return_logprob=True" + input_ids = payload.get("input_ids", []) + + prompt_str = self.tokenizer.decode(input_ids, skip_special_tokens=False) + process_result = self.process_fn(prompt_str) + output_ids = self.tokenizer.encode(process_result.text, add_special_tokens=False) + + prompt_tokens = len(input_ids) + completion_tokens = len(output_ids) + + finish_reason_dict = {"type": process_result.finish_reason} + if process_result.finish_reason == "length": + finish_reason_dict["length"] = completion_tokens + + output_token_logprobs = [(-1 / 128 * i, token_id) for i, token_id in enumerate(output_ids)] + + meta_info = { + "finish_reason": finish_reason_dict, + "prompt_tokens": prompt_tokens, + "cached_tokens": process_result.cached_tokens, + "completion_tokens": completion_tokens, + "output_token_logprobs": output_token_logprobs, + **process_result.meta_info.to_dict(), + } + + return {"text": process_result.text, "meta_info": meta_info} + + def _compute_chat_completions_response(self, payload: dict) -> dict: + messages = payload.get("messages", []) + tools = payload.get("tools") + + prompt_str = self.tokenizer.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True, tools=tools + ) + + prompt_ids = None + if payload.get("return_prompt_token_ids"): + input_ids = payload.get("input_ids") + if input_ids is not None: + prompt_ids = list(input_ids) + else: + prompt_ids = self.tokenizer.encode(prompt_str, add_special_tokens=False) + + process_result = self.process_fn(prompt_str) + output_ids = self.tokenizer.encode(process_result.text, add_special_tokens=False) + + logprobs_content = [ + { + "token": self.tokenizer.convert_ids_to_tokens(tid), + "token_id": tid, + "logprob": -1 / 128 * i, + } + for i, tid in enumerate(output_ids) + ] + + finish_reason = process_result.finish_reason + tool_calls = None + if tools and finish_reason == "stop": + parser = FunctionCallParser( + tools=TypeAdapter(list[Tool]).validate_python(tools), + tool_call_parser="qwen25", + ) + message_content, parsed_calls = parser.parse_non_stream(process_result.text) + if parsed_calls: + finish_reason = "tool_calls" + tool_calls = [ + { + "id": f"call{i:05d}", + "type": "function", + "function": {"name": call.name, "arguments": call.parameters or "{}"}, + } + for i, call in enumerate(parsed_calls) + ] + else: + message_content = process_result.text + + output_token_logprobs = [(-1 / 128 * i, tid) for i, tid in enumerate(output_ids)] + + choice = { + "index": 0, + "message": { + "role": "assistant", + "content": message_content, + "tool_calls": tool_calls, + }, + "logprobs": {"content": logprobs_content}, + "finish_reason": finish_reason, + "meta_info": { + "output_token_logprobs": output_token_logprobs, + "completion_tokens": len(output_ids), + **process_result.meta_info.to_dict(), + }, + } + if prompt_ids is not None: + choice["prompt_token_ids"] = prompt_ids + + return { + "id": f"chatcmpl-{uuid.uuid4().hex[:8]}", + "object": "chat.completion", + "created": int(time.time()), + "model": "mock-model", + "choices": [choice], + } + + +class Counter: + def __init__(self): + self._current = 0 + self._max = 0 + + @property + def max_value(self) -> int: + return self._max + + def reset(self): + self._current = 0 + self._max = 0 + + @contextmanager + def track(self): + self._current += 1 + self._max = max(self._max, self._current) + try: + yield + finally: + self._current -= 1 + + +def default_process_fn(prompt: str) -> ProcessResult: + match = re.search(r"What is 1\+(\d+)\?", prompt) + if match: + num = int(match.group(1)) + ans = 1 + num + return ProcessResult(text=f"\\boxed{{{ans}}}", finish_reason="stop") + return ProcessResult(text="I don't understand.", finish_reason="stop") + + +@contextmanager +def with_mock_server( + model_name: str = "Qwen/Qwen3-0.6B", + process_fn: ProcessFn = default_process_fn, + host: str = "127.0.0.1", + port: int | None = None, + latency: float = 0.0, +): + server = MockSGLangServer( + model_name=model_name, + process_fn=process_fn, + host=host, + port=port, + latency=latency, + ) + try: + server.start() + yield server + finally: + server.stop() diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/mock_trajectories.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/mock_trajectories.py new file mode 100644 index 0000000..a077fcc --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/mock_trajectories.py @@ -0,0 +1,1198 @@ +"""Multi-turn trajectory definitions for testing. + +Each trajectory class defines a complete multi-turn conversation with tool calls. +Used by: +- tests/fast/rollout/generate_hub/test_pretokenized_chat.py (chat template verification) +- tests/fast/router/test_session_pretokenized_e2e.py (session proxy e2e) + +Class attributes consumed by chat_template_verify: + +- ``APPEND_ROLES: frozenset[str]`` — non-assistant roles that appear *after* + the first assistant message (``tool`` / ``user`` / ``system``). These are + the roles the session must allow to be appended on top of an assistant- + stopped prefix; drives ``--tito-allowed-append-roles`` filtering. +- ``IS_THINKING: bool`` — ``True`` iff at least one assistant message carries + ``reasoning_content``. Drives ``--thinking`` filtering and whether the + ``enable_thinking`` chat-template kwarg is passed. + +Both are declared explicitly on each class so readers can see a trajectory's +verify-layer classification without having to execute the module. +""" + +from __future__ import annotations + +from copy import deepcopy +from dataclasses import dataclass +from typing import Any + +from tito_gateway.vendor.miles_compat.utils.test_utils.mock_sglang_server import ProcessFn, ProcessResult + +# --------------------------------------------------------------------------- +# Shared tool definitions +# --------------------------------------------------------------------------- + +WEATHER_TOOLS = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather information for a location", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string", "description": "City name"}, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "Temperature unit", + }, + }, + "required": ["city"], + }, + }, + } +] + +DATE_TOOL = { + "type": "function", + "function": { + "name": "get_date", + "description": "Get the current date and time for a timezone", + "parameters": { + "type": "object", + "properties": { + "timezone": { + "type": "string", + "description": "Timezone name (e.g. Asia/Shanghai, UTC)", + }, + }, + "required": ["timezone"], + }, + }, +} + +ALL_TOOLS = WEATHER_TOOLS + [DATE_TOOL] + + +# --------------------------------------------------------------------------- +# Data structures +# --------------------------------------------------------------------------- + + +@dataclass +class Turn: + """One turn in a multi-turn trajectory.""" + + request_messages: list[dict[str, Any]] + assistant_message: dict[str, Any] + response_text: str = "" # raw text returned by process_fn (computed by build_trajectory) + + +@dataclass +class Trajectory: + """A fully resolved multi-turn trajectory ready for testing.""" + + tools: list[dict[str, Any]] | None + turns: list[Turn] + full_messages: list[dict[str, Any]] + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def last_user_index(messages: list[dict[str, Any]]) -> int: + """Return the index of the last user message in *messages*.""" + for i in range(len(messages) - 1, -1, -1): + if messages[i]["role"] == "user": + return i + raise ValueError("No user message found") + + +# --------------------------------------------------------------------------- +# Trajectory classes (by scenario) +# --------------------------------------------------------------------------- + + +class SingleToolTrajectory: + """sys, user, assistant(tool_call), tool — 1 turn""" + + TOOLS = WEATHER_TOOLS + PRETOKENIZE_POSITIONS = [3] + APPEND_ROLES = frozenset({"tool"}) + IS_THINKING = False + MESSAGES = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What's the weather in Beijing?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "Beijing", "unit": "celsius"}, + }, + } + ], + }, + { + "role": "tool", + "content": '{"temperature": 25, "condition": "sunny"}', + "tool_call_id": "call_1", + }, + ] + + +class MultiTurnTrajectory: + """sys, user, ass(tool), tool, ass(tool), tool — 2 turns""" + + TOOLS = WEATHER_TOOLS + PRETOKENIZE_POSITIONS = [3, 5] + APPEND_ROLES = frozenset({"tool"}) + IS_THINKING = False + MESSAGES = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What's the weather in Beijing and Shanghai?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "Beijing"}, + }, + } + ], + }, + { + "role": "tool", + "content": '{"temperature": 25, "condition": "sunny"}', + "tool_call_id": "call_1", + }, + { + "role": "assistant", + "content": "Beijing is 25C. Let me check Shanghai.", + "tool_calls": [ + { + "id": "call_2", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "Shanghai"}, + }, + } + ], + }, + { + "role": "tool", + "content": '{"temperature": 30, "condition": "cloudy"}', + "tool_call_id": "call_2", + }, + ] + + +class MultiToolSingleTurnTrajectory: + """sys, user, assistant(2 tool_calls: weather+date), tool, tool — 1 turn""" + + TOOLS = ALL_TOOLS + PRETOKENIZE_POSITIONS = [3] + APPEND_ROLES = frozenset({"tool"}) + IS_THINKING = False + MESSAGES = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What's the weather in Beijing and what date is it?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "Beijing"}, + }, + }, + { + "id": "call_2", + "type": "function", + "function": { + "name": "get_date", + "arguments": {"timezone": "Asia/Shanghai"}, + }, + }, + ], + }, + { + "role": "tool", + "content": '{"temperature": 25}', + "tool_call_id": "call_1", + }, + { + "role": "tool", + "content": '{"date": "2025-03-15", "time": "14:30:00"}', + "tool_call_id": "call_2", + }, + ] + + +class ParallelToolsTrajectory: + """sys, user, assistant(3 parallel tool_calls), tool, tool, tool — 1 turn""" + + TOOLS = WEATHER_TOOLS + PRETOKENIZE_POSITIONS = [3] + APPEND_ROLES = frozenset({"tool"}) + IS_THINKING = False + MESSAGES = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Compare weather in Beijing, Shanghai, and Guangzhou"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "Beijing"}, + }, + }, + { + "id": "call_2", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "Shanghai"}, + }, + }, + { + "id": "call_3", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "Guangzhou"}, + }, + }, + ], + }, + { + "role": "tool", + "content": '{"temperature": 25, "condition": "sunny"}', + "tool_call_id": "call_1", + }, + { + "role": "tool", + "content": '{"temperature": 30, "condition": "cloudy"}', + "tool_call_id": "call_2", + }, + { + "role": "tool", + "content": '{"temperature": 35, "condition": "rainy"}', + "tool_call_id": "call_3", + }, + ] + + +class LongChainTrajectory: + """sys, user, ass(tool), tool, ass(tool:date), tool, ass(tool), tool — 3 turns""" + + TOOLS = ALL_TOOLS + PRETOKENIZE_POSITIONS = [3, 5, 7] + APPEND_ROLES = frozenset({"tool"}) + IS_THINKING = False + MESSAGES = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Do a multi-step task"}, + { + "role": "assistant", + "content": "Step 1", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "Beijing"}, + }, + } + ], + }, + { + "role": "tool", + "content": '{"temperature": 25}', + "tool_call_id": "call_1", + }, + { + "role": "assistant", + "content": "Step 2", + "tool_calls": [ + { + "id": "call_2", + "type": "function", + "function": { + "name": "get_date", + "arguments": {"timezone": "UTC"}, + }, + } + ], + }, + { + "role": "tool", + "content": '{"date": "2025-03-15", "time": "12:00:00"}', + "tool_call_id": "call_2", + }, + { + "role": "assistant", + "content": "Step 3", + "tool_calls": [ + { + "id": "call_3", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "Guangzhou"}, + }, + } + ], + }, + { + "role": "tool", + "content": '{"temperature": 35}', + "tool_call_id": "call_3", + }, + ] + + +class RetrySystemTrajectory: + """sys, user, ass(tool), tool, system_retry, ass(tool), tool — 2 turns with mid-conversation system message. + + Simulates an agent that injects a system-level retry prompt when the model + fails to produce a useful tool call on the first attempt. + """ + + TOOLS = WEATHER_TOOLS + PRETOKENIZE_POSITIONS = [3, 6] + APPEND_ROLES = frozenset({"tool", "system"}) + IS_THINKING = False + MESSAGES = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What's the weather in Beijing and Shanghai?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "Beijing"}, + }, + } + ], + }, + { + "role": "tool", + "content": '{"temperature": 25, "condition": "sunny"}', + "tool_call_id": "call_1", + }, + {"role": "system", "content": "You still need to check Shanghai. Please call get_weather for Shanghai."}, + { + "role": "assistant", + "content": "Let me check Shanghai.", + "tool_calls": [ + { + "id": "call_2", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "Shanghai"}, + }, + } + ], + }, + { + "role": "tool", + "content": '{"temperature": 30, "condition": "cloudy"}', + "tool_call_id": "call_2", + }, + ] + + +class MultiUserToolChainTrajectory: + """sys, user1, ass(tool), tool, ass, user2, ass(tool), tool, ass(tool:date), tool + + NOTE: LinearTrajectory can carry multiple user messages when + ``allowed_append_roles`` includes ``"user"``; this trajectory exercises + that path. The distribution may still deviate from the chat template + behavior, causing high tito_session_mismatch_rate. + """ + + TOOLS = ALL_TOOLS + PRETOKENIZE_POSITIONS = [3, 5, 7, 9] + APPEND_ROLES = frozenset({"tool", "user"}) + IS_THINKING = False + MESSAGES = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What's the weather in Beijing?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "Beijing"}, + }, + } + ], + }, + { + "role": "tool", + "content": '{"temperature": 25, "condition": "sunny"}', + "tool_call_id": "call_1", + }, + { + "role": "assistant", + "content": "Beijing is 25C and sunny.", + }, + {"role": "user", "content": "Now check Shanghai and what date is it?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_2", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "Shanghai"}, + }, + } + ], + }, + { + "role": "tool", + "content": '{"temperature": 30, "condition": "cloudy"}', + "tool_call_id": "call_2", + }, + { + "role": "assistant", + "content": "Shanghai is 30C. Let me check the date.", + "tool_calls": [ + { + "id": "call_3", + "type": "function", + "function": { + "name": "get_date", + "arguments": {"timezone": "Asia/Shanghai"}, + }, + } + ], + }, + { + "role": "tool", + "content": '{"date": "2025-03-15", "time": "22:30:00"}', + "tool_call_id": "call_3", + }, + ] + + +class SimpleNoToolTrajectory: + """sys, user, asst, system_reminder, user2, asst2 — no tools, with system/user append boundaries. + + Codifies the synthetic 'single_system' case the old CI used to manually + append: at N=3 prefix ends at the first asst, append starts with the + system_reminder, exercising the system-append boundary on a no-tool model. + """ + + TOOLS = None + PRETOKENIZE_POSITIONS = [3, 4, 5, 6] + APPEND_ROLES = frozenset({"user", "system"}) + IS_THINKING = False + MESSAGES = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello!"}, + {"role": "assistant", "content": "Hi there! How can I help?"}, + {"role": "system", "content": "Please answer in one short sentence."}, + {"role": "user", "content": "What's 2+2?"}, + {"role": "assistant", "content": "Four."}, + ] + + +class MultiTurnNoToolTrajectory: + """sys, user, assistant, user (no tools) — multi-turn plain conversation""" + + TOOLS = None + PRETOKENIZE_POSITIONS = [3, 5] + APPEND_ROLES = frozenset({"user"}) + IS_THINKING = False + MESSAGES = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is the capital of France?"}, + {"role": "assistant", "content": "The capital of France is Paris."}, + {"role": "user", "content": "And what about Germany?"}, + ] + + +class MultiTurnNoToolThinkingTrajectory: + """sys, user, assistant(reasoning_content), user (no tools) — multi-turn with thinking""" + + TOOLS = None + PRETOKENIZE_POSITIONS = [3, 5] + APPEND_ROLES = frozenset({"user"}) + IS_THINKING = True + MESSAGES = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is the capital of France?"}, + { + "role": "assistant", + "reasoning_content": "The user is asking about geography. The capital of France is Paris.", + "content": "The capital of France is Paris.", + }, + {"role": "user", "content": "And what about Germany?"}, + ] + + +# --------------------------------------------------------------------------- +# Thinking variants +# --------------------------------------------------------------------------- + + +class SingleToolThinkingTrajectory: + """sys, user, ass(think+tool), tool, user2, ass(think+tool), tool, user3, ass — multi-role alternating with thinking. + + Codifies the synthetic 'alternating_user_tool' case the old CI used to + manually append: prefix cuts at N=4/N=7 exercise the user-after-tool + boundary on a thinking model. + """ + + TOOLS = WEATHER_TOOLS + PRETOKENIZE_POSITIONS = [3, 4, 5, 6, 7, 8] + APPEND_ROLES = frozenset({"tool", "user"}) + IS_THINKING = True + MESSAGES = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What's the weather in Beijing?"}, + { + "role": "assistant", + "reasoning_content": "The user wants to know the weather in Beijing. I should call the get_weather function.", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "Beijing", "unit": "celsius"}, + }, + } + ], + }, + { + "role": "tool", + "content": '{"temperature": 25, "condition": "sunny"}', + "tool_call_id": "call_1", + }, + {"role": "user", "content": "Now check Shanghai too."}, + { + "role": "assistant", + "reasoning_content": "Now the user wants Shanghai's weather. Calling get_weather again.", + "content": None, + "tool_calls": [ + { + "id": "call_2", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "Shanghai", "unit": "celsius"}, + }, + } + ], + }, + { + "role": "tool", + "content": '{"temperature": 30, "condition": "cloudy"}', + "tool_call_id": "call_2", + }, + {"role": "user", "content": "And tell me the date as well."}, + { + "role": "assistant", + "reasoning_content": "The user is asking for the date. I'll answer based on what I know.", + "content": "Beijing is 25°C and sunny; Shanghai is 30°C and cloudy. I don't have access to the current date.", + }, + ] + + +class MultiTurnThinkingTrajectory: + """sys, user, ass(thinking+tool), tool, ass(thinking+tool), tool — 2 turns""" + + TOOLS = WEATHER_TOOLS + PRETOKENIZE_POSITIONS = [3, 5] + APPEND_ROLES = frozenset({"tool"}) + IS_THINKING = True + MESSAGES = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What's the weather in Beijing and Shanghai?"}, + { + "role": "assistant", + "reasoning_content": "Let me check Beijing first.", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "Beijing"}, + }, + } + ], + }, + { + "role": "tool", + "content": '{"temperature": 25, "condition": "sunny"}', + "tool_call_id": "call_1", + }, + { + "role": "assistant", + "reasoning_content": "Beijing is 25C. Now let me check Shanghai.", + "content": "Beijing is 25C. Let me check Shanghai.", + "tool_calls": [ + { + "id": "call_2", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "Shanghai"}, + }, + } + ], + }, + { + "role": "tool", + "content": '{"temperature": 30, "condition": "cloudy"}', + "tool_call_id": "call_2", + }, + ] + + +class LongChainThinkingTrajectory: + """sys, user, ass(thinking+tool), tool, ass(thinking+tool), tool, ass(thinking+tool), tool — 3 turns""" + + TOOLS = ALL_TOOLS + PRETOKENIZE_POSITIONS = [3, 5, 7] + APPEND_ROLES = frozenset({"tool"}) + IS_THINKING = True + MESSAGES = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Do a multi-step task"}, + { + "role": "assistant", + "reasoning_content": "Starting step 1, checking Beijing weather.", + "content": "Step 1", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "Beijing"}, + }, + } + ], + }, + { + "role": "tool", + "content": '{"temperature": 25}', + "tool_call_id": "call_1", + }, + { + "role": "assistant", + "reasoning_content": "Got Beijing result. Now step 2, checking the date.", + "content": "Step 2", + "tool_calls": [ + { + "id": "call_2", + "type": "function", + "function": { + "name": "get_date", + "arguments": {"timezone": "UTC"}, + }, + } + ], + }, + { + "role": "tool", + "content": '{"date": "2025-03-15", "time": "12:00:00"}', + "tool_call_id": "call_2", + }, + { + "role": "assistant", + "reasoning_content": "Got date result. Now step 3, checking Guangzhou.", + "content": "Step 3", + "tool_calls": [ + { + "id": "call_3", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "Guangzhou"}, + }, + } + ], + }, + { + "role": "tool", + "content": '{"temperature": 35}', + "tool_call_id": "call_3", + }, + ] + + +class MultiUserTurnThinkingTrajectory: + """sys, user1, ass(thinking+tool), tool, ass(thinking), user2, ass(thinking+tool), tool + + Cross-user-turn with thinking. Tests that thinking content from user turn 1 + is not compressed/modified when rendering the full conversation including + user turn 2. + """ + + TOOLS = WEATHER_TOOLS + PRETOKENIZE_POSITIONS = [7] + APPEND_ROLES = frozenset({"tool", "user"}) + IS_THINKING = True + MESSAGES = [ + {"role": "system", "content": "You are a helpful assistant."}, + # --- user turn 1 --- + {"role": "user", "content": "What's the weather in Beijing?"}, + { + "role": "assistant", + "reasoning_content": "User wants Beijing weather, let me check.", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "Beijing"}, + }, + } + ], + }, + { + "role": "tool", + "content": '{"temperature": 25, "condition": "sunny"}', + "tool_call_id": "call_1", + }, + { + "role": "assistant", + "reasoning_content": "Beijing is 25C and sunny. I should tell the user.", + "content": "Beijing is 25°C and sunny!", + }, + # --- user turn 2 --- + {"role": "user", "content": "Now check Shanghai too."}, + { + "role": "assistant", + "reasoning_content": "User wants Shanghai weather now. Let me call the tool.", + "content": None, + "tool_calls": [ + { + "id": "call_2", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "Shanghai"}, + }, + } + ], + }, + { + "role": "tool", + "content": '{"temperature": 30, "condition": "cloudy"}', + "tool_call_id": "call_2", + }, + ] + + +# --------------------------------------------------------------------------- +# Intermediate system message variants +# --------------------------------------------------------------------------- + + +class IntermediateSystemTrajectory: + """sys, user, ass(tool), tool, system, ass(tool:date), tool, system, ass(tool), tool — 3 turns with system""" + + TOOLS = ALL_TOOLS + PRETOKENIZE_POSITIONS = [3, 6, 9] + APPEND_ROLES = frozenset({"tool", "system"}) + IS_THINKING = False + MESSAGES = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Do a multi-step task"}, + { + "role": "assistant", + "content": "Step 1", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "Beijing"}, + }, + } + ], + }, + { + "role": "tool", + "content": '{"temperature": 25}', + "tool_call_id": "call_1", + }, + {"role": "system", "content": "Step 1 complete. Proceed to step 2."}, + { + "role": "assistant", + "content": "Step 2", + "tool_calls": [ + { + "id": "call_2", + "type": "function", + "function": { + "name": "get_date", + "arguments": {"timezone": "UTC"}, + }, + } + ], + }, + { + "role": "tool", + "content": '{"date": "2025-03-15", "time": "12:00:00"}', + "tool_call_id": "call_2", + }, + {"role": "system", "content": "Step 2 complete. Proceed to step 3."}, + { + "role": "assistant", + "content": "Step 3", + "tool_calls": [ + { + "id": "call_3", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "Guangzhou"}, + }, + } + ], + }, + { + "role": "tool", + "content": '{"temperature": 35}', + "tool_call_id": "call_3", + }, + ] + + +class IntermediateSystemThinkingTrajectory: + """sys, user, ass(t+tool), tool, system, ass(t+tool:date), tool, system, ass(t+tool), tool — 3 turns with system+thinking""" + + TOOLS = ALL_TOOLS + PRETOKENIZE_POSITIONS = [3, 6, 9] + APPEND_ROLES = frozenset({"tool", "system"}) + IS_THINKING = True + MESSAGES = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Do a multi-step task"}, + { + "role": "assistant", + "reasoning_content": "Starting step 1, checking Beijing weather.", + "content": "Step 1", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "Beijing"}, + }, + } + ], + }, + { + "role": "tool", + "content": '{"temperature": 25}', + "tool_call_id": "call_1", + }, + {"role": "system", "content": "Step 1 complete. Proceed to step 2."}, + { + "role": "assistant", + "reasoning_content": "Got Beijing result. Now step 2, checking the date.", + "content": "Step 2", + "tool_calls": [ + { + "id": "call_2", + "type": "function", + "function": { + "name": "get_date", + "arguments": {"timezone": "UTC"}, + }, + } + ], + }, + { + "role": "tool", + "content": '{"date": "2025-03-15", "time": "12:00:00"}', + "tool_call_id": "call_2", + }, + {"role": "system", "content": "Step 2 complete. Proceed to step 3."}, + { + "role": "assistant", + "reasoning_content": "Got date result. Now step 3, checking Guangzhou.", + "content": "Step 3", + "tool_calls": [ + { + "id": "call_3", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "Guangzhou"}, + }, + } + ], + }, + { + "role": "tool", + "content": '{"temperature": 35}', + "tool_call_id": "call_3", + }, + ] + + +class MultiRoleSequenceTrajectory: + """sys, user, asst+tool, tool, user2, asst+tool, system_reminder, tool, asst-final. + + Fills the {thinking=False, append_roles={tool, user, system}} matrix cell + that GLM47's tool+user+system SUPPORTED_TEMPLATES row otherwise has no + fixture for. Cuts exercise three boundaries: tool-append (N=3), user-after-tool + (N=4), system-after-asst (N=6). Cuts at N=5/N=7/N=8 are intentionally not + listed — they only exercise generation-prompt-only or repeat tool-append + which other trajectories already cover. + """ + + TOOLS = ALL_TOOLS + PRETOKENIZE_POSITIONS = [3, 4, 6] + APPEND_ROLES = frozenset({"tool", "user", "system"}) + IS_THINKING = False + MESSAGES = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What's the weather in Beijing today?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_w1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "Beijing", "unit": "celsius"}, + }, + } + ], + }, + { + "role": "tool", + "content": '{"temperature": 22, "condition": "sunny"}', + "tool_call_id": "call_w1", + }, + {"role": "user", "content": "Also tell me the date in Beijing."}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_d1", + "type": "function", + "function": { + "name": "get_date", + "arguments": {"timezone": "Asia/Shanghai"}, + }, + } + ], + }, + {"role": "system", "content": "Please answer in one short sentence."}, + { + "role": "tool", + "content": '{"date": "2026-04-28"}', + "tool_call_id": "call_d1", + }, + { + "role": "assistant", + "content": "Beijing is 22°C and sunny on 2026-04-28.", + }, + ] + + +# --------------------------------------------------------------------------- +# Helpers: build Trajectory and process_fn from a trajectory class +# --------------------------------------------------------------------------- + + +def _split_turns(messages: list[dict[str, Any]]) -> list[tuple[list[dict], dict]]: + """Split a full message sequence into (request_messages, assistant_message) pairs. + + Each assistant message marks the end of one turn. + """ + turns: list[tuple[list[dict], dict]] = [] + accumulated: list[dict] = [] + + i = 0 + # Collect non-assistant prefix (system, user) + while i < len(messages) and messages[i]["role"] != "assistant": + accumulated.append(messages[i]) + i += 1 + + while i < len(messages): + assert messages[i]["role"] == "assistant", f"Expected assistant at index {i}, got {messages[i]['role']}" + assistant_msg = messages[i] + turns.append((list(accumulated), assistant_msg)) + accumulated.append(assistant_msg) + i += 1 + + # Collect subsequent tool (or user for multi-user) messages + while i < len(messages) and messages[i]["role"] != "assistant": + accumulated.append(messages[i]) + i += 1 + + return turns + + +def build_trajectory( + tokenizer: Any, + trajectory_cls: type, + chat_template: str | None = None, +) -> Trajectory: + """Build a Trajectory from a trajectory class. + + Splits messages into turns and computes response_text for each turn + using the diff method: render(msgs + [assistant], gen_prompt=False) - render(msgs, gen_prompt=True). + + Args: + tokenizer: HuggingFace tokenizer. + trajectory_cls: A trajectory class with MESSAGES and TOOLS attributes. + chat_template: Optional chat template string. If None, uses tokenizer's default. + """ + messages = deepcopy(trajectory_cls.MESSAGES) + tools = trajectory_cls.TOOLS + + raw_turns = _split_turns(messages) + + def _render(msgs: list[dict], add_generation_prompt: bool) -> str: + if chat_template is not None: + from tito_gateway.vendor.miles_compat.utils.chat_template_utils.template import apply_chat_template_from_str + + return apply_chat_template_from_str( + chat_template, msgs, add_generation_prompt=add_generation_prompt, tools=tools + ) + return tokenizer.apply_chat_template( + msgs, tokenize=False, add_generation_prompt=add_generation_prompt, tools=tools + ) + + turns: list[Turn] = [] + for request_msgs, assistant_msg in raw_turns: + prompt_text = _render(request_msgs, add_generation_prompt=True) + with_assistant_text = _render(request_msgs + [assistant_msg], add_generation_prompt=False) + assert with_assistant_text.startswith(prompt_text), ( + f"Assistant text does not extend prompt text.\n" + f"prompt[-100:]: {prompt_text[-100:]!r}\n" + f"with_ass[:len(prompt)+50]: {with_assistant_text[:len(prompt_text)+50]!r}" + ) + response_text = with_assistant_text[len(prompt_text) :] + + turns.append( + Turn( + request_messages=request_msgs, + assistant_message=assistant_msg, + response_text=response_text, + ) + ) + + return Trajectory(tools=tools, turns=turns, full_messages=messages) + + +def build_process_fn( + trajectory: Trajectory, + tokenizer: Any, + chat_template: str | None = None, +) -> ProcessFn: + """Build a process_fn that maps rendered prompt strings to response texts. + + Pre-computes the exact prompt string for each turn and returns the + corresponding response_text. + """ + tools = trajectory.tools + + def _render(msgs: list[dict]) -> str: + if chat_template is not None: + from tito_gateway.vendor.miles_compat.utils.chat_template_utils.template import apply_chat_template_from_str + + return apply_chat_template_from_str(chat_template, msgs, add_generation_prompt=True, tools=tools) + return tokenizer.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True, tools=tools) + + # Build prompt → response mapping + prompt_response_map: dict[str, str] = {} + for turn in trajectory.turns: + prompt_str = _render(turn.request_messages) + prompt_response_map[prompt_str] = turn.response_text + + def process_fn(prompt: str) -> ProcessResult: + for expected_prompt, response_text in prompt_response_map.items(): + if prompt == expected_prompt: + return ProcessResult(text=response_text, finish_reason="stop") + raise ValueError( + f"Unexpected prompt (length={len(prompt)}).\n" + f"Known prompts: {[len(p) for p in prompt_response_map]}\n" + f"Prompt tail: {prompt[-200:]!r}" + ) + + return process_fn + + +class SequentialProcessFn: + """A process_fn that returns response texts in turn order. + + Unlike build_process_fn which does exact prompt matching, this version + simply returns the next turn's response_text on each call. Useful for + e2e tests where the actual prompt may differ from pre-computed prompts + (e.g., tool_call IDs and argument formats differ between trajectory + definitions and mock server responses). + + Call reset() between test runs to restart from the first turn. + """ + + def __init__(self, trajectory: Trajectory): + self._response_texts = [turn.response_text for turn in trajectory.turns] + self._call_count = 0 + + def reset(self): + self._call_count = 0 + + def __call__(self, prompt: str) -> ProcessResult: + idx = self._call_count + if idx >= len(self._response_texts): + raise ValueError( + f"Sequential process_fn exhausted: called {idx + 1} times " + f"but only {len(self._response_texts)} turns defined" + ) + self._call_count += 1 + return ProcessResult(text=self._response_texts[idx], finish_reason="stop") diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/session_verify_agent.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/session_verify_agent.py new file mode 100644 index 0000000..5b3c55f --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/session_verify_agent.py @@ -0,0 +1,460 @@ +"""Custom-generate / custom-agent driver for TITO session-server verification. + +Wired through ``--custom-generate-function-path`` / +``--custom-agent-function-path``; consumed by +``tests/e2e/sglang/test_session_server_multi_role/`` (one test file per +model family) and ``scripts/tools/verify_session_tito_tokenizer.py``. +""" + +from __future__ import annotations + +import json +import logging +import os +from enum import Enum +try: + from enum import StrEnum +except ImportError: + class StrEnum(str, Enum): + pass + +import httpx + +from miles.rollout.base_types import GenerateFnInput, GenerateFnOutput +from miles.rollout.generate_hub.agentic_tool_call import generate as _base_generate + +logger = logging.getLogger(__name__) + + +class DriverAction(Enum): + TOOL_RESULT = "tool_result" + USER_FOLLOWUP = "user_followup" + SYSTEM_REMINDER = "system_reminder" + ROLLBACK = "rollback" + FORCE_FINAL = "force_final" + + +_T = DriverAction.TOOL_RESULT +_U = DriverAction.USER_FOLLOWUP +_S = DriverAction.SYSTEM_REMINDER +_R = DriverAction.ROLLBACK +_F = DriverAction.FORCE_FINAL + + +class ToolCallFailureMode(StrEnum): + """Recovery strategy when a TOOL_RESULT step finds the assistant emitted no tool_calls. + + APPEND_TOOL : Splice a sentinel ``tool`` message and continue. Works on + lenient templates; strict templates that hard-assert any + ``tool`` role must follow an assistant with ``tool_calls`` + (e.g. MiniMax-M2.7) will reject the next request at server-side. + APPEND_USER : Splice a ``user`` message carrying the same failure text as + APPEND_TOOL. Requires "user" in ``allowed_append_roles`` — + raises ValueError at agent start otherwise, so misconfig is + immediately visible instead of silently downgrading. + ROLLBACK : Pop the offending assistant and let the loop's chat call at + the bottom re-inference. Universal — no role-surface + dependency — and the default. + """ + + APPEND_TOOL = "append_tool" + APPEND_USER = "append_user" + ROLLBACK = "rollback" + + +DEFAULT_TOOL_CALL_FAILURE_MODE = ToolCallFailureMode.ROLLBACK + +# Cap consecutive ROLLBACK retries — same context every time, so a model that +# never tool-calls would loop forever. +MAX_CONSECUTIVE_TOOL_CALL_FAILURE_ROLLBACKS = 3 + +# Same body for both APPEND_TOOL and APPEND_USER fallbacks; only the role of +# the spliced message differs between the two modes. +TOOL_CALL_PARSE_FAILURE_TEXT = ( + "Tool call parsing failed: the previous assistant turn did not emit a " + "parseable tool_call. Please retry with a valid tool invocation." +) + +# Mismatch tiers reported by the session-server's per-sample comparator +# (sessions.py:83). Any occurrence of these "hard" types in a sample's +# tito_session_mismatch indicates a TITO bug and fails the sample. The +# soft `assistant_text` tier is excluded — it is aggregated across samples +# and gated by a ratio threshold instead. +_FORBIDDEN_MISMATCH_TYPES: frozenset[str] = frozenset( + {"special_token_count", "special_token_type", "non_assistant_text"} +) + +# Override per call: ``--session-verify-cycles N`` (CLI) or ``cycles=N`` +# (pytest via ``run_session_verify``). Smaller-context models with a 4K +# response budget should drop to 2 to avoid context overflow. +DEFAULT_CYCLES = 3 + +_SUPPORTED_ROLE_SURFACES: tuple[frozenset[str], ...] = ( + frozenset({"tool"}), + frozenset({"tool", "user"}), + frozenset({"tool", "user", "system"}), +) + + +def _build_cycle(role_surface: frozenset[str]) -> list[DriverAction]: + cycle: list[DriverAction] = [_T] + if "user" in role_surface: + cycle.append(_U) + cycle.append(_T) + if "system" in role_surface: + cycle.append(_S) + cycle.append(_R) + return cycle + + +# English-only on purpose: matches the production agentic flows tokenization +# and tool-call parsing are tuned against. +USER_FOLLOWUP_TEXT = "Now check the weather in Shanghai." +SYSTEM_REMINDER_TEXT = "Note: from now on, answer in a single sentence; skip all pleasantries." +FORCE_FINAL_TEXT = "Please summarize all results inside ... tags." + +TOOLS = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a given city.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city name, e.g. Beijing", + }, + }, + "required": ["location"], + }, + }, + }, +] + +MOCK_TOOL_RESULTS = [ + '{"temperature_celsius": 22, "condition": "sunny"}', + '{"temperature_celsius": 15, "condition": "cloudy"}', + '{"temperature_celsius": 30, "condition": "rainy"}', + '{"temperature_celsius": 8, "condition": "snowy"}', +] + + +INITIAL_SYSTEM_PROMPT = ( + "You are a weather assistant. Use the get_weather tool when the user asks " + "about a city's weather. Answer one question at a time and wait for the " + "next user message; do not summarize until the user explicitly asks you " + "to. When asked to summarize, wrap the final summary in " + "... tags." +) +INITIAL_USER_PROMPT = "What's the weather in Beijing?" + + +def select_schedule(allowed_roles, *, cycles: int = DEFAULT_CYCLES) -> list[DriverAction]: + """Pick the schedule for ``frozenset(allowed_roles)``; raises on unregistered.""" + key = frozenset(allowed_roles) + if key not in _SUPPORTED_ROLE_SURFACES: + registered = sorted(sorted(k) for k in _SUPPORTED_ROLE_SURFACES) + raise ValueError(f"No schedule registered for allowed_roles={sorted(key)}. Registered: {registered}") + if cycles < 1: + raise ValueError(f"cycles must be >= 1, got {cycles}") + cycle = _build_cycle(key) + # Extra R after the first cycle exercises consecutive-rollback adjacency, + # which cycle-repeat alone never produces. + schedule = list(cycle) + [_R] + cycle * (cycles - 1) + if "user" in key: + schedule.append(_F) + return schedule + + +def build_initial_messages() -> list[dict]: + """The fixed (system, user) prompt all schedules start from.""" + return [ + {"role": "system", "content": INITIAL_SYSTEM_PROMPT}, + {"role": "user", "content": INITIAL_USER_PROMPT}, + ] + + +async def _chat(client, base_url, messages, request_kwargs, *, label): + payload = {"messages": messages, "tools": TOOLS, **request_kwargs} + resp = await client.post(f"{base_url}/v1/chat/completions", json=payload) + assert resp.status_code == 200, f"{label} failed ({resp.status_code}): {resp.text}" + return resp.json() + + +async def run_agent(base_url, prompt, request_kwargs, metadata, **kwargs): + """Custom-agent entry point. Returns ``{"driver_events": [...], **counters}``. + + ``allowed_append_roles`` must be present in ``metadata`` (the ``generate`` + wrapper below injects it from ``args.tito_allowed_append_roles``). + ``prompt`` is ignored — the driver synthesizes its own initial conversation + from ``build_initial_messages`` so runs are reproducible. + """ + allowed_roles = metadata.get("allowed_append_roles") + if allowed_roles is None: + raise ValueError( + "session_verify_agent.run_agent requires allowed_append_roles in metadata; " + "the generate wrapper should inject it from args.tito_allowed_append_roles." + ) + cycles = metadata.get("session_verify_cycles", DEFAULT_CYCLES) + schedule = select_schedule(allowed_roles, cycles=cycles) + + failure_mode = ToolCallFailureMode(metadata.get("tool_call_failure_mode", DEFAULT_TOOL_CALL_FAILURE_MODE)) + # APPEND_USER injects a user message — only valid if 'user' is in + # allowed_append_roles. Refuse up front instead of silently downgrading. + if failure_mode is ToolCallFailureMode.APPEND_USER and "user" not in allowed_roles: + raise ValueError( + f"tool_call_failure_mode=APPEND_USER requires 'user' in allowed_append_roles, " + f"got {sorted(allowed_roles)}. Pick ROLLBACK (universal) or APPEND_TOOL " + "(lenient-template) for tool-only surfaces." + ) + + rk = {k: v for k, v in request_kwargs.items() if k not in ("tools",)} + messages = build_initial_messages() + events: list[str] = [] + counters = { + "rollback_count": 0, + "user_count": 0, + "system_count": 0, + "tool_result_count": 0, + "tool_call_count": 0, + } + # Streak of TOOL_RESULT steps that fell into the ROLLBACK fallback without + # the model recovering to a real tool_call. Reset on any successful + # tool_call; gated by MAX_CONSECUTIVE_TOOL_CALL_FAILURE_ROLLBACKS to keep + # silently-stuck samples from burning wall-time. + consecutive_failure_rollbacks = 0 + + async with httpx.AsyncClient(timeout=180) as client: + # Initial completion — no driver action yet. + resp = await _chat(client, base_url, messages, rk, label="Initial") + assistant = resp["choices"][0]["message"] + messages.append(assistant) + events.append("initial") + counters["tool_call_count"] += len(assistant.get("tool_calls") or []) + + for step_idx, action in enumerate(schedule): + label = f"Step {step_idx + 1} {action.value}" + + if action is DriverAction.TOOL_RESULT: + tool_calls = assistant.get("tool_calls") or [] + if tool_calls: + consecutive_failure_rollbacks = 0 + for i, tc in enumerate(tool_calls): + result_idx = (counters["tool_result_count"] + i) % len(MOCK_TOOL_RESULTS) + messages.append( + { + "role": "tool", + "content": MOCK_TOOL_RESULTS[result_idx], + "tool_call_id": tc["id"], + } + ) + counters["tool_result_count"] += len(tool_calls) + events.append("append_tool") + else: + # Model emitted no tool_calls — apply the configured fallback. + # Templates differ on what role may follow a "no tool_calls" + # assistant: + # - GLM / Nemotron (lenient): a tool message is fine -> APPEND_TOOL. + # - Kimi: a tool message must carry the id from a valid + # tool_call, which we don't have -> APPEND_TOOL not OK. + # - MiniMax: a tool message must follow an assistant with + # non-empty tool_calls -> APPEND_TOOL not OK. + # If APPEND_TOOL not ok, use APPEND_USER as instead. + match failure_mode: + case ToolCallFailureMode.APPEND_TOOL: + messages.append( + { + "role": "tool", + "tool_call_id": "none", + "content": TOOL_CALL_PARSE_FAILURE_TEXT, + } + ) + events.append("tool_call_failure_append_tool") + case ToolCallFailureMode.APPEND_USER: + messages.append({"role": "user", "content": TOOL_CALL_PARSE_FAILURE_TEXT}) + counters["user_count"] += 1 + events.append("tool_call_failure_append_user") + case ToolCallFailureMode.ROLLBACK: + # Same as the schedule's ROLLBACK + assert messages and messages[-1]["role"] == "assistant", ( + f"tool_call_failure_mode=ROLLBACK: tail role is " + f"{messages[-1]['role'] if messages else 'EMPTY'}, expected assistant" + ) + consecutive_failure_rollbacks += 1 + if consecutive_failure_rollbacks > MAX_CONSECUTIVE_TOOL_CALL_FAILURE_ROLLBACKS: + raise AssertionError( + f"ROLLBACK fallback hit {consecutive_failure_rollbacks} consecutive " + f"tool_call failures (limit={MAX_CONSECUTIVE_TOOL_CALL_FAILURE_ROLLBACKS}). " + "Model is not tool-calling on this prompt — check sampling temperature, " + "the tool spec, or switch tool_call_failure_mode to APPEND_TOOL/APPEND_USER " + "if sentinel-driven retry is preferred." + ) + messages.pop() + counters["rollback_count"] += 1 + events.append("tool_call_failure_rollback") + case _: + raise AssertionError(f"Unknown ToolCallFailureMode {failure_mode!r}") + + elif action is DriverAction.USER_FOLLOWUP: + messages.append({"role": "user", "content": USER_FOLLOWUP_TEXT}) + counters["user_count"] += 1 + events.append("append_user") + + elif action is DriverAction.SYSTEM_REMINDER: + messages.append({"role": "system", "content": SYSTEM_REMINDER_TEXT}) + counters["system_count"] += 1 + events.append("append_system") + + elif action is DriverAction.ROLLBACK: + # Pop the last assistant from our local copy. The next + # request therefore has one fewer message than what the + # server has stored, which is the trigger for its + # ``_detect_and_rollback`` path — the server rewinds its + # state, then re-inferences. + if not messages or messages[-1]["role"] != "assistant": + raise AssertionError( + f"Cannot rollback at step {step_idx}: tail role is " + f"{messages[-1]['role'] if messages else 'EMPTY'}, expected assistant" + ) + messages.pop() + counters["rollback_count"] += 1 + events.append("rollback") + + elif action is DriverAction.FORCE_FINAL: + messages.append({"role": "user", "content": FORCE_FINAL_TEXT}) + events.append("force_final") + + else: + raise AssertionError(f"Unknown DriverAction {action!r}") + + resp = await _chat(client, base_url, messages, rk, label=label) + assistant = resp["choices"][0]["message"] + messages.append(assistant) + counters["tool_call_count"] += len(assistant.get("tool_calls") or []) + + logger.info("Agent done: events=%s counters=%s", events, counters) + + return {"driver_events": events, **counters} + + +async def generate(input: GenerateFnInput) -> GenerateFnOutput: + """Custom-generate wrapper that asserts driver-action coverage. + + - Per-sample: every sample must contain ``rollback``, plus ``append_user`` + / ``append_system`` when those roles are allowed. + - Cross-sample: at least one sample must contain ``append_tool`` + (model-dependent on emitting a tool_call). + """ + allowed_roles = list(input.args.tito_allowed_append_roles) + cycles = getattr(input.args, "session_verify_cycles", DEFAULT_CYCLES) + failure_mode = getattr(input.args, "tool_call_failure_mode", DEFAULT_TOOL_CALL_FAILURE_MODE) + # Sample.metadata is mutable even when the outer dataclass is frozen. + input.sample.metadata["allowed_append_roles"] = allowed_roles + input.sample.metadata["session_verify_cycles"] = cycles + input.sample.metadata["tool_call_failure_mode"] = failure_mode + + output = await _base_generate(input) + + samples = output.samples if isinstance(output.samples, list) else [output.samples] + events_per_sample = [s.metadata.get("driver_events", []) for s in samples] + metrics_path = os.environ.get("MILES_SESSION_VERIFY_METRICS_PATH") + + required_per_sample = ["rollback"] + if "user" in allowed_roles: + required_per_sample.append("append_user") + if "system" in allowed_roles: + required_per_sample.append("append_system") + + for i, events in enumerate(events_per_sample): + missing = [req for req in required_per_sample if req not in events] + if missing: + raise AssertionError( + f"Session multi-role e2e: sample {i} missing required driver events " + f"{missing}. allowed_roles={allowed_roles}, events={events}" + ) + + if not metrics_path and not any("append_tool" in events for events in events_per_sample): + raise AssertionError( + "Session multi-role e2e: no sample produced an append_tool action — " + f"the model may not be tool-calling. events_per_sample={events_per_sample}" + ) + + for i, sample in enumerate(samples): + mismatches = sample.metadata.get("tito_session_mismatch") + if mismatches is None: + raise AssertionError( + f"Session multi-role e2e: sample {i} has no tito_session_mismatch " + f"in metadata. The session-server's compute_session_mismatch raised " + f"TokenizationError (sessions.py:83 swallows it) — this always " + f"indicates a TITO subclass / setup bug, not a real PASS." + ) + forbidden = [m for m in mismatches if m.get("type") in _FORBIDDEN_MISMATCH_TYPES] + if forbidden: + raise AssertionError( + f"Session multi-role e2e: sample {i} has forbidden mismatches " + f"{forbidden}. allowed_roles={allowed_roles}. These types must be 0 " + f"for any TITO-correct setup." + ) + if metrics_path: + assistant_mismatches = [m for m in mismatches if m.get("type") == "assistant_text"] + had_assistant_mismatch = bool(assistant_mismatches) + example = None + if assistant_mismatches: + first = assistant_mismatches[0] + example = { + "segment_index": first.get("segment_index"), + "expected_text": (first.get("expected_text") or "")[:300], + "actual_text": (first.get("actual_text") or "")[:300], + } + with open(metrics_path, "a") as f: + f.write( + json.dumps( + { + "sample_index": i, + "driver_events": events_per_sample[i], + "had_assistant_mismatch": had_assistant_mismatch, + "total_mismatches": len(mismatches), + "assistant_mismatch_count": len(assistant_mismatches), + "assistant_mismatch_example": example, + } + ) + + "\n" + ) + + logger.info( + "Multi-role coverage verified: per_sample=%s, samples=%d, events=%s", + required_per_sample, + len(samples), + events_per_sample, + ) + return output + + +def _add_arguments(parser): + _base_generate.add_arguments(parser) + parser.add_argument( + "--session-verify-cycles", + type=int, + default=DEFAULT_CYCLES, + help="Number of driver schedule cycles per sample for session-server " + "TITO verification. Each cycle exercises every action in the role " + "surface plus a rollback; more cycles stress the TITO accumulator " + "longer but expand context length. Drop to 2 on tighter-context " + "models (e.g. Qwen3 32K with 4K response budget).", + ) + parser.add_argument( + "--tool-call-failure-mode", + type=str, + default=DEFAULT_TOOL_CALL_FAILURE_MODE.value, + choices=[m.value for m in ToolCallFailureMode], + help="Recovery mode when a TOOL_RESULT step sees no tool_calls on the " + "assistant. 'rollback' (default, universal) pops the assistant and " + "re-inferences. 'append_tool' splices a sentinel tool message (only " + "works on lenient templates). 'append_user' splices a user message " + "with the same failure text — requires 'user' in allowed_append_roles.", + ) + + +generate.add_arguments = _add_arguments diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/session_verify_runner.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/session_verify_runner.py new file mode 100644 index 0000000..cea07cf --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/session_verify_runner.py @@ -0,0 +1,332 @@ +"""Boot a real ``miles`` rollout pipeline + run the multi-role TITO driver. + +Used by both consumers: + +- pytest e2e: ``tests/e2e/sglang/test_session_server_multi_role/`` (one test + file per model family) +- CLI: ``scripts/tools/verify_session_tito_tokenizer.py`` + +Both forms run the same ``execute_train(--debug-rollout-only)`` path: full miles +pipeline (sglang + miles-router with session support) is launched, ``train`` is +skipped, and the rollout drives ``session_verify_agent.run_agent`` against the +session server. + +Args flow through miles' canonical ``parse_args`` Namespace. + +# Backend choice + +``execute_train`` asserts ``("--train-backend fsdp" in train_args) == (megatron_model_type is None)``, +so the ``fsdp`` + ``None`` pair is the only consistent way to skip megatron init +in ``--debug-rollout-only`` mode. We use that. +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import shutil +import tempfile +from typing import Any + +from miles.utils.chat_template_utils import resolve_reasoning_and_tool_call_parser + +logger = logging.getLogger(__name__) + +# Soft cap on how many samples may report any assistant_text mismatch. Hard +# mismatch types (special_token_count / special_token_type / non_assistant_text) +# are asserted per-sample inside the agent wrapper — those must be 0. +ASSISTANT_TEXT_MISMATCH_RATIO_THRESHOLD = 0.2 + +PROMPT_DATA_PATH = "/root/datasets/session_multi_role_verify.jsonl" +LOCAL_MODELS_ROOT = "/root/models" + +# The driver agent synthesizes its own initial conversation, but the rollout +# pipeline still needs a non-empty prompt-data file as input. This placeholder +# matches the agent's own initial prompt so the prompt is well-formed even if +# something downstream inspects it. +_PLACEHOLDER_PROMPT_RECORD = { + "messages": [ + {"role": "system", "content": "You are a weather assistant."}, + {"role": "user", "content": "What's the weather in Beijing?"}, + ], +} + +_PROXY_ENV_VARS = ("http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY") + +SESSION_VERIFY_INVARIANT_ARGS: dict[str, Any] = { + "prompt_data": PROMPT_DATA_PATH, + "input_key": "messages", + "num_rollout": 1, + "rollout_batch_size": 16, + "rollout_max_response_len": 8192, + "rollout_temperature": 0.7, + "global_batch_size": 64, + "rm_type": "random", + "custom_generate_function_path": "miles.utils.test_utils.session_verify_agent.generate", + "custom_agent_function_path": "miles.utils.test_utils.session_verify_agent.run_agent", + "use_session_server": True, + "debug_rollout_only": True, + "ci_test": True, + "colocate": True, + "train_backend": "fsdp", + "sglang_expert_parallel_size": 1, +} + + +def _command_utils() -> Any: + """Load Miles training command helpers only for the optional e2e path.""" + try: + import miles.utils.external_utils.command_utils as command_utils + except Exception as exc: + raise RuntimeError( + "verify-session-tito-tokenizer requires Miles training/e2e command " + "utilities. Install the optional Miles/SGLang training stack before " + "running the session verifier." + ) from exc + return command_utils + + +def session_verify_extras(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: + """``add_custom_arguments`` hook for ``miles.utils.arguments.parse_args``. + + Adds the wrapper-only ``--assistant-text-threshold`` knob (a post-process + gate on the per-sample metrics JSONL, NOT in ``train_args``) and applies + session-verify invariants as parser defaults — user CLI still overrides + these via the canonical miles flags. + """ + parser.add_argument( + "--assistant-text-threshold", + type=float, + default=ASSISTANT_TEXT_MISMATCH_RATIO_THRESHOLD, + help=( + f"Soft threshold for assistant_text mismatch ratio. Default {ASSISTANT_TEXT_MISMATCH_RATIO_THRESHOLD}. " + "Raise to 1.0 for families whose upstream sglang reasoning parser " + "is known to roundtrip imperfectly (e.g. nemotron_3 keeps a " + "trailing newline in reasoning_content) — hard mismatches still " + "gate. Post-process gate on per-sample JSONL metrics; not " + "forwarded to ``train_args``." + ), + ) + parser.set_defaults(**SESSION_VERIFY_INVARIANT_ARGS) + return parser + + +def _ensure_prompt_data() -> str: + os.makedirs(os.path.dirname(PROMPT_DATA_PATH), exist_ok=True) + with open(PROMPT_DATA_PATH, "w") as f: + f.write(json.dumps(_PLACEHOLDER_PROMPT_RECORD) + "\n") + return PROMPT_DATA_PATH + + +def _ensure_model_downloaded(hf_checkpoint: str) -> str: + """Return a local model path, downloading HF repos when needed. + + Lets callers pass either a HuggingFace repo id (downloaded under + ``/root/models/``) or an existing local checkpoint path + (returned as-is, no download). + """ + if os.path.exists(hf_checkpoint): + return hf_checkpoint + + short = hf_checkpoint.split("/")[-1] + local_dir = os.path.join(LOCAL_MODELS_ROOT, short) + os.makedirs(LOCAL_MODELS_ROOT, exist_ok=True) + _command_utils().exec_command(f"hf download {hf_checkpoint} --local-dir {local_dir}") + return local_dir + + +def _clear_proxy_env() -> dict[str, str | None]: + previous = {proxy_var: os.environ.get(proxy_var) for proxy_var in _PROXY_ENV_VARS} + for proxy_var in _PROXY_ENV_VARS: + os.environ.pop(proxy_var, None) + return previous + + +def _restore_proxy_env(previous: dict[str, str | None]) -> None: + for proxy_var, value in previous.items(): + if value is None: + os.environ.pop(proxy_var, None) + else: + os.environ[proxy_var] = value + + +def namespace_to_train_args(ns: argparse.Namespace) -> str: + """Serialize a fully-shaped Namespace into the ``train_args`` string. + + Reads miles-canonical field names off ``ns``; emits the exact flag set + ``execute_train`` re-parses downstream. ``actor_num_nodes`` is written + explicitly from ``ns.actor_num_nodes`` (NOT defaulted at the serializer + level) so the runner stays pinned to whatever the caller's Namespace + declared, regardless of any drift in miles' upstream default. + """ + allowed_roles_arg = " ".join(ns.tito_allowed_append_roles) + parts: list[str] = [ + f"--hf-checkpoint {ns.hf_checkpoint}", + f"--prompt-data {ns.prompt_data}", + f"--input-key {ns.input_key}", + f"--num-rollout {ns.num_rollout}", + f"--rollout-batch-size {ns.rollout_batch_size}", + f"--n-samples-per-prompt {ns.n_samples_per_prompt}", + f"--rollout-max-response-len {ns.rollout_max_response_len}", + f"--rollout-temperature {ns.rollout_temperature}", + f"--global-batch-size {ns.global_batch_size}", + f"--custom-generate-function-path {ns.custom_generate_function_path}", + f"--custom-agent-function-path {ns.custom_agent_function_path}", + f"--session-verify-cycles {ns.session_verify_cycles}", + f"--tool-call-failure-mode {ns.tool_call_failure_mode}", + f"--tito-model {ns.tito_model}", + f"--tito-allowed-append-roles {allowed_roles_arg}", + f"--rollout-num-gpus-per-engine {ns.rollout_num_gpus_per_engine}", + f"--sglang-reasoning-parser {ns.sglang_reasoning_parser}", + f"--rm-type {ns.rm_type}", + f"--actor-num-nodes {ns.actor_num_nodes}", + f"--actor-num-gpus-per-node {ns.actor_num_gpus_per_node}", + f"--train-backend {ns.train_backend}", + ] + if ns.sglang_tool_call_parser: + parts.append(f"--sglang-tool-call-parser {ns.sglang_tool_call_parser}") + # DeepSeek V3.2 (and other NSA/MoE archs) requires expert-parallel > 1 in + # sglang; the default is 1, which is fatal at engine init. Only emit the + # flag when the caller asks for ep>1 so single-expert models stay untouched. + if ns.sglang_expert_parallel_size > 1: + parts.append(f"--sglang-expert-parallel-size {ns.sglang_expert_parallel_size}") + if ns.use_session_server: + parts.append("--use-session-server") + if ns.debug_rollout_only: + parts.append("--debug-rollout-only") + if ns.ci_test: + parts.append("--ci-test") + if ns.colocate: + parts.append("--colocate") + return " ".join(parts) + " " + + +def run_session_verify(args: argparse.Namespace) -> None: + """Boot ``miles`` rollout pipeline and run the multi-role driver. + + Returns nothing on success; raises ``AssertionError`` on TITO mismatch + (HTTP 500 from server-side prefix check) or coverage shortfall (raised by + ``session_verify_agent.generate``). + + ``args`` MUST be a fully-shaped Namespace carrying miles-canonical field + names plus the session-verify-specific fields (``session_verify_cycles``, + ``tool_call_failure_mode``, ``assistant_text_threshold``). Build it via + ``parse_args(add_custom_arguments=session_verify_extras)`` for the CLI + path or by spreading ``SESSION_VERIFY_INVARIANT_ARGS`` into + ``argparse.Namespace(...)`` for tests. + + Mutates ``args`` in three places before composing train_args: + - ``args.sglang_reasoning_parser`` / ``args.sglang_tool_call_parser`` are + resolved against the TITO subclass's bound values via + ``resolve_reasoning_and_tool_call_parser`` — caller-passed values that + disagree with the bound values raise ``ValueError`` here, before any + GPU work starts. + - ``args.hf_checkpoint`` is replaced with the local download path so the + composed train_args points at the downloaded model, not the HF id. + - ``args.tito_allowed_append_roles`` is normalized (lowercase, dedup, + ensure ``'tool'`` is in) to match the schedule contract in + ``session_verify_agent._SUPPORTED_ROLE_SURFACES``. + """ + args.sglang_reasoning_parser, args.sglang_tool_call_parser = resolve_reasoning_and_tool_call_parser( + args.tito_model, args.sglang_reasoning_parser, args.sglang_tool_call_parser + ) + args.tito_allowed_append_roles = sorted(set(r.lower() for r in args.tito_allowed_append_roles) | {"tool"}) + + _ensure_prompt_data() + proxy_env = _clear_proxy_env() + try: + args.hf_checkpoint = _ensure_model_downloaded(args.hf_checkpoint) + + train_args = namespace_to_train_args(args) + + # Per-sample token-seq metrics file: rollout workers append one JSONL line + # per sample inside session_verify_agent.generate; we aggregate after + # execute_train returns to apply the assistant_text soft threshold. + metrics_fd, metrics_path = tempfile.mkstemp(prefix="session_verify_metrics_", suffix=".jsonl") + os.close(metrics_fd) + + try: + _command_utils().execute_train( + train_args=train_args, + num_gpus_per_node=args.actor_num_gpus_per_node, + megatron_model_type=None, + extra_env_vars={ + "MILES_EXPERIMENTAL_ROLLOUT_REFACTOR": "1", + "MILES_TITO_MODEL": args.tito_model, + "MILES_SESSION_VERIFY_METRICS_PATH": metrics_path, + }, + ) + try: + assert_session_verify_metrics(metrics_path, assistant_text_threshold=args.assistant_text_threshold) + except AssertionError: + preserved_metrics_path = metrics_path + ".failed" + shutil.copy(metrics_path, preserved_metrics_path) + logger.error("Preserved per-sample mismatch payloads at %s for post-mortem", preserved_metrics_path) + raise + finally: + try: + os.unlink(metrics_path) + except OSError: + pass + finally: + _restore_proxy_env(proxy_env) + + +def assert_session_verify_metrics(metrics_path: str, *, assistant_text_threshold: float) -> None: + """Read per-sample JSONL metrics and assert cross-sample verifier gates. + + Forbidden mismatch types (special_*, non_assistant_text) are caught + per-sample in the agent wrapper and would have already raised by now. + Here we only cross-check the soft assistant_text rate against the + caller-provided threshold (per-model: some upstream sglang reasoning + parsers — notably ``nemotron_3`` — leave a trailing ``\\n`` in + ``reasoning_content`` that breaks the canonical roundtrip until the + parser is patched, so those families ride at threshold=1.0). + """ + samples_with_mismatch = 0 + total_samples = 0 + has_append_tool = False + with open(metrics_path) as f: + for line in f: + line = line.strip() + if not line: + continue + entry = json.loads(line) + total_samples += 1 + has_append_tool = has_append_tool or "append_tool" in entry.get("driver_events", []) + if entry.get("had_assistant_mismatch"): + samples_with_mismatch += 1 + + if total_samples == 0: + raise AssertionError( + f"Session multi-role e2e: no per-sample metrics found at {metrics_path}. " + "Either the rollout produced 0 samples, or the agent wrapper failed to " + "run before any sample completed. Check rollout logs." + ) + + if not has_append_tool: + raise AssertionError( + "Session multi-role e2e: no sample produced an append_tool action — " + "the model may not be tool-calling. Check sampling temperature, " + "the tool spec, or parser configuration." + ) + + ratio = samples_with_mismatch / total_samples + logger.info( + "Token-seq metric summary: samples=%d, with_assistant_text_mismatch=%d, ratio=%.3f, threshold=%.3f", + total_samples, + samples_with_mismatch, + ratio, + assistant_text_threshold, + ) + if ratio > assistant_text_threshold: + raise AssertionError( + f"Session multi-role e2e: assistant_text mismatch ratio " + f"{samples_with_mismatch}/{total_samples}={ratio:.3f} exceeds " + f"threshold {assistant_text_threshold}. TITO " + "tokenization for assistant content has drifted from the chat " + "template's canonical render — investigate via " + "verify_session_tito_tokenizer.py + sample-level mismatch logs." + ) diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/uvicorn_thread_server.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/uvicorn_thread_server.py new file mode 100644 index 0000000..904343c --- /dev/null +++ b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/uvicorn_thread_server.py @@ -0,0 +1,49 @@ +import asyncio +import socket +import threading +import time + +import uvicorn + + +class UvicornThreadServer: + def __init__(self, app, host: str, port: int): + self._app = app + self.host = host + self.port = port + self._server: uvicorn.Server | None = None + self._thread: threading.Thread | None = None + + @property + def url(self) -> str: + return f"http://{self.host}:{self.port}" + + def start(self) -> None: + config = uvicorn.Config(self._app, host=self.host, port=self.port, log_level="info") + self._server = uvicorn.Server(config) + + def run() -> None: + asyncio.run(self._server.serve()) + + self._thread = threading.Thread(target=run, daemon=True) + self._thread.start() + self._wait_for_port_open() + + def stop(self) -> None: + if self._server is not None: + self._server.should_exit = True + if self._thread is not None and self._thread.is_alive(): + self._thread.join(timeout=2.0) + + def _wait_for_port_open(self) -> None: + for _ in range(50): + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + result = sock.connect_ex((self.host, self.port)) + sock.close() + if result == 0: + return + except Exception: + pass + time.sleep(0.1) + raise RuntimeError(f"Failed to start server on {self.url}") diff --git a/sidecars/tito/tito_gateway/verify_chat_template.py b/sidecars/tito/tito_gateway/verify_chat_template.py new file mode 100644 index 0000000..5e4ffc0 --- /dev/null +++ b/sidecars/tito/tito_gateway/verify_chat_template.py @@ -0,0 +1,125 @@ +"""CLI implementation for Miles chat-template append-only verification.""" + +from __future__ import annotations + +from typing import Any + + +def run_from_args(args: Any) -> int: + """Run the vendored Miles chat-template verifier from an argparse namespace.""" + if args.model is None and args.template is None: + raise ValueError("one of --model or --template is required") + if args.tito_model is not None and args.model is None: + raise ValueError("--tito-model requires --model so the TITO verifier can load the tokenizer") + + extra_template_kwargs = dict(args.chat_template_kwargs or {}) + allowed_roles = set(args.tito_allowed_append_roles) | {"tool"} + use_tito_instance = args.tito_model is not None + + if use_tito_instance: + from miles.utils.chat_template_utils import resolve_fixed_chat_template + from miles.utils.processing_utils import load_tokenizer + + fixed_path, resolved_kwargs = resolve_fixed_chat_template(args.tito_model, sorted(allowed_roles)) + for key, value in resolved_kwargs.items(): + if key in extra_template_kwargs: + continue + extra_template_kwargs[key] = value + print(f"Auto-set --chat-template-kwargs {key}={value!r} (from --tito-model={args.tito_model})") + + template_path = args.template or fixed_path + tokenizer = load_tokenizer(args.model, chat_template_path=template_path, trust_remote_code=True) + if args.template: + source_desc = f"template override via TITO: {args.template}" + elif fixed_path: + source_desc = f"fixed template via TITO: {fixed_path}" + elif getattr(tokenizer, "chat_template", None) is not None: + source_desc = f"HuggingFace via TITO: {args.model}" + else: + source_desc = f"TITO encoder: {args.tito_model}" + chat_template = None + elif args.template: + with open(args.template) as f: + chat_template = f.read() + source_desc = f"file: {args.template}" + tokenizer = None + else: + from miles.utils.chat_template_utils.template import load_hf_chat_template + + chat_template = load_hf_chat_template(args.model) + source_desc = f"HuggingFace: {args.model}" + tokenizer = None + + from miles.utils.test_utils.chat_template_verify import ( + ALL_CASES, + check_coverage, + run_all_checks, + run_all_checks_via_tito, + select_cases, + ) + + is_thinking_filter = {"off": False, "on": True, "both": None}[args.thinking] + selected = select_cases(allowed_append_roles=allowed_roles, is_thinking=is_thinking_filter) + + print(f"Template source: {source_desc}") + print(f"Allowed append roles: {sorted(allowed_roles)}") + print(f"Thinking mode: {args.thinking}") + if extra_template_kwargs: + print(f"Template kwargs: {extra_template_kwargs}") + print(f"Selected trajectories: {len(selected)} of {len(ALL_CASES)} (after filtering)") + print() + + coverage = check_coverage() + if coverage.missing: + print("Trajectory coverage gaps ((thinking, append_roles \\ {tool}) with no trajectory):") + for is_thinking, roles in coverage.missing: + label = "thinking " if is_thinking else "non-thinking" + roles_str = "{" + ", ".join(roles) + "}" if roles else "{}" + print(f" - {label} x {roles_str}") + print() + + if use_tito_instance: + results = run_all_checks_via_tito( + tokenizer, + args.tito_model, + allowed_append_roles=allowed_roles, + thinking=args.thinking, + extra_template_kwargs=extra_template_kwargs, + ) + else: + results = run_all_checks( + chat_template, + allowed_append_roles=allowed_roles, + thinking=args.thinking, + extra_template_kwargs=extra_template_kwargs, + ) + + passed = sum(1 for r in results if r.passed) + failed = sum(1 for r in results if not r.passed) + max_name_len = max((len(r.case_name) for r in results), default=0) + + for r in results: + status = "PASS" if r.passed else "FAIL" + line = f" [{status}] {r.case_name:<{max_name_len}}" + if r.error: + first_line = r.error.split("\n")[0] + if len(first_line) > 80: + first_line = first_line[:77] + "..." + line += f" -- {first_line}" + print(line) + + print() + print(f"Results: {passed}/{len(results)} passed, {failed} failed") + + if failed: + if use_tito_instance: + print("\nVerdict: FAIL - TITO incremental tokenization did NOT match standard render") + else: + print("\nVerdict: FAIL - template is NOT append-only after last user message") + return 1 + + if use_tito_instance: + print("\nVerdict: PASS - TITO incremental tokenization matched standard render") + else: + print("\nVerdict: PASS - template IS append-only after last user message") + return 0 diff --git a/sidecars/tito/tito_gateway/verify_session_tito_tokenizer.py b/sidecars/tito/tito_gateway/verify_session_tito_tokenizer.py new file mode 100644 index 0000000..79de7cb --- /dev/null +++ b/sidecars/tito/tito_gateway/verify_session_tito_tokenizer.py @@ -0,0 +1,33 @@ +"""Dependency-gated entrypoint for Miles session-server TITO e2e verification.""" + +from __future__ import annotations + +from typing import Any + + +def run_from_args(args: Any) -> int: + """Run the optional session TITO verifier when its e2e stack is available.""" + try: + from miles.utils.test_utils.session_verify_runner import run_session_verify + except Exception as exc: + print( + "verify-session-tito-tokenizer requires the optional Miles/SGLang " + "session e2e runner.", + ) + print(f"Missing runner detail: {type(exc).__name__}: {exc}") + return 2 + + try: + run_session_verify(args=args) + except AssertionError as exc: + print("verify-session-tito-tokenizer failed TITO/session verification.") + print(f"Verification detail: {exc}") + return 1 + except (ImportError, ModuleNotFoundError, RuntimeError, FileNotFoundError, OSError, ValueError) as exc: + print( + "verify-session-tito-tokenizer requires the optional Miles/SGLang " + "training stack and a valid verifier configuration.", + ) + print(f"Dependency/runtime detail: {type(exc).__name__}: {exc}") + return 2 + return 0 From b81f2fc523ee1c3fdf7893ac31ab63bd7ee0ea4b Mon Sep 17 00:00:00 2001 From: FatPigeorz Date: Mon, 29 Jun 2026 06:18:42 +0000 Subject: [PATCH 02/11] runtime+abridge: reliability refactor checkpoint + architecture-review fixes Checkpoint of the in-progress single-transport reliability work together with the fixes from the multi-agent architecture review. Verified at this state: pyright clean + 296 core/abridge tests pass. Architecture-review fixes folded in: - H1 result-emit task is retained (set + done-callback) so a GC'd task can't drop a successful call's terminal state. - H2 the worker's own agentix.* logs are routed off the /log capture pipe (propagate=False) to kill the broken-pipe feedback loop; user stdlib logging stays captured (ray-style, no agentix import needed). - H3 session() always deletes the container even if aclose() raises. - H4 upstream HTTP status is carried through the abridge tunnel (RemoteSioError .status_code) instead of collapsing to 502. - M1 _ensure_sio disconnects a stale client before rebuilding (no duplicate /rpc socket on a reconnect gap). - M2 abridge request_timeout is threaded into ns.request (was capped at 300s). - M4 + doc drift: PROTOCOL.md gains resume/ack + correct test path; codec docstrings drop ext-types; dead AGENTIX_LOG_BUFFER refs removed; REFACTOR log wording fixed. - L cleanups: drop dead make_sio ASGIApp + boot_error frame plumbing; typed CallCancelled terminal state (no bare CancelledError escaping Ok|Failed); call:error KeyError guarded; worker cancel enqueued sync; worker drain join bounded; proxy session teardown no longer masks the body error; sidecar _proc cleared / unreachable raise documented. Co-Authored-By: Claude Opus 4.8 --- REFACTOR.md | 132 ++++++++ agentix/__init__.py | 16 +- agentix/provider/base.py | 28 +- agentix/runtime/PROTOCOL.md | 31 +- agentix/runtime/client/__init__.py | 6 + agentix/runtime/client/client.py | 167 ++++------ agentix/runtime/client/result.py | 41 +++ agentix/runtime/server/app.py | 79 +---- agentix/runtime/server/sio.py | 106 ++++--- agentix/runtime/server/worker/client.py | 9 - agentix/runtime/server/worker/process.py | 197 +++++++----- agentix/runtime/shared/__init__.py | 2 +- agentix/runtime/shared/codec.py | 91 +----- agentix/runtime/shared/framing.py | 3 +- agentix/sio.py | 19 +- agentix/utils/log/__init__.py | 58 +--- agentix/utils/log/_bridge.py | 285 ++---------------- docs/concepts/plugins.mdx | 42 +-- docs/concepts/remote-calls.mdx | 8 +- docs/reference/architecture.mdx | 5 +- docs/reference/public-api.mdx | 7 +- .../abridge/agentix/bridge/clients/openai.py | 7 +- plugins/abridge/agentix/bridge/proxy.py | 35 ++- plugins/abridge/agentix/bridge/sidecar.py | 9 +- tests/e2e/test_reconnect.py | 103 +------ tests/runtime/client/test_client_options.py | 22 -- tests/runtime/client/test_robustness.py | 19 -- tests/runtime/test_protocol.py | 87 ++++-- tests/test_sio_namespace.py | 231 ++++---------- tests/test_stream_respawn_resets_dedup.py | 18 -- tests/utils/log/test_bridge.py | 59 ++-- 31 files changed, 768 insertions(+), 1154 deletions(-) create mode 100644 REFACTOR.md create mode 100644 agentix/runtime/client/result.py delete mode 100644 tests/runtime/client/test_client_options.py diff --git a/REFACTOR.md b/REFACTOR.md new file mode 100644 index 0000000..f7baac0 --- /dev/null +++ b/REFACTOR.md @@ -0,0 +1,132 @@ +# Convergence refactor — what we're building + +Design baseline. Some items done, the rest planned (see Work below). + +## Principles + +- **Programming framework.** Make `async def rollout(sandbox, task) -> R` + pleasant and typed. The hero is the user's rollout function. +- **Two surfaces.** Typed code (rollout authors: `Sandbox` / + `SandboxConfig` / `SandboxProvider` / `remote()` / plugins / `trace`) + and a string CLI (`agentix build` / `agentix deploy` + registries). + `deploy` is a CLI process; the bundle ref enters code as a `str`; + providers are constructed by typed import. +- **Type what we own.** Public API, `Result[T]`, plugin contracts, and + build-generated stubs are fully typed by construction. + +## Reliability contract + +The framework guarantees the integrity of *information about* each call. + +- Every `remote()` resolves to `Result` = **`Ok | Failed`** — a + truthful, unique terminal state. It never hangs and runs `fn` + at-most-once. +- **`resume`** re-fetches a terminal state (no re-run). **`retry`** is a + new `call_id` (a new rollout); the caller owns retry. + +Per channel: + +| Channel | Guarantee | +|---|---| +| **RPC** | at-most-once execution; terminal state delivered exactly once; an undeliverable result is a `Failed` | +| **trace** (the dataset) | at-least-once + host-side dedup + durable sink | +| **log** | durable on disk + best-effort live stream | + +## Target architecture + +- **One transport:** Socket.IO `/rpc`. HTTP serves `/health`. +- **Namespaces** stay pipe-forwarded and plugin-agnostic. +- **RPC and trace** each carry their reliability; share one retain/ack + buffer where it is genuinely cleaner. +- **log** is stdout/stderr line capture (per-sandbox file + best-effort + stream on the `/log` namespace) — a plain forwarded namespace, no + `ReliableStream` and no structured `LogRecord` bridge. +- **One typed plugin primitive** (below). +- **Typing is Python-native:** ParamSpec + `Result[T]` + build-time + codegen. The build step is our compile step. + +## Plugin primitive + +One declarative, typed surface: define the contract once, implement one +side, call typed from the other. Default is request/response; streaming +is an explicit opt-in. + +```python +# contract.py — one typed interface, shared by both sides +class Solve(BaseModel): task: str +class Solution(BaseModel): patch: str + +# host side — implement, fully typed +class Solver(Plugin): + name = "solver" # -> /solver + @on + async def solve(self, req: Solve) -> Solution: # typed in/out + return Solution(patch=await call_model(req.task)) + + @stream # at-least-once opt-in + async def span(self, ev: SpanEvent) -> None: ... + +# sandbox target fn — typed call +async def rollout(): + sol = await plugins.solver.solve(Solve(task="...")) + return sol.patch +``` + +- One base (`Plugin`), one registration + (`provider.session(..., plugins=[Solver()])`). +- The method *is* the op (decorator-discovered); pydantic payloads; + framework-managed `request_id` and error envelope. +- abridge is a specialization on top (op names = HTTP paths + an + in-sandbox FastAPI tunnel). +- Decide: the sandbox-side caller handle stays typed via a + contract-typed handle (preferred) over a metaclass-routed one class. + +## Build-time typed remote + +`@remote` annotation → `agentix build` emits typed client stubs + a +closed dispatch manifest + a schema'd codec. The generated boundary is +fully typed by construction. `sandbox.remote(fn, ...)` keeps its +ParamSpec `(P) -> R`; codegen adds closed-set validation, a real schema, +and (when wanted) non-Python clients. + +## Work + +### Done — branch `refactor/single-transport` + +1. **Single transport.** Removed the HTTP `/call` fast-path; every + `c.remote()` rides Socket.IO `/rpc`. ruff + pyright clean; 285 tests + pass. +2. **No silent loss.** A `resume` for an evicted/unknown `call_id` + returns a definite `call:error` (`ResultUnavailable`); terminal + states are `{result, error}`. +3. **Never-hang guarantee satisfied.** The worker already fails every + in-flight call on death (`WorkerProcessExited`), fails fast on a + closed worker, turns an oversized result frame into a `FrameTooLarge` + error, and cancels idempotently — `remote()` always reaches a + terminal state (or `CallTimeout`). +4. **Narrowed the top-level surface.** Moved `providers`, + `register_provider`, `BundleDeployer`, `DeployedBundle` off + `agentix.__all__` (still in `agentix.provider.base`); stripped the + codec to plain msgpack; collapsed the reconnection options to + socketio's defaults. +5. **`Result[T]` API.** `Ok | Failed` in `agentix.runtime.client.result`, + exported at top level; `remote()` still raises, `try_remote()` + returns `Result[R]` for exhaustive `match`. +6. **log → Ray-style capture.** The worker captures its stdout *and* + stderr (stdlib `logging` writes to stderr, so it's captured too), + appends to a sandbox-side `sandbox.log`, and streams each line + best-effort on `/log`; the host replays under `agentix.sandbox.{stdout, + stderr}`. Deleted the structured `LogRecord` bridge (`WorkerLogHandler`, + `emit_worker_record`, the host `_replay_record`) and `/log`'s + `ReliableStream` use. `configure_logging` stays as the local-logging + helper (host/runtime/worker). + +### To do — in order + +1. **trace reshape.** Prompt per-span at-least-once emit + host durable + sink. +2. **Shared retain/ack buffer** for RPC + trace, where it is cleaner. +3. **Plugin primitive.** Typed `Plugin` + `@on` / `@stream`; abridge as + a specialization. +4. **Build-time typed remote bindings.** `@remote` → build emits stubs + + manifest + schema codec. diff --git a/agentix/__init__.py b/agentix/__init__.py index 418495b..1cf50fa 100644 --- a/agentix/__init__.py +++ b/agentix/__init__.py @@ -10,20 +10,20 @@ __path__ = pkgutil.extend_path(__path__, __name__) from agentix.provider.base import ( - BundleDeployer, - DeployedBundle, Sandbox, SandboxConfig, SandboxId, SandboxInfo, SandboxProvider, SandboxResource, - providers, - register_provider, ) from agentix.runtime.client import ( + CallCancelled, CallTimeout, + Failed, + Ok, RemoteCallError, + Result, RuntimeClient, RuntimeUnreachable, WorkerExited, @@ -38,13 +38,15 @@ __all__ = [ "AsyncClientNamespace", - "BundleDeployer", + "CallCancelled", "CallTimeout", - "DeployedBundle", + "Failed", "Namespace", + "Ok", "RemoteCallable", "RemoteCallError", "RemoteSioError", + "Result", "RuntimeClient", "RuntimeUnreachable", "Sandbox", @@ -58,9 +60,7 @@ "configure_logging", "context", "log", - "providers", "register_namespace", - "register_provider", "request_handler", "trace", ] diff --git a/agentix/provider/base.py b/agentix/provider/base.py index 4790353..3cff659 100644 --- a/agentix/provider/base.py +++ b/agentix/provider/base.py @@ -40,7 +40,7 @@ async def get(self, sandbox_id): ... from agentix.provider._plugin import Registry if TYPE_CHECKING: - from agentix.runtime.client import RuntimeClient + from agentix.runtime.client import Result, RuntimeClient from agentix.runtime.shared.models import HealthResponse P = ParamSpec("P") @@ -215,6 +215,16 @@ async def remote( """Execute `fn(*args, **kwargs)` in this sandbox and return its result.""" return await self._runtime_client().remote(fn, *args, **kwargs) + async def try_remote( + self, + fn: Callable[P, R] | Callable[P, Awaitable[R]], + *args: P.args, + **kwargs: P.kwargs, + ) -> Result[R]: + """Execute `fn` and return a `Result[R]` (`Ok | Failed`) instead of + raising on a terminal error — see `RuntimeClient.try_remote`.""" + return await self._runtime_client().try_remote(fn, *args, **kwargs) + async def health(self) -> HealthResponse: return await self._runtime_client().health() @@ -264,12 +274,24 @@ async def session( result = await sandbox.remote(agent.run, task=task) """ sandbox = await self.create(config) + # Contract: `create()` only provisions the sandbox; it must NOT + # materialize the RuntimeClient (the lazy `_runtime_client()` reads + # `call_deadline` at first `remote()`). That is what lets us stamp the + # deadline here, post-create. A provider that eagerly connected inside + # `create()` would bake in `call_deadline=None` — such a provider must + # accept the deadline through `create()` instead. sandbox.call_deadline = call_deadline try: yield sandbox finally: - await sandbox.aclose() - await self.delete(sandbox.sandbox_id) + # `delete()` must run even if `aclose()` raises (e.g. an httpx + # pool error or CancelledError during shutdown); otherwise the + # container and its reserved port leak — the exact failure the + # never-leak contract targets. + try: + await sandbox.aclose() + finally: + await self.delete(sandbox.sandbox_id) @runtime_checkable diff --git a/agentix/runtime/PROTOCOL.md b/agentix/runtime/PROTOCOL.md index 784d4b7..2c4fa76 100644 --- a/agentix/runtime/PROTOCOL.md +++ b/agentix/runtime/PROTOCOL.md @@ -1,7 +1,7 @@ # Agentix RPC Protocol The runtime wire contract for `RuntimeClient.remote(fn, *args, **kwargs)`. -Tests in `tests/test_rpc_protocol.py` enforce these rules. +Tests in `tests/runtime/test_protocol.py` enforce these rules. ## Callable Reference @@ -35,17 +35,14 @@ await client.remote(run, seed=42) | Path | Carries | Wire | | --- | --- | --- | | `GET /health` | health probe | HTTP JSON | -| `POST /call` | internal short-call fast path | HTTP msgpack | -| Socket.IO `/rpc` | `c.remote()` RPC | msgpack-wrapped `call` / `call:result` / `call:error` / `cancel` | +| Socket.IO `/rpc` | `c.remote()` RPC | msgpack-wrapped `call` / `call:result` / `call:error` / `cancel` / `resume` / `ack` | | Socket.IO `/trace`, `/log`, `/` | side channels | plugin-defined events (msgpack payloads) | | worker private pipe | runtime ↔ worker | length-prefixed msgpack frames | -HTTP covers health plus the internal `/call` fast path for short -remote calls. Socket.IO `/rpc` remains the RPC event channel when a -call is submitted over SIO or an accepted HTTP call completes -asynchronously. The worker pipe is the runtime-to-worker edge inside -the sandbox. The current implementation uses one worker subprocess per -runtime. +Every `c.remote()` rides one transport: Socket.IO `/rpc`. HTTP serves +only the `/health` probe. The worker pipe is the runtime-to-worker edge +inside the sandbox. The current implementation uses one worker +subprocess per runtime. ## Socket.IO Events (RPC on `/rpc`) @@ -54,11 +51,21 @@ call {call_id, callable, arguments} call:result {call_id, value} # value is pickle.dumps(result) call:error {call_id, error} cancel {call_id} +resume {call_ids} # host → server on (re)connect +ack {call_id} # host → server, frees the retained result ``` `call_id` correlates request ↔ response. Cancellation produces a `call:error` with `error.cancelled=True`. +`resume` and `ack` carry the reliability contract. On every (re)connect the +host emits `resume` with the `call_ids` it is still awaiting; the server +replays each one's terminal state as a `call:result` / `call:error` — and for +an evicted or unknown `call_id`, a definite `call:error` +(`error.type="ResultUnavailable"`) rather than silence, so a reconnecting host +never hangs. After consuming a result the host emits `ack`, which lets the +server drop it from its bounded retain buffer (`pending_results`). + Trace, log, and plugin traffic use their own namespaces on the same Socket.IO connection. Sandbox plugins emit through `agentix.sio`; the worker forwards `sio_emit` / `sio_open` frames to the server, which @@ -77,7 +84,6 @@ away from user subprocesses). | server → worker | `shutdown` | — | | server → worker | `sio_inbound` | `namespace`, `event`, `data` | | worker → server | `ready` | — | -| worker → server | `boot_error` | `error` | | worker → server | `result` | `call_id`, `value` | | worker → server | `error` | `call_id`, `error` | | worker → server | `sio_emit` | `namespace`, `event`, `data` | @@ -95,6 +101,11 @@ away from user subprocesses). 4. **Worker death closes calls.** If the worker subprocess exits, the runtime fails every in-flight call with `WorkerExited` so the client never hangs. +5. **No silent loss.** A `resume` for a `call_id` the runtime no longer + holds (its result was evicted under cap, or the id is unknown) gets a + `call:error` (`type="ResultUnavailable"`), never silence. An + undeliverable result is a failure, not a separate "lost" state — the + caller decides whether to retry as a new call. ## Error Model diff --git a/agentix/runtime/client/__init__.py b/agentix/runtime/client/__init__.py index ba8ffcc..68a1a56 100644 --- a/agentix/runtime/client/__init__.py +++ b/agentix/runtime/client/__init__.py @@ -14,16 +14,22 @@ """ from agentix.runtime.client.client import ( + CallCancelled, CallTimeout, RemoteCallError, RuntimeClient, RuntimeUnreachable, WorkerExited, ) +from agentix.runtime.client.result import Failed, Ok, Result __all__ = [ + "CallCancelled", "CallTimeout", + "Failed", + "Ok", "RemoteCallError", + "Result", "RuntimeClient", "RuntimeUnreachable", "WorkerExited", diff --git a/agentix/runtime/client/client.py b/agentix/runtime/client/client.py index 51178e8..e48d683 100644 --- a/agentix/runtime/client/client.py +++ b/agentix/runtime/client/client.py @@ -26,12 +26,13 @@ import pickle import uuid from collections.abc import Awaitable, Callable -from typing import Any, Literal, ParamSpec, TypeVar, cast +from typing import Any, ParamSpec, TypeVar, cast import httpx import socketio from socketio.exceptions import ConnectionError as SioConnectionError +from agentix.runtime.client.result import Failed, Ok, Result from agentix.runtime.shared import MAX_MESSAGE_BYTES from agentix.runtime.shared.callables import RemoteCallable, display_name_for from agentix.runtime.shared.codec import pack, unpack @@ -90,9 +91,18 @@ def returncode(self) -> int | None: return self.error.returncode +class CallCancelled(RemoteCallError): + """The runtime reported the call as cancelled — a terminal *server-side* + state, distinct from local `asyncio` task cancellation. Subclasses + `RemoteCallError` so it rides the normal terminal-state path: `remote()` + raises it and `try_remote()` surfaces it as `Failed`. (A bare + `asyncio.CancelledError` here would escape the `Ok | Failed` sum type and + read as local cancellation.)""" + + def _raise_remote_error(display_name: str, error: RemoteError): if error.cancelled: - raise asyncio.CancelledError(error.message) + raise CallCancelled(display_name=display_name, error=error) if error.type == "WorkerDied": raise WorkerExited(display_name=display_name, error=error) raise RemoteCallError(display_name=display_name, error=error) @@ -120,32 +130,22 @@ def __init__( base_url: str, timeout: float = 300, *, - http_sync_ms: int | None = 1000, call_deadline: float | None = None, - reconnection: bool | None = None, - reconnection_attempts: int | None = None, - reconnection_delay: float | None = None, - reconnection_delay_max: float | None = None, - randomization_factor: float | None = None, ): """Connect to a runtime server at `base_url`. `timeout` is the per-request HTTP/WebSocket timeout in seconds; raise it for long agent calls (e.g. `RuntimeClient(url, timeout=1800)`). - `http_sync_ms` is the inline HTTP fast-path budget for short calls, - sent as the RFC 7240 `Prefer: respond-async, wait=N` header: a call - that finishes within this many milliseconds returns over HTTP (200), - otherwise the server replies 202 and the result follows on Socket.IO. - Set `http_sync_ms=None` to disable the fast path and send every call - over Socket.IO. + Every call rides one transport: Socket.IO on `/rpc`. HTTP is used + only for the `/health` probe. Reconnection uses socketio's defaults + (on, infinite attempts, 1–5s backoff). `call_deadline` (seconds, None = unbounded) is the cheap catch-all upper bound for any single `remote(...)`: whatever the cause — worker hang, silent sandbox loss, network black hole — the caller gets a `CallTimeout` (and the call is cancelled server-side) instead of - hanging. The `reconnection*` knobs override socketio's defaults for - long-lived sessions; left as None they use socketio's own defaults. + hanging. """ self._base_url = base_url self._client = httpx.AsyncClient(base_url=base_url, timeout=timeout) @@ -157,27 +157,9 @@ def __init__( # Namespaces queued for registration on connect. self._namespaces: list[socketio.AsyncClientNamespace] = [] self._register_core_namespaces() - # HTTP fast-path budget in ms (None disables it), sent as the - # RFC 7240 `Prefer: respond-async, wait=N` header (converted to - # seconds by `_try_http_fast_path`). - self._http_sync_budget_ms: int | None = http_sync_ms # Upper bound for any single `remote(...)` (seconds). None = no # deadline. The cheap catch-all so the caller never hangs. self._call_deadline = call_deadline - # Socket.IO reconnection knobs. Left out (None) → socketio's own - # defaults (reconnection on, infinite attempts, 1–5s backoff); - # override at construction for long-lived (tens-of-hours) sessions. - self._sio_options: dict[str, Any] = { - key: value - for key, value in ( - ("reconnection", reconnection), - ("reconnection_attempts", reconnection_attempts), - ("reconnection_delay", reconnection_delay), - ("reconnection_delay_max", reconnection_delay_max), - ("randomization_factor", randomization_factor), - ) - if value is not None - } def _register_core_namespaces(self) -> None: """Register agentix-core's built-in `/trace` and `/log` handlers.""" @@ -237,46 +219,6 @@ def register_namespace(self, ns: socketio.AsyncClientNamespace) -> None: raise ValueError(f"namespace {path!r} already registered") self._namespaces.append(ns) - async def _try_http_fast_path( - self, - *, - sio: socketio.AsyncClient, - payload: dict[str, Any], - ) -> tuple[Literal["fallback", "accepted", "result", "error"], Any]: - if self._http_sync_budget_ms is None: - # Fast path disabled — go straight to the Socket.IO channel. - return "fallback", None - sid = getattr(sio, "sid", None) - if not (isinstance(sid, str) and sid): - return "fallback", None - - wait = self._http_sync_budget_ms / 1000 # ms → seconds for `Prefer: wait=` - wait_token = str(int(wait)) if wait == int(wait) else str(wait) - r = await self._client.post( - "/call", - content=pack(payload), - headers={ - "content-type": "application/msgpack", - "prefer": f"respond-async, wait={wait_token}", - }, - ) - r.raise_for_status() - - # 202: not done within the budget — the result follows on SIO. - if r.status_code == 202: - return "accepted", None - - # 200: completed — success or remote exception, per the `ok` flag. - reply = unpack(r.content) if r.content else {} - if not isinstance(reply, dict): - raise RuntimeError("invalid /call reply payload") - if reply.get("ok") is True: - return "result", _unpickle_value(reply.get("value")) - if reply.get("ok") is False: - err = RemoteError.model_validate(reply.get("error") or {}) - return "error", err - raise RuntimeError("invalid /call reply fields") - async def remote( self, fn: Callable[P, R] | Callable[P, Awaitable[R]], @@ -309,32 +251,23 @@ async def remote( # (worker hang, silent sandbox loss, network black hole), the # await below cannot block past the deadline. None = no bound. async with asyncio.timeout(self._call_deadline): - # Fast path: try HTTP first for short-running calls. If the - # call exceeds the sync budget, the server returns `accepted` - # and completes via the normal SIO result channel. - kind, value = await self._try_http_fast_path(sio=sio, payload=payload) - if kind == "fallback": - await sio.emit("call", pack(payload), namespace=RPC_NAMESPACE) - elif kind == "result": - terminated = True - return cast(R, value) - elif kind == "error": - terminated = True - _raise_remote_error(display_name, cast(RemoteError, value)) + await sio.emit("call", pack(payload), namespace=RPC_NAMESPACE) while True: kind, data = await q.get() if kind == "result": terminated = True return cast(R, _unpickle_value(data.get("value"))) if kind == "error": - err = RemoteError.model_validate(data["error"]) + # Defensive: a malformed frame with no `error` payload + # must still resolve to a typed terminal error, not a + # bare KeyError escaping `remote()` / `try_remote()`. + raw_err = data.get("error") or { + "type": "MalformedError", + "message": "runtime sent a call:error with no error payload", + } + err = RemoteError.model_validate(raw_err) terminated = True _raise_remote_error(display_name, err) - if kind == "fatal": - # The connection was terminally lost (reconnection - # disabled) — no result will ever arrive on this queue. - terminated = True - raise data except TimeoutError: raise CallTimeout( f"remote call '{display_name}' exceeded deadline of {self._call_deadline}s" @@ -349,6 +282,21 @@ async def remote( namespace=RPC_NAMESPACE, ) + async def try_remote( + self, + fn: Callable[P, R] | Callable[P, Awaitable[R]], + *args: P.args, + **kwargs: P.kwargs, + ) -> Result[R]: + """Like `remote()`, but returns a `Result[R]` (`Ok | Failed`) instead + of raising on a terminal error — for callers that branch on the + outcome with `match`. Misuse (a non-importable callable) and + cancellation still raise.""" + try: + return Ok(await self.remote(fn, *args, **kwargs)) + except (RemoteCallError, CallTimeout, RuntimeUnreachable) as exc: + return Failed(exc) + # ── Socket.IO connection management ───────────────────────── async def _ensure_sio(self) -> socketio.AsyncClient: @@ -357,13 +305,24 @@ async def _ensure_sio(self) -> socketio.AsyncClient: async with self._sio_lock: if self._sio is not None and self._sio.connected: return self._sio + if self._sio is not None: + # A handle exists but is disconnected (a transport drop mid + # reconnect). Tear it down before building a fresh client — + # overwriting it without disconnecting leaks its aiohttp session + # and background reconnect task, and would leave a second live + # `/rpc` socket once the abandoned client reconnects on its own. + # In-flight calls are not lost: their `self._pending` entries + # persist and the fresh client's `connect` handler re-emits + # `resume` to recover their results. + with contextlib.suppress(BaseException): + await self._sio.disconnect() + self._sio = None # `max_msg_size` lifts the websocket's receive cap (engineio's # client rides aiohttp, default 4 MB) — large `c.remote` # payloads / plugin events otherwise kill the connection. # Matches the server's `max_http_buffer_size`. sio = socketio.AsyncClient( websocket_extra_options={"max_msg_size": MAX_MESSAGE_BYTES}, - **self._sio_options, ) async def _on_call_result(data): @@ -390,16 +349,10 @@ async def _on_connect(*_args): ) async def _on_disconnect(*_args): - # With reconnection on (the default), tasks survive server-side - # and `_on_connect` re-emits `resume` to recover results, so we - # just wait. With reconnection explicitly disabled, this - # disconnect is terminal — no resume will ever deliver the - # pending results, so fail them now instead of hanging until - # `call_deadline` (which defaults to unbounded). - if self._sio_options.get("reconnection") is False: - self._fail_pending(RuntimeUnreachable(f"runtime server connection lost: {self._base_url}")) - else: - logger.debug("sio disconnect; will resume after reconnect") + # Reconnection is on, so server-side tasks survive and + # `_on_connect` re-emits `resume` to recover results — just + # wait. `call_deadline` bounds the wait if it never reconnects. + logger.debug("sio disconnect; will resume after reconnect") async def _on_connect_error(*args): # Previously unobserved: surface (re)connection failures so @@ -450,14 +403,6 @@ async def _ack(self, call_id: str) -> None: with contextlib.suppress(BaseException): await sio.emit("ack", pack({"call_id": call_id}), namespace=RPC_NAMESPACE) - def _fail_pending(self, exc: BaseException) -> None: - """Drain every in-flight call's queue with a fatal error so each - waiting `remote(...)` stops and raises, instead of blocking forever on - a connection that will never deliver a result.""" - for q in self._pending.values(): - with contextlib.suppress(BaseException): - q.put_nowait(("fatal", exc)) - __all__ = [ "CallTimeout", diff --git a/agentix/runtime/client/result.py b/agentix/runtime/client/result.py new file mode 100644 index 0000000..b8dbdae --- /dev/null +++ b/agentix/runtime/client/result.py @@ -0,0 +1,41 @@ +"""`Result[T]` — the typed outcome of a remote call. + +`remote()` raises on failure (idiomatic Python, clean happy path). +`try_remote()` returns this `Ok | Failed` sum type instead, for callers +that branch on the outcome at scale (a rollout harness) and want +exhaustive matching: + + match await sandbox.try_remote(solve, task=t): + case Ok(patch): use(patch) + case Failed(WorkerExited() as e): retry_with_more_memory(e.returncode) + case Failed(error): record_failure(error) +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Generic, TypeVar + +T = TypeVar("T") + + +@dataclass(frozen=True) +class Ok(Generic[T]): + """A remote call that returned a value.""" + + value: T + + +@dataclass(frozen=True) +class Failed: + """A remote call that ended in a terminal error — carries the same + exception `remote()` would have raised (`RemoteCallError` / + `WorkerExited` / `CallTimeout` / `RuntimeUnreachable`).""" + + error: Exception + + +# `Ok[T] | Failed`, subscriptable as `Result[R]` (a generic union alias). +Result = Ok[T] | Failed + +__all__ = ["Failed", "Ok", "Result"] diff --git a/agentix/runtime/server/app.py b/agentix/runtime/server/app.py index 4d372f0..66d3a48 100644 --- a/agentix/runtime/server/app.py +++ b/agentix/runtime/server/app.py @@ -5,17 +5,11 @@ Endpoints: - `GET /health` -- `POST /call` — internal fast-path used by `RuntimeClient.remote`; - msgpack request/response. The caller sends RFC 7240 - `Prefer: respond-async, wait=N` (N seconds; fractional accepted). - Returns **200** with the result if it lands within that budget; - otherwise **202** with `{call_id}` + a `Location` header, and the - result follows on Socket.IO (`call:result` / `call:error`). The - honored budget is echoed in `Preference-Applied: wait=N`. - Socket.IO at `/socket.io/` — unary RPC on `/rpc` (`call` / `call:result` / `call:error`, `cancel`, plus `resume`/`ack` for reconnect-safe delivery), and side-channel namespaces (`/trace`, `/log`, and - plugin paths registered via `agentix.sio`). + plugin paths registered via `agentix.sio`). Every `c.remote(...)` + rides this one transport. Remote requests carry a `RemoteCallable` import path plus a pickle of the (args, kwargs) tuple. Only importable top-level functions and @@ -25,15 +19,13 @@ from __future__ import annotations import logging -import re from contextlib import asynccontextmanager -from fastapi import FastAPI, Request, Response +from fastapi import FastAPI from agentix import __version__ from agentix.runtime.server.sio import make_sio from agentix.runtime.server.worker import RuntimeWorkerClient -from agentix.runtime.shared.codec import pack, unpack from agentix.runtime.shared.models import HealthResponse from agentix.utils.log import configure_logging @@ -66,69 +58,6 @@ async def health() -> HealthResponse: return HealthResponse(version=__version__) -_MSGPACK = "application/msgpack" -# RFC 7240 `wait` is delta-seconds (integer); we accept fractional too -# since we own both ends — a benign superset that keeps sub-second -# budgets expressible. -_WAIT_RE = re.compile(r"(?:^|[,;\s])wait\s*=\s*\"?([0-9]+(?:\.[0-9]+)?)", re.IGNORECASE) -_DEFAULT_WAIT_S = 1.0 - - -def _parse_prefer_wait(prefer: str, *, default: float) -> float: - match = _WAIT_RE.search(prefer or "") - if not match: - return default - try: - return max(float(match.group(1)), 0.0) - except ValueError: - return default - - -def _format_wait(seconds: float) -> str: - return str(int(seconds)) if seconds == int(seconds) else str(seconds) - - -@_fastapi_app.post("/call") -async def call(request: Request) -> Response: - """Internal fast-path endpoint used by `RuntimeClient.remote`. - - Request/response payloads are msgpack bytes, not JSON. The sync - budget is the RFC 7240 `Prefer: respond-async, wait=N` header. - """ - raw = await request.body() - payload = unpack(raw) if raw else {} - if not isinstance(payload, dict): - payload = {} - - if not isinstance(payload.get("call_id"), str): - error = {"type": "BadRequest", "message": "missing or invalid call_id"} - return Response( - content=pack({"ok": False, "error": error}), - status_code=400, - media_type=_MSGPACK, - ) - - wait_s = _parse_prefer_wait(request.headers.get("prefer", ""), default=_DEFAULT_WAIT_S) - applied = {"Preference-Applied": f"wait={_format_wait(wait_s)}"} - - submit = getattr(_sio, "submit_http_call") - result = await submit(payload, wait_s=wait_s) - - if result.get("accepted") is True: - call_id = result.get("call_id") - return Response( - content=pack({"call_id": call_id}), - status_code=202, - media_type=_MSGPACK, - headers={**applied, "Location": f"/call/{call_id}"}, - ) - - # Completed within the budget (200), success or remote exception — - # both are a delivered RPC outcome, distinguished by the `ok` flag. - body = {key: result[key] for key in ("ok", "value", "error") if key in result} - return Response(content=pack(body), status_code=200, media_type=_MSGPACK, headers=applied) - - # ── Compose ASGI app: FastAPI health + Socket.IO remote calls ── # # The combined ASGI app is what uvicorn runs as @@ -137,7 +66,7 @@ async def call(request: Request) -> Response: import socketio as _socketio # noqa: E402 -_sio, _ = make_sio(_worker) +_sio = make_sio(_worker) app = _socketio.ASGIApp(_sio, _fastapi_app, socketio_path="/socket.io") app.fastapi = _fastapi_app # type: ignore[attr-defined] app.state = _fastapi_app.state # type: ignore[attr-defined] diff --git a/agentix/runtime/server/sio.py b/agentix/runtime/server/sio.py index 75834e2..9f7223c 100644 --- a/agentix/runtime/server/sio.py +++ b/agentix/runtime/server/sio.py @@ -38,7 +38,9 @@ # Cap on the unacked-result cache. A host that completes calls and never acks # (a crashed or buggy client) would otherwise pin every result — each holding a # full pickled return value, up to MAX_MESSAGE_BYTES — in memory forever. Past -# the cap, the oldest unacked entry is evicted. +# the cap, the oldest unacked entry is evicted. Eviction is not a silent loss: +# a later `resume` for that call_id gets a definite `call:error` (see +# `on_resume` / `_unavailable_error`), so the host fails rather than hangs. _MAX_PENDING_RESULTS = 4096 @@ -79,6 +81,19 @@ def _cancelled_error(call_id: str) -> dict[str, Any]: } +def _unavailable_error(call_id: str) -> dict[str, Any]: + return { + "call_id": call_id, + "error": RemoteError( + type="ResultUnavailable", + message=( + "result is no longer held by the runtime (evicted or unknown call_id); " + "retry as a new call" + ), + ).model_dump(), + } + + def _store_pending_result( cache: dict[str, tuple[str, dict[str, Any]]], call_id: str, @@ -101,7 +116,7 @@ def _store_pending_result( def make_sio( worker: RuntimeWorkerClient, -) -> tuple[socketio.AsyncServer, socketio.ASGIApp]: +) -> socketio.AsyncServer: # `namespaces='*'` accepts connects on any namespace path. Plugin # namespaces are registered lazily by the worker (`sio_open` frame # in response to `agentix.register_namespace(...)`); the host may @@ -127,14 +142,13 @@ def make_sio( max_http_buffer_size=MAX_MESSAGE_BYTES, ) # ── execution-once invariant ───────────────────────────────── - # `_start_call` is the only place a task is created. Every site - # that may call it (`on_call`, `submit_http_call`) gates on - # `call_id in calls or call_id in pending_results` first, so a - # given call_id starts at most one task. Combined with the host - # generating a fresh call_id per `c.remote(...)`, this guarantees - # the user-facing contract: each `c.remote(fn, ...)` runs `fn` at - # most once on the runtime, even across reconnects, replays, and - # mixed HTTP/SIO submission paths. + # `_start_call` is the only place a task is created, and `on_call` + # gates on `call_id in calls or call_id in pending_results` first, + # so a given call_id starts at most one task. Combined with the + # host generating a fresh call_id per `c.remote(...)`, this + # guarantees the user-facing contract: each `c.remote(fn, ...)` + # runs `fn` at most once on the runtime, even across reconnects and + # duplicate `call` / `resume` submissions. calls: dict[str, asyncio.Task] = {} # Completed tasks waiting for the host to ack receipt. The host # acks via the `ack` SIO event after consuming the result; only @@ -145,6 +159,12 @@ def make_sio( pending_results: dict[str, tuple[str, dict[str, Any]]] = {} evictions = 0 # count of cap evictions, for throttled warning opened_namespaces: set[str] = set() # paths the worker has opened + # Strong refs to the result-delivery tasks. `asyncio.create_task` only + # registers a weak reference with the loop, so without this set a + # delivery task could be GC'd before it stores the result in + # `pending_results` — turning a successful call into a `Failed` on the + # next `resume`. Mirrors the `calls` tracking above. + emit_tasks: set[asyncio.Task] = set() async def _execute_call(payload: dict[str, Any], call_id: str) -> tuple[str, dict[str, Any]]: try: @@ -223,34 +243,12 @@ def _start_call(payload: dict[str, Any], call_id: str) -> asyncio.Task: _track_call(call_id, task) return task - async def submit_http_call(payload: dict[str, Any], *, wait_s: float = 1.0) -> dict[str, Any]: - call_id = payload.get("call_id") - if not isinstance(call_id, str): - _event, frame = _missing_call_id() - return {"accepted": False, "ok": False, **frame} - - if call_id in calls or call_id in pending_results: - # Already in flight or sitting unacked. The host will pick - # the result up via SIO (either fresh emit or `resume`). - return {"accepted": True, "call_id": call_id} - - task = _start_call(payload, call_id) - - timeout_s = max(wait_s, 0.0) - try: - event, frame = await asyncio.wait_for(asyncio.shield(task), timeout=timeout_s) - except TimeoutError: - task.add_done_callback( - lambda t, cid=call_id: asyncio.create_task(_emit_task_result(t, cid)) - ) - return {"accepted": True, "call_id": call_id} - - if event == "call:result": - return {"accepted": False, "ok": True, **frame} - return {"accepted": False, "ok": False, **frame} - - # Runtime internal hook used by the HTTP fast-path endpoint. - setattr(sio, "submit_http_call", submit_http_call) + def _schedule_emit(task: asyncio.Task, call_id: str) -> None: + # Retain a strong ref until the delivery task finishes; otherwise the + # loop's weak ref lets it be collected before it caches the result. + emit_task = asyncio.create_task(_emit_task_result(task, call_id)) + emit_tasks.add(emit_task) + emit_task.add_done_callback(emit_tasks.discard) async def on_connect(sid: str, environ: dict, auth: Any = None) -> None: logger.debug("sio connect %s", sid) @@ -282,9 +280,7 @@ async def on_call(sid: str, data: Any) -> None: return task = _start_call(payload, call_id) - task.add_done_callback( - lambda t, cid=call_id: asyncio.create_task(_emit_task_result(t, cid)) - ) + task.add_done_callback(lambda t, cid=call_id: _schedule_emit(t, cid)) async def on_cancel(sid: str, data: Any) -> None: payload = _u(data) @@ -306,8 +302,15 @@ async def on_cancel(sid: str, data: Any) -> None: ) async def on_resume(sid: str, data: Any) -> None: - """Replay cached results for the call_ids the host is still - waiting on. Called by the host right after (re)connect.""" + """Resolve the call_ids the host is still waiting on. Called by the + host right after (re)connect. Each id reaches a definite terminal + state — never silence: + + - a cached result → replayed; + - still running → left alone (its result arrives on completion); + - no record (evicted under cap, or unknown) → a `call:error` so the + host's `remote()` fails instead of hanging forever. + """ payload = _u(data) ids = payload.get("call_ids") if not isinstance(ids, list): @@ -316,10 +319,18 @@ async def on_resume(sid: str, data: Any) -> None: if not isinstance(cid, str): continue cached = pending_results.get(cid) - if cached is None: + if cached is not None: + event, frame = cached + await sio.emit(event, pack(frame), to=sid, namespace=RPC_NAMESPACE) continue - event, frame = cached - await sio.emit(event, pack(frame), to=sid, namespace=RPC_NAMESPACE) + if cid in calls: + continue + await sio.emit( + "call:error", + pack(_unavailable_error(cid)), + to=sid, + namespace=RPC_NAMESPACE, + ) async def on_ack(sid: str, data: Any) -> None: """Host confirms it has consumed the result. Free the slot.""" @@ -386,5 +397,4 @@ async def trigger_event(self, event: str, *args: Any) -> Any: opened_namespaces.add(core_ns) _register_namespace(core_ns) - asgi_app = socketio.ASGIApp(sio, socketio_path="/socket.io") - return sio, asgi_app + return sio diff --git a/agentix/runtime/server/worker/client.py b/agentix/runtime/server/worker/client.py index 200d899..35935b5 100644 --- a/agentix/runtime/server/worker/client.py +++ b/agentix/runtime/server/worker/client.py @@ -210,7 +210,6 @@ def __init__( self._outbound: asyncio.Queue[dict[str, Any]] = asyncio.Queue() self._drainer: asyncio.Task | None = None self._ready = asyncio.Event() - self._boot_error: dict[str, Any] | None = None self._read_task: asyncio.Task | None = None self._closed = asyncio.Event() @@ -267,11 +266,6 @@ async def start(self) -> None: for task in (ready_task, closed_task, proc_task): if not task.done(): task.cancel() - if self._boot_error is not None: - await self.shutdown() - raise RuntimeError( - f"runtime worker failed to boot: {self._boot_error.get('type')}: {self._boot_error.get('message')}" - ) async def _read_loop(self) -> None: assert self._proc is not None and self._proc.stdout is not None @@ -306,9 +300,6 @@ async def _on_frame(self, frame: dict[str, Any]) -> None: kind = frame.get("type") if kind == "ready": self._ready.set() - elif kind == "boot_error": - self._boot_error = frame.get("error") or {"type": "Unknown", "message": ""} - self._ready.set() elif kind == "result": cid = frame.get("call_id", "") fut = self._pending.pop(cid, None) diff --git a/agentix/runtime/server/worker/process.py b/agentix/runtime/server/worker/process.py index b08e3b0..8250903 100644 --- a/agentix/runtime/server/worker/process.py +++ b/agentix/runtime/server/worker/process.py @@ -14,8 +14,8 @@ import logging import os import sys -import time import traceback +from pathlib import Path from typing import Any from agentix import sio as _sio @@ -26,8 +26,7 @@ from agentix.runtime.shared.idents import CallId from agentix.runtime.shared.models import RemoteError, RemoteRequest from agentix.utils import log as _log -from agentix.utils.log._bridge import emit_worker_record -from agentix.utils.log._config import LOG_CONTEXT_ATTR, get_log_context +from agentix.utils.log._bridge import LOG_EVENT, LOG_NAMESPACE from agentix.utils.trace._bridge import install_worker_bridge logger = logging.getLogger("agentix.runtime.server.worker.process") @@ -53,6 +52,10 @@ def __init__(self) -> None: self._outbound_q: asyncio.Queue[dict[str, Any]] = asyncio.Queue() self._drainer: asyncio.Task | None = None self._stdio_tasks: list[asyncio.Task] = [] + # Durable, best-effort sandbox-side capture file (Ray-style). Opened + # lazily on first line; failures disable it without touching the loop. + self._log_file: Any = None + self._log_file_off = False async def run(self) -> None: loop = asyncio.get_running_loop() @@ -64,18 +67,30 @@ async def run(self) -> None: # desyncing the protocol and hanging every later call. # # Move the framing onto private fds and point fd 0 at /dev/null, so - # inherited stdin is harmless. fd 1 becomes a user-output pipe: - # `print()` and child-process stdout are drained separately and - # forwarded through the `/log` side channel instead of corrupting - # the control frame stream. + # inherited stdin is harmless. fd 1 / fd 2 become user-output pipes: + # `print()`, child-process output, and stdlib `logging` (which writes + # to stderr) are drained separately and forwarded through the `/log` + # side channel — Ray-style raw capture — instead of corrupting the + # control frame stream. frame_in_fd = os.dup(0) frame_out_fd = os.dup(1) + # Save the real stderr before fd 2 becomes the capture pipe — the + # worker's OWN stdlib logging is repointed here so its diagnostics go + # to the container/Ray log and are NOT re-captured by the stderr pipe. + # Without this, a worker log line emitted while draining /log (e.g. an + # "outbound frame write failed" on a broken pipe) loops back through + # _emit_log_line -> _drain_outbound -> fails -> logs again. + real_stderr_fd = os.dup(2) stdout_read_fd, stdout_write_fd = os.pipe() + stderr_read_fd, stderr_write_fd = os.pipe() devnull = os.open(os.devnull, os.O_RDWR) os.dup2(devnull, 0) os.dup2(stdout_write_fd, 1) + os.dup2(stderr_write_fd, 2) os.close(stdout_write_fd) + os.close(stderr_write_fd) os.close(devnull) + _redirect_internal_logging(real_stderr_fd) _make_stdout_eager() reader = asyncio.StreamReader() @@ -95,11 +110,12 @@ async def run(self) -> None: # `agentix.sio.emit/on/request`; the bridge ferries frames over # the pipe to the server, which puts them on the real SIO. _sio._install(self._enqueue_frame) - # Built-in /trace and /log namespaces — both are agentix-core - # extensions registered on top of agentix.sio. + # Built-in /trace namespace (agentix-core extension on agentix.sio). + # /log is no longer a structured bridge — stdout/stderr are captured + # raw below. install_worker_bridge() - _log.install_worker_bridge() - self._stdio_tasks.append(loop.create_task(self._drain_stdout(stdout_read_fd))) + self._stdio_tasks.append(loop.create_task(self._drain_stream(stdout_read_fd, "stdout"))) + self._stdio_tasks.append(loop.create_task(self._drain_stream(stderr_read_fd, "stderr"))) await self._send({"type": "ready"}) while not self._shutdown.is_set(): @@ -122,15 +138,22 @@ async def run(self) -> None: if self._calls: await asyncio.gather(*self._calls.values(), return_exceptions=True) if self._stdio_tasks: - _close_stdout_pipe() + _close_stdio_pipes() _, pending = await asyncio.wait(self._stdio_tasks, timeout=1.0) for task in pending: task.cancel() if pending: await asyncio.gather(*pending, return_exceptions=True) - await self._outbound_q.join() + # Bound the drain: a wedged outbound pipe (writer.drain() blocked on a + # full OS pipe) would hang join() forever — task_done() never fires for + # the stuck frame. Mirror the server-side bounded join. + with contextlib.suppress(TimeoutError): + await asyncio.wait_for(self._outbound_q.join(), timeout=2.0) if self._drainer is not None: self._drainer.cancel() + if self._log_file is not None: + with contextlib.suppress(Exception): + self._log_file.close() async def _drain_outbound(self) -> None: assert self._writer is not None @@ -174,7 +197,7 @@ def _recover_failed_frame(self, frame: dict[str, Any]) -> None: async def _send(self, payload: dict[str, Any]) -> None: await self._outbound_q.put(payload) - async def _drain_stdout(self, fd: int) -> None: + async def _drain_stream(self, fd: int, stream: str) -> None: loop = asyncio.get_running_loop() reader = asyncio.StreamReader() await loop.connect_read_pipe( @@ -183,10 +206,10 @@ async def _drain_stdout(self, fd: int) -> None: ) # Read fixed-size chunks and split into lines ourselves. `readline()` # raises on a line longer than the StreamReader limit (64 KiB); that - # error was swallowed and KILLED this loop, so fd 1 stopped draining - # and the next `print()` blocked on a full pipe — deadlocking the - # in-flight call. Chunked reads can never overflow, so the pipe is - # always drained regardless of line length. + # error was swallowed and KILLED this loop, so the fd stopped draining + # and the next write blocked on a full pipe — deadlocking the in-flight + # call. Chunked reads can never overflow, so the pipe is always drained + # regardless of line length. buf = bytearray() try: while True: @@ -196,20 +219,56 @@ async def _drain_stdout(self, fd: int) -> None: buf.extend(chunk) *lines, buf_rest = bytes(buf).split(b"\n") for line in lines: - _emit_stdio_line("stdout", line) + self._emit_log_line(stream, line) buf = bytearray(buf_rest) # A newline-less spew (e.g. a binary blob) must not grow `buf` # without bound — flush it as a partial line. if len(buf) >= 65536: - _emit_stdio_line("stdout", bytes(buf)) + self._emit_log_line(stream, bytes(buf)) buf.clear() except asyncio.CancelledError: pass except Exception: - logger.debug("stdout drain failed", exc_info=True) + logger.debug("%s drain failed", stream, exc_info=True) finally: if buf: - _emit_stdio_line("stdout", bytes(buf)) + self._emit_log_line(stream, bytes(buf)) + + def _emit_log_line(self, stream: str, raw: bytes) -> None: + """Ferry one captured stdout/stderr line: append to the durable + sandbox-side file, then best-effort stream it to the host on `/log`. + + Both steps are silent — this path must never write to stdout/stderr + itself (it would be re-captured here, looping), so failures are + swallowed rather than logged.""" + text = raw.decode("utf-8", "replace").rstrip("\r\n") + self._write_log_file(stream, text) + try: + self._outbound_q.put_nowait( + { + "type": "sio_emit", + "namespace": LOG_NAMESPACE, + "event": LOG_EVENT, + "data": {"stream": stream, "line": text}, + } + ) + except Exception: + pass + + def _write_log_file(self, stream: str, text: str) -> None: + if self._log_file_off: + return + try: + if self._log_file is None: + log_dir = Path(os.environ.get("AGENTIX_LOG_DIR", "/tmp/agentix")) + log_dir.mkdir(parents=True, exist_ok=True) + self._log_file = (log_dir / "sandbox.log").open("a", encoding="utf-8") + self._log_file.write(f"[{stream}] {text}\n") + self._log_file.flush() + except Exception: + # Durability is best-effort; if the file can't be written, keep + # streaming and stop retrying the file. + self._log_file_off = True def _enqueue_frame(self, frame: dict[str, Any]) -> None: """Sync put for the agentix.sio bridge — must never block.""" @@ -276,18 +335,20 @@ def _cancel(self, call_id: str) -> None: task = self._calls.get(call_id) if task is not None: task.cancel() - asyncio.create_task( - self._send( - { - "type": "error", - "call_id": call_id, - "error": RemoteError( - type="Cancelled", - message="remote call cancelled", - cancelled=True, - ).model_dump(), - } - ) + # Enqueue synchronously (the outbound queue is unbounded) instead of + # spawning an untracked `create_task`, which the loop only weakly + # references and could GC before it runs — dropping the Cancelled + # frame. + self._enqueue_frame( + { + "type": "error", + "call_id": call_id, + "error": RemoteError( + type="Cancelled", + message="remote call cancelled", + cancelled=True, + ).model_dump(), + } ) @@ -296,6 +357,28 @@ async def _amain() -> None: await worker.run() +def _redirect_internal_logging(real_stderr_fd: int) -> None: + """Keep the worker's OWN ``agentix.*`` diagnostics off the capture pipe, + WITHOUT diverting user logging. + + fd 2 is the capture pipe — its lines are replayed on the host's ``/log`` and + appended to ``sandbox.log``. User stdlib logging is meant to ride that pipe + (REFACTOR.md: "stdlib logging writes to stderr, so it's captured too"), so + the root handler ``configure_logging`` installed is left untouched. But the + worker's own ``agentix.*`` infra logs must NOT be re-captured: on a broken + outbound pipe that self-amplifies into a hot loop (a write failure logs to + stderr → the line is captured → re-enqueued → the write fails again → …). + Route only the ``agentix`` logger to the real stderr (saved before fd 2 + became the pipe) and stop it propagating to the captured root handler.""" + with contextlib.suppress(Exception): + real_stderr = os.fdopen(real_stderr_fd, "w", buffering=1) + handler = logging.StreamHandler(real_stderr) + handler.setFormatter(logging.Formatter("%(asctime)s [%(name)s] %(levelname)s %(message)s")) + agentix_logger = logging.getLogger("agentix") + agentix_logger.handlers = [handler] + agentix_logger.propagate = False + + def _make_stdout_eager() -> None: """Make regular `print()` visible without requiring `flush=True`.""" with contextlib.suppress(Exception): @@ -304,40 +387,18 @@ def _make_stdout_eager() -> None: reconfigure(line_buffering=True, write_through=True) -def _close_stdout_pipe() -> None: - """Flush fd 1 and detach it from the capture pipe so the drainer reaches EOF.""" - with contextlib.suppress(Exception): - sys.stdout.flush() - with contextlib.suppress(Exception): - devnull = os.open(os.devnull, os.O_WRONLY) - try: - os.dup2(devnull, 1) - finally: - os.close(devnull) - - -def _emit_stdio_line(stream: str, raw: bytes) -> None: - text = raw.decode("utf-8", "replace").rstrip("\r\n") - emit_worker_record( - { - "name": f"agentix.sandbox.{stream}", - "level": "INFO", - "levelno": logging.INFO, - "message": text, - "created": time.time(), - "pathname": "", - "lineno": 0, - "funcName": "", - "module": "stdio", - "exc_text": None, - "stack_info": None, - LOG_CONTEXT_ATTR: get_log_context(), - "extras": { - "agentix_stream": stream, - "worker_id": os.environ.get("AGENTIX_WORKER_ID"), - }, - } - ) +def _close_stdio_pipes() -> None: + """Flush fd 1 / fd 2 and detach them from the capture pipes so the + drainers reach EOF.""" + for stream, fd in ((sys.stdout, 1), (sys.stderr, 2)): + with contextlib.suppress(Exception): + stream.flush() + with contextlib.suppress(Exception): + devnull = os.open(os.devnull, os.O_WRONLY) + try: + os.dup2(devnull, fd) + finally: + os.close(devnull) def main() -> None: diff --git a/agentix/runtime/shared/__init__.py b/agentix/runtime/shared/__init__.py index 0842a17..438f2f9 100644 --- a/agentix/runtime/shared/__init__.py +++ b/agentix/runtime/shared/__init__.py @@ -14,7 +14,7 @@ - `callables` — `RemoteCallable` import-path encoding - `idents` — branded NewType ids on the wire (`CallId`) - - `codec` — msgpack pack/unpack + ext types (numpy, pydantic) + - `codec` — plain msgpack pack/unpack (no ext types) - `framing` — length-prefixed msgpack framing for worker stdio - `models` — pydantic wire types (`RemoteRequest`, `RemoteResponse`, …) - `env` — bundle runtime contract: runtime paths, env vars, diff --git a/agentix/runtime/shared/codec.py b/agentix/runtime/shared/codec.py index 4b5101a..ddbed37 100644 --- a/agentix/runtime/shared/codec.py +++ b/agentix/runtime/shared/codec.py @@ -1,97 +1,26 @@ -"""Wire codec — msgpack with extension types. +"""Wire codec — msgpack. -Every worker frame and Socket.IO event payload flows -through `pack(obj)` / `unpack(bytes)`. The goal is: cross-language wire -format, native binary types (no base64), small + fast, and -round-trippable Python types via msgpack extension types. - -Extension types registered: - - * `_EXT_NDARRAY` (1) — numpy arrays. Header (`dtype_str|shape_csv`) - + null byte + raw `tobytes()`. Cross-language consumers replicate - the same header format. - * `_EXT_PYDANTIC` (2) — pydantic `BaseModel` instances. Encoded as - `(qualname, model_dump(mode="python") packed)`. On the receiving - side the qualname is informational; the decoded dict is returned - as a plain mapping for callers to interpret. - -Numpy is optional — if it's not installed, the ndarray hook is just -skipped (the type never appears on the wire). pydantic is a hard dep -because the rest of the framework uses it. +Every worker frame and Socket.IO event payload flows through `pack(obj)` +/ `unpack(bytes)`. Payloads are plain msgpack-native types: dicts, lists, +strings, numbers, and bytes. RPC args/returns travel as pickle bytes +*inside* frames; pydantic models are `model_dump()`-ed to dicts before +packing. Cross-language, native binary (no base64), small and fast. """ from __future__ import annotations -import importlib.util from typing import Any import msgpack -from pydantic import BaseModel - -# numpy is an optional dep. Importing it eagerly costs ~400 ms (it -# pulls in a sizeable C-extension graph) and the framework's hot path -# never needs it unless an ndarray actually shows up on the wire — so -# we check for the dist via `find_spec` (no heavy work) and defer the -# real import to first ndarray encode/decode. -_HAS_NUMPY = importlib.util.find_spec("numpy") is not None -_np: Any = None # populated lazily by `_numpy()` - -_EXT_NDARRAY = 1 -_EXT_PYDANTIC = 2 - - -def _numpy() -> Any: - """Lazy numpy import. Cached on the module.""" - global _np - if _np is None: - import numpy # type: ignore[reportMissingImports] # noqa: PLC0415 - - _np = numpy - return _np - - -def _encode_ext(obj: Any) -> msgpack.ExtType: - if _HAS_NUMPY: - np = _numpy() - if isinstance(obj, np.ndarray): - header = f"{obj.dtype.str}|{','.join(map(str, obj.shape))}".encode() - return msgpack.ExtType(_EXT_NDARRAY, header + b"\x00" + obj.tobytes()) - if isinstance(obj, BaseModel): - payload = msgpack.packb( - obj.model_dump(mode="python"), - default=_encode_ext, - use_bin_type=True, - ) - return msgpack.ExtType(_EXT_PYDANTIC, payload) - raise TypeError(f"agentix.codec: cannot encode {type(obj).__name__}") - - -def _decode_ext(code: int, data: bytes) -> Any: - if code == _EXT_NDARRAY: - if not _HAS_NUMPY: - raise RuntimeError("ndarray ext received but numpy not installed") - np = _numpy() - header, raw = data.split(b"\x00", 1) - dtype_str, shape_str = header.decode().split("|") - shape = tuple(int(s) for s in shape_str.split(",") if s) - return np.frombuffer(raw, dtype=np.dtype(dtype_str)).reshape(shape) - if code == _EXT_PYDANTIC: - # Decoded as a plain dict for callers to interpret. - return msgpack.unpackb(data, ext_hook=_decode_ext, raw=False) - return msgpack.ExtType(code, data) - # Module-level `Packer` reused across `pack()` calls. `autoreset=True` # means each `.pack()` returns a complete frame and resets internal -# state — safe for the single-threaded asyncio loop. Re-entrant -# packing (e.g. `_encode_ext` packing a pydantic model) still goes -# through `msgpack.packb`, which creates its own short-lived Packer -# so the module-level one's state is not clobbered. -_PACKER = msgpack.Packer(default=_encode_ext, use_bin_type=True, autoreset=True) +# state — safe for the single-threaded asyncio loop. +_PACKER = msgpack.Packer(use_bin_type=True, autoreset=True) def pack(obj: Any) -> bytes: - """Serialize an arbitrary Python object to msgpack bytes.""" + """Serialize a msgpack-native Python object to bytes.""" return _PACKER.pack(obj) @@ -101,7 +30,7 @@ def unpack(blob: bytes | bytearray | memoryview) -> Any: memoryview natively; widening the signature lets callers pass Socket.IO payloads (often `bytearray` after framing) through without copying.""" - return msgpack.unpackb(blob, ext_hook=_decode_ext, raw=False) + return msgpack.unpackb(blob, raw=False) __all__ = ["pack", "unpack"] diff --git a/agentix/runtime/shared/framing.py b/agentix/runtime/shared/framing.py index c9820c2..e488f19 100644 --- a/agentix/runtime/shared/framing.py +++ b/agentix/runtime/shared/framing.py @@ -7,7 +7,7 @@ +--------+-------------------+ The msgpack blob is a dict — see frame schemas below. `agentix.runtime.shared.codec` -handles encode/decode, including ext types for ndarray + pydantic models. +handles encode/decode (plain msgpack, no ext types). Frame schemas (`{"type": "...", ...}` — extra fields per type): @@ -18,7 +18,6 @@ ─── worker → runtime ───────────────────────────────────── ready {} — sent once after worker startup - boot_error {error} — sent once if startup fails result {call_id, value} — call succeeded (value is pickle bytes) error {call_id, error} — call failed sio_open {namespace} — open a side-channel namespace diff --git a/agentix/sio.py b/agentix/sio.py index 9c5638a..c725b2d 100644 --- a/agentix/sio.py +++ b/agentix/sio.py @@ -9,7 +9,7 @@ - `/rpc` — RPC (call / cancel / call:result / call:error) - `/trace` — Trace/Span lifecycle - - `/log` — stdlib `logging` records + - `/log` — captured stdout/stderr lines (best-effort) Plugins MUST use their own namespace path (convention: `/`), typically registered via `agentix.register_namespace(MyNs())`. Two @@ -56,10 +56,14 @@ async def fetch_remote(self, payload): class RemoteSioError(RuntimeError): """Raised by `Namespace.request()` when the reply carries an `:error`.""" - def __init__(self, type_: str, message: str) -> None: + def __init__(self, type_: str, message: str, status_code: int | None = None) -> None: super().__init__(f"{type_}: {message}") self.type = type_ self.message = message + # Upstream HTTP status the host handler chose (e.g. AbridgeError 429/400); + # None when the error envelope carried no status. Lets the in-sandbox + # tunnel reply with the real status instead of collapsing to 502. + self.status_code = status_code # ── module-level bridge state ────────────────────────────────────── @@ -241,10 +245,12 @@ async def _on_reply_error(self, payload: Any) -> None: fut = self._pending_requests.get(req_id) if isinstance(req_id, str) else None if fut is not None and not fut.done(): err = payload.get("error") or {"type": "Unknown", "message": ""} + status_code = err.get("status_code") fut.set_exception( RemoteSioError( err.get("type", "Unknown"), err.get("message", ""), + status_code if isinstance(status_code, int) else None, ) ) @@ -311,9 +317,10 @@ async def _swallow_exc(coro: Awaitable[None], namespace: str, event: str) -> Non def _env_buffer(env_var: str, default: int = 10_000) -> int: """Positive int from `env_var`, or `default` when unset/invalid. - Lets the `/log` and `/trace` bridges size their `ReliableStream` - buffers (`AGENTIX_LOG_BUFFER` / `AGENTIX_TRACE_BUFFER`) without - forking agentix when a high-volume workload needs a deeper buffer. + Lets the `/trace` bridge size its `ReliableStream` buffer + (`AGENTIX_TRACE_BUFFER`) without forking agentix when a high-volume + workload needs a deeper buffer. (`/log` no longer uses `ReliableStream` + — it is best-effort line capture, so there is no `AGENTIX_LOG_BUFFER`.) """ raw = os.environ.get(env_var) if raw is None: @@ -413,7 +420,7 @@ def _wrap(self, event: str, data: Any) -> tuple[int, dict[str, Any]]: logger.warning( "ReliableStream %s buffer full (max_buffer=%d): dropping oldest " "unacked events; at-least-once delivery degraded (dropped=%d). " - "Raise AGENTIX_LOG_BUFFER / AGENTIX_TRACE_BUFFER or ack faster.", + "Raise AGENTIX_TRACE_BUFFER or ack faster.", self._ns.namespace, self._buffer.maxlen, dropped_total, diff --git a/agentix/utils/log/__init__.py b/agentix/utils/log/__init__.py index b8b2718..5424beb 100644 --- a/agentix/utils/log/__init__.py +++ b/agentix/utils/log/__init__.py @@ -1,57 +1,21 @@ -"""agentix.utils.log — sandbox-side logs ferried to the host. +"""agentix.utils.log — sandbox stdout/stderr captured and ferried to the host. -This module is a thin bridge for the *third* observability pillar -(distinct from `agentix.utils.trace`). Workers don't need a custom API: -stdout is captured by the runtime, and stdlib logging is bridged directly: +Workers need no logging API. The runtime captures the worker's stdout and +stderr (stdlib `logging` writes to stderr, so it is captured too) and streams +each line best-effort on the `/log` namespace, replayed on the host under +`agentix.sandbox.{stdout,stderr}`. A durable copy is written to a sandbox-side +file. import logging - logger = logging.getLogger(__name__) - logger.info("hello from sandbox") + logging.getLogger(__name__).info("hello from sandbox") # -> host logs + print("hello from stdout") # -> host logs - print("hello from stdout") - -At worker boot, `install_worker_bridge()` adds a `logging.Handler` to -the root logger that emits each `LogRecord` on the `/log` SIO -namespace. The host's `RuntimeClient` auto-registers a consumer that -forwards records into the host's own `logging` system, so they appear -in host logs untouched. The worker runtime also captures stdout and sends -each line through the same `/log` stream as `agentix.sandbox.stdout`. - -## Delivery contract - -`/log` is a side channel, separate from the `c.remote(...)` result -path. The contract is: - - - **Ordering**: records emitted on a single connection arrive in - FIFO order. - - **Eventual delivery**: under a healthy connection, every emitted - record reaches the host. - - **No happens-before with `remote()`**: a log record emitted from - inside `fn` may arrive on the host *after* `c.remote(fn, ...)` - has already returned. Treat side-channel observability as - eventually-consistent telemetry, not as a synchronization barrier. +`configure_logging` sets up local stdlib logging for the host, runtime, and +worker processes (level / format / context from `AGENTIX_LOG_*` env vars). """ from __future__ import annotations -import logging - from agentix.utils.log._config import configure_logging -__all__ = ["configure_logging", "install_worker_bridge"] - - -def install_worker_bridge(level: int = logging.NOTSET) -> logging.Handler: - """Install the bridge handler on the root logger. Idempotent.""" - from agentix.utils.log._bridge import WorkerLogHandler - - root = logging.getLogger() - for h in root.handlers: - if isinstance(h, WorkerLogHandler): - return h - handler = WorkerLogHandler() - handler.setLevel(level) - root.addHandler(handler) - if root.level == logging.NOTSET or root.level > logging.INFO: - root.setLevel(logging.INFO) - return handler +__all__ = ["configure_logging"] diff --git a/agentix/utils/log/_bridge.py b/agentix/utils/log/_bridge.py index 6c4a610..150c067 100644 --- a/agentix/utils/log/_bridge.py +++ b/agentix/utils/log/_bridge.py @@ -1,9 +1,13 @@ -"""`/log` SIO namespace — worker handler + host replayer. +"""`/log` — best-effort raw stdout/stderr capture from the sandbox. -`/log` is a reconnect-safe stream: events carry monotonic `_seq`, the -sandbox buffers them until the host acks, and on reconnect the host -emits `_resume` to re-receive everything since its last ack. See -`agentix.sio.ReliableStream` for the wire envelope and contract. +The worker captures its own stdout and stderr (Ray-style; stdlib +`logging` writes to stderr, so it is captured too) and streams each line +best-effort on the `/log` namespace as a `line` event carrying +`{stream, line}`. The host replays each line into its own `logging` tree +under `agentix.sandbox.{stdout,stderr}`, so it shows up in host logs. + +This channel is the live, lossy stream — no acks, no replay. Durable +capture is the sandbox-side file the worker also writes. """ from __future__ import annotations @@ -13,269 +17,38 @@ import socketio -from agentix import sio as _sio -from agentix.utils.log._config import LOG_CONTEXT_ATTR - -NAMESPACE = "/log" -RECORD_EVENT = "record" - - -# ── worker side ─────────────────────────────────────────────────── - - -class _WorkerLogNamespace(_sio.Namespace): - namespace = NAMESPACE - _allow_reserved = True - - -_namespace_singleton: _WorkerLogNamespace | None = None -_stream_singleton: _sio.ReliableStream | None = None - - -def _get_worker_stream() -> _sio.ReliableStream: - global _namespace_singleton, _stream_singleton - if _namespace_singleton is None: - _namespace_singleton = _WorkerLogNamespace() - _sio.register_namespace(_namespace_singleton) - if _stream_singleton is None: - _stream_singleton = _sio.ReliableStream( - _namespace_singleton, - max_buffer=_sio._env_buffer("AGENTIX_LOG_BUFFER"), - ) - return _stream_singleton - - -class WorkerLogHandler(logging.Handler): - """Translate `LogRecord`s into `/log:record` events. - - Records ride a `ReliableStream` so the host receives every record - even across SIO disconnects, with FIFO ordering. - - Avoids self-recursion: `agentix.utils.log` is excluded from forwarding to - prevent feedback if our own debug logs were ever enabled. - """ - - _EXCLUDED_LOGGERS = ("agentix.sio", "agentix.utils.log") - - def emit(self, record: logging.LogRecord) -> None: - if any(record.name.startswith(prefix) for prefix in self._EXCLUDED_LOGGERS): - return - if not _sio._is_installed(): - return - try: - payload = _record_payload(record) - stream = _get_worker_stream() - stream.emit_nowait(RECORD_EVENT, payload) - except Exception: - self.handleError(record) - - -def emit_worker_record(payload: dict[str, Any]) -> None: - """Emit a pre-built log payload on the worker `/log` stream. - - This is for runtime-owned sources such as captured stdout where routing - through stdlib logging would recurse back into stderr/stdout handlers. - """ - if not _sio._is_installed(): - return - stream = _get_worker_stream() - stream.emit_nowait(RECORD_EVENT, payload) - - -# Fields LogRecord defines natively; everything else on `record.__dict__` -# is treated as a user-provided `extra={...}` field and forwarded. -_STD_RECORD_KEYS = frozenset( - { - "name", - "msg", - "args", - "levelname", - "levelno", - "pathname", - "filename", - "module", - "exc_info", - "exc_text", - "stack_info", - "lineno", - "funcName", - "created", - "msecs", - "relativeCreated", - "thread", - "threadName", - "processName", - "process", - "message", - "asctime", - # Added to LogRecord in Python 3.12; absent on 3.11. Listed - # unconditionally so a record produced on 3.12+ doesn't try to - # smuggle `taskName` through `extra=` into a fresh record. - "taskName", - LOG_CONTEXT_ATTR, - } -) - - -def _coerce_extra(value: Any) -> Any: - """Make a user-supplied `extra` value safe to msgpack-encode. - - A non-encodable value (an arbitrary object, `Decimal`, …) would otherwise - fail to pack on the outbound drainer, which drops the whole frame and loses - the record. Reduce anything that isn't a msgpack-native scalar/container to - `repr()` so the record always survives as text. - """ - if value is None or isinstance(value, (str, bool, int, float, bytes)): - return value - if isinstance(value, (list, tuple)): - return [_coerce_extra(v) for v in value] - if isinstance(value, dict): - return {str(k): _coerce_extra(v) for k, v in value.items()} - return repr(value) - - -def _record_payload(record: logging.LogRecord) -> dict[str, Any]: - extras = { - k: _coerce_extra(v) - for k, v in record.__dict__.items() - if k not in _STD_RECORD_KEYS and not k.startswith("_") - } - return { - "name": record.name, - "level": record.levelname, - "levelno": record.levelno, - "message": record.getMessage(), - "created": record.created, - "pathname": record.pathname, - "lineno": record.lineno, - "funcName": record.funcName, - "module": record.module, - "exc_text": record.exc_text - or (logging.Formatter().formatException(record.exc_info) if record.exc_info else None), - "stack_info": record.stack_info, - LOG_CONTEXT_ATTR: getattr(record, LOG_CONTEXT_ATTR, None), - "extras": extras or None, - } - - -# ── host side ───────────────────────────────────────────────────── +LOG_NAMESPACE = "/log" +LOG_EVENT = "line" class HostLogNamespace(socketio.AsyncClientNamespace): - """Replays inbound `/log:record` events into the host's `logging` tree. + """Replays inbound `/log:line` events into `agentix.sandbox.{stream}`. - Each forwarded record is dispatched against the same logger name it - had in the sandbox, so existing host-side handlers/formatters pick it - up naturally. - - Reconnect safety: tracks `_last_seq` per stream; on (re)connect emits - `_resume {since_seq}` so the sandbox replays anything missed; after - each delivery emits `_ack {seq}` so the sandbox can release its - buffer. - """ + Replay runs INLINE in the receive loop (we override `trigger_event` + directly rather than inheriting agentix's detached-dispatch + `AsyncClientNamespace`). This is deliberate and mirrors the sibling + `HostTraceNamespace`: the handler is a single non-blocking + `logging.getLogger(...).info(line)`, and inline replay preserves strict + FIFO line order for free. The only way it could stall the loop is a + user-installed *slow* handler on the `agentix.sandbox.*` loggers — an + unusual setup; route such handlers through a `QueueHandler` if needed.""" def __init__(self) -> None: - super().__init__(NAMESPACE) - self._last_seq = 0 - self._sid: str | None = None + super().__init__(LOG_NAMESPACE) async def trigger_event(self, event: str, *args: Any) -> Any: - if event == "connect": - # Initial connect AND every reconnect goes through here. - await self._emit_resume() - return - if event in ("disconnect", "connect_error"): - return - if event != RECORD_EVENT: + if event != LOG_EVENT: return - from agentix.runtime.client._sio_facade import _decode - envelope = _decode(args[0]) if args else None - if not isinstance(envelope, dict): - return - - seq = envelope.get("_seq") - sid = envelope.get("_sid") - payload = envelope.get("data") - if not isinstance(seq, int) or not isinstance(payload, dict): - # Legacy / malformed payload — fall through without dedup. - if isinstance(envelope, dict) and isinstance(payload, dict): - _replay_record(payload) + payload = _decode(args[0]) if args else None + if not isinstance(payload, dict): return - - if sid != self._sid: - # A new stream id means the sandbox-side stream restarted — e.g. - # the worker subprocess crashed and was respawned with a fresh - # ReliableStream whose `_seq` counter starts back at 1. Adopt the - # new stream and reset the cursor so its early records aren't - # mistaken for duplicates of the old stream and silently dropped. - self._sid = sid - self._last_seq = 0 - - if seq <= self._last_seq: - # Duplicate from a resume + already-delivered race. Re-ack - # so the sandbox can move on. - await self._emit_ack(seq) + line = payload.get("line") + if not isinstance(line, str): return - self._last_seq = seq - _replay_record(payload) - await self._emit_ack(seq) - - async def _emit_resume(self) -> None: - with _suppress(): - await self.emit(_sio._STREAM_RESUME_EVENT, _pack({"since_seq": self._last_seq})) - - async def _emit_ack(self, seq: int) -> None: - with _suppress(): - await self.emit(_sio._STREAM_ACK_EVENT, _pack({"seq": seq})) - - -def _pack(data: Any) -> bytes: - from agentix.runtime.shared.codec import pack as _msgpack - - return _msgpack(data) - - -def _suppress(): - import contextlib - - return contextlib.suppress(BaseException) - - -def _replay_record(payload: dict[str, Any]) -> None: - logger = logging.getLogger(str(payload.get("name", "agentix.sandbox"))) - levelno = int(payload.get("levelno", logging.INFO)) - if not logger.isEnabledFor(levelno): - return - # `makeRecord` rejects any `extra` key that collides with a standard - # LogRecord attribute. Sender and receiver may run different Python - # versions (the sandbox could add a field this version doesn't have, - # or vice versa), so filter defensively rather than trusting the - # sender's `_STD_RECORD_KEYS`. - extras = { - k: v for k, v in (payload.get("extras") or {}).items() - if k not in _STD_RECORD_KEYS - } - record = logger.makeRecord( - name=logger.name, - level=levelno, - fn=str(payload.get("pathname", "")), - lno=int(payload.get("lineno", 0)), - msg=str(payload.get("message", "")), - args=(), - exc_info=None, - extra=extras, - ) - record.funcName = str(payload.get("funcName", "")) - record.module = str(payload.get("module", "")) - if payload.get("exc_text"): - record.exc_text = str(payload["exc_text"]) - if payload.get("stack_info"): - record.stack_info = str(payload["stack_info"]) - if payload.get(LOG_CONTEXT_ATTR): - setattr(record, LOG_CONTEXT_ATTR, str(payload[LOG_CONTEXT_ATTR])) - logger.handle(record) + stream = str(payload.get("stream", "stdout")) + logging.getLogger(f"agentix.sandbox.{stream}").info(line) -__all__ = ["HostLogNamespace", "WorkerLogHandler", "emit_worker_record"] +__all__ = ["LOG_EVENT", "LOG_NAMESPACE", "HostLogNamespace"] diff --git a/docs/concepts/plugins.mdx b/docs/concepts/plugins.mdx index f3ff879..eb36b90 100644 --- a/docs/concepts/plugins.mdx +++ b/docs/concepts/plugins.mdx @@ -39,7 +39,7 @@ Agentix core owns three reserved Socket.IO namespaces: | Namespace | System | User-facing API | Extension point | | --- | --- | --- | --- | | `/rpc` | RPC | `await sandbox.remote(fn, *args, **kwargs)` | expose a normal importable Python callable | -| `/log` | logging | stdlib `logging` in sandbox code | configure host logging handlers, levels, and formatters | +| `/log` | logging | `print(...)` / stdlib `logging` in sandbox code | configure host logging handlers, levels, and formatters | | `/trace` | tracing | `agentix.trace.trace(...)`, `agentix.trace.span(...)` | register `agentix.trace.Processor` implementations | Plugins must not claim these namespaces. A plugin that needs its own event @@ -68,10 +68,10 @@ The sandbox serializes the target as `fn.__module__ + "::" + fn.__qualname__`, pickles args and kwargs, and the worker imports the same callable inside the sandbox. -## Logging: Extend With stdlib logging +## Logging: Captured stdout/stderr -The `/log` namespace is the logging bridge. Sandbox code uses standard Python -logging: +The `/log` namespace ferries the sandbox's captured output to the host, +Ray-style. Sandbox code just prints or logs — no API: ```python import logging @@ -79,39 +79,43 @@ import logging logger = logging.getLogger(__name__) async def run() -> None: - logger.info("starting rollout") + logger.info("starting rollout") # stdlib logging writes to stderr + print("done") # stdout ``` -At worker boot, Agentix installs a root `logging.Handler` that forwards -`LogRecord` data over `/log`. The sandbox automatically registers the host -consumer and replays those records into the host logging tree. +The worker captures its own stdout *and* stderr (stdlib `logging` writes to +stderr, so it is captured too), appends each line to a durable sandbox-side +`sandbox.log`, and streams it best-effort on `/log`. The host replays each +line under `agentix.sandbox.stdout` / `agentix.sandbox.stderr`, so it flows +into the host logging tree. ```mermaid actions={false} flowchart LR - SandboxLogger["Sandbox code
logging.getLogger(...)"] - WorkerHandler["Worker root logging.Handler"] - LogNamespace["/log namespace"] + SandboxOut["Sandbox code
print(...) / logging → stderr"] + Capture["Worker stdout/stderr capture
+ durable sandbox.log"] + LogNamespace["/log namespace
best-effort"] HostConsumer["HostLogNamespace
auto-registered"] - HostLogging["Host logging tree
handlers + formatters"] + HostLogging["Host logging tree
agentix.sandbox.{stdout,stderr}"] - SandboxLogger -->|"logger.info(...)"| WorkerHandler - WorkerHandler -->|"LogRecord payload"| LogNamespace + SandboxOut --> Capture + Capture -->|"{stream, line}"| LogNamespace LogNamespace --> HostConsumer - HostConsumer -->|"logger.handle(record)"| HostLogging + HostConsumer -->|"logger.info(line)"| HostLogging ``` -Users customize logging with normal logging configuration on the host: +`/log` is a best-effort live stream — no acks or replay. Durable capture is +the sandbox-side `sandbox.log`. Customize host logging normally: ```python import logging from agentix.utils.log import configure_logging configure_logging(default_context="host") -logging.getLogger("my_eval").setLevel(logging.INFO) +logging.getLogger("agentix.sandbox.stdout").setLevel(logging.INFO) ``` -Do not register your own `/log` namespace. If a plugin needs structured events -that are not log records, give the plugin its own namespace. +Do not register your own `/log` namespace. If a plugin needs structured +events, give the plugin its own namespace. ## Tracing: Extend With Processors diff --git a/docs/concepts/remote-calls.mdx b/docs/concepts/remote-calls.mdx index 8511788..417da04 100644 --- a/docs/concepts/remote-calls.mdx +++ b/docs/concepts/remote-calls.mdx @@ -72,9 +72,9 @@ becomes this wire payload: runtime uses it to correlate `call:result` / `call:error` responses and to support cancellation. -Remote calls use Socket.IO events on the `/rpc` namespace. The sandbox -may use the internal HTTP `/call` fast path for short calls, but accepted -long-running calls and replayed results still complete over `/rpc`. +Remote calls use Socket.IO events on the `/rpc` namespace — one +transport for every call, short or long-running. HTTP serves only the +`/health` probe. ## Example @@ -119,7 +119,7 @@ ride their own namespaces, separate from `sandbox.remote()`: | --- | --- | --- | | `/rpc` | host ↔ sandbox | `sandbox.remote()` | | `/trace` | sandbox → host | span lifecycle (auto-registered) | -| `/log` | sandbox → host | stdlib logging records (auto-registered) | +| `/log` | sandbox → host | captured stdout/stderr lines, best-effort (auto-registered) | | `/` | both | plugin-defined events via `agentix.sio` | Register a host-side handler before the first remote call: diff --git a/docs/reference/architecture.mdx b/docs/reference/architecture.mdx index 3b562a8..042e505 100644 --- a/docs/reference/architecture.mdx +++ b/docs/reference/architecture.mdx @@ -41,7 +41,7 @@ flowchart LR Server -->|call:result or call:error| Client Client -->|unpickle result| App - Worker -.->|/log records| Server + Worker -.->|/log stdout+stderr lines| Server Worker -.->|/trace spans| Server Worker <-->|plugin namespace events| Server Server -.->|side-channel events| Client @@ -54,7 +54,7 @@ Agentix core owns three reserved Socket.IO namespaces: | Namespace | System | Public API | | --- | --- | --- | | `/rpc` | RPC | `RuntimeClient.remote(fn, *args, **kwargs)` | -| `/log` | logging | standard `logging` records forwarded sandbox -> host | +| `/log` | logging | captured stdout/stderr lines forwarded sandbox -> host (best-effort) | | `/trace` | tracing | `agentix.trace.trace(...)`, `agentix.trace.span(...)`, `trace.Processor` | Plugin-specific protocols use their own namespace, conventionally @@ -111,7 +111,6 @@ blobs inside `call:result`. | Path | Carries | Wire | | --- | --- | --- | | `GET /health` | health probe | HTTP JSON | -| `POST /call` | internal short-call fast path | HTTP msgpack | | Socket.IO `/rpc` | `c.remote()` RPC | `call` / `call:result` / `call:error` / `cancel` | | Socket.IO `/trace`, `/log`, `/` | side channels | plugin-defined events (msgpack payloads) | | worker private pipe | runtime ↔ worker | length-prefixed msgpack frames | diff --git a/docs/reference/public-api.mdx b/docs/reference/public-api.mdx index 0f8c966..8a0d4be 100644 --- a/docs/reference/public-api.mdx +++ b/docs/reference/public-api.mdx @@ -107,10 +107,9 @@ Also public: The timeout knobs apply to both `sandbox.remote` and `RuntimeClient`. `RuntimeClient(url, timeout=...)` sets the per-request timeout in seconds. The default is `300`; raise it for agent workloads (roughly `600`–`1800`s), e.g. -`RuntimeClient(url, timeout=1800)`. `http_sync_ms` (default `1000`) tunes the -inline HTTP fast-path budget for short calls; set `http_sync_ms=None` to disable -it so every call goes over Socket.IO. To bound a single call independently of -the request timeout, wrap it in `asyncio.wait_for(sandbox.remote(...), deadline)`. +`RuntimeClient(url, timeout=1800)`. Every call rides one transport — +Socket.IO on `/rpc`. To bound a single call independently of the request +timeout, wrap it in `asyncio.wait_for(sandbox.remote(...), deadline)`. ## SandboxProvider API diff --git a/plugins/abridge/agentix/bridge/clients/openai.py b/plugins/abridge/agentix/bridge/clients/openai.py index 537c3c2..b8e7227 100644 --- a/plugins/abridge/agentix/bridge/clients/openai.py +++ b/plugins/abridge/agentix/bridge/clients/openai.py @@ -7,9 +7,10 @@ at our tunnel. The SDK accepts the OpenAI Chat Completions request shape via typed -kwargs. Agents that send non-standard fields the SDK doesn't accept -will see an `UpstreamError`; for arbitrary-shape forwarding write your -own `@on("/v1/chat/completions")` handler with raw httpx. +kwargs. Upstream failures surface as `AbridgeError` carrying the +upstream HTTP status (so the agent sees the real 429/400/... instead of +a blanket 502); for arbitrary-shape forwarding write your own +`@on("/v1/chat/completions")` handler with raw httpx. """ from __future__ import annotations diff --git a/plugins/abridge/agentix/bridge/proxy.py b/plugins/abridge/agentix/bridge/proxy.py index f057746..5ec5bea 100644 --- a/plugins/abridge/agentix/bridge/proxy.py +++ b/plugins/abridge/agentix/bridge/proxy.py @@ -54,6 +54,7 @@ import json import logging import socket +import sys import time from collections.abc import AsyncIterator, Awaitable, Callable from dataclasses import dataclass @@ -376,9 +377,11 @@ async def forward(request: FastAPIRequest) -> Response: # matching handler registered under the same name. The wire # payload is just the decoded object — no wrapping envelope # beyond request correlation and no HTTP metadata. - result = await asyncio.wait_for( - ns.request(path, body), timeout=request_timeout - ) + # Single timeout source: thread the configured value into + # `ns.request` itself. A redundant outer `wait_for` here was a no-op + # above `ns.request`'s own (smaller) default, silently capping the + # configurable `request_timeout` at that default. + result = await ns.request(path, body, timeout=request_timeout) except TimeoutError: message = "tunnel timed out waiting for host" logger.warning("abridge tunnel %s: %s", path, message) @@ -420,14 +423,12 @@ def _to_http_response(result: object) -> Response: def _status_from_remote_error(exc: RemoteSioError) -> int: - """`RemoteSioError(type, message)` carries no status code. We map - well-known exception type names to HTTP statuses; everything else - becomes 502.""" - if exc.type == "UpstreamError": - # The client raised UpstreamError. The message format may include - # the status code, but it's not structured. Default 502. - return 502 - return 502 + """Use the upstream HTTP status the host handler chose. `AbridgeError` + carries `status_code` (429/400/404/...), which the host threads through the + wire error envelope and the sandbox preserves on `RemoteSioError`. Fall back + to 502 only when the remote error carried no status.""" + status = getattr(exc, "status_code", None) + return status if isinstance(status, int) else 502 # ── host-side: Proxy ───────────────────────────────────────────────────── @@ -652,7 +653,17 @@ async def session(self, sandbox: Sandbox) -> AsyncIterator[TunnelHandle]: try: yield handle finally: - await self.stop(sandbox) + # If the body is already raising, a teardown failure must not mask + # it (the finally's exception would demote the real error to + # __context__). Log-and-swallow stop() errors in that case; surface + # them normally when the body succeeded. + body_failed = sys.exc_info()[1] is not None + try: + await self.stop(sandbox) + except Exception: + if not body_failed: + raise + logger.exception("abridge: session teardown failed after body error (suppressed)") # ── handy property ──────────────────────────────────────────────── diff --git a/plugins/abridge/agentix/bridge/sidecar.py b/plugins/abridge/agentix/bridge/sidecar.py index 78f98ba..0005136 100644 --- a/plugins/abridge/agentix/bridge/sidecar.py +++ b/plugins/abridge/agentix/bridge/sidecar.py @@ -210,7 +210,11 @@ async def __aenter__(self) -> str: finally: self._starting = False - raise SidecarError(f"sidecar exhausted port retries on {self._host}") + # Unreachable: the bounded retry loop always returns (success) or + # re-raises (the last attempt can't `continue` — that guard requires + # `attempt < attempts`). Kept as an assert so it documents the invariant + # and satisfies the `-> str` contract without masking a real error. + raise AssertionError("unreachable: sidecar start loop must return or raise") async def __aexit__(self, *exc: object) -> None: await self._terminate() @@ -330,6 +334,9 @@ async def _terminate(self) -> None: raise finally: await self._finish_drainers() + # Clear the handle so a re-entered/inspected sidecar doesn't read + # the dead process as "still running". + self._proc = None __all__ = ["Command", "Sidecar", "SidecarError"] diff --git a/tests/e2e/test_reconnect.py b/tests/e2e/test_reconnect.py index 3127d28..37cfe61 100644 --- a/tests/e2e/test_reconnect.py +++ b/tests/e2e/test_reconnect.py @@ -1,31 +1,26 @@ -"""End-to-end tests for the runtime's "reconnect and lose nothing" -contract. - -Both the RPC channel (`c.remote(...)`) and the side-channel streams -(`/log`, `/trace`) are designed to recover transparently when the SIO -transport drops, as long as the underlying server process stays alive. -These tests simulate an involuntary disconnect by force-closing the -EngineIO transport (without going through the voluntary-disconnect -codepath that socketio uses for `client.disconnect()`), then assert -that: - - - In-flight `c.remote(...)` calls still return their result via the - `resume` / `ack` protocol once socketio's auto-reconnect succeeds. - - Records emitted on `/log` while no host was connected still arrive - after reconnect, in original order, exactly once. +"""End-to-end test for the runtime's "reconnect and lose nothing" +contract on the RPC channel. + +An in-flight `c.remote(...)` must still return its result when the SIO +transport drops, as long as the server process stays alive: the server +keeps the task running across the disconnect, caches the terminal result +in `pending_results`, and the reconnecting client emits `resume` to pick +it up. We simulate an involuntary disconnect by force-closing the +EngineIO transport (not the voluntary-disconnect codepath socketio uses +for `client.disconnect()`). + +(`/log` is a best-effort live stream — no resume/replay — so it is not +part of this contract; durable capture is the sandbox-side file.) """ from __future__ import annotations import asyncio -import logging import pytest from agentix import RuntimeClient -from agentix.utils.log._config import LOG_CONTEXT_ATTR from tests import _worker_target as target -from tests._namespace_target import emit_log_burst pytestmark = pytest.mark.asyncio @@ -42,15 +37,6 @@ async def _force_disconnect(sio) -> None: await ws.close() -async def _wait_until(predicate, *, timeout: float = 5.0, step: float = 0.05) -> bool: - deadline = asyncio.get_event_loop().time() + timeout - while asyncio.get_event_loop().time() < deadline: - if predicate(): - return True - await asyncio.sleep(step) - return predicate() - - async def test_in_flight_remote_call_resumes_after_disconnect(use_inprocess_worker, live_server): """A `c.remote(...)` that's mid-flight when the transport drops must still return its result once the client auto-reconnects. @@ -80,66 +66,3 @@ async def test_in_flight_remote_call_resumes_after_disconnect(use_inprocess_work result = await asyncio.wait_for(remote_task, timeout=15) assert result == 1, "fn must have run exactly once across the disconnect" - - -async def test_log_stream_resumes_records_buffered_during_disconnect( - use_inprocess_worker, live_server -): - """`/log` records the worker emits while the host is offline must - arrive after reconnect, with no duplicates and in FIFO order.""" - use_inprocess_worker() - base_url = await live_server() - - captured: list[logging.LogRecord] = [] - - class _Capture(logging.Handler): - def emit(self, record: logging.LogRecord) -> None: - if record.name == "namespace_target": - captured.append(record) - - target_logger = logging.getLogger("namespace_target") - target_logger.setLevel(logging.INFO) - handler = _Capture() - target_logger.addHandler(handler) - - burst_count = 30 - try: - async with RuntimeClient(base_url) as c: - # Kick off the burst; it returns once the worker has - # finished emitting all records (queued on the worker's - # outbound pipe). The host SIO loop drains them - # asynchronously, so dropping the connection right after - # remote() returns leaves at least some records still in - # the sandbox-side `ReliableStream` buffer. - burst = asyncio.create_task(c.remote(emit_log_burst, "burst", burst_count)) - - # Give the worker a moment to start producing records. - await asyncio.sleep(0.05) - await _force_disconnect(c._sio) - # Make sure the burst remote() call itself finishes (its - # result rides the same resume protocol). - assert await asyncio.wait_for(burst, timeout=15) == burst_count - - # Wait for every record to land on the host. - ok = await _wait_until( - lambda: sum(1 for r in captured if r.getMessage().startswith("burst-")) - >= burst_count, - timeout=15, - ) - assert ok, ( - f"only {sum(1 for r in captured if r.getMessage().startswith('burst-'))}" - f" of {burst_count} records arrived" - ) - finally: - target_logger.removeHandler(handler) - - messages = [r.getMessage() for r in captured if r.getMessage().startswith("burst-")] - expected = [f"burst-{i:03d}" for i in range(burst_count)] - assert messages == expected, f"out-of-order or duplicate delivery: {messages[:5]}..." - - # Sanity: every record carries the same sandbox-side log context, - # confirming they all came over the same `/log` stream rather than - # bypassing the bridge. - contexts = {getattr(r, LOG_CONTEXT_ATTR, "") for r in captured if r.getMessage().startswith("burst-")} - assert len(contexts) == 1 - assert next(iter(contexts)).startswith("sandbox-") diff --git a/tests/runtime/client/test_client_options.py b/tests/runtime/client/test_client_options.py deleted file mode 100644 index b2d00df..0000000 --- a/tests/runtime/client/test_client_options.py +++ /dev/null @@ -1,22 +0,0 @@ -"""RuntimeClient construction options.""" - -from __future__ import annotations - -import socketio - -from agentix import RuntimeClient - - -def test_http_sync_ms_default() -> None: - client = RuntimeClient("http://localhost:0") - assert client._http_sync_budget_ms == 1000 - - -async def test_http_sync_ms_none_disables_fast_path() -> None: - client = RuntimeClient("http://localhost:0", http_sync_ms=None) - try: - kind, value = await client._try_http_fast_path(sio=socketio.AsyncClient(), payload={}) - assert kind == "fallback" - assert value is None - finally: - await client.close() diff --git a/tests/runtime/client/test_robustness.py b/tests/runtime/client/test_robustness.py index b505642..0594ddc 100644 --- a/tests/runtime/client/test_robustness.py +++ b/tests/runtime/client/test_robustness.py @@ -61,22 +61,3 @@ async def test_worker_death_surfaces_as_typed_error(live_server): # structured process exit status, so callers branch on OOM without string-matching. assert isinstance(excinfo.value, WorkerExited) assert excinfo.value.returncode == -9 - - -@pytest.mark.asyncio -async def test_fail_pending_drains_queues_with_fatal_error(): - """On a terminal disconnect the client hands every in-flight call a fatal - error so `remote(...)` stops waiting instead of hanging.""" - import asyncio - - client = RuntimeClient("http://127.0.0.1:1") - try: - q: asyncio.Queue = asyncio.Queue() - client._pending["c1"] = q - err = RuntimeUnreachable("connection lost") - client._fail_pending(err) - kind, data = q.get_nowait() - assert kind == "fatal" - assert data is err - finally: - await client._client.aclose() diff --git a/tests/runtime/test_protocol.py b/tests/runtime/test_protocol.py index c099f9e..7a696f5 100644 --- a/tests/runtime/test_protocol.py +++ b/tests/runtime/test_protocol.py @@ -14,7 +14,7 @@ import pytest import socketio -from agentix import RemoteCallError, RuntimeClient +from agentix import Failed, Ok, RemoteCallError, RuntimeClient from agentix.runtime.shared.codec import pack, unpack from agentix.runtime.shared.models import RemoteRequest from tests import _worker_target as target @@ -27,13 +27,14 @@ # ── basics ───────────────────────────────────────────────────────────── -async def test_http_remote_endpoint_is_not_registered(runtime_module): +async def test_http_rpc_endpoints_are_not_registered(runtime_module): + """Only `/health` is served over HTTP — RPC has no HTTP endpoint; + every `c.remote()` rides Socket.IO `/rpc`.""" server, _, _ = runtime_module transport = httpx.ASGITransport(app=server.app) async with httpx.AsyncClient(transport=transport, base_url="http://test") as http: - r = await http.post("/_remote", content=b"") - - assert r.status_code == 404 + assert (await http.post("/_remote", content=b"")).status_code == 404 + assert (await http.post("/call", content=b"")).status_code == 404 async def test_socketio_call_serialized_callable(use_inprocess_worker, live_server): @@ -100,18 +101,36 @@ async def test_client_remote_round_trip(use_inprocess_worker, live_server): assert result.msg == "echo:hello" -async def test_client_remote_http_fast_path_falls_back_to_sio(use_inprocess_worker, live_server): +async def test_try_remote_returns_ok(use_inprocess_worker, live_server): + use_inprocess_worker() + base_url = await live_server() + async with RuntimeClient(base_url) as c: + result = await c.try_remote(target.add, 2, 3) + assert isinstance(result, Ok) + assert result.value == 5 + + +async def test_try_remote_returns_failed(use_inprocess_worker, live_server): + use_inprocess_worker() + base_url = await live_server() + async with RuntimeClient(base_url) as c: + result = await c.try_remote(target.boom) + assert isinstance(result, Failed) + assert isinstance(result.error, RemoteCallError) + + +async def test_client_remote_long_call_round_trip(use_inprocess_worker, live_server): use_inprocess_worker() base_url = await live_server() async with RuntimeClient(base_url) as c: - # Exceeds the 1s HTTP sync budget, so result should arrive on SIO. + # A multi-second call round-trips over the single SIO transport. assert await c.remote(asyncio.sleep, 1.2) is None -async def test_same_call_id_via_mixed_paths_runs_fn_exactly_once(use_inprocess_worker, live_server): +async def test_same_call_id_runs_fn_exactly_once(use_inprocess_worker, live_server): """The runtime must execute `fn` exactly once per `call_id`, even - when the same id is submitted through every path we expose: - HTTP fast-path, raw SIO `call`, and SIO `resume`. + when the same id arrives over every SIO submission path: a duplicate + `call` and a `resume`. """ use_inprocess_worker() base_url = await live_server() @@ -131,25 +150,13 @@ async def _on_result(data): sio.on("call:result", _on_result, namespace=RPC_NAMESPACE) await sio.connect(base_url, namespaces=[RPC_NAMESPACE]) try: - # Three submissions in quick succession on three paths. - async with httpx.AsyncClient(base_url=base_url) as http: - r = await http.post( - "/call", - content=pack(req.model_dump()), - headers={ - "content-type": "application/msgpack", - "prefer": "respond-async, wait=0.05", - }, - ) - r.raise_for_status() - + # Same call_id submitted three times across the SIO paths. await sio.emit("call", payload_bytes, namespace=RPC_NAMESPACE) await sio.emit( "resume", pack({"call_ids": [call_id]}), namespace=RPC_NAMESPACE, ) - # And a second SIO `call` for good measure. await sio.emit("call", payload_bytes, namespace=RPC_NAMESPACE) payload = await asyncio.wait_for(results.get(), timeout=5) @@ -214,13 +221,41 @@ async def _on_result(data): assert _pickle.loads(payload["value"]) == 1, "fn must run exactly once" -async def test_client_remote_http_fallback_does_not_double_execute(use_inprocess_worker, live_server): +async def test_resume_for_unknown_call_id_fails_definitively(use_inprocess_worker, live_server): + """A `resume` for a call_id the runtime no longer holds (evicted + under cap, or never seen) must return a definite `call:error` — the + contract forbids silence, which would hang the host's `remote()`. + """ + use_inprocess_worker() + base_url = await live_server() + + sio = socketio.AsyncClient() + errors: asyncio.Queue = asyncio.Queue() + + async def _on_error(data): + await errors.put(unpack(data)) + + sio.on("call:error", _on_error, namespace=RPC_NAMESPACE) + await sio.connect(base_url, namespaces=[RPC_NAMESPACE]) + try: + await sio.emit( + "resume", + pack({"call_ids": ["never-existed"]}), + namespace=RPC_NAMESPACE, + ) + payload = await asyncio.wait_for(errors.get(), timeout=5) + finally: + await sio.disconnect() + + assert payload["call_id"] == "never-existed" + assert payload["error"]["type"] == "ResultUnavailable" + + +async def test_client_remote_runs_fn_exactly_once(use_inprocess_worker, live_server): use_inprocess_worker() base_url = await live_server() async with RuntimeClient(base_url) as c: await c.remote(target.reset_exec_counter) - # Must execute exactly once even when request returns 202 then - # completes via SIO. result = await c.remote(target.count_exec_and_sleep, 1.2) assert result == 1 diff --git a/tests/test_sio_namespace.py b/tests/test_sio_namespace.py index cbdb048..769b76a 100644 --- a/tests/test_sio_namespace.py +++ b/tests/test_sio_namespace.py @@ -10,14 +10,12 @@ import pytest from agentix import AsyncClientNamespace, RuntimeClient -from agentix.utils.log._config import LOG_CONTEXT_ATTR from tests._namespace_target import ( echo_via_namespace, emit_formatted_log, emit_log_burst, emit_log_line, emit_log_with_exception, - emit_log_with_extra, fire_namespace_event, ) from tests._worker_target import print_stdout @@ -99,218 +97,105 @@ async def test_slow_namespace_handler_does_not_block_runtime(live_server): assert slow_host.started, "slow handler never ran" -@pytest.mark.asyncio -async def test_log_records_arrive_on_host(live_server): - """Verify the full /log experience: plain messages, %-format args, - extras dicts, and exception tracebacks all reach the host intact. - Logger names + levelno round-trip so host filters see the sandbox - record as if it had originated locally. - """ - base_url = await live_server() +# ── /log: raw stdout/stderr capture (Ray-style) ──────────────────────── +# +# The worker captures its stdout and stderr (stdlib `logging` writes to +# stderr, so it is captured too) and streams each line best-effort on +# `/log`. The host replays each line under `agentix.sandbox.{stdout,stderr}`. - captured: list[logging.LogRecord] = [] - class _Capture(logging.Handler): - def emit(self, record: logging.LogRecord) -> None: - if record.name == "namespace_target": - captured.append(record) +def _capture(logger_name: str) -> tuple[list[str], logging.Logger, logging.Handler]: + captured: list[str] = [] - target_logger = logging.getLogger("namespace_target") - target_logger.setLevel(logging.INFO) - handler = _Capture() - target_logger.addHandler(handler) - try: - async with RuntimeClient(base_url) as c: - await c.remote(emit_log_line, "from sandbox", "INFO") - await c.remote(emit_formatted_log, "user %s acted on %s", "alice", "doc-7") - await c.remote(emit_log_with_extra, "with extras", request_id="r-42", attempt=3) - await c.remote(emit_log_with_exception, "caught one") - # Let the /log pipe drain. - await asyncio.sleep(0.5) - finally: - target_logger.removeHandler(handler) - - messages = {r.getMessage(): r for r in captured} - - # Side-channel ordering: records emitted in this order from the - # sandbox arrive on the host in the same order. The contract is - # NOT that they arrive before the matching `c.remote()` returns, - # only that the `/log` stream itself is FIFO. - expected_order = [ - "from sandbox", - "user alice acted on doc-7", - "with extras", - "caught one", - ] - arrival = [r.getMessage() for r in captured if r.getMessage() in expected_order] - assert arrival == expected_order, f"out-of-order log delivery: {arrival}" - - # Plain log line. - assert "from sandbox" in messages - context = getattr(messages["from sandbox"], LOG_CONTEXT_ATTR, "") - assert context.startswith("sandbox-") - assert "-worker-" in context - - # %-style formatting: getMessage() already ran in the sandbox. - assert "user alice acted on doc-7" in messages - - # extras kwargs survive — they show up as attributes on the record. - extras_rec = messages.get("with extras") - assert extras_rec is not None - assert getattr(extras_rec, "request_id", None) == "r-42" - assert getattr(extras_rec, "attempt", None) == 3 - - # logger.exception() ships the formatted traceback in exc_text. - exc_rec = messages.get("caught one") - assert exc_rec is not None - assert exc_rec.exc_text and "ValueError: kaboom" in exc_rec.exc_text + class _Cap(logging.Handler): + def emit(self, record: logging.LogRecord) -> None: + captured.append(record.getMessage()) + lg = logging.getLogger(logger_name) + lg.setLevel(logging.INFO) + handler = _Cap() + lg.addHandler(handler) + return captured, lg, handler -@pytest.mark.asyncio -async def test_log_record_carries_worker_context(live_server): - """`/log` is a side channel independent of `c.remote(...)` result - delivery. The contract is: log records eventually arrive on the - host with the worker's context attached. There is no - happens-before relationship between a log record from inside `fn` - and the return of the corresponding `remote()` call — the two - travel on different transports. - """ - base_url = await live_server() - captured: list[logging.LogRecord] = [] +async def _await_line(captured: list[str], needle: str, *, timeout: float = 3.0) -> bool: + deadline = asyncio.get_event_loop().time() + timeout + while asyncio.get_event_loop().time() < deadline: + if any(needle in m for m in captured): + return True + await asyncio.sleep(0.05) + return False - class _Capture(logging.Handler): - def emit(self, record: logging.LogRecord) -> None: - if record.name == "namespace_target": - captured.append(record) - target_logger = logging.getLogger("namespace_target") - target_logger.setLevel(logging.INFO) - handler = _Capture() - target_logger.addHandler(handler) +@pytest.mark.asyncio +async def test_user_logging_arrives_on_host_via_stderr(live_server): + """Stdlib `logging` inside the sandbox writes to stderr, which the + runtime captures and replays on the host under `agentix.sandbox.stderr` + — including %-formatted messages and exception tracebacks.""" + base_url = await live_server() + captured, lg, h = _capture("agentix.sandbox.stderr") try: async with RuntimeClient(base_url) as c: - await c.remote(emit_log_line, "from sandbox worker", "INFO") - record = await _await_record(captured, "from sandbox worker") - assert record is not None - context = getattr(record, LOG_CONTEXT_ATTR, "") - assert context.startswith("sandbox-") - assert "-worker-" in context + await c.remote(emit_log_line, "from sandbox", "INFO") + await c.remote(emit_formatted_log, "user %s acted on %s", "alice", "doc-7") + await c.remote(emit_log_with_exception, "caught one") + assert await _await_line(captured, "from sandbox") + assert await _await_line(captured, "user alice acted on doc-7") + # logger.exception() writes the traceback to stderr too. + assert await _await_line(captured, "ValueError: kaboom") finally: - target_logger.removeHandler(handler) + lg.removeHandler(h) @pytest.mark.asyncio async def test_remote_print_stdout_arrives_on_host(live_server): base_url = await live_server() - - captured: list[logging.LogRecord] = [] - - class _Capture(logging.Handler): - def emit(self, record: logging.LogRecord) -> None: - if record.name == "agentix.sandbox.stdout": - captured.append(record) - - target_logger = logging.getLogger("agentix.sandbox.stdout") - target_logger.setLevel(logging.INFO) - handler = _Capture() - target_logger.addHandler(handler) + captured, lg, h = _capture("agentix.sandbox.stdout") try: async with RuntimeClient(base_url) as c: result = await c.remote(print_stdout, "hello from print") assert result == "printed" - record = await _await_record(captured, "hello from print") - assert record is not None - assert getattr(record, "agentix_stream", None) == "stdout" - context = getattr(record, LOG_CONTEXT_ATTR, "") - assert context.startswith("sandbox-") - assert "-worker-" in context + assert await _await_line(captured, "hello from print") finally: - target_logger.removeHandler(handler) - - -async def _await_record( - captured: list[logging.LogRecord], - message: str, - *, - timeout: float = 2.0, -) -> logging.LogRecord | None: - """Drain the `/log` side channel for up to `timeout` seconds, - waiting for a record matching `message` to arrive.""" - deadline = asyncio.get_event_loop().time() + timeout - while asyncio.get_event_loop().time() < deadline: - match = next((r for r in captured if r.getMessage() == message), None) - if match is not None: - return match - await asyncio.sleep(0.05) - return None + lg.removeHandler(h) @pytest.mark.asyncio -async def test_log_stream_preserves_order_and_envelope(live_server): - """Records emitted under a burst arrive on the host wrapped in the - `ReliableStream` envelope (`_seq`, `data`), with monotonic `_seq` - and FIFO delivery order. This is the same envelope that lets the - host resume after a disconnect — see the ReliableStream unit - tests for the disconnect/replay path itself. - """ +async def test_captured_log_stream_preserves_order(live_server): + """Captured stderr lines arrive on the host in FIFO order — the pipe and + drain are ordered. Best-effort: no acks, no replay.""" base_url = await live_server() - - captured: list[logging.LogRecord] = [] - - class _Capture(logging.Handler): - def emit(self, record: logging.LogRecord) -> None: - if record.name == "namespace_target": - captured.append(record) - - target_logger = logging.getLogger("namespace_target") - target_logger.setLevel(logging.INFO) - handler = _Capture() - target_logger.addHandler(handler) - + captured, lg, h = _capture("agentix.sandbox.stderr") burst_count = 50 try: async with RuntimeClient(base_url) as c: await c.remote(emit_log_burst, "burst", burst_count) - - # Drain the side channel until every record has landed. deadline = asyncio.get_event_loop().time() + 5 while asyncio.get_event_loop().time() < deadline: - if sum(1 for r in captured if r.getMessage().startswith("burst-")) >= burst_count: + if sum(1 for m in captured if "burst-" in m) >= burst_count: break await asyncio.sleep(0.05) finally: - target_logger.removeHandler(handler) + lg.removeHandler(h) - messages = [r.getMessage() for r in captured if r.getMessage().startswith("burst-")] - expected = [f"burst-{i:03d}" for i in range(burst_count)] - assert messages == expected, ( - f"log stream lost or reordered events: got {len(messages)} of {burst_count}" - ) + seq = [int(m.split("burst-")[1][:3]) for m in captured if "burst-" in m] + assert seq == sorted(seq), f"out-of-order capture: {seq}" + assert seq == list(range(burst_count)), f"lost lines: got {len(seq)} of {burst_count}" @pytest.mark.asyncio async def test_worker_log_context_can_be_configured_with_env(live_server, monkeypatch): + """`AGENTIX_WORKER_LOG_CONTEXT` labels the worker's log lines; the label + rides along in the captured text.""" monkeypatch.setenv("AGENTIX_WORKER_LOG_CONTEXT", "custom-worker-{id}") base_url = await live_server() - - captured: list[logging.LogRecord] = [] - - class _Capture(logging.Handler): - def emit(self, record: logging.LogRecord) -> None: - if record.name == "namespace_target": - captured.append(record) - - target_logger = logging.getLogger("namespace_target") - target_logger.setLevel(logging.INFO) - handler = _Capture() - target_logger.addHandler(handler) + captured, lg, h = _capture("agentix.sandbox.stderr") try: async with RuntimeClient(base_url) as c: - await c.remote(emit_log_line, "custom context", "INFO") - record = await _await_record(captured, "custom context") - assert record is not None - context = getattr(record, LOG_CONTEXT_ATTR, "") - assert context.startswith("custom-worker-") + await c.remote(emit_log_line, "ctx-check", "INFO") + assert await _await_line(captured, "ctx-check") finally: - target_logger.removeHandler(handler) + lg.removeHandler(h) + + line = next(m for m in captured if "ctx-check" in m) + assert "custom-worker-" in line diff --git a/tests/test_stream_respawn_resets_dedup.py b/tests/test_stream_respawn_resets_dedup.py index c1a2bcc..945164d 100644 --- a/tests/test_stream_respawn_resets_dedup.py +++ b/tests/test_stream_respawn_resets_dedup.py @@ -16,7 +16,6 @@ from __future__ import annotations from agentix.runtime.shared.codec import pack -from agentix.utils.log import _bridge as log_bridge from agentix.utils.trace import _bridge as trace_bridge @@ -57,20 +56,3 @@ async def test_trace_same_stream_still_dedups(monkeypatch) -> None: await ns.trigger_event("span_start", _env("aaaa", 2, {"span_id": "dup"})) # resume replay assert [p["span_id"] for _, p in dispatched] == ["a1", "a2", "a3"] # no "dup" - - -async def test_log_respawn_resets_dedup_cursor(monkeypatch) -> None: - replayed: list[dict] = [] - monkeypatch.setattr(log_bridge, "_replay_record", replayed.append) - - ns = log_bridge.HostLogNamespace() - record_event = log_bridge.RECORD_EVENT - - for seq in (1, 2, 3): - await ns.trigger_event(record_event, _env("aaaa", seq, {"msg": f"a{seq}"})) - assert [r["msg"] for r in replayed] == ["a1", "a2", "a3"] - - await ns.trigger_event(record_event, _env("bbbb", 1, {"msg": "b1"})) - assert replayed[-1] == {"msg": "b1"} - assert ns._sid == "bbbb" - assert ns._last_seq == 1 diff --git a/tests/utils/log/test_bridge.py b/tests/utils/log/test_bridge.py index 337c9b3..ba3f72e 100644 --- a/tests/utils/log/test_bridge.py +++ b/tests/utils/log/test_bridge.py @@ -1,48 +1,37 @@ -"""Tests for the worker→host logging bridge payload.""" +"""Tests for the host-side `/log` raw-line replayer.""" from __future__ import annotations import logging -from decimal import Decimal - -from agentix.runtime.shared.codec import pack -from agentix.utils.log._bridge import _coerce_extra, _record_payload +import pytest -def _record(**extras: object) -> logging.LogRecord: - record = logging.LogRecord("test", logging.INFO, "p.py", 10, "hello %s", ("world",), None) - for key, value in extras.items(): - setattr(record, key, value) - return record - +from agentix.runtime.shared.codec import pack +from agentix.utils.log._bridge import LOG_EVENT, HostLogNamespace -def test_coerce_extra_keeps_native_types() -> None: - assert _coerce_extra("s") == "s" - assert _coerce_extra(3) == 3 - assert _coerce_extra(True) is True - assert _coerce_extra(None) is None - assert _coerce_extra([1, "a"]) == [1, "a"] - assert _coerce_extra({"k": 2}) == {"k": 2} +pytestmark = pytest.mark.asyncio -def test_coerce_extra_reprs_unencodable() -> None: - class Weird: - def __repr__(self) -> str: - return "" +async def test_replays_line_into_sandbox_logger(caplog) -> None: + ns = HostLogNamespace() + with caplog.at_level(logging.INFO, logger="agentix.sandbox.stdout"): + await ns.trigger_event(LOG_EVENT, pack({"stream": "stdout", "line": "hello from sandbox"})) + assert any( + r.name == "agentix.sandbox.stdout" and r.getMessage() == "hello from sandbox" + for r in caplog.records + ) - assert _coerce_extra(Weird()) == "" - assert _coerce_extra(Decimal("1.5")) == "Decimal('1.5')" - assert _coerce_extra({"obj": Weird()}) == {"obj": ""} +async def test_stderr_lines_go_to_stderr_logger(caplog) -> None: + ns = HostLogNamespace() + with caplog.at_level(logging.INFO, logger="agentix.sandbox.stderr"): + await ns.trigger_event(LOG_EVENT, pack({"stream": "stderr", "line": "boom"})) + assert any(r.name == "agentix.sandbox.stderr" for r in caplog.records) -def test_record_payload_is_always_packable() -> None: - class Weird: - def __repr__(self) -> str: - return "" - payload = _record_payload(_record(obj=Weird(), count=3, label="x")) - extras = payload["extras"] - assert extras == {"obj": "", "count": 3, "label": "x"} - # The whole frame must now msgpack-encode (regression: a non-serializable - # extra previously made the drainer drop the record). - assert pack(payload) +async def test_ignores_non_line_and_malformed_events(caplog) -> None: + ns = HostLogNamespace() + with caplog.at_level(logging.INFO): + await ns.trigger_event("connect") + await ns.trigger_event(LOG_EVENT, pack({"stream": "stdout"})) # no line + assert not [r for r in caplog.records if r.name.startswith("agentix.sandbox")] From 8353997f16033c54fa6f5ce6d6694aae7656526e Mon Sep 17 00:00:00 2001 From: FatPigeorz Date: Mon, 29 Jun 2026 06:19:19 +0000 Subject: [PATCH 03/11] tito-gateway: wire BackendPool for multi-backend sticky routing (M3) + /healthz MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Architecture-review M3: the gateway now routes inference across N backends (sglang/vLLM replicas) via BackendPool — sticky by session_id for prefix-cache locality, report_down on a transport error, forget on session DELETE. Done entirely in non-vendored gateway code (a MilesSessionServer subclass overriding do_proxy + a forget middleware), so VENDORED_MILES_AUDIT stays valid — no vendored file is touched. - config: add backend_urls / routing_policy (validated). - gateway: build the pool (single discovered backend by default, or the explicit backend_urls list); add a /healthz alias so abridge's default Sidecar health probe works (L10). - tests: pool routing/wiring (test_pool_routing.py) + the pool unit tests. 13 tito tests + pyright clean. Co-Authored-By: Claude Opus 4.8 --- sidecars/tito/tests/test_pool.py | 67 +++++++++++ sidecars/tito/tests/test_pool_routing.py | 137 +++++++++++++++++++++++ sidecars/tito/tito_gateway/config.py | 10 ++ sidecars/tito/tito_gateway/gateway.py | 39 +++++-- sidecars/tito/tito_gateway/pool.py | 87 ++++++++++++++ sidecars/tito/tito_gateway/server.py | 91 ++++++++++++++- 6 files changed, 417 insertions(+), 14 deletions(-) create mode 100644 sidecars/tito/tests/test_pool.py create mode 100644 sidecars/tito/tests/test_pool_routing.py create mode 100644 sidecars/tito/tito_gateway/pool.py diff --git a/sidecars/tito/tests/test_pool.py b/sidecars/tito/tests/test_pool.py new file mode 100644 index 0000000..da47d77 --- /dev/null +++ b/sidecars/tito/tests/test_pool.py @@ -0,0 +1,67 @@ +"""Unit tests for the Gateway backend pool routing (no model in the loop).""" + +from __future__ import annotations + +import pytest +from tito_gateway.pool import BackendPool + +A, B, C = "http://h1:8000", "http://h2:8000", "http://h3:8000" + + +def test_requires_backends() -> None: + with pytest.raises(ValueError): + BackendPool([]) + + +def test_bad_policy() -> None: + with pytest.raises(ValueError): + BackendPool([A], policy="nope") + + +def test_single_backend_always() -> None: + pool = BackendPool([A]) + assert pool.pick("s1") == A + assert pool.pick() == A + + +def test_sticky_pins_session_to_one_backend() -> None: + pool = BackendPool([A, B, C], policy="sticky") + first = pool.pick("rollout-1") + # Same session keeps hitting the same backend across many turns. + assert all(pool.pick("rollout-1") == first for _ in range(10)) + + +def test_sticky_spreads_distinct_sessions_round_robin() -> None: + pool = BackendPool([A, B, C], policy="sticky") + assigned = [pool.pick(f"s{i}") for i in range(3)] + assert sorted(assigned) == sorted([A, B, C]) # 3 sessions → 3 distinct backends + + +def test_round_robin_cycles_every_request() -> None: + pool = BackendPool([A, B], policy="round_robin") + assert [pool.pick("ignored") for _ in range(4)] == [A, B, A, B] + + +def test_down_backend_is_skipped() -> None: + pool = BackendPool([A, B], policy="round_robin") + pool.report_down(A) + assert {pool.pick() for _ in range(6)} == {B} + pool.report_up(A) + assert A in {pool.pick() for _ in range(6)} + + +def test_sticky_session_reassigned_when_backend_down() -> None: + pool = BackendPool([A, B], policy="sticky") + pinned = pool.pick("r1") + pool.report_down(pinned) + reassigned = pool.pick("r1") + assert reassigned != pinned + assert reassigned not in pool._down + + +def test_all_down_falls_back_not_fails() -> None: + pool = BackendPool([A, B], policy="round_robin") + pool.report_down(A) + pool.report_down(B) + # Better to attempt a (maybe-recovered) backend than fail routing outright. + assert pool.pick() in (A, B) diff --git a/sidecars/tito/tests/test_pool_routing.py b/sidecars/tito/tests/test_pool_routing.py new file mode 100644 index 0000000..454ab5d --- /dev/null +++ b/sidecars/tito/tests/test_pool_routing.py @@ -0,0 +1,137 @@ +"""Wiring tests for BackendPool routing in the SessionServer (no model/GPU). + +Uses ``hf_checkpoint=None`` so the vendored session server skips tokenizer/route +setup — we drive the pool-aware ``do_proxy`` / forget hook directly. +""" + +from __future__ import annotations + +import sys +import types + +import pytest + + +def _install_sglang_stub() -> None: + """Stub `sglang...Tool` so the vendored template module imports without the + sglang runtime (this routing test needs no model). Mirrors tito_experiment.""" + if "sglang" in sys.modules: + return + from typing import Any, Optional + + from pydantic import BaseModel + + class _Function(BaseModel): + name: str + description: Optional[str] = None + parameters: Optional[dict[str, Any]] = None + + class Tool(BaseModel): + type: str = "function" + function: _Function + + names = ["sglang", "sglang.srt", "sglang.srt.entrypoints", "sglang.srt.entrypoints.openai"] + mods = {n: types.ModuleType(n) for n in names} + protocol = types.ModuleType("sglang.srt.entrypoints.openai.protocol") + protocol.Tool = Tool # type: ignore[attr-defined] + mods["sglang.srt.entrypoints.openai"].protocol = protocol # type: ignore[attr-defined] + mods["sglang.srt.entrypoints"].openai = mods["sglang.srt.entrypoints.openai"] # type: ignore[attr-defined] + mods["sglang.srt"].entrypoints = mods["sglang.srt.entrypoints"] # type: ignore[attr-defined] + mods["sglang"].srt = mods["sglang.srt"] # type: ignore[attr-defined] + for n, m in mods.items(): + sys.modules[n] = m + sys.modules["sglang.srt.entrypoints.openai.protocol"] = protocol + + +_install_sglang_stub() + +from tito_gateway.pool import BackendPool # noqa: E402 +from tito_gateway.server import SessionServer, _session_id_from_path # noqa: E402 + +A = "http://a:8000" +B = "http://b:8000" + + +def _args(): + return types.SimpleNamespace(hf_checkpoint=None, miles_router_timeout=600.0) + + +class _URL: + def __init__(self, path: str, query: str = "") -> None: + self.path = path + self.query = query + + +class _Request: + def __init__(self, path: str, method: str = "POST", body: bytes = b"{}") -> None: + self.url = _URL(path) + self.method = method + self.headers = {} + self._body = body + + async def body(self) -> bytes: + return self._body + + +class _Resp: + def __init__(self, status: int = 200) -> None: + self.status_code = status + self.headers = {} + + async def aread(self) -> bytes: + return b"{}" + + +def test_session_id_from_path(): + assert _session_id_from_path("/sessions/abc/v1/chat/completions") == "abc" + assert _session_id_from_path("/sessions/xyz") == "xyz" + assert _session_id_from_path("/health") is None + assert _session_id_from_path("/") is None + + +@pytest.mark.asyncio +async def test_sticky_routing_pins_session(monkeypatch): + pool = BackendPool([A, B], policy="sticky") + srv = SessionServer(_args(), pool) + seen: list[str] = [] + + async def fake_request(method, url, content=None, headers=None): + seen.append(url) + return _Resp() + + monkeypatch.setattr(srv._impl.client, "request", fake_request) + for _ in range(3): + await srv._impl.do_proxy(_Request("/sessions/s1/v1/chat/completions"), "v1/chat/completions") + # all three turns of one session hit the same backend (prefix-cache locality) + assert len({u.split("/v1/")[0] for u in seen}) == 1 + + +@pytest.mark.asyncio +async def test_transport_error_reports_backend_down(monkeypatch): + import httpx + + pool = BackendPool([A, B], policy="sticky") + srv = SessionServer(_args(), pool) + + async def boom(method, url, content=None, headers=None): + raise httpx.ConnectError("refused") + + monkeypatch.setattr(srv._impl.client, "request", boom) + result = await srv._impl.do_proxy(_Request("/sessions/s9/v1/chat/completions"), "v1/chat/completions") + assert result["status_code"] == 502 + # the picked backend was marked down + assert pool._down # noqa: SLF001 - asserting routing side effect + + +@pytest.mark.asyncio +async def test_forget_on_delete_drops_pin(): + pool = BackendPool([A, B], policy="sticky") + pool.pick("s2") + assert "s2" in pool._assigned # noqa: SLF001 + srv = SessionServer(_args(), pool) + + async def call_next(_req): + return _Resp(status=204) + + await srv._impl._forget_on_delete(_Request("/sessions/s2", method="DELETE"), call_next) + assert "s2" not in pool._assigned # noqa: SLF001 diff --git a/sidecars/tito/tito_gateway/config.py b/sidecars/tito/tito_gateway/config.py index 801ecd0..cfc823d 100644 --- a/sidecars/tito/tito_gateway/config.py +++ b/sidecars/tito/tito_gateway/config.py @@ -18,6 +18,11 @@ class TITOGatewayConfig: hf_checkpoint: str backend_url: str | None = None + # Explicit multi-backend pool (sglang/vLLM replicas). When set, these are + # used as-is and single-URL discovery is skipped; `backend_url` is left as + # the first entry for callers that read it. + backend_urls: tuple[str, ...] = () + routing_policy: str = "sticky" chat_template_path: str | None = None apply_chat_template_kwargs: dict[str, Any] = field(default_factory=dict) tito_model: str = "default" @@ -38,6 +43,11 @@ def __post_init__(self) -> None: raise ValueError(f"unsupported tito append roles: {invalid}") object.__setattr__(self, "tito_allowed_append_roles", normalized_roles or ("tool",)) + if self.routing_policy not in ("sticky", "round_robin"): + raise ValueError( + f"routing_policy must be 'sticky' or 'round_robin'; got {self.routing_policy!r}" + ) + @classmethod def from_cli_values( cls, diff --git a/sidecars/tito/tito_gateway/gateway.py b/sidecars/tito/tito_gateway/gateway.py index f573504..b5aec8f 100644 --- a/sidecars/tito/tito_gateway/gateway.py +++ b/sidecars/tito/tito_gateway/gateway.py @@ -5,26 +5,47 @@ from dataclasses import replace from tito_gateway.config import TITOGatewayConfig -from tito_gateway.discovery import discover_backend_url +from tito_gateway.discovery import discover_backend_url, normalize_backend_url +from tito_gateway.pool import BackendPool from tito_gateway.server import SessionServer class TITOGateway: - """Small wrapper that resolves a backend and owns a session server app.""" + """Small wrapper that resolves backend(s) and owns a session server app. + + Routes inference across a :class:`BackendPool` — a single backend (resolved + by discovery) by default, or several when ``config.backend_urls`` is set. + """ def __init__(self, config: TITOGatewayConfig): - backend_url = discover_backend_url( - config.backend_url, - probe_candidates=config.backend_probe_candidates, - probe_timeout=config.backend_probe_timeout, - ) - self.config = replace(config, backend_url=backend_url) - self.server = SessionServer(self.config.as_miles_namespace(), backend_url) + if config.backend_urls: + urls = [normalize_backend_url(u) for u in config.backend_urls] + self.config = replace(config, backend_url=urls[0]) + else: + backend_url = discover_backend_url( + config.backend_url, + probe_candidates=config.backend_probe_candidates, + probe_timeout=config.backend_probe_timeout, + ) + self.config = replace(config, backend_url=backend_url) + urls = [backend_url] + self.pool = BackendPool(urls, policy=config.routing_policy) + self.server = SessionServer(self.config.as_miles_namespace(), self.pool) + self._register_health_alias() @classmethod def from_server(cls, *, hf_checkpoint: str, backend_url: str | None = None, **kwargs) -> "TITOGateway": return cls(TITOGatewayConfig(hf_checkpoint=hf_checkpoint, backend_url=backend_url, **kwargs)) + def _register_health_alias(self) -> None: + # abridge's Sidecar probes `/healthz` by default; the vendored session + # server only exposes `/health`. Add a thin alias so a default Sidecar + # wiring works without overriding `health_path`. + async def healthz() -> dict[str, str]: + return {"status": "ok"} + + self.app.add_api_route("/healthz", healthz, methods=["GET"]) + @property def app(self): return self.server.app diff --git a/sidecars/tito/tito_gateway/pool.py b/sidecars/tito/tito_gateway/pool.py new file mode 100644 index 0000000..43228d8 --- /dev/null +++ b/sidecars/tito/tito_gateway/pool.py @@ -0,0 +1,87 @@ +"""Backend pool — route OpenAI-compatible requests across N base URLs. + +The TITO Gateway accepts one *or more* OpenAI-compatible backend URLs +(sglang/vLLM replicas) and forwards each request to one of them. This is +the routing layer, independent of TITO tokenization, so it is unit-tested +on its own with no model in the loop. + +Policy: + - ``sticky`` (default): each ``session_id`` is pinned to one backend, + chosen round-robin among healthy backends on first sight and then + remembered. A multi-turn rollout reuses one replica's prefix KV-cache, + which is the right default for TITO. (TITO sends explicit ``input_ids``, + so any replica *can* serve any turn — stickiness is a cache-locality + optimization, not a correctness requirement.) + - ``round_robin``: spread every request across healthy backends. + +Backends reported down via ``report_down`` are skipped until ``report_up``; +a sticky session whose backend goes down is reassigned on its next pick. +""" + +from __future__ import annotations + +import threading +from collections.abc import Sequence + +_POLICIES = ("sticky", "round_robin") + + +class BackendPool: + def __init__(self, backends: Sequence[str], *, policy: str = "sticky") -> None: + urls = [b.rstrip("/") for b in backends if b] + if not urls: + raise ValueError("BackendPool requires at least one backend url") + if policy not in _POLICIES: + raise ValueError(f"policy must be one of {_POLICIES}; got {policy!r}") + self._backends = urls + self._policy = policy + self._rr = 0 + self._assigned: dict[str, str] = {} + self._down: set[str] = set() + self._lock = threading.Lock() + + @property + def backends(self) -> tuple[str, ...]: + return tuple(self._backends) + + def _healthy(self) -> list[str]: + healthy = [b for b in self._backends if b not in self._down] + # All down → fall back to the full set rather than fail the request; + # the forward attempt surfaces the real error. + return healthy or list(self._backends) + + def _next_round_robin(self, healthy: list[str]) -> str: + chosen = healthy[self._rr % len(healthy)] + self._rr += 1 + return chosen + + def pick(self, session_id: str | None = None) -> str: + """Choose a backend for a request. With the sticky policy and a + `session_id`, return that session's pinned backend (assigning one + the first time, or reassigning if the pinned one is down).""" + with self._lock: + healthy = self._healthy() + if self._policy == "sticky" and session_id is not None: + current = self._assigned.get(session_id) + if current is not None and current not in self._down: + return current + chosen = self._next_round_robin(healthy) + self._assigned[session_id] = chosen + return chosen + return self._next_round_robin(healthy) + + def report_down(self, backend: str) -> None: + with self._lock: + self._down.add(backend.rstrip("/")) + + def report_up(self, backend: str) -> None: + with self._lock: + self._down.discard(backend.rstrip("/")) + + def forget(self, session_id: str) -> None: + """Drop a session's sticky assignment (call when the rollout ends).""" + with self._lock: + self._assigned.pop(session_id, None) + + +__all__ = ["BackendPool"] diff --git a/sidecars/tito/tito_gateway/server.py b/sidecars/tito/tito_gateway/server.py index 10fc79c..9ec08e3 100644 --- a/sidecars/tito/tito_gateway/server.py +++ b/sidecars/tito/tito_gateway/server.py @@ -1,19 +1,100 @@ -"""Session server wrapper around the vendored Miles implementation.""" +"""Session server wrapper around the vendored Miles implementation. + +Adds multi-backend routing on top of the vendored single-backend +``SessionServer`` WITHOUT modifying any vendored file: a thin subclass +overrides ``do_proxy`` to pick a backend from a :class:`BackendPool` per +request (sticky by ``session_id`` for prefix-cache locality), reports a +backend down on a transport error, and forgets a session's pin when the +session is deleted. +""" from __future__ import annotations +import json +import logging from typing import Any +import httpx + +from tito_gateway.pool import BackendPool + +logger = logging.getLogger(__name__) + +_HOP_BY_HOP = ("content-length", "transfer-encoding", "host") + + +def _session_id_from_path(path: str) -> str | None: + """Extract ``{session_id}`` from ``/sessions/{session_id}[/...]``.""" + parts = path.strip("/").split("/") + if len(parts) >= 2 and parts[0] == "sessions": + return parts[1] + return None + class SessionServer: - """Lazy wrapper for Miles' standalone FastAPI session server.""" + """Wrapper for Miles' standalone FastAPI session server, routing proxied + inference across a :class:`BackendPool`.""" - def __init__(self, args: Any, backend_url: str): + def __init__(self, args: Any, pool: BackendPool): from tito_gateway.vendor.miles_compat.rollout.session.session_server import ( SessionServer as MilesSessionServer, ) - self._impl = MilesSessionServer(args, backend_url) + class _PooledSessionServer(MilesSessionServer): + def __init__(self, args: Any, pool: BackendPool) -> None: + self._pool = pool + # Nominal backend_url for any vendored code that reads it; the + # per-request route is chosen in `do_proxy` below. + super().__init__(args, pool.backends[0]) + self.app.middleware("http")(self._forget_on_delete) + + async def do_proxy(self, request, path, body=None, headers=None) -> dict: # type: ignore[override] + session_id = _session_id_from_path(request.url.path) + backend_url = self._pool.pick(session_id) + url = f"{backend_url}/{path}" + if request.url.query: + url = f"{url}?{request.url.query}" + if body is None: + body = await request.body() + if headers is None: + headers = dict(request.headers) + headers = {k: v for k, v in headers.items() if k.lower() not in _HOP_BY_HOP} + try: + response = await self.client.request(request.method, url, content=body, headers=headers) + except httpx.TransportError as exc: + # Mark this replica down so the session re-pins on its next + # request; surface the error to the agent unchanged. + self._pool.report_down(backend_url) + logger.warning("pooled proxy transport error %s -> %s: %s", path, backend_url, exc) + error_body = json.dumps( + {"error": f"backend transport error: {type(exc).__name__}: {exc}"} + ).encode() + return { + "request_body": body, + "response_body": error_body, + "status_code": 502, + "headers": {"content-type": "application/json"}, + } + content = await response.aread() + return { + "request_body": body, + "response_body": content, + "status_code": response.status_code, + "headers": dict(response.headers), + } + + async def _forget_on_delete(self, request, call_next): + response = await call_next(request) + if request.method == "DELETE" and response.status_code < 300: + session_id = _session_id_from_path(request.url.path) + if session_id is not None: + # Drop the sticky pin so `_assigned` doesn't grow without + # bound across a long-lived gateway. + self._pool.forget(session_id) + return response + + self._impl = _PooledSessionServer(args, pool) self.args = args - self.backend_url = backend_url + self.pool = pool + self.backend_url = pool.backends[0] self.app = self._impl.app From efedacfb85d41ed0262c4abf6f3e2e67024cd319 Mon Sep 17 00:00:00 2001 From: FatPigeorz Date: Mon, 29 Jun 2026 13:16:35 +0000 Subject: [PATCH 04/11] provider: add uv SandboxProvider (runs the runtime from a uv venv, no Docker/Nix) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A lightweight SandboxProvider at plugins/providers/uv that materializes a venv with uv and launches the runtime server as a local subprocess — for dev / eval / CI where container isolation isn't needed. - create(): `uv venv` + `uv pip install -e ` (or reuse_venv=), then `/bin/python -m uvicorn agentix.runtime.server.app:app`; raw-socket health wait; delete()/get()/aclose(). - entry-point agentix.provider -> uv = agentix.provider.uv:UvProvider, so the string registry resolves "uv" on an editable workspace install. - The worker imports user code only from site-packages (PYTHONPATH stripped), so the rollout module is pip-installed into the venv, like the Docker bundle. Verified on the cluster: 4 tests + pyright green; drove a full agentic RL rollout (UvProvider -> runtime -> TITO gateway -> sglang) end to end. Co-Authored-By: Claude Opus 4.8 --- plugins/providers/uv/README.md | 31 +++ plugins/providers/uv/agentix/provider/uv.py | 246 ++++++++++++++++++ plugins/providers/uv/pyproject.toml | 33 +++ .../providers/uv/tests/test_uv_provider.py | 63 +++++ 4 files changed, 373 insertions(+) create mode 100644 plugins/providers/uv/README.md create mode 100644 plugins/providers/uv/agentix/provider/uv.py create mode 100644 plugins/providers/uv/pyproject.toml create mode 100644 plugins/providers/uv/tests/test_uv_provider.py diff --git a/plugins/providers/uv/README.md b/plugins/providers/uv/README.md new file mode 100644 index 0000000..eed5ce1 --- /dev/null +++ b/plugins/providers/uv/README.md @@ -0,0 +1,31 @@ +# agentix-provider-uv + +A lightweight Agentix `SandboxProvider` that runs the runtime from a +**uv-materialized virtualenv** — no Docker image, no Nix bundle. + +`uv` builds a venv for the target project (so its importable callables + +`agentixx` core are present), then the runtime server is launched as a local +subprocess (`python -m uvicorn agentix.runtime.server.app:app`). The worker the +server spawns inherits that interpreter, so `await sandbox.remote(fn, ...)` runs +against the project's real dependencies. + +```python +from agentix.provider.base import SandboxConfig +from agentix.provider.uv import UvProvider, UvProviderConfig + +# materialize from a project (must depend on agentixx) +provider = UvProvider(UvProviderConfig(project=".")) +# ...or reuse a prebuilt env and skip materialization +provider = UvProvider(UvProviderConfig(reuse_venv="/path/to/venv")) + +async with provider.session(SandboxConfig(image="uv", bundle="uv")) as sandbox: + result = await sandbox.remote(my_rollout, task=task) +``` + +`SandboxConfig.image` / `bundle` are unused (placeholders); only `env` is +honored. This backend runs on the host with **no container isolation** — use it +for fast local dev / eval / CI, and a container provider (`docker` / +`apptainer`) or managed backend for untrusted code or hard resource limits. + +`providers().get("uv")` resolves after `uv sync`. There is no `agentix deploy +uv` — the runtime is materialized from source, so there is no bundle artifact. diff --git a/plugins/providers/uv/agentix/provider/uv.py b/plugins/providers/uv/agentix/provider/uv.py new file mode 100644 index 0000000..3bba796 --- /dev/null +++ b/plugins/providers/uv/agentix/provider/uv.py @@ -0,0 +1,246 @@ +"""uv SandboxProvider — run the Agentix runtime from a uv-materialized venv. + +A lightweight provider that skips the Docker/Nix bundle entirely. `uv` +materializes a virtualenv for the target project (so its importable callables +plus `agentixx` core are present), then the runtime server is launched as a +local subprocess (`python -m uvicorn agentix.runtime.server.app:app`). The +worker subprocess the server spawns inherits that interpreter +(`sys.executable`), so `await sandbox.remote(fn, ...)` runs `fn` against the +project's real dependencies — no container, no rebuild. + +Aimed at local dev / eval / CI where Docker is unavailable or too slow. It +trades isolation for speed: the runtime runs on the host, not in a sandboxed +container. For untrusted code or hard resource limits, use a container +provider (`docker` / `apptainer`) or a managed backend instead. + + from agentix.provider.uv import UvProvider, UvProviderConfig + + provider = UvProvider(UvProviderConfig(project=".")) # uv pip install -e . + async with provider.session(SandboxConfig(image="uv", bundle="uv")) as sandbox: + result = await sandbox.remote(my_rollout, task=task) + +`SandboxConfig.image` / `bundle` are unused here (there is no image or bundle); +pass any placeholder. Only `SandboxConfig.env` is honored — merged into the +runtime server's environment. Backend settings live in `UvProviderConfig`, +mirroring how other providers take a backend config object. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import logging +import os +import shutil +import socket +import tempfile +import uuid +from dataclasses import dataclass +from pathlib import Path + +from agentix.provider.base import ( + Sandbox, + SandboxConfig, + SandboxId, + SandboxInfo, + SandboxProvider, +) + +logger = logging.getLogger("agentix.provider.uv") + +_RUNTIME_APP = "agentix.runtime.server.app:app" + + +@dataclass +class UvProviderConfig: + """Backend config for `UvProvider`. + + Either point at a `project` to materialize a fresh venv (`uv venv` + + `uv pip install -e ` — the project must depend on `agentixx`), or + point `reuse_venv` at an existing interpreter env to skip materialization + (fast iteration / CI where the env is prebuilt). + """ + + project: str | None = None + python: str = "3.12" + index_url: str | None = None + extra_index_url: tuple[str, ...] = () + install: tuple[str, ...] = () + reuse_venv: str | None = None + uv_bin: str = "uv" + host: str = "127.0.0.1" + ws: str = "auto" + health_timeout: float = 60.0 + + def __post_init__(self) -> None: + if self.project is None and self.reuse_venv is None: + raise ValueError("UvProviderConfig needs either `project` or `reuse_venv`") + + +@dataclass +class _Running: + proc: asyncio.subprocess.Process + port: int + + +def _free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +async def _run(*argv: str, timeout: float = 1800.0) -> None: + proc = await asyncio.create_subprocess_exec( + *argv, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT + ) + try: + out, _ = await asyncio.wait_for(proc.communicate(), timeout=timeout) + except TimeoutError: + proc.kill() + raise + if proc.returncode != 0: + tail = out.decode(errors="replace")[-2000:] if out else "" + raise RuntimeError(f"command failed (rc={proc.returncode}): {' '.join(argv)}\n{tail}") + + +class UvProvider(SandboxProvider): + """Provision sandboxes as a runtime server launched from a uv venv.""" + + def __init__(self, config: UvProviderConfig | None = None) -> None: + if config is None: + config = UvProviderConfig(project=".") + self.config = config + self._running: dict[SandboxId, _Running] = {} + self._venv: Path | None = None + self._owned_venv_root: Path | None = None + self._venv_lock = asyncio.Lock() + + async def _ensure_venv(self) -> Path: + """Materialize (once) and return the venv whose `python` runs the + runtime. Reused across every `create()` on this provider.""" + if self.config.reuse_venv is not None: + return Path(self.config.reuse_venv) + async with self._venv_lock: + if self._venv is not None: + return self._venv + root = Path(tempfile.mkdtemp(prefix="agentix-uv-")) + venv = root / "venv" + await _run(self.config.uv_bin, "venv", "--python", self.config.python, str(venv)) + py = str(venv / "bin" / "python") + idx: list[str] = [] + if self.config.index_url: + idx += ["--index-url", self.config.index_url] + for extra in self.config.extra_index_url: + idx += ["--extra-index-url", extra] + targets: list[str] = [] + if self.config.project is not None: + targets += ["-e", self.config.project] + targets += list(self.config.install) + if targets: + await _run(self.config.uv_bin, "pip", "install", "--python", py, *idx, *targets) + self._venv = venv + self._owned_venv_root = root + return venv + + async def create(self, config: SandboxConfig) -> Sandbox: + venv = await self._ensure_venv() + python = str(venv / "bin" / "python") + port = _free_port() + + env = dict(os.environ) + env.setdefault("AGENTIX_LOG_CONTEXT", "uv-sandbox-{uname}") + if config.env: + env.update(config.env) + + cmd = [ + python, "-m", "uvicorn", _RUNTIME_APP, + "--host", self.config.host, "--port", str(port), + "--log-level", "error", "--ws", self.config.ws, "--lifespan", "on", + ] + proc = await asyncio.create_subprocess_exec( + *cmd, env=env, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT + ) + sandbox_id = SandboxId(f"uv-{uuid.uuid4().hex[:12]}") + self._running[sandbox_id] = _Running(proc=proc, port=port) + try: + await self._wait_healthy(sandbox_id, port, proc) + except BaseException: + await self.delete(sandbox_id) + raise + return Sandbox( + sandbox_id=sandbox_id, + runtime_url=f"http://{self.config.host}:{port}", + status="running", + ) + + async def _wait_healthy(self, sandbox_id: SandboxId, port: int, proc: asyncio.subprocess.Process) -> None: + # Raw TCP GET /health — never via an HTTP client that honors proxy env + # vars, which would hang a loopback probe behind a corp proxy. + attempts = max(1, int(self.config.health_timeout / 0.5)) + for _ in range(attempts): + if proc.returncode is not None: + out = (await proc.stdout.read()) if proc.stdout else b"" + raise RuntimeError( + f"runtime server (uv) exited rc={proc.returncode} before health: " + f"{out.decode(errors='replace')[-2000:]}" + ) + try: + reader, writer = await asyncio.wait_for( + asyncio.open_connection(self.config.host, port), timeout=2 + ) + except (TimeoutError, OSError): + await asyncio.sleep(0.5) + continue + try: + writer.write(b"GET /health HTTP/1.0\r\nHost: localhost\r\n\r\n") + await writer.drain() + status_line = await asyncio.wait_for(reader.readline(), timeout=2) + if status_line.startswith(b"HTTP/1.") and b" 200 " in status_line: + return + except (TimeoutError, OSError): + pass + finally: + writer.close() + with contextlib.suppress(OSError): + await writer.wait_closed() + await asyncio.sleep(0.5) + raise TimeoutError(f"runtime server (uv) not healthy on :{port}") + + async def get(self, sandbox_id: SandboxId) -> SandboxInfo: + running = self._running.get(sandbox_id) + if running is None: + raise KeyError(f"Sandbox not found: {sandbox_id}") + status = "running" if running.proc.returncode is None else "exited" + return SandboxInfo( + sandbox_id=sandbox_id, + runtime_url=f"http://{self.config.host}:{running.port}", + status=status, + ) + + async def delete(self, sandbox_id: SandboxId) -> None: + running = self._running.pop(sandbox_id, None) + if running is None: + return + await self._terminate(running.proc, sandbox_id) + + async def _terminate(self, proc: asyncio.subprocess.Process, sandbox_id: SandboxId) -> None: + if proc.returncode is not None: + return + proc.terminate() + try: + await asyncio.wait_for(proc.wait(), timeout=10.0) + except TimeoutError: + logger.warning("uv runtime %s did not exit after SIGTERM; SIGKILL", sandbox_id) + proc.kill() + with contextlib.suppress(Exception): + await asyncio.wait_for(proc.wait(), timeout=5.0) + + async def aclose(self) -> None: + """Terminate every running sandbox and remove a venv this provider + materialized. An externally supplied `reuse_venv` is left untouched.""" + for sandbox_id in list(self._running): + await self.delete(sandbox_id) + if self._owned_venv_root is not None: + shutil.rmtree(self._owned_venv_root, ignore_errors=True) + self._owned_venv_root = None + self._venv = None diff --git a/plugins/providers/uv/pyproject.toml b/plugins/providers/uv/pyproject.toml new file mode 100644 index 0000000..285cc89 --- /dev/null +++ b/plugins/providers/uv/pyproject.toml @@ -0,0 +1,33 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "agentix-provider-uv" +version = "0.1.0" +description = "uv-materialized local runtime provider for Agentix (no Docker/Nix bundle)" +requires-python = ">=3.11" +dependencies = [ + # Protocol + dataclasses (`SandboxProvider`, `Sandbox`, `SandboxConfig`, + # `SandboxInfo`, `SandboxId`) all live in core agentix. + "agentixx", + # The provider shells out to `uv` to materialize the runtime venv; depend + # on it so the backend works without a system-wide uv install. + "uv>=0.5", +] + +# `agentixx` is the monorepo workspace root — used editable, never from PyPI. +[tool.uv.sources] +agentixx = { workspace = true } + +# `uv sync` makes `providers().get("uv")` resolve — the registry walks this +# entry-point group. There is no `agentix deploy uv`: this backend materializes +# the runtime from source via uv, so there is no bundle artifact to deploy. +[project.entry-points."agentix.provider"] +uv = "agentix.provider.uv:UvProvider" + +[tool.hatch.build.targets.wheel] +# One file at `agentix/provider/uv.py`. The `agentix` and `agentix/provider` +# dirs carry no __init__.py here — those belong to core agentix; this wheel +# installs a sibling into the same namespace. +packages = ["agentix"] diff --git a/plugins/providers/uv/tests/test_uv_provider.py b/plugins/providers/uv/tests/test_uv_provider.py new file mode 100644 index 0000000..461e64d --- /dev/null +++ b/plugins/providers/uv/tests/test_uv_provider.py @@ -0,0 +1,63 @@ +"""uv provider: launch the runtime from a venv and drive a real remote() call. + +Uses `reuse_venv` pointed at the interpreter running the tests (it already has +`agentixx` + uvicorn), so the test needs no uv materialization. Remote targets +are stdlib functions (`math.*`) — always importable by the worker, so the test +exercises the provider's runtime wiring without packaging a fixture module. (A +user's own rollout module is reached the same way every provider does it: +installed into the venv via `UvProviderConfig.project` / `install`.) +""" + +from __future__ import annotations + +import math +import sys + +import pytest + +from agentix.provider.base import SandboxConfig, SandboxProvider +from agentix.provider.uv import UvProvider, UvProviderConfig + + +def _reuse_venv() -> str: + # venv root of the interpreter running the tests. Use sys.prefix, NOT a + # resolved sys.executable: the venv's bin/python is a symlink, and resolving + # it jumps to the base interpreter (whose env lacks agentixx). + return sys.prefix + + +def test_config_requires_project_or_venv(): + with pytest.raises(ValueError): + UvProviderConfig() + + +def test_is_sandboxprovider(): + provider = UvProvider(UvProviderConfig(reuse_venv=_reuse_venv())) + assert isinstance(provider, SandboxProvider) + + +@pytest.mark.asyncio +async def test_remote_roundtrip(): + provider = UvProvider(UvProviderConfig(reuse_venv=_reuse_venv())) + try: + async with provider.session(SandboxConfig(image="uv", bundle="uv")) as sandbox: + assert (await sandbox.health()).version + assert await sandbox.remote(math.factorial, 5) == 120 + assert await sandbox.remote(math.gcd, 12, 8) == 4 + finally: + await provider.aclose() + + +@pytest.mark.asyncio +async def test_get_and_delete(): + provider = UvProvider(UvProviderConfig(reuse_venv=_reuse_venv())) + try: + sandbox = await provider.create(SandboxConfig(image="uv", bundle="uv")) + info = await provider.get(sandbox.sandbox_id) + assert info.status == "running" + await sandbox.aclose() + await provider.delete(sandbox.sandbox_id) + with pytest.raises(KeyError): + await provider.get(sandbox.sandbox_id) + finally: + await provider.aclose() From 56ba436ac1a501487f851fe68b66caa01b0792ff Mon Sep 17 00:00:00 2001 From: FatPigeorz Date: Mon, 29 Jun 2026 13:29:34 +0000 Subject: [PATCH 05/11] =?UTF-8?q?abridge:=20add=20SessionForward=20?= =?UTF-8?q?=E2=80=94=20route=20/v1/...=20into=20a=20session-scoped=20sidec?= =?UTF-8?q?ar?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The TITO gateway records a trajectory only under /sessions/{id}/v1/chat/completions, with the id assigned by the gateway at POST /sessions. A plain Forward posts straight to {target}{path}, so it can't reach that route. SessionForward closes the gap: it lazily creates the session (or eagerly via open()), remembers the assigned id, and rewrites every inbound path to {create_path}/{id}{path} — so an in-sandbox black-box agent keeps calling an unmodified /v1/chat/completions and the whole rollout lands in one session. Read .session_id afterward to harvest the trajectory (GET /sessions/{id}); the session is not deleted on aclose(). - Forward gains a tiny _url_for(path) seam (no behavior change) that SessionForward overrides; session creation reuses Forward's httpx pool, the x-session-id / x-request-id stamping, and the error handling. - Generic, not TITO-specific (create_path / session_id_field configurable); the vendored Miles session server is untouched, so the M3 audit stays valid. Tests: 6 new (mock httpx + a live stdlib sidecar mounting /sessions and /sessions/{id}/v1/chat/completions, proving the real path rewrite). Full abridge suite 49 passed; pyright clean on the changed source. Co-Authored-By: Claude Opus 4.8 --- plugins/abridge/agentix/bridge/__init__.py | 2 +- plugins/abridge/agentix/bridge/forward.py | 99 ++++++++++++- plugins/abridge/tests/test_sidecar_forward.py | 134 ++++++++++++++++++ 3 files changed, 232 insertions(+), 3 deletions(-) diff --git a/plugins/abridge/agentix/bridge/__init__.py b/plugins/abridge/agentix/bridge/__init__.py index 053617b..330e332 100644 --- a/plugins/abridge/agentix/bridge/__init__.py +++ b/plugins/abridge/agentix/bridge/__init__.py @@ -33,7 +33,7 @@ from __future__ import annotations -from .forward import Forward +from .forward import Forward, SessionForward from .proxy import ( NAMESPACE, AbridgeError, diff --git a/plugins/abridge/agentix/bridge/forward.py b/plugins/abridge/agentix/bridge/forward.py index eada27e..3b000b1 100644 --- a/plugins/abridge/agentix/bridge/forward.py +++ b/plugins/abridge/agentix/bridge/forward.py @@ -27,6 +27,7 @@ from __future__ import annotations +import asyncio import logging import uuid from collections.abc import Mapping @@ -89,7 +90,7 @@ async def _forward(self, path: str, request: Request) -> ClientResponse: "x-request-id": record_id, "content-type": "application/json", } - url = self._target + path + url = self._url_for(path) try: resp = await self._get_client().post(url, json=request.body, headers=headers) except httpx.HTTPError as exc: @@ -103,6 +104,11 @@ async def _forward(self, path: str, request: Request) -> ClientResponse: status_code=resp.status_code, ) + def _url_for(self, path: str) -> str: + """Upstream URL for an inbound `path`. Override to remap — e.g. a + session-scoped sidecar prefixes `/sessions/{id}`.""" + return self._target + path + def _get_client(self) -> httpx.AsyncClient: client = self._client if client is None or client.is_closed: @@ -118,4 +124,93 @@ async def aclose(self) -> None: await client.aclose() -__all__ = ["Forward"] +class SessionForward(Forward): + """Forward to a *session-scoped* sidecar that keys a trajectory by URL path. + + Some sidecars don't accept a bare `/v1/chat/completions` — they require a + session created up front and then addressed by path: `POST {create_path}` + returns `{"session_id": ...}`, and every later call goes to + `{create_path}/{session_id}{path}`. The TITO gateway is the motivating case: + its recording route is `/sessions/{id}/v1/chat/completions`, so a plain + `Forward` (which posts straight to `{target}{path}`) can't reach it, and the + id isn't known until the gateway assigns it. + + `SessionForward` creates the session lazily on the first forwarded request + (or eagerly via `open()`), remembers the assigned id, and rewrites every + inbound `path` to the session-scoped URL. So the in-sandbox agent keeps + calling an unmodified `/v1/chat/completions` and the whole rollout still + lands in one session. Read `.session_id` afterward to harvest the trajectory + (`GET {create_path}/{session_id}`). + + fwd = SessionForward(gateway_url, paths=["/v1/chat/completions"]) + async with Proxy(fwd).session(sandbox) as handle: + await sandbox.remote(agent, env=openai_env_for(handle)) + trajectory = (await httpx.AsyncClient().get( + f"{gateway_url}/sessions/{fwd.session_id}")).json() + + The session is intentionally *not* deleted on `aclose()` — the trajectory is + the point, and reusing the forwarder keeps the same session. + """ + + def __init__( + self, + target_url: str, + *, + paths: list[str], + create_path: str = "/sessions", + session_id_field: str = "session_id", + timeout: float = 600.0, + headers: Mapping[str, str] | None = None, + ) -> None: + super().__init__(target_url, paths=paths, timeout=timeout, headers=headers) + self._create_path = "/" + create_path.strip("/") + self._session_field = session_id_field + self._session_ready = False + self._session_lock = asyncio.Lock() + + async def open(self) -> str: + """Create the sidecar session now (idempotent) and return its id. + + Lazy creation also happens on the first forwarded request, so `open()` + is only needed when the host wants the id before the agent runs. + """ + await self._ensure_session() + return self.session_id + + async def _ensure_session(self) -> None: + if self._session_ready: + return + async with self._session_lock: + if self._session_ready: + return + url = self._target + self._create_path + headers = {**self._headers, "content-type": "application/json"} + try: + resp = await self._get_client().post(url, json={}, headers=headers) + except httpx.HTTPError as exc: + logger.warning("abridge session create %s: %s", url, exc) + raise AbridgeError(f"create session at {url}: {exc}", status_code=502) from exc + if resp.status_code != 200: + raise AbridgeError( + f"create session at {url}: HTTP {resp.status_code}", status_code=502 + ) + try: + session_id = resp.json()[self._session_field] + except (ValueError, KeyError, TypeError) as exc: + raise AbridgeError( + f"create session at {url}: response missing {self._session_field!r}", + status_code=502, + ) from exc + self.session_id = str(session_id) + self._session_ready = True + logger.info("abridge session created at %s: %s", url, self.session_id) + + def _url_for(self, path: str) -> str: + return f"{self._target}{self._create_path}/{self.session_id}{path}" + + async def _forward(self, path: str, request: Request) -> ClientResponse: + await self._ensure_session() + return await super()._forward(path, request) + + +__all__ = ["Forward", "SessionForward"] diff --git a/plugins/abridge/tests/test_sidecar_forward.py b/plugins/abridge/tests/test_sidecar_forward.py index e882928..471c586 100644 --- a/plugins/abridge/tests/test_sidecar_forward.py +++ b/plugins/abridge/tests/test_sidecar_forward.py @@ -19,6 +19,7 @@ Forward, Proxy, Request, + SessionForward, Sidecar, SidecarError, TunnelHandle, @@ -284,3 +285,136 @@ async def test_forward_through_live_sidecar(tmp_path) -> None: assert resp.media_type == "application/json" finally: await fwd.aclose() + + +# ── SessionForward (unit, mocked httpx) ─────────────────────────────── + + +async def test_session_forward_creates_session_then_rewrites_path(monkeypatch) -> None: + fwd = SessionForward("http://gw", paths=["/v1/chat/completions"]) + calls: list = [] + + async def fake_post(url, *, json, headers): + calls.append((url, json, headers)) + if url.endswith("/sessions"): + return httpx.Response(200, content=b'{"session_id": "S9"}', headers={"content-type": "application/json"}) + return httpx.Response(200, content=b'{"ok": true}', headers={"content-type": "application/json"}) + + assert fwd._client is not None + monkeypatch.setattr(fwd._client, "post", fake_post) + resp = await fwd.abridge_routes()["/v1/chat/completions"]( + _req("/v1/chat/completions", {"model": "qwen3-4b"}) + ) + + assert isinstance(resp, ClientResponse) + assert resp.status_code == 200 and resp.body == b'{"ok": true}' + assert fwd.session_id == "S9" + # First upstream call created the session; second routed into it by path. + assert calls[0][0] == "http://gw/sessions" + assert calls[1][0] == "http://gw/sessions/S9/v1/chat/completions" + assert calls[1][1] == {"model": "qwen3-4b"} + assert calls[1][2]["x-session-id"] == "S9" + + +async def test_session_forward_creates_session_once(monkeypatch) -> None: + fwd = SessionForward("http://gw", paths=["/v1/chat/completions"]) + creates = 0 + + async def fake_post(url, *, json, headers): + nonlocal creates + if url.endswith("/sessions"): + creates += 1 + return httpx.Response(200, content=b'{"session_id": "S"}', headers={"content-type": "application/json"}) + return httpx.Response(200, content=b"{}", headers={"content-type": "application/json"}) + + assert fwd._client is not None + monkeypatch.setattr(fwd._client, "post", fake_post) + handler = fwd.abridge_routes()["/v1/chat/completions"] + await handler(_req("/v1/chat/completions", {})) + await handler(_req("/v1/chat/completions", {})) + assert creates == 1 + + +async def test_session_forward_open_precreates_session(monkeypatch) -> None: + fwd = SessionForward("http://gw", paths=["/v1/chat/completions"]) + + async def fake_post(url, *, json, headers): + return httpx.Response(200, content=b'{"session_id": "PRE"}', headers={"content-type": "application/json"}) + + assert fwd._client is not None + monkeypatch.setattr(fwd._client, "post", fake_post) + assert await fwd.open() == "PRE" + assert fwd.session_id == "PRE" + + +async def test_session_forward_create_failure_is_502(monkeypatch) -> None: + fwd = SessionForward("http://gw", paths=["/v1/chat/completions"]) + + async def fake_post(url, *, json, headers): + return httpx.Response(500, content=b"boom") + + assert fwd._client is not None + monkeypatch.setattr(fwd._client, "post", fake_post) + with pytest.raises(AbridgeError) as ei: + await fwd.open() + assert ei.value.status_code == 502 + + +async def test_session_forward_missing_id_field_is_502(monkeypatch) -> None: + fwd = SessionForward("http://gw", paths=["/v1/chat/completions"]) + + async def fake_post(url, *, json, headers): + return httpx.Response(200, content=b'{"nope": 1}', headers={"content-type": "application/json"}) + + assert fwd._client is not None + monkeypatch.setattr(fwd._client, "post", fake_post) + with pytest.raises(AbridgeError) as ei: + await fwd.open() + assert ei.value.status_code == 502 + + +# ── SessionForward (integration, real sidecar) ──────────────────────── + +SESSION_SERVER = """ +import sys, json +from http.server import BaseHTTPRequestHandler, HTTPServer + +SID = "sess-LIVE" + +class H(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200); self.end_headers(); self.wfile.write(b"ok") + + def do_POST(self): + n = int(self.headers.get("content-length", 0)) + body = self.rfile.read(n) + self.send_response(200) + self.send_header("content-type", "application/json") + self.end_headers() + if self.path == "/sessions": + self.wfile.write(json.dumps({"session_id": SID}).encode()) + else: + self.wfile.write(json.dumps({"path": self.path, "got": json.loads(body or b"{}")}).encode()) + + def log_message(self, *a): + pass + +HTTPServer(("127.0.0.1", int(sys.argv[1])), H).serve_forever() +""" + + +async def test_session_forward_through_live_sidecar(tmp_path) -> None: + script = tmp_path / "sess.py" + script.write_text(SESSION_SERVER) + async with Sidecar(command=[sys.executable, str(script), "{port}"]) as url: + fwd = SessionForward(url, paths=["/v1/chat/completions"]) + try: + resp = await fwd.abridge_routes()["/v1/chat/completions"]( + _req("/v1/chat/completions", {"model": "m"}) + ) + assert resp.status_code == 200 + assert fwd.session_id == "sess-LIVE" + assert b'"path": "/sessions/sess-LIVE/v1/chat/completions"' in resp.body + assert b'"model": "m"' in resp.body + finally: + await fwd.aclose() From 6a463d093a2c2f3a422b314f5ccb428a38431f5b Mon Sep 17 00:00:00 2001 From: FatPigeorz Date: Mon, 29 Jun 2026 16:33:20 +0000 Subject: [PATCH 06/11] abridge: address API-review findings on SessionForward MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From the multi-lens API design review of the abridge surface: - `.session_id` is now unavailable until the gateway session exists — reading it before `open()`/the first request raises instead of returning Forward's throwaway uuid, so a premature harvest fails loudly rather than hitting a fabricated id. - Fix the class docstring example: it referenced a non-existent `openai_env_for` helper; show the real harvest pattern via `.session_id` and the new `delete_session()`, and make the agent-wiring line runnable. - Document the one-instance-per-rollout reuse contract (all of an instance's calls accumulate into one gateway session; use a fresh instance per rollout). - Add `delete_session()` to reap the server-side session after harvesting (no-op if none; kept separate from `aclose()`/`Proxy.stop` so the default preserves the trajectory; resets so the next call opens a fresh session). Tests: +2 (premature `.session_id` read raises; `delete_session` reaps + resets). Full abridge suite 51 passed; pyright clean on the changed source. Co-Authored-By: Claude Opus 4.8 --- plugins/abridge/agentix/bridge/forward.py | 55 +++++++++++++++++-- plugins/abridge/tests/test_sidecar_forward.py | 29 ++++++++++ 2 files changed, 78 insertions(+), 6 deletions(-) diff --git a/plugins/abridge/agentix/bridge/forward.py b/plugins/abridge/agentix/bridge/forward.py index 3b000b1..c093dde 100644 --- a/plugins/abridge/agentix/bridge/forward.py +++ b/plugins/abridge/agentix/bridge/forward.py @@ -28,6 +28,7 @@ from __future__ import annotations import asyncio +import contextlib import logging import uuid from collections.abc import Mapping @@ -139,17 +140,25 @@ class SessionForward(Forward): (or eagerly via `open()`), remembers the assigned id, and rewrites every inbound `path` to the session-scoped URL. So the in-sandbox agent keeps calling an unmodified `/v1/chat/completions` and the whole rollout still - lands in one session. Read `.session_id` afterward to harvest the trajectory - (`GET {create_path}/{session_id}`). + lands in one session. fwd = SessionForward(gateway_url, paths=["/v1/chat/completions"]) async with Proxy(fwd).session(sandbox) as handle: - await sandbox.remote(agent, env=openai_env_for(handle)) + # the agent just POSTs /v1/chat/completions at handle.url; it never + # sees a session id — SessionForward creates + scopes it host-side. + await sandbox.remote(agent, base_url=handle.url) + sid = fwd.session_id # assigned after the run (or `await fwd.open()` up front) trajectory = (await httpx.AsyncClient().get( - f"{gateway_url}/sessions/{fwd.session_id}")).json() - + f"{gateway_url}/sessions/{sid}")).json() + await fwd.delete_session() # optional: reap the server-side session + + `.session_id` is only valid once the session exists — reading it before the + first request (or `open()`) raises, so a premature harvest fails loudly rather + than hitting a fabricated id. One instance == one gateway session: all of an + instance's calls accumulate into the same session, so use a fresh + `SessionForward` per rollout (or call `delete_session()` to reap and reset). The session is intentionally *not* deleted on `aclose()` — the trajectory is - the point, and reusing the forwarder keeps the same session. + the point and must survive the proxy teardown for harvesting. """ def __init__( @@ -163,11 +172,29 @@ def __init__( headers: Mapping[str, str] | None = None, ) -> None: super().__init__(target_url, paths=paths, timeout=timeout, headers=headers) + # Forward.__init__ stamped a throwaway uuid via the setter below; discard + # it — the gateway assigns the real id on open()/first call. Until then a + # read of `.session_id` raises instead of returning a meaningless value. + self._session_id: str | None = None self._create_path = "/" + create_path.strip("/") self._session_field = session_id_field self._session_ready = False self._session_lock = asyncio.Lock() + @property + def session_id(self) -> str: + if self._session_id is None: + raise RuntimeError( + "SessionForward.session_id is unavailable until the gateway session " + "is created — call `await fwd.open()` (or make one forwarded request) " + "before harvesting." + ) + return self._session_id + + @session_id.setter + def session_id(self, value: str) -> None: + self._session_id = value + async def open(self) -> str: """Create the sidecar session now (idempotent) and return its id. @@ -177,6 +204,22 @@ async def open(self) -> str: await self._ensure_session() return self.session_id + async def delete_session(self) -> None: + """Reap the server-side session (e.g. after the trajectory is harvested). + + No-op if no session was created. Kept separate from `aclose()` / `Proxy.stop` + so the default flow preserves the trajectory; after this the next request + opens a fresh session. Transport errors are suppressed — best-effort reap. + """ + sid = self._session_id + if sid is None: + return + url = f"{self._target}{self._create_path}/{sid}" + with contextlib.suppress(httpx.HTTPError): + await self._get_client().delete(url, headers=dict(self._headers)) + self._session_ready = False + self._session_id = None + async def _ensure_session(self) -> None: if self._session_ready: return diff --git a/plugins/abridge/tests/test_sidecar_forward.py b/plugins/abridge/tests/test_sidecar_forward.py index 471c586..57d28ba 100644 --- a/plugins/abridge/tests/test_sidecar_forward.py +++ b/plugins/abridge/tests/test_sidecar_forward.py @@ -373,6 +373,35 @@ async def fake_post(url, *, json, headers): assert ei.value.status_code == 502 +def test_session_forward_session_id_before_open_raises() -> None: + fwd = SessionForward("http://gw", paths=["/v1/chat/completions"]) + with pytest.raises(RuntimeError): + _ = fwd.session_id + + +async def test_session_forward_delete_session_reaps_and_resets(monkeypatch) -> None: + fwd = SessionForward("http://gw", paths=["/v1/chat/completions"]) + deleted: list = [] + + async def fake_post(url, *, json, headers): + return httpx.Response(200, content=b'{"session_id": "S"}', headers={"content-type": "application/json"}) + + async def fake_delete(url, *, headers): + deleted.append(url) + return httpx.Response(204) + + assert fwd._client is not None + monkeypatch.setattr(fwd._client, "post", fake_post) + monkeypatch.setattr(fwd._client, "delete", fake_delete) + + assert await fwd.open() == "S" + await fwd.delete_session() + assert deleted == ["http://gw/sessions/S"] + # after reaping, the id is gone again — a premature read raises. + with pytest.raises(RuntimeError): + _ = fwd.session_id + + # ── SessionForward (integration, real sidecar) ──────────────────────── SESSION_SERVER = """ From 7ae51b6e4ea64d7883a7baa434d858d1aae00ced Mon Sep 17 00:00:00 2001 From: FatPigeorz Date: Mon, 29 Jun 2026 17:40:21 +0000 Subject: [PATCH 07/11] =?UTF-8?q?abridge:=20API-review=20batch=202=20?= =?UTF-8?q?=E2=80=94=20consistency=20+=20boundary=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Acting on the multi-lens API design review (pre-existing / consistency findings): - README five-minute example used `upstream_model=`, which the AnthropicFromOpenAIClient constructor doesn't accept (only `model=`) — fixed so the first runnable block doesn't TypeError. - Surface the load-bearing ordering constraint on Proxy.start: registering the /abridge namespace must precede the runtime client's connect. start() now catches the low-level RuntimeError and re-raises in this surface's vocabulary ("open the proxy before any other sandbox.remote()/health()"), and the docstring states it. - Give OpenAIClient an environ(handle) mirroring the Anthropic clients, with the OpenAI SDK's required /v1 suffix baked into OPENAI_BASE_URL so callers can't drop it. All three bundled clients now wire uniformly via env=client.environ(handle). - Promote the dynamic-route seam to a typed, exported DynamicRoutes Protocol; _collect_handlers gates on isinstance(client, DynamicRoutes) instead of a duck-typed getattr, and Client documents both registration paths. - Distinguish "couldn't reach the sidecar" (now AbridgeError 503) from a relayed upstream 502, so agent retry logic can tell sidecar-down from bad-gateway. - The tunnel returns 400 for a body that is present but not a JSON object (array/string/unparseable) instead of silently coercing it to {}. - Drop @runtime_checkable from the empty Client marker Protocol (isinstance was always-true, a false validation signal); construction-time validation stays the single source of truth. Tests: +2 (non-object body -> 400; OpenAIClient.environ bakes in /v1); network error assertion updated to 503. Full abridge suite 53 passed; pyright clean. Co-Authored-By: Claude Opus 4.8 --- plugins/abridge/README.md | 2 +- plugins/abridge/agentix/bridge/__init__.py | 2 + .../agentix/bridge/clients/__init__.py | 13 +-- .../abridge/agentix/bridge/clients/openai.py | 14 ++- plugins/abridge/agentix/bridge/forward.py | 9 +- plugins/abridge/agentix/bridge/proxy.py | 85 ++++++++++++++----- plugins/abridge/tests/test_sidecar_forward.py | 36 +++++++- 7 files changed, 127 insertions(+), 34 deletions(-) diff --git a/plugins/abridge/README.md b/plugins/abridge/README.md index 795aa24..e260c91 100644 --- a/plugins/abridge/README.md +++ b/plugins/abridge/README.md @@ -49,7 +49,7 @@ from agentix.bridge.clients import AnthropicFromOpenAIClient client = AnthropicFromOpenAIClient( base_url="https://api.openai.com/v1", # OpenAI / OpenRouter / vLLM / your gateway api_key="sk-...", - upstream_model="gpt-4o", # the agent keeps sending claude-* model ids + model="gpt-4o", # the agent keeps sending claude-* model ids ) proxy = Proxy(client) diff --git a/plugins/abridge/agentix/bridge/__init__.py b/plugins/abridge/agentix/bridge/__init__.py index 330e332..ef0095c 100644 --- a/plugins/abridge/agentix/bridge/__init__.py +++ b/plugins/abridge/agentix/bridge/__init__.py @@ -39,6 +39,7 @@ AbridgeError, Client, ClientResponse, + DynamicRoutes, Handler, Proxy, Request, @@ -54,6 +55,7 @@ "Client", "ClientResponse", "Command", + "DynamicRoutes", "Forward", "Handler", "NAMESPACE", diff --git a/plugins/abridge/agentix/bridge/clients/__init__.py b/plugins/abridge/agentix/bridge/clients/__init__.py index 3869ba7..930d1c2 100644 --- a/plugins/abridge/agentix/bridge/clients/__init__.py +++ b/plugins/abridge/agentix/bridge/clients/__init__.py @@ -13,12 +13,13 @@ OpenAI-compatible (translation lives here). Same path set as `AnthropicClient`. -The two Anthropic-side classes also expose `environ(handle)` (instance -method) — the env-var bundle (`ANTHROPIC_BASE_URL` + placeholder -`ANTHROPIC_API_KEY`) an in-sandbox Anthropic SDK needs to route through -the tunnel. The OpenAI client doesn't ship an `environ`; agents that -use the OpenAI SDK typically construct the client with -`base_url=handle.url + "/v1"` directly. +All three classes expose `environ(handle)` (instance method) — the env-var +bundle an in-sandbox SDK needs to route through the tunnel, so the wiring step +is `env=client.environ(handle)` uniformly. `OpenAIClient` returns +`{OPENAI_BASE_URL: handle.url + "/v1", OPENAI_API_KEY: placeholder}` — the `/v1` +suffix the OpenAI SDK expects is baked in so a caller can't drop it; the two +Anthropic-side classes return `{ANTHROPIC_BASE_URL: handle.url, ANTHROPIC_API_KEY: +placeholder}` (no `/v1` — the Anthropic SDK appends it itself). The two `populate_*_span` helpers are exposed at this level so user- written clients can stamp the same OTel GenAI attrs the bundled diff --git a/plugins/abridge/agentix/bridge/clients/openai.py b/plugins/abridge/agentix/bridge/clients/openai.py index b8e7227..b906882 100644 --- a/plugins/abridge/agentix/bridge/clients/openai.py +++ b/plugins/abridge/agentix/bridge/clients/openai.py @@ -21,7 +21,7 @@ from agentix.utils import trace -from ..proxy import AbridgeError, ClientResponse, Request, on +from ..proxy import AbridgeError, ClientResponse, Request, TunnelHandle, on from ._genai_span import populate_openai_span if TYPE_CHECKING: @@ -114,5 +114,17 @@ async def chat(self, request: Request) -> ClientResponse: populate_openai_span(request=request.body, response=response_dict) return ClientResponse.json(response_dict) + def environ(self, handle: TunnelHandle) -> dict[str, str]: + """Env-var bundle an in-sandbox OpenAI SDK needs to route through `handle`, + mirroring `AnthropicClient.environ`. `OPENAI_BASE_URL` carries the `/v1` + suffix the OpenAI SDK expects (it appends only `/chat/completions`), so the + load-bearing suffix is baked in here instead of left to the caller. + `OPENAI_API_KEY` is a non-secret placeholder whose shape passes the SDK's + local format check — the real upstream key lives on the host (this client).""" + return { + "OPENAI_BASE_URL": handle.url + "/v1", + "OPENAI_API_KEY": PLACEHOLDER_API_KEY, + } + __all__ = ["PLACEHOLDER_API_KEY", "OpenAIClient"] diff --git a/plugins/abridge/agentix/bridge/forward.py b/plugins/abridge/agentix/bridge/forward.py index c093dde..af3a6ab 100644 --- a/plugins/abridge/agentix/bridge/forward.py +++ b/plugins/abridge/agentix/bridge/forward.py @@ -53,7 +53,10 @@ class Forward: Responses, including 4xx and 5xx responses, remain normal `ClientResponse` values so their status and body survive the tunnel. - `AbridgeError(502)` is reserved for failures to obtain an HTTP response. + `AbridgeError(503)` signals a failure to obtain any HTTP response from the + sidecar (connection refused, DNS, timeout) — a distinct code from a real + upstream 502 the sidecar relays, so the agent can tell "sidecar down" from + "sidecar returned bad gateway". """ def __init__( @@ -96,7 +99,7 @@ async def _forward(self, path: str, request: Request) -> ClientResponse: resp = await self._get_client().post(url, json=request.body, headers=headers) except httpx.HTTPError as exc: logger.warning("abridge forward %s: %s", url, exc) - raise AbridgeError(f"forward to {url}: {exc}", status_code=502) from exc + raise AbridgeError(f"forward to {url}: {exc}", status_code=503) from exc media_type = resp.headers.get("content-type", "application/json").split(";")[0].strip() return ClientResponse( @@ -232,7 +235,7 @@ async def _ensure_session(self) -> None: resp = await self._get_client().post(url, json={}, headers=headers) except httpx.HTTPError as exc: logger.warning("abridge session create %s: %s", url, exc) - raise AbridgeError(f"create session at {url}: {exc}", status_code=502) from exc + raise AbridgeError(f"create session at {url}: {exc}", status_code=503) from exc if resp.status_code != 200: raise AbridgeError( f"create session at {url}: HTTP {resp.status_code}", status_code=502 diff --git a/plugins/abridge/agentix/bridge/proxy.py b/plugins/abridge/agentix/bridge/proxy.py index 5ec5bea..8ea121f 100644 --- a/plugins/abridge/agentix/bridge/proxy.py +++ b/plugins/abridge/agentix/bridge/proxy.py @@ -139,21 +139,36 @@ def sse(cls, body: bytes, *, status_code: int = 200) -> ClientResponse: Handler = Callable[[Request], Awaitable[ClientResponse]] -@runtime_checkable class Client(Protocol): - """Marker protocol for any class with at least one `@on(path)`-decorated - method. - - There's nothing for the protocol to require structurally — `@on` is a - method-level attribute, not a class-level signature, so `isinstance` - against `Client` doesn't validate handler presence (that's - `Proxy.__init__`'s job at construction time). The name exists so - `Proxy(*clients: Client)` reads as "pass handler classes here" rather - than `*clients: object`. A client may additionally implement async + """Marker protocol for a handler object passed to `Proxy(...)`. + + Two kinds qualify, and they compose: a class with at least one + `@on(path)`-decorated method, and/or a class implementing + `DynamicRoutes.abridge_routes()` (paths chosen at construction, e.g. + `Forward`). The protocol requires nothing structurally — `@on` is a + method-level attribute, not a class-level signature — so it is deliberately + NOT `runtime_checkable`: `isinstance(x, Client)` would be true for anything + and is meaningless. Handler presence is validated by `Proxy.__init__` at + construction time. The name just makes `Proxy(*clients: Client)` read as + "pass handler objects here". A client may additionally implement async `aclose()`; `Proxy.stop()` closes such clients once per lifecycle. """ +@runtime_checkable +class DynamicRoutes(Protocol): + """A client that contributes routes chosen at construction time — paths the + class-level `@on` tag can't express, e.g. `Forward(target, paths=[...])`. + + `abridge_routes()` returns `{path: handler}`; `Proxy` merges it alongside any + `@on` handlers under the same duplicate-path rule. This is the typed, blessed + second registration seam (vs. `@on`): a handler returning dynamic routes has a + checkable contract instead of an undocumented duck-typed method. + """ + + def abridge_routes(self) -> dict[str, Handler]: ... + + @runtime_checkable class _AsyncCloseable(Protocol): def aclose(self) -> Awaitable[None]: ... @@ -230,13 +245,12 @@ def _collect_handlers(client: Client) -> dict[str, Handler]: ) handlers[path] = getattr(client, name) - # Dynamic routes: a client may expose `abridge_routes() -> dict[str, - # Handler]` for paths chosen at construction time (e.g. `Forward(target, - # paths=[...])`), which the class-level `@on` tag can't express. They - # compose with `@on` handlers under the same duplicate-path rule. - dynamic = getattr(client, "abridge_routes", None) - if callable(dynamic): - routes = dynamic() + # Dynamic routes: a client implementing `DynamicRoutes` contributes paths + # chosen at construction time (e.g. `Forward(target, paths=[...])`), which + # the class-level `@on` tag can't express. They compose with `@on` handlers + # under the same duplicate-path rule. + if isinstance(client, DynamicRoutes): + routes: object = client.abridge_routes() if not isinstance(routes, dict): raise TypeError( f"{type(client).__name__}.abridge_routes() must return a dict[str, handler]" @@ -371,6 +385,14 @@ def _make_forwarder( async def forward(request: FastAPIRequest) -> Response: body = await _read_json(request) + if body is None: + # Body was present but not a JSON object (unparseable or a non-object + # like an array/string). Fail at the boundary with a precise error + # instead of silently coercing to {} and confusing the upstream. + return JSONResponse( + {"error": {"message": "request body must be a JSON object"}}, + status_code=400, + ) try: # SIO event name IS the path; the host's `Proxy` has a @@ -398,16 +420,20 @@ async def forward(request: FastAPIRequest) -> Response: return forward -async def _read_json(request: FastAPIRequest) -> dict[str, Any]: +async def _read_json(request: FastAPIRequest) -> dict[str, Any] | None: + """Decode the body as a JSON object. `{}` for a genuinely empty body; `None` + when the body is present but not a JSON object (unparseable, or a valid + non-object like an array/string). The caller turns `None` into a 400 so the + failure surfaces at the boundary instead of as a silently-coerced `{}`.""" raw = await request.body() if not raw: return {} try: parsed = json.loads(raw) except ValueError: - return {} + return None if not isinstance(parsed, dict): - return {} + return None return parsed @@ -584,13 +610,29 @@ async def start(self, sandbox: Sandbox) -> TunnelHandle: Calling `start` again while this proxy is active returns the current handle instead of leaking another tunnel. If startup fails, clients with `aclose()` are still closed. + + ORDERING: open the proxy before any other `sandbox.remote()` / health + call. The `/abridge` namespace must be registered before the runtime + client connects, so `proxy.start` / `proxy.session` has to run first; a + prior remote call that already connected the client makes this raise. """ if self._handle is not None: return self._handle self._clients_closed = False try: - sandbox.register_namespace(self) + try: + sandbox.register_namespace(self) + except RuntimeError as exc: + # register_namespace only raises RuntimeError once the runtime + # client has connected — i.e. a remote()/health() ran first. + # Re-raise in this surface's vocabulary instead of the low-level + # "before entering the async context" message. + raise RuntimeError( + "open the abridge proxy (proxy.session/start) before any other " + "sandbox.remote()/health() call — its /abridge namespace must be " + "registered before the runtime client connects" + ) from exc handle = await sandbox.remote(_start_tunnel, paths=list(self.paths)) except BaseException: try: @@ -682,6 +724,7 @@ def url(self) -> str: "AbridgeError", "Client", "ClientResponse", + "DynamicRoutes", "Handler", "NAMESPACE", "Proxy", diff --git a/plugins/abridge/tests/test_sidecar_forward.py b/plugins/abridge/tests/test_sidecar_forward.py index 57d28ba..f296917 100644 --- a/plugins/abridge/tests/test_sidecar_forward.py +++ b/plugins/abridge/tests/test_sidecar_forward.py @@ -122,7 +122,8 @@ async def fake_post(url, *, json, headers): assert resp.body == b'{"error":"down"}' -async def test_forward_network_error_is_502(monkeypatch) -> None: +async def test_forward_network_error_is_503(monkeypatch) -> None: + """Failure to reach the sidecar is 503 — distinct from a relayed upstream 502.""" fwd = Forward("http://side.car", paths=["/v1/messages"]) async def fake_post(url, *, json, headers): @@ -131,7 +132,7 @@ async def fake_post(url, *, json, headers): monkeypatch.setattr(fwd._client, "post", fake_post) with pytest.raises(AbridgeError) as ei: await fwd.abridge_routes()["/v1/messages"](_req("/v1/messages", {})) - assert ei.value.status_code == 502 + assert ei.value.status_code == 503 async def test_forward_http_status_survives_tunnel_and_sio(monkeypatch) -> None: @@ -447,3 +448,34 @@ async def test_session_forward_through_live_sidecar(tmp_path) -> None: assert b'"model": "m"' in resp.body finally: await fwd.aclose() + + +# ── tunnel boundary + client env helpers ────────────────────────────── + + +async def test_tunnel_rejects_non_object_body(monkeypatch) -> None: + """A present-but-non-object JSON body (an array) is a 400 at the tunnel, + not a silent coercion to {}.""" + import agentix as agentix_mod + import agentix.bridge.proxy as proxy_mod + + monkeypatch.setattr(agentix_mod, "register_namespace", lambda ns: None) + monkeypatch.setattr(proxy_mod, "_namespace_singleton", None) + + handle = await proxy_mod._start_tunnel(paths=["/v1/messages"]) + try: + async with httpx.AsyncClient(base_url=handle.url, timeout=10) as client: + r = await client.post("/v1/messages", json=["not", "an", "object"]) + assert r.status_code == 400 + finally: + await proxy_mod._stop_tunnel(handle=handle) + + +def test_openai_client_environ_bakes_in_v1() -> None: + """OpenAIClient.environ mirrors the Anthropic ones and bakes in the /v1 suffix.""" + from agentix.bridge.clients import OpenAIClient + + c = OpenAIClient(base_url="https://up.stream/v1", api_key="real-key", model="gpt-4o") + env = c.environ(TunnelHandle(url="http://127.0.0.1:9", port=9)) + assert env["OPENAI_BASE_URL"] == "http://127.0.0.1:9/v1" + assert env["OPENAI_API_KEY"].startswith("sk-") From 6fd8c67b90f05b099f8982c1a490d4304f064ad3 Mon Sep 17 00:00:00 2001 From: FatPigeorz Date: Tue, 30 Jun 2026 03:21:43 +0000 Subject: [PATCH 08/11] =?UTF-8?q?abridge:=20compose=20Convert=20=E2=88=98?= =?UTF-8?q?=20Session=20=E2=80=94=20Anthropic=20agent=20=E2=86=92=20TITO?= =?UTF-8?q?=20(Claude=20Code=20training)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make protocol-conversion and session-routing two ORTHOGONAL, composable capabilities instead of one fused class: - Forward.handler(path) exposes a Forward/SessionForward as a plain downstream Handler — the composition seam (an OpenAI body in, a ClientResponse out). - AnthropicToOpenAI(downstream, model=...) is the Convert capability and nothing else: it translates Anthropic /v1/messages ⇄ OpenAI chat-completions and hands the OpenAI body to `downstream`. Transport-blind. Compose: tito = SessionForward(gateway_url, paths=["/v1/chat/completions"]) # protocol-blind proxy = Proxy(AnthropicToOpenAI(tito.handler(), model="qwen3-4b")) SessionForward never sees Anthropic; AnthropicToOpenAI never sees a session. Fidelity for strict session backends (the hard part): a session recorder like the TITO gateway matches each turn's resent assistant message byte-for-byte (role, content, reasoning_content, tool_calls), but the Anthropic round-trip is lossy — it drops reasoning_content (Qwen3 emits "\n\n" even with thinking off) and the per-tool-call `index` sglang stamps. So AnthropicToOpenAI REMEMBERS the exact assistant message the downstream returned, keyed by the tool-call ids that survive the round-trip, and replays it verbatim on later turns — making history identical from the backend's POV and immune to whatever private fields it keeps. Works for real Claude Code (tool_use ids are stable) and keeps the converter's own concern (translation fidelity), not the session layer's. No OpenAI SDK dependency (raw httpx via the downstream Forward). Verified on GPU (B30Z): a Claude-Code-shaped Anthropic agent with reasoning ON, driving Qwen3-4B through AnthropicToOpenAI ∘ SessionForward → TITO gateway → sglang: 3 real tool calls (982), 2 records, 1432-token trajectory, 0 hard mismatch, reward 1.0. Tests: +4 (handler() seam; convert over a fake downstream; convert ∘ SessionForward end-to-end; verbatim assistant replay). Full abridge suite 57 passed; pyright clean. Co-Authored-By: Claude Opus 4.8 --- .../agentix/bridge/clients/__init__.py | 2 + .../bridge/clients/anthropic_to_openai.py | 138 ++++++++++++++++++ plugins/abridge/agentix/bridge/forward.py | 17 +++ plugins/abridge/tests/test_sidecar_forward.py | 130 +++++++++++++++++ 4 files changed, 287 insertions(+) create mode 100644 plugins/abridge/agentix/bridge/clients/anthropic_to_openai.py diff --git a/plugins/abridge/agentix/bridge/clients/__init__.py b/plugins/abridge/agentix/bridge/clients/__init__.py index 930d1c2..8086215 100644 --- a/plugins/abridge/agentix/bridge/clients/__init__.py +++ b/plugins/abridge/agentix/bridge/clients/__init__.py @@ -32,6 +32,7 @@ from .anthropic import PLACEHOLDER_API_KEY as ANTHROPIC_PLACEHOLDER_API_KEY from .anthropic import AnthropicClient from .anthropic_from_openai import AnthropicFromOpenAIClient +from .anthropic_to_openai import AnthropicToOpenAI from .openai import PLACEHOLDER_API_KEY as OPENAI_PLACEHOLDER_API_KEY from .openai import OpenAIClient @@ -39,6 +40,7 @@ "ANTHROPIC_PLACEHOLDER_API_KEY", "AnthropicClient", "AnthropicFromOpenAIClient", + "AnthropicToOpenAI", "OPENAI_PLACEHOLDER_API_KEY", "OpenAIClient", "populate_anthropic_span", diff --git a/plugins/abridge/agentix/bridge/clients/anthropic_to_openai.py b/plugins/abridge/agentix/bridge/clients/anthropic_to_openai.py new file mode 100644 index 0000000..3d7a81c --- /dev/null +++ b/plugins/abridge/agentix/bridge/clients/anthropic_to_openai.py @@ -0,0 +1,138 @@ +"""`AnthropicToOpenAI` — the Convert capability: an Anthropic-Messages agent over +ANY OpenAI-compatible downstream. + +This is one orthogonal capability — protocol translation — and nothing else. It +translates the agent's Anthropic `/v1/messages` to an OpenAI chat-completions body, +hands that body to a `downstream` (any abridge `Handler`), then translates the +OpenAI completion back to Anthropic. It is deliberately transport-blind: the +downstream owns the HTTP, the session, and any recording. + +Compose it with a transport/routing capability: + + # plain OpenAI gateway (no session) + Proxy(AnthropicToOpenAI(Forward(base_url, paths=["/v1/chat/completions"]).handler())) + + # session-scoped recorder (the TITO gateway) — session stays transparent + tito = SessionForward(gateway_url, paths=["/v1/chat/completions"]) + proxy = Proxy(AnthropicToOpenAI(tito.handler(), model="qwen3-4b")) + ... # the agent only ever speaks Anthropic + harvest(tito.session_id) # the session lives in the SessionForward, not here + +`AnthropicToOpenAI` knows nothing about sessions; `SessionForward` knows nothing +about Anthropic. They compose because the seam between them is just abridge's +`Handler` (an OpenAI chat body in, a `ClientResponse` out). No OpenAI SDK +dependency — the upstream hop is whatever `Handler` you pass. +""" + +from __future__ import annotations + +import json +import logging +from typing import Any + +from agentix.utils import trace + +from ..proxy import ClientResponse, Handler, Request, TunnelHandle, on +from ._anthropic_transforms import ( + anthropic_messages_to_openai, + anthropic_sse, + count_anthropic_tokens, + openai_to_anthropic_messages, +) +from ._genai_span import populate_anthropic_span +from .anthropic import PLACEHOLDER_API_KEY + +logger = logging.getLogger(__name__) + + +class AnthropicToOpenAI: + """Anthropic Messages agent → any OpenAI-compatible `downstream` Handler. + + `downstream` is the transport/routing capability this converter sits on top of + — a `Forward(...).handler()` for a direct OpenAI gateway, or a + `SessionForward(...).handler()` for a session-scoped recorder like the TITO + gateway. `model`, when set, overrides the agent's model id in the OpenAI body. + `count_tokens` is answered locally (character estimate, no downstream call). + + This layer does not own the downstream's lifecycle: hold the forwarder yourself + to read `.session_id` / call `delete_session()` / `aclose()`. + """ + + def __init__(self, downstream: Handler, *, model: str | None = None) -> None: + self._downstream = downstream + self._model = model + # Per-tool-call-id memory of the EXACT assistant message the downstream + # returned. The Anthropic round-trip is lossy (it drops reasoning_content + # and the per-tool-call `index`), so a reconstructed assistant won't match + # what a session-recording backend (TITO) stored byte-for-byte. On the next + # turn we replay the remembered original verbatim, keyed by the tool-call + # ids that survive the round-trip — making the history identical from the + # backend's POV and immune to whatever private fields the backend keeps. + self._assistant_by_ids: dict[tuple[str, ...], dict[str, Any]] = {} + + @on("/v1/messages") + async def messages(self, request: Request) -> ClientResponse: + openai_body = anthropic_messages_to_openai(request.body, upstream_model=self._model) + # The downstream produces a non-streaming OpenAI completion (a TITO + # recorder needs output_token_logprobs); we re-render SSE locally below if + # the agent asked for streaming. + openai_body["stream"] = False + self._replay_remembered_assistants(openai_body) + with trace.span(f"anthropic messages {request.body.get('model') or ''}"): + resp = await self._downstream(Request(path="/v1/messages", body=openai_body)) + if resp.status_code != 200: + # Pass the downstream's error (4xx/5xx) straight through with its + # status; the agent sees a non-200, not a malformed body. + return resp + openai_resp = json.loads(resp.body) + self._remember_assistant(openai_resp) + anthropic_resp = openai_to_anthropic_messages( + openai_resp, response_model=str(request.body.get("model") or "") + ) + populate_anthropic_span(request=request.body, response=anthropic_resp) + if request.body.get("stream"): + return ClientResponse.sse(anthropic_sse(anthropic_resp)) + return ClientResponse.json(anthropic_resp) + + @staticmethod + def _ids(message: dict[str, Any]) -> tuple[str, ...]: + return tuple( + tc["id"] + for tc in (message.get("tool_calls") or []) + if isinstance(tc, dict) and tc.get("id") + ) + + def _remember_assistant(self, openai_resp: dict[str, Any]) -> None: + choice = (openai_resp.get("choices") or [{}])[0] + message = choice.get("message") or {} + ids = self._ids(message) + if ids: + self._assistant_by_ids[ids] = message + + def _replay_remembered_assistants(self, openai_body: dict[str, Any]) -> None: + messages = openai_body.get("messages") + if not isinstance(messages, list): + return + for i, msg in enumerate(messages): + if not isinstance(msg, dict) or msg.get("role") != "assistant": + continue + remembered = self._assistant_by_ids.get(self._ids(msg)) + if remembered is not None: + messages[i] = remembered + + @on("/v1/messages/count_tokens") + async def count_tokens(self, request: Request) -> ClientResponse: + return ClientResponse.json( + {"input_tokens": count_anthropic_tokens(request.body).input_tokens} + ) + + def environ(self, handle: TunnelHandle) -> dict[str, str]: + """Anthropic env-var bundle — from the agent's POV the wire is Anthropic, + regardless of the OpenAI downstream.""" + return { + "ANTHROPIC_BASE_URL": handle.url, + "ANTHROPIC_API_KEY": PLACEHOLDER_API_KEY, + } + + +__all__ = ["AnthropicToOpenAI"] diff --git a/plugins/abridge/agentix/bridge/forward.py b/plugins/abridge/agentix/bridge/forward.py index af3a6ab..ae5cb16 100644 --- a/plugins/abridge/agentix/bridge/forward.py +++ b/plugins/abridge/agentix/bridge/forward.py @@ -86,6 +86,23 @@ async def handler(request: Request) -> ClientResponse: return handler + def handler(self, path: str | None = None) -> Handler: + """The bound forwarder for `path` (or the sole path if there's exactly + one) as a plain `Handler`. This is the composition seam: it lets a + converter/wrapper use this Forward as a transparent downstream — + `AnthropicToOpenAI(SessionForward(gw).handler())` — without knowing it's a + Forward, a SessionForward, or anything else.""" + routes = self.abridge_routes() + if path is None: + if len(routes) != 1: + raise ValueError( + f"handler() needs an explicit path; forwarder has {sorted(routes)}" + ) + (path,) = routes + if path not in routes: + raise ValueError(f"no route for {path!r}; have {sorted(routes)}") + return routes[path] + async def _forward(self, path: str, request: Request) -> ClientResponse: record_id = uuid.uuid4().hex headers = { diff --git a/plugins/abridge/tests/test_sidecar_forward.py b/plugins/abridge/tests/test_sidecar_forward.py index f296917..fa489ed 100644 --- a/plugins/abridge/tests/test_sidecar_forward.py +++ b/plugins/abridge/tests/test_sidecar_forward.py @@ -479,3 +479,133 @@ def test_openai_client_environ_bakes_in_v1() -> None: env = c.environ(TunnelHandle(url="http://127.0.0.1:9", port=9)) assert env["OPENAI_BASE_URL"] == "http://127.0.0.1:9/v1" assert env["OPENAI_API_KEY"].startswith("sk-") + + +# ── composition: Convert (AnthropicToOpenAI) ∘ transport (Forward/SessionForward) ── + +_OPENAI_COMPLETION = ( + b'{"id":"c1","object":"chat.completion","model":"qwen3-4b",' + b'"choices":[{"index":0,"finish_reason":"stop",' + b'"message":{"role":"assistant","content":"hi there"}}],' + b'"usage":{"prompt_tokens":3,"completion_tokens":2,"total_tokens":5}}' +) + + +def test_forward_handler_accessor() -> None: + fwd = Forward("http://x", paths=["/v1/chat/completions"]) + assert callable(fwd.handler()) # sole path, no arg needed + assert callable(fwd.handler("/v1/chat/completions")) + with pytest.raises(ValueError): + Forward("http://x", paths=["/a", "/b"]).handler() # ambiguous → must name a path + + +async def test_anthropic_to_openai_translates_over_any_downstream() -> None: + """The Convert layer is transport-blind: it translates Anthropic→OpenAI, calls + the downstream Handler, and translates the OpenAI completion back to Anthropic.""" + from agentix.bridge.clients import AnthropicToOpenAI + + captured: dict = {} + + async def fake_downstream(request: Request) -> ClientResponse: + captured["body"] = request.body + return ClientResponse(body=_OPENAI_COMPLETION, media_type="application/json") + + conv = AnthropicToOpenAI(fake_downstream, model="qwen3-4b") + resp = await conv.messages( + _req("/v1/messages", { + "model": "claude-3-5-sonnet", + "max_tokens": 64, + "messages": [{"role": "user", "content": "say hi"}], + }) + ) + assert resp.status_code == 200 + assert b"hi there" in resp.body + assert b'"role": "assistant"' in resp.body # Anthropic response shape + # the downstream saw an OpenAI-shaped, non-streaming body with the model override + assert captured["body"]["model"] == "qwen3-4b" + assert captured["body"]["stream"] is False + assert captured["body"]["messages"][0]["role"] == "user" + + +async def test_anthropic_to_openai_composes_with_session_forward(monkeypatch) -> None: + """End-to-end composition: Anthropic agent → AnthropicToOpenAI → SessionForward + creates the session and rewrites the path → OpenAI completion → back to Anthropic. + The converter never touches a session; the SessionForward never sees Anthropic.""" + from agentix.bridge.clients import AnthropicToOpenAI + + tito = SessionForward("http://gw", paths=["/v1/chat/completions"]) + calls: list = [] + + async def fake_post(url, *, json, headers): + calls.append(url) + if url.endswith("/sessions"): + return httpx.Response(200, content=b'{"session_id": "S1"}', headers={"content-type": "application/json"}) + return httpx.Response(200, content=_OPENAI_COMPLETION, headers={"content-type": "application/json"}) + + assert tito._client is not None + monkeypatch.setattr(tito._client, "post", fake_post) + + conv = AnthropicToOpenAI(tito.handler(), model="qwen3-4b") + resp = await conv.messages( + _req("/v1/messages", { + "model": "claude", + "max_tokens": 64, + "messages": [{"role": "user", "content": "say hi"}], + }) + ) + assert resp.status_code == 200 + assert b"hi there" in resp.body + assert tito.session_id == "S1" + assert calls[0] == "http://gw/sessions" + assert calls[1] == "http://gw/sessions/S1/v1/chat/completions" + await tito.aclose() + + +async def test_anthropic_to_openai_replays_remembered_assistant() -> None: + """The lossy Anthropic round-trip drops reasoning_content + tool-call `index`, so + a reconstructed assistant won't byte-match what a session backend stored. The + converter remembers the exact downstream assistant and replays it verbatim on the + next turn, keyed by the surviving tool-call ids.""" + from agentix.bridge.clients import AnthropicToOpenAI + + stored_assistant = { + "role": "assistant", + "content": "", + "reasoning_content": "\n\n", + "tool_calls": [{ + "id": "call_X", "index": 0, "type": "function", + "function": {"name": "python", "arguments": "{\"expression\": \"1+1\"}"}, + }], + } + sent: list = [] + + async def fake_downstream(request: Request) -> ClientResponse: + sent.append(request.body.get("messages")) + return ClientResponse.json({ + "choices": [{"index": 0, "finish_reason": "tool_calls", "message": stored_assistant}], + "model": "m", "usage": {}, + }) + + conv = AnthropicToOpenAI(fake_downstream, model="m") + await conv.messages(_req("/v1/messages", { + "model": "c", "max_tokens": 64, "messages": [{"role": "user", "content": "hi"}], + })) + # turn 1: the agent resends the round-tripped assistant (no index, no reasoning_content) + await conv.messages(_req("/v1/messages", { + "model": "c", "max_tokens": 64, + "messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "call_X", "name": "python", "input": {"expression": "1+1"}}]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "call_X", "content": "2"}]}, + ], + })) + + asst = [m for m in sent[1] if m.get("role") == "assistant"] + assert len(asst) == 1 + # value-equal to the stored assistant (incl. reasoning_content + index), not the + # lossy reconstruction — so the backend's byte-match would pass. + assert asst[0] == stored_assistant + assert asst[0]["reasoning_content"] == "\n\n" + assert asst[0]["tool_calls"][0]["index"] == 0 From 4cbf197facbf0481cdf5cccaca595f9a475d346b Mon Sep 17 00:00:00 2001 From: FatPigeorz Date: Tue, 30 Jun 2026 13:38:50 +0000 Subject: [PATCH 09/11] tito: native TITO engine + plugin-ize as agentix.tito (drop vendored Miles) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reimplement the token-in-token-out alignment engine natively under agentix/tito/engine and move the gateway out of sidecars/tito into the plugins/ workspace as the agentix-tito plugin (import agentix.tito). Engine (agentix/tito/engine), all model-agnostic except a small per-model delta: - pretokenize: TITOTokenizer + Qwen3TITOTokenizer — the only Qwen3 difference is a fixed jinja template plus re-inserting the `\n` after `<|im_end|>` the model omits when it stops - compare: special-token-segment mismatch audit - trajectory: LinearTrajectory + SessionRegistry (single-step rollback, from-scratch-vs-accumulated mismatch report) - session_app: FastAPI session routes (3-phase pretokenize -> proxy -> checkpoint) - messages / render / processing / errors No vendored training-framework code and no sglang dependency: the engine tokenizes with transformers + tokenizers + jinja2, isolated to this plugin so agentix core / abridge never pull them. Verified CPU-only with a tiny in-memory tokenizer — pyright clean, 38 tests pass, no model download or GPU. Also: - wire plugins/tito into the root pyright include + extraPaths - drop the inherited standalone-repo scaffolding (docs/, plan.md, vendored-Miles audit, upstream pin, standalone CI, zh README) and rewrite the README + the sidecars/ index for the native plugin - rename lingering miles-isms: config.as_session_args, router_timeout Co-Authored-By: Claude Opus 4.8 --- {sidecars => plugins}/tito/.gitignore | 0 {sidecars => plugins}/tito/LICENSE | 0 plugins/tito/README.md | 110 ++ plugins/tito/agentix/tito/__init__.py | 21 + plugins/tito/agentix/tito/cli.py | 95 ++ .../tito/agentix/tito}/config.py | 29 +- .../tito/agentix/tito}/discovery.py | 0 plugins/tito/agentix/tito/engine/__init__.py | 7 + plugins/tito/agentix/tito/engine/compare.py | 189 +++ plugins/tito/agentix/tito/engine/errors.py | 33 + plugins/tito/agentix/tito/engine/messages.py | 72 + .../tito/agentix/tito/engine/pretokenize.py | 244 ++++ .../tito/agentix/tito/engine/processing.py | 16 + plugins/tito/agentix/tito/engine/render.py | 94 ++ .../tito/agentix/tito/engine/session_app.py | 180 +++ .../tito/engine}/templates/qwen3_fixed.jinja | 0 .../tito/agentix/tito/engine/trajectory.py | 222 +++ .../tito/agentix/tito}/gateway.py | 14 +- .../tito/agentix/tito}/pool.py | 0 plugins/tito/agentix/tito/server.py | 113 ++ plugins/tito/agentix/tito/tokenizer.py | 29 + plugins/tito/pyproject.toml | 71 + plugins/tito/tests/package/test_cli.py | 41 + .../tests/package/test_config_discovery.py | 11 +- plugins/tito/tests/package/test_engine.py | 141 ++ .../tito/tests/package/test_import_surface.py | 21 +- {sidecars => plugins}/tito/tests/test_pool.py | 2 +- .../tito/tests/test_pool_routing.py | 55 +- pyproject.toml | 3 + sidecars/README.md | 19 +- .../tito/.github/workflows/python-package.yml | 61 - sidecars/tito/README.md | 83 -- sidecars/tito/README.zh-CN.md | 81 -- sidecars/tito/docs/api.md | 50 - sidecars/tito/docs/api.zh-CN.md | 49 - sidecars/tito/docs/cli.md | 50 - sidecars/tito/docs/cli.zh-CN.md | 50 - sidecars/tito/docs/concepts.md | 55 - sidecars/tito/docs/concepts.zh-CN.md | 53 - sidecars/tito/docs/development.md | 66 - sidecars/tito/docs/development.zh-CN.md | 64 - sidecars/tito/docs/guide.md | 13 - sidecars/tito/docs/guide.zh-CN.md | 13 - sidecars/tito/docs/index.md | 22 - sidecars/tito/docs/index.zh-CN.md | 20 - sidecars/tito/docs/quickstart.md | 70 - sidecars/tito/docs/quickstart.zh-CN.md | 70 - sidecars/tito/docs/verification.md | 46 - sidecars/tito/docs/verification.zh-CN.md | 45 - sidecars/tito/miles/__init__.py | 5 - sidecars/tito/miles/_upstream_loader.py | 75 -- sidecars/tito/miles/rollout/__init__.py | 5 - sidecars/tito/miles/rollout/base_types.py | 9 - .../miles/rollout/generate_hub/__init__.py | 5 - .../rollout/generate_hub/agentic_tool_call.py | 9 - .../tito/miles/rollout/session/__init__.py | 1 - .../rollout/session/linear_trajectory.py | 3 - .../miles/rollout/session/session_errors.py | 3 - .../miles/rollout/session/session_server.py | 3 - .../miles/rollout/session/session_types.py | 3 - .../tito/miles/rollout/session/sessions.py | 3 - sidecars/tito/miles/utils/__init__.py | 5 - .../utils/chat_template_utils/__init__.py | 3 - .../utils/chat_template_utils/deepseek_v32.py | 3 - .../utils/chat_template_utils/deepseek_v4.py | 3 - .../utils/chat_template_utils/template.py | 3 - .../chat_template_utils/tito_tokenizer.py | 4 - .../token_seq_comparator.py | 3 - .../miles/utils/external_utils/__init__.py | 5 - .../utils/external_utils/command_utils.py | 9 - sidecars/tito/miles/utils/hf_config.py | 3 - sidecars/tito/miles/utils/http_utils.py | 3 - sidecars/tito/miles/utils/processing_utils.py | 3 - .../tito/miles/utils/test_utils/__init__.py | 1 - .../utils/test_utils/chat_template_verify.py | 3 - .../utils/test_utils/mock_sglang_server.py | 3 - .../utils/test_utils/mock_trajectories.py | 3 - .../utils/test_utils/session_verify_agent.py | 3 - .../utils/test_utils/session_verify_runner.py | 3 - .../utils/test_utils/uvicorn_thread_server.py | 3 - sidecars/tito/plan.md | 228 ---- sidecars/tito/pyproject.toml | 91 -- .../scripts/prepare_test_tokenizer_cache.py | 83 -- sidecars/tito/tests/ci/__init__.py | 1 - sidecars/tito/tests/ci/ci_register.py | 12 - sidecars/tito/tests/fast/__init__.py | 1 - sidecars/tito/tests/fast/router/__init__.py | 1 - .../router/session_pretokenized_test_utils.py | 202 --- .../qwen3_thinking_2507_and_next_fixed.jinja | 82 -- sidecars/tito/tests/package/test_cli.py | 212 --- .../tests/package/test_gateway_integration.py | 82 -- .../package/test_session_verifier_plumbing.py | 74 - .../tests/package/test_upstream_delegation.py | 121 -- .../tito/tests/package/test_vendored_miles.py | 31 - .../router/test_session_pretokenized_e2e.py | 174 --- .../router/test_session_race_conditions.py | 422 ------ .../upstream/fast/router/test_sessions.py | 140 -- .../test_pretokenized_via_tito.py | 161 --- .../test_tito_tokenizer.py | 582 -------- .../test_utils/test_session_verify_runner.py | 85 -- .../tito/tito_gateway/VENDORED_MILES_AUDIT.md | 29 - sidecars/tito/tito_gateway/__init__.py | 16 - sidecars/tito/tito_gateway/cli.py | 241 ---- sidecars/tito/tito_gateway/server.py | 100 -- sidecars/tito/tito_gateway/tokenizer.py | 30 - sidecars/tito/tito_gateway/upstream.json | 7 - sidecars/tito/tito_gateway/vendor/__init__.py | 0 .../vendor/miles_compat/__init__.py | 0 .../vendor/miles_compat/rollout/__init__.py | 0 .../vendor/miles_compat/rollout/base_types.py | 29 - .../rollout/generate_hub/__init__.py | 1 - .../rollout/generate_hub/agentic_tool_call.py | 29 - .../miles_compat/rollout/session/__init__.py | 0 .../rollout/session/linear_trajectory.py | 285 ---- .../rollout/session/session_errors.py | 51 - .../rollout/session/session_server.py | 111 -- .../rollout/session/session_types.py | 16 - .../miles_compat/rollout/session/sessions.py | 251 ---- .../vendor/miles_compat/utils/__init__.py | 0 .../utils/chat_template_utils/__init__.py | 39 - .../utils/chat_template_utils/deepseek_v32.py | 97 -- .../utils/chat_template_utils/deepseek_v4.py | 100 -- .../utils/chat_template_utils/template.py | 256 ---- .../templates/kimi_k25_fixed.jinja | 111 -- .../templates/minimax_m25_fixed.jinja | 159 --- .../templates/minimax_m27_fixed.jinja | 159 --- .../templates/qwen3.5_fixed.jinja | 151 --- .../qwen3_thinking_2507_and_next_fixed.jinja | 82 -- .../chat_template_utils/tito_tokenizer.py | 1014 -------------- .../token_seq_comparator.py | 289 ---- .../utils/external_utils/__init__.py | 1 - .../utils/external_utils/command_utils.py | 33 - .../vendor/miles_compat/utils/hf_config.py | 108 -- .../vendor/miles_compat/utils/http_utils.py | 315 ----- .../miles_compat/utils/processing_utils.py | 175 --- .../miles_compat/utils/test_utils/__init__.py | 0 .../utils/test_utils/chat_template_verify.py | 602 --------- .../utils/test_utils/mock_sglang_server.py | 270 ---- .../utils/test_utils/mock_trajectories.py | 1198 ----------------- .../utils/test_utils/session_verify_agent.py | 460 ------- .../utils/test_utils/session_verify_runner.py | 332 ----- .../utils/test_utils/uvicorn_thread_server.py | 49 - .../tito/tito_gateway/verify_chat_template.py | 125 -- .../verify_session_tito_tokenizer.py | 33 - 144 files changed, 1733 insertions(+), 11389 deletions(-) rename {sidecars => plugins}/tito/.gitignore (100%) rename {sidecars => plugins}/tito/LICENSE (100%) create mode 100644 plugins/tito/README.md create mode 100644 plugins/tito/agentix/tito/__init__.py create mode 100644 plugins/tito/agentix/tito/cli.py rename {sidecars/tito/tito_gateway => plugins/tito/agentix/tito}/config.py (74%) rename {sidecars/tito/tito_gateway => plugins/tito/agentix/tito}/discovery.py (100%) create mode 100644 plugins/tito/agentix/tito/engine/__init__.py create mode 100644 plugins/tito/agentix/tito/engine/compare.py create mode 100644 plugins/tito/agentix/tito/engine/errors.py create mode 100644 plugins/tito/agentix/tito/engine/messages.py create mode 100644 plugins/tito/agentix/tito/engine/pretokenize.py create mode 100644 plugins/tito/agentix/tito/engine/processing.py create mode 100644 plugins/tito/agentix/tito/engine/render.py create mode 100644 plugins/tito/agentix/tito/engine/session_app.py rename {sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils => plugins/tito/agentix/tito/engine}/templates/qwen3_fixed.jinja (100%) create mode 100644 plugins/tito/agentix/tito/engine/trajectory.py rename {sidecars/tito/tito_gateway => plugins/tito/agentix/tito}/gateway.py (80%) rename {sidecars/tito/tito_gateway => plugins/tito/agentix/tito}/pool.py (100%) create mode 100644 plugins/tito/agentix/tito/server.py create mode 100644 plugins/tito/agentix/tito/tokenizer.py create mode 100644 plugins/tito/pyproject.toml create mode 100644 plugins/tito/tests/package/test_cli.py rename {sidecars => plugins}/tito/tests/package/test_config_discovery.py (93%) create mode 100644 plugins/tito/tests/package/test_engine.py rename {sidecars => plugins}/tito/tests/package/test_import_surface.py (60%) rename {sidecars => plugins}/tito/tests/test_pool.py (97%) rename {sidecars => plugins}/tito/tests/test_pool_routing.py (51%) delete mode 100644 sidecars/tito/.github/workflows/python-package.yml delete mode 100644 sidecars/tito/README.md delete mode 100644 sidecars/tito/README.zh-CN.md delete mode 100644 sidecars/tito/docs/api.md delete mode 100644 sidecars/tito/docs/api.zh-CN.md delete mode 100644 sidecars/tito/docs/cli.md delete mode 100644 sidecars/tito/docs/cli.zh-CN.md delete mode 100644 sidecars/tito/docs/concepts.md delete mode 100644 sidecars/tito/docs/concepts.zh-CN.md delete mode 100644 sidecars/tito/docs/development.md delete mode 100644 sidecars/tito/docs/development.zh-CN.md delete mode 100644 sidecars/tito/docs/guide.md delete mode 100644 sidecars/tito/docs/guide.zh-CN.md delete mode 100644 sidecars/tito/docs/index.md delete mode 100644 sidecars/tito/docs/index.zh-CN.md delete mode 100644 sidecars/tito/docs/quickstart.md delete mode 100644 sidecars/tito/docs/quickstart.zh-CN.md delete mode 100644 sidecars/tito/docs/verification.md delete mode 100644 sidecars/tito/docs/verification.zh-CN.md delete mode 100644 sidecars/tito/miles/__init__.py delete mode 100644 sidecars/tito/miles/_upstream_loader.py delete mode 100644 sidecars/tito/miles/rollout/__init__.py delete mode 100644 sidecars/tito/miles/rollout/base_types.py delete mode 100644 sidecars/tito/miles/rollout/generate_hub/__init__.py delete mode 100644 sidecars/tito/miles/rollout/generate_hub/agentic_tool_call.py delete mode 100644 sidecars/tito/miles/rollout/session/__init__.py delete mode 100644 sidecars/tito/miles/rollout/session/linear_trajectory.py delete mode 100644 sidecars/tito/miles/rollout/session/session_errors.py delete mode 100644 sidecars/tito/miles/rollout/session/session_server.py delete mode 100644 sidecars/tito/miles/rollout/session/session_types.py delete mode 100644 sidecars/tito/miles/rollout/session/sessions.py delete mode 100644 sidecars/tito/miles/utils/__init__.py delete mode 100644 sidecars/tito/miles/utils/chat_template_utils/__init__.py delete mode 100644 sidecars/tito/miles/utils/chat_template_utils/deepseek_v32.py delete mode 100644 sidecars/tito/miles/utils/chat_template_utils/deepseek_v4.py delete mode 100644 sidecars/tito/miles/utils/chat_template_utils/template.py delete mode 100644 sidecars/tito/miles/utils/chat_template_utils/tito_tokenizer.py delete mode 100644 sidecars/tito/miles/utils/chat_template_utils/token_seq_comparator.py delete mode 100644 sidecars/tito/miles/utils/external_utils/__init__.py delete mode 100644 sidecars/tito/miles/utils/external_utils/command_utils.py delete mode 100644 sidecars/tito/miles/utils/hf_config.py delete mode 100644 sidecars/tito/miles/utils/http_utils.py delete mode 100644 sidecars/tito/miles/utils/processing_utils.py delete mode 100644 sidecars/tito/miles/utils/test_utils/__init__.py delete mode 100644 sidecars/tito/miles/utils/test_utils/chat_template_verify.py delete mode 100644 sidecars/tito/miles/utils/test_utils/mock_sglang_server.py delete mode 100644 sidecars/tito/miles/utils/test_utils/mock_trajectories.py delete mode 100644 sidecars/tito/miles/utils/test_utils/session_verify_agent.py delete mode 100644 sidecars/tito/miles/utils/test_utils/session_verify_runner.py delete mode 100644 sidecars/tito/miles/utils/test_utils/uvicorn_thread_server.py delete mode 100644 sidecars/tito/plan.md delete mode 100644 sidecars/tito/pyproject.toml delete mode 100644 sidecars/tito/scripts/prepare_test_tokenizer_cache.py delete mode 100644 sidecars/tito/tests/ci/__init__.py delete mode 100644 sidecars/tito/tests/ci/ci_register.py delete mode 100644 sidecars/tito/tests/fast/__init__.py delete mode 100644 sidecars/tito/tests/fast/router/__init__.py delete mode 100644 sidecars/tito/tests/fast/router/session_pretokenized_test_utils.py delete mode 100644 sidecars/tito/tests/miles/utils/chat_template_utils/templates/qwen3_thinking_2507_and_next_fixed.jinja delete mode 100644 sidecars/tito/tests/package/test_cli.py delete mode 100644 sidecars/tito/tests/package/test_gateway_integration.py delete mode 100644 sidecars/tito/tests/package/test_session_verifier_plumbing.py delete mode 100644 sidecars/tito/tests/package/test_upstream_delegation.py delete mode 100644 sidecars/tito/tests/package/test_vendored_miles.py delete mode 100644 sidecars/tito/tests/upstream/fast/router/test_session_pretokenized_e2e.py delete mode 100644 sidecars/tito/tests/upstream/fast/router/test_session_race_conditions.py delete mode 100644 sidecars/tito/tests/upstream/fast/router/test_sessions.py delete mode 100644 sidecars/tito/tests/upstream/fast/utils/chat_template_utils/test_pretokenized_via_tito.py delete mode 100644 sidecars/tito/tests/upstream/fast/utils/chat_template_utils/test_tito_tokenizer.py delete mode 100644 sidecars/tito/tests/upstream/fast/utils/test_utils/test_session_verify_runner.py delete mode 100644 sidecars/tito/tito_gateway/VENDORED_MILES_AUDIT.md delete mode 100644 sidecars/tito/tito_gateway/__init__.py delete mode 100644 sidecars/tito/tito_gateway/cli.py delete mode 100644 sidecars/tito/tito_gateway/server.py delete mode 100644 sidecars/tito/tito_gateway/tokenizer.py delete mode 100644 sidecars/tito/tito_gateway/upstream.json delete mode 100644 sidecars/tito/tito_gateway/vendor/__init__.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/__init__.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/rollout/__init__.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/rollout/base_types.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/rollout/generate_hub/__init__.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/rollout/generate_hub/agentic_tool_call.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/__init__.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/linear_trajectory.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/session_errors.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/session_server.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/session_types.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/sessions.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/__init__.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/__init__.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/deepseek_v32.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/deepseek_v4.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/template.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/kimi_k25_fixed.jinja delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/minimax_m25_fixed.jinja delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/minimax_m27_fixed.jinja delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/qwen3.5_fixed.jinja delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/qwen3_thinking_2507_and_next_fixed.jinja delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/tito_tokenizer.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/token_seq_comparator.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/external_utils/__init__.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/external_utils/command_utils.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/hf_config.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/http_utils.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/processing_utils.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/__init__.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/chat_template_verify.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/mock_sglang_server.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/mock_trajectories.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/session_verify_agent.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/session_verify_runner.py delete mode 100644 sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/uvicorn_thread_server.py delete mode 100644 sidecars/tito/tito_gateway/verify_chat_template.py delete mode 100644 sidecars/tito/tito_gateway/verify_session_tito_tokenizer.py diff --git a/sidecars/tito/.gitignore b/plugins/tito/.gitignore similarity index 100% rename from sidecars/tito/.gitignore rename to plugins/tito/.gitignore diff --git a/sidecars/tito/LICENSE b/plugins/tito/LICENSE similarity index 100% rename from sidecars/tito/LICENSE rename to plugins/tito/LICENSE diff --git a/plugins/tito/README.md b/plugins/tito/README.md new file mode 100644 index 0000000..f1076d4 --- /dev/null +++ b/plugins/tito/README.md @@ -0,0 +1,110 @@ +# agentix-tito — TITO Gateway + +An Agentix plugin (`import agentix.tito`) that records **token-aligned** +agent↔model trajectories. It sits between an agent and an OpenAI-compatible +inference backend (e.g. sglang) as a session-scoped proxy, and accumulates the +exact `input_ids` / completion token IDs of every turn — the trajectory an RL +trainer needs, with no host-side re-tokenization. + +This is a **native implementation** of the TITO token-alignment engine +(`agentix.tito.engine`): no vendored training-framework code and no `sglang` +dependency. The engine tokenizes prompts itself with `transformers` + +`tokenizers` + a fixed Jinja chat template. + +## TITO in one paragraph + +TITO = *token-in, token-out*. Instead of re-tokenizing the rendered chat +transcript host-side (which can drift from what the model actually saw), the +gateway: + +1. **pretokenizes** each request's messages to `input_ids` and sends those to + the backend (token-in), +2. reads the **exact completion token IDs** back from + `meta_info.output_token_logprobs` (token-out), +3. reuses the **byte-identical token prefix** across turns, tokenizing only the + newly-appended non-assistant messages (tool/user/system) as a suffix in a + synthetic context, and +4. on read, **audits** the accumulated trajectory against a from-scratch render + (`compute_session_mismatch`) so any tokenizer drift is detected, not hidden. + +The algorithm is **model-agnostic** (base `TITOTokenizer`); a model family is a +fixed chat template plus a tiny boundary fixup — e.g. `Qwen3TITOTokenizer` +re-inserts the `\n` after `<|im_end|>` that the model omits when it stops. + +## Install + +It is a member of the Agentix uv workspace, installed editable with the rest: + +```bash +uv sync --all-packages --all-extras +``` + +Its runtime deps (`transformers`, `tokenizers`, `jinja2`, …) are isolated to +this plugin — agentix core and other plugins never pull them. + +## CLI + +```bash +agentix-tito serve \ + --hf-checkpoint Qwen/Qwen3-4B \ + --backend-url http://127.0.0.1:30000 \ + --tito-model qwen3 \ + --session-server-port 30001 +``` + +`--tito-model` selects the tokenizer family (`qwen3`, or `default` for the +tokenizer's own template). `--backend-url` may be omitted to auto-discover a +local backend (see `agentix.tito.discovery`). Run `agentix-tito serve -h` for +the full list. + +## HTTP surface + +- `POST /sessions` → `{session_id}` +- `POST /sessions/{id}/v1/chat/completions` — proxied chat completion; the + gateway forces `logprobs`/`return_meta_info`, injects the pretokenized + `input_ids`, and appends a token-aligned checkpoint. +- `GET /sessions/{id}` — records + metadata, incl. `accumulated_token_ids` and + `tito_session_mismatch` (empty list ⇒ byte-identical to a fresh render). +- `DELETE /sessions/{id}` — close the session and forget its pool pin. + +Multiple backend replicas are supported via `BackendPool`: requests are pinned +sticky-by-`session_id` for prefix-cache locality, and a replica is marked down +on a transport error. + +## Python API + +```python +from agentix.tito import TITOGateway, TITOGatewayConfig + +TITOGateway(TITOGatewayConfig( + hf_checkpoint="Qwen/Qwen3-4B", + backend_url="http://127.0.0.1:30000", + tito_model="qwen3", +)).run() +``` + +`agentix.tito.get_tito_tokenizer(tokenizer, "qwen3")` builds the engine +tokenizer directly if you only want incremental pretokenization. + +## Layout + +```text +agentix/tito/ +├── gateway.py / server.py / pool.py / discovery.py / config.py / cli.py +└── engine/ — the native TITO token-alignment engine + ├── pretokenize.py — TITOTokenizer (+ Qwen3TITOTokenizer) + ├── compare.py — special-token-segment mismatch audit + ├── trajectory.py — LinearTrajectory + SessionRegistry + ├── session_app.py — FastAPI session routes + ├── messages.py / render.py / processing.py / errors.py + └── templates/qwen3_fixed.jinja +``` + +## Tests + +```bash +pytest plugins/tito/tests +``` + +The engine tests are self-contained — they build a tiny in-memory tokenizer, so +no model download or GPU is required. diff --git a/plugins/tito/agentix/tito/__init__.py b/plugins/tito/agentix/tito/__init__.py new file mode 100644 index 0000000..1c76b06 --- /dev/null +++ b/plugins/tito/agentix/tito/__init__.py @@ -0,0 +1,21 @@ +"""Agentix TITO plugin — token-in-token-out session-recording gateway. + +A native reimplementation of the TITO token-alignment engine (see +`agentix.tito.engine`); no vendored training-framework code and no sglang +dependency. +""" + +from .config import TITOGatewayConfig +from .discovery import discover_backend_url +from .gateway import TITOGateway +from .server import SessionServer +from .tokenizer import TITOTokenizerType, get_tito_tokenizer + +__all__ = [ + "TITOGateway", + "TITOGatewayConfig", + "SessionServer", + "TITOTokenizerType", + "discover_backend_url", + "get_tito_tokenizer", +] diff --git a/plugins/tito/agentix/tito/cli.py b/plugins/tito/agentix/tito/cli.py new file mode 100644 index 0000000..c31885e --- /dev/null +++ b/plugins/tito/agentix/tito/cli.py @@ -0,0 +1,95 @@ +"""Command-line entrypoint for the Agentix TITO gateway.""" + +from __future__ import annotations + +import argparse +import sys + +from .config import TITOGatewayConfig +from .gateway import TITOGateway +from .tokenizer import TITOTokenizerType + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="agentix-tito", + description="Agentix TITO gateway — token-in-token-out session-recording proxy.", + ) + subparsers = parser.add_subparsers(dest="command") + _add_serve_parser(subparsers) + return parser + + +def _add_serve_parser(subparsers: argparse._SubParsersAction) -> argparse.ArgumentParser: + serve = subparsers.add_parser("serve", help="Start the TITO gateway server.") + _add_serve_arguments(serve) + return serve + + +def _add_serve_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--hf-checkpoint", required=True, help="HuggingFace model ID or local checkpoint path.") + parser.add_argument("--backend-url", default=None, help="OpenAI-compatible backend URL to proxy to.") + parser.add_argument("--chat-template-path", default=None, help="Optional fixed chat template path.") + parser.add_argument( + "--tito-model", + choices=[item.value for item in TITOTokenizerType], + default=TITOTokenizerType.DEFAULT.value, + help="TITO tokenizer family (qwen3, or default for the tokenizer's own template).", + ) + parser.add_argument( + "--tito-allowed-append-roles", + nargs="+", + choices=["tool", "user", "system"], + default=["tool"], + help="Roles allowed after an assistant turn; tool is the default.", + ) + parser.add_argument("--session-server-ip", default="127.0.0.1", help="Gateway bind host.") + parser.add_argument("--session-server-port", type=int, default=30000, help="Gateway bind port.") + parser.add_argument("--router-timeout", type=float, default=600.0, help="Proxy timeout in seconds.") + parser.add_argument( + "--backend-probe-candidate", + action="append", + default=None, + metavar="URL", + help="Local backend URL candidate to probe after explicit and environment URLs; repeatable.", + ) + parser.add_argument( + "--backend-probe-timeout", + type=float, + default=0.25, + help="Per-endpoint backend probe timeout in seconds.", + ) + + +def _serve(args: argparse.Namespace) -> int: + config = TITOGatewayConfig.from_cli_values( + hf_checkpoint=args.hf_checkpoint, + backend_url=args.backend_url, + chat_template_path=args.chat_template_path, + tito_model=args.tito_model, + tito_allowed_append_roles=args.tito_allowed_append_roles, + session_server_ip=args.session_server_ip, + session_server_port=args.session_server_port, + router_timeout=args.router_timeout, + backend_probe_candidates=args.backend_probe_candidate, + backend_probe_timeout=args.backend_probe_timeout, + ) + TITOGateway(config).run() + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = build_parser() + raw_args = list(sys.argv[1:] if argv is None else argv) + if not raw_args or raw_args[0] not in {"serve", "-h", "--help"}: + raw_args.insert(0, "serve") + args = parser.parse_args(raw_args) + try: + return _serve(args) + except Exception as exc: # noqa: BLE001 + print(f"agentix-tito: error: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/sidecars/tito/tito_gateway/config.py b/plugins/tito/agentix/tito/config.py similarity index 74% rename from sidecars/tito/tito_gateway/config.py rename to plugins/tito/agentix/tito/config.py index cfc823d..1431835 100644 --- a/sidecars/tito/tito_gateway/config.py +++ b/plugins/tito/agentix/tito/config.py @@ -2,11 +2,9 @@ from __future__ import annotations -import json from dataclasses import dataclass, field -from typing import Any -from tito_gateway.discovery import DEFAULT_BACKEND_PROBE_CANDIDATES +from .discovery import DEFAULT_BACKEND_PROBE_CANDIDATES _VALID_APPEND_ROLES = frozenset({"tool", "user", "system"}) @@ -14,7 +12,7 @@ @dataclass(frozen=True) class TITOGatewayConfig: - """Miles-compatible configuration for the standalone gateway wrapper.""" + """Configuration for the standalone TITO gateway wrapper.""" hf_checkpoint: str backend_url: str | None = None @@ -24,12 +22,11 @@ class TITOGatewayConfig: backend_urls: tuple[str, ...] = () routing_policy: str = "sticky" chat_template_path: str | None = None - apply_chat_template_kwargs: dict[str, Any] = field(default_factory=dict) tito_model: str = "default" tito_allowed_append_roles: tuple[str, ...] = ("tool",) session_server_ip: str = "127.0.0.1" session_server_port: int = 30000 - miles_router_timeout: float = 600.0 + router_timeout: float = 600.0 backend_probe_candidates: tuple[str, ...] = field(default_factory=lambda: DEFAULT_BACKEND_PROBE_CANDIDATES) backend_probe_timeout: float = 0.25 @@ -55,47 +52,37 @@ def from_cli_values( hf_checkpoint: str, backend_url: str | None, chat_template_path: str | None, - apply_chat_template_kwargs: str | None, tito_model: str, tito_allowed_append_roles: list[str], session_server_ip: str, session_server_port: int, - miles_router_timeout: float, + router_timeout: float, backend_probe_candidates: list[str] | None = None, backend_probe_timeout: float = 0.25, ) -> "TITOGatewayConfig": - kwargs: dict[str, Any] = {} - if apply_chat_template_kwargs: - parsed = json.loads(apply_chat_template_kwargs) - if not isinstance(parsed, dict): - raise ValueError("--apply-chat-template-kwargs must decode to a JSON object") - kwargs = parsed - return cls( hf_checkpoint=hf_checkpoint, backend_url=backend_url, chat_template_path=chat_template_path, - apply_chat_template_kwargs=kwargs, tito_model=tito_model, tito_allowed_append_roles=tuple(tito_allowed_append_roles), session_server_ip=session_server_ip, session_server_port=session_server_port, - miles_router_timeout=miles_router_timeout, + router_timeout=router_timeout, backend_probe_candidates=tuple(backend_probe_candidates or DEFAULT_BACKEND_PROBE_CANDIDATES), backend_probe_timeout=backend_probe_timeout, ) - def as_miles_namespace(self): - """Return an argparse-like namespace for vendored Miles session code.""" + def as_session_args(self): + """Return an argparse-like namespace consumed by the engine session routes.""" from types import SimpleNamespace return SimpleNamespace( hf_checkpoint=self.hf_checkpoint, chat_template_path=self.chat_template_path, - apply_chat_template_kwargs=self.apply_chat_template_kwargs, tito_model=self.tito_model, tito_allowed_append_roles=list(self.tito_allowed_append_roles), session_server_ip=self.session_server_ip, session_server_port=self.session_server_port, - miles_router_timeout=self.miles_router_timeout, + router_timeout=self.router_timeout, ) diff --git a/sidecars/tito/tito_gateway/discovery.py b/plugins/tito/agentix/tito/discovery.py similarity index 100% rename from sidecars/tito/tito_gateway/discovery.py rename to plugins/tito/agentix/tito/discovery.py diff --git a/plugins/tito/agentix/tito/engine/__init__.py b/plugins/tito/agentix/tito/engine/__init__.py new file mode 100644 index 0000000..1c69946 --- /dev/null +++ b/plugins/tito/agentix/tito/engine/__init__.py @@ -0,0 +1,7 @@ +"""Agentix's native TITO engine — token-in token-out pretokenization, session +trajectory, and mismatch-audit logic. + +Model-agnostic core lives here; per-model behavior is a small amount of data +(a fixed chat template) plus a couple of constants and an optional boundary +fixup. See `pretokenize.TITOTokenizer` for the algorithm. +""" diff --git a/plugins/tito/agentix/tito/engine/compare.py b/plugins/tito/agentix/tito/engine/compare.py new file mode 100644 index 0000000..8c20ed3 --- /dev/null +++ b/plugins/tito/agentix/tito/engine/compare.py @@ -0,0 +1,189 @@ +"""Token-sequence comparator: segment by special tokens, classify mismatches. + +Used to check that an incrementally-accumulated trajectory tokenizes identically +to a from-scratch render. The comparison is structural: the special-token skeleton +and non-assistant content must match exactly; assistant content may differ (the +model's own tokens) and is reported as a soft mismatch. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +from typing import Any + + +class MismatchType(str, Enum): + # Segment count or special/content pattern differs — structural break. + SPECIAL_TOKEN_COUNT = "special_token_count" + # Aligned special-token segment holds a different special token. + SPECIAL_TOKEN_TYPE = "special_token_type" + # Non-assistant content (system/user/tool) differs — the prompt drifted. + NON_ASSISTANT_TEXT = "non_assistant_text" + # Assistant content differs — expected and non-severe (model's own tokens). + ASSISTANT_TEXT = "assistant_text" + + +@dataclass +class Segment: + token_ids: list[int] + is_special: bool = False + + +@dataclass +class Mismatch: + type: MismatchType + segment_index: int + expected_text: str = "" + actual_text: str = "" + detail: str = "" + + def to_dict(self) -> dict[str, Any]: + return { + "type": self.type.value, + "segment_index": self.segment_index, + "expected_text": self.expected_text, + "actual_text": self.actual_text, + "detail": self.detail, + } + + +class TokenSeqComparator: + """Segment two token-ID sequences at special-token boundaries and compare. + + `assistant_start_str` (e.g. ``"<|im_start|>assistant"``) classifies a content + segment as assistant vs non-assistant. `special_token_ids`, if given, overrides + the set collected from the tokenizer. `trim_trailing_ids` are stripped from both + tails before comparison (a stop token the model emits but the template doesn't). + """ + + def __init__( + self, + tokenizer: Any, + *, + assistant_start_str: str | None, + special_token_ids: set[int] | None = None, + trim_trailing_ids: frozenset[int] | set[int] | None = None, + ) -> None: + self.tokenizer = tokenizer + self._assistant_start_str = assistant_start_str + self._special_ids = set(special_token_ids) if special_token_ids is not None else self.collect_special_ids(tokenizer) + self._trim_trailing_ids = set(trim_trailing_ids) if trim_trailing_ids else None + + @staticmethod + def collect_special_ids(tokenizer: Any) -> set[int]: + """Token IDs flagged ``special=True`` by the tokenizer. Content tokens a role + produces (e.g. ````) are NOT special, so they aren't collected here.""" + ids: set[int] = set(getattr(tokenizer, "all_special_ids", []) or []) + decoder = getattr(tokenizer, "added_tokens_decoder", None) + if decoder: + ids |= {k for k, v in decoder.items() if getattr(v, "special", False)} + return ids + + def segment_by_special_tokens(self, token_ids: list[int]) -> list[Segment]: + """Each special token is its own single-ID segment; consecutive non-special + tokens group into one content segment.""" + segments: list[Segment] = [] + current: list[int] = [] + for tid in token_ids: + if tid in self._special_ids: + if current: + segments.append(Segment(token_ids=current)) + current = [] + segments.append(Segment(token_ids=[tid], is_special=True)) + else: + current.append(tid) + if current: + segments.append(Segment(token_ids=current)) + return segments + + def compare_sequences( + self, + expected_ids: list[int], + actual_ids: list[int], + trim_trailing_ids: frozenset[int] | set[int] | None = None, + ) -> list[Mismatch]: + trim = self._trim_trailing_ids or set() + if trim_trailing_ids: + trim = trim | trim_trailing_ids + if trim: + expected_ids = _trim_trailing(expected_ids, trim) + actual_ids = _trim_trailing(actual_ids, trim) + + exp_segs = self.segment_by_special_tokens(expected_ids) + act_segs = self.segment_by_special_tokens(actual_ids) + + structural = self._check_segment_structure(exp_segs, act_segs) + if structural is not None: + return [structural] + + mismatches: list[Mismatch] = [] + for idx, (exp, act) in enumerate(zip(exp_segs, act_segs, strict=True)): + is_assistant = self._is_assistant_content(exp_segs, idx) and self._is_assistant_content(act_segs, idx) + m = self._compare_single_segment(idx, exp, act, is_assistant_content=is_assistant) + if m is not None: + mismatches.append(m) + return mismatches + + def _check_segment_structure(self, exp_segs: list[Segment], act_segs: list[Segment]) -> Mismatch | None: + if len(exp_segs) != len(act_segs): + detail = f"segment count differs: expected {len(exp_segs)}, got {len(act_segs)}" + elif [s.is_special for s in exp_segs] != [s.is_special for s in act_segs]: + detail = "segment structure (special/content pattern) differs" + else: + return None + return Mismatch( + type=MismatchType.SPECIAL_TOKEN_COUNT, + segment_index=-1, + expected_text=self._describe_structure(exp_segs), + actual_text=self._describe_structure(act_segs), + detail=detail, + ) + + def _compare_single_segment(self, idx: int, exp: Segment, act: Segment, *, is_assistant_content: bool) -> Mismatch | None: + if exp.is_special: + if exp.token_ids != act.token_ids: + return Mismatch( + type=MismatchType.SPECIAL_TOKEN_TYPE, + segment_index=idx, + expected_text=self._decode(exp.token_ids), + actual_text=self._decode(act.token_ids), + ) + return None + exp_text = self._decode(exp.token_ids) + act_text = self._decode(act.token_ids) + if exp_text == act_text: + return None + return Mismatch( + type=MismatchType.ASSISTANT_TEXT if is_assistant_content else MismatchType.NON_ASSISTANT_TEXT, + segment_index=idx, + expected_text=exp_text, + actual_text=act_text, + ) + + def _is_assistant_content(self, segments: list[Segment], idx: int) -> bool: + if self._assistant_start_str is None: + return False + if segments[idx].is_special or idx == 0: + return False + prev = segments[idx - 1] + if not prev.is_special: + return False + special_text = self._decode(prev.token_ids) + content_prefix = self._decode(segments[idx].token_ids[:20]) + return (special_text + content_prefix).startswith(self._assistant_start_str) + + def _decode(self, token_ids: list[int]) -> str: + return self.tokenizer.decode(token_ids, skip_special_tokens=False) + + def _describe_structure(self, segments: list[Segment]) -> str: + return " ".join( + f"[{self._decode(s.token_ids)}]" if s.is_special else f"({len(s.token_ids)} tokens)" for s in segments + ) + + +def _trim_trailing(ids: list[int], to_remove: set[int]) -> list[int]: + end = len(ids) + while end > 0 and ids[end - 1] in to_remove: + end -= 1 + return ids[:end] diff --git a/plugins/tito/agentix/tito/engine/errors.py b/plugins/tito/agentix/tito/engine/errors.py new file mode 100644 index 0000000..d93aaa8 --- /dev/null +++ b/plugins/tito/agentix/tito/engine/errors.py @@ -0,0 +1,33 @@ +"""Session error hierarchy. Each carries the HTTP status the gateway returns.""" + +from __future__ import annotations + + +class SessionError(Exception): + """Base class for all session-related errors.""" + + status_code: int = 500 + + +class SessionNotFoundError(SessionError): + """The requested session ID does not exist.""" + + status_code: int = 404 + + +class MessageValidationError(SessionError): + """Request messages aren't a valid append-only extension (or a rollback failed).""" + + status_code: int = 400 + + +class TokenizationError(SessionError): + """A TITO tokenization invariant was violated (e.g. pretokenized prefix mismatch).""" + + status_code: int = 500 + + +class UpstreamResponseError(SessionError): + """The upstream sglang response is invalid or unexpected (missing meta_info, etc.).""" + + status_code: int = 502 diff --git a/plugins/tito/agentix/tito/engine/messages.py b/plugins/tito/agentix/tito/engine/messages.py new file mode 100644 index 0000000..2035dfb --- /dev/null +++ b/plugins/tito/agentix/tito/engine/messages.py @@ -0,0 +1,72 @@ +"""Message-level helpers used by the session state machine. + +`message_matches` compares only the fields that affect chat-template tokenization, +so a stored assistant message and a resent one are considered equal iff they +tokenize identically. `assert_messages_append_only_with_allowed_role` enforces that +each turn extends the stored history without rewriting it. +""" + +from __future__ import annotations + +from typing import Any + +# Keys a chat template actually reads. Extra client-injected keys +# (provider_specific_fields, etc.) don't affect tokenization, so we ignore them. +TEMPLATE_RELEVANT_KEYS = ("role", "content", "reasoning_content", "tool_calls") + +DEFAULT_APPEND_ROLES: list[str] = ["tool"] + + +def normalize_value(value: Any) -> Any: + """Collapse the falsy sentinels that render identically in Jinja2 (None, "", []) + to None. Non-falsy content — including whitespace like trailing newlines — is + returned as-is, because boundary characters must tokenize identically.""" + if value is None or value == "" or value == []: + return None + return value + + +def message_matches(stored: dict[str, Any], new: dict[str, Any]) -> bool: + for key in TEMPLATE_RELEVANT_KEYS: + if normalize_value(stored.get(key)) != normalize_value(new.get(key)): + return False + return True + + +def assert_messages_append_only_with_allowed_role( + stored_messages: list[dict[str, Any]], + new_messages: list[dict[str, Any]], + allowed_append_roles: list[str] = DEFAULT_APPEND_ROLES, +) -> None: + """Assert *new_messages* is an append-only extension of *stored_messages*: the + stored prefix matches (by template-relevant keys) and each appended message's + role is in *allowed_append_roles*. Raises ValueError otherwise.""" + if not stored_messages: + return + + if len(new_messages) < len(stored_messages): + raise ValueError( + f"new messages ({len(new_messages)}) are fewer than stored messages ({len(stored_messages)})", + new_messages, + stored_messages, + ) + + for i, stored_msg in enumerate(stored_messages): + if not message_matches(stored_msg, new_messages[i]): + diffs = { + key: {"stored": repr(stored_msg.get(key))[:200], "new": repr(new_messages[i].get(key))[:200]} + for key in TEMPLATE_RELEVANT_KEYS + if stored_msg.get(key) != new_messages[i].get(key) + } + raise ValueError( + f"message mismatch at index {i} " + f"(role: stored={stored_msg.get('role')}, new={new_messages[i].get('role')}). " + f"Diffs: {diffs}" + ) + + for j, msg in enumerate(new_messages[len(stored_messages):]): + if msg.get("role") not in allowed_append_roles: + raise ValueError( + f"appended message at index {len(stored_messages) + j} " + f"has role={msg.get('role')!r}, allowed={allowed_append_roles}" + ) diff --git a/plugins/tito/agentix/tito/engine/pretokenize.py b/plugins/tito/agentix/tito/engine/pretokenize.py new file mode 100644 index 0000000..4d22aa0 --- /dev/null +++ b/plugins/tito/agentix/tito/engine/pretokenize.py @@ -0,0 +1,244 @@ +"""TITO tokenizer — incremental tokenization for pretokenized-prefix reuse. + +The base `TITOTokenizer` holds the whole model-agnostic algorithm: it computes the +token IDs for non-assistant messages (tool/user/system) appended after the +assistant's generated tokens, by rendering each segment in a minimal synthetic +context and taking the suffix, then merges them onto the stored prefix. A model +subclass only fixes boundary tokens at the junction (e.g. Qwen3's missing newline +after `<|im_end|>`) and points at its fixed chat template. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from .compare import TokenSeqComparator +from .messages import assert_messages_append_only_with_allowed_role +from .render import apply_chat_template + +TEMPLATE_DIR = Path(__file__).parent / "templates" +_VALID_ROLES = frozenset({"tool", "user", "system"}) +_DUMMY_SYSTEM: dict[str, Any] = {"role": "system", "content": "dummy system"} + + +def _build_dummy_assistant(tool_responses: list[dict[str, Any]]) -> dict[str, Any]: + """A dummy assistant whose tool_calls match *tool_responses*, so the template + renders the following tool-response turn boundaries correctly.""" + return { + "role": "assistant", + "content": "", + "reasoning_content": " ", + "tool_calls": [ + { + "id": resp.get("tool_call_id") or f"call0000{i}", + "type": "function", + "function": {"name": resp.get("name") or "dummy_func", "arguments": {}}, + } + for i, resp in enumerate(tool_responses) + ], + } + + +class TITOTokenizer: + """Incremental tokenization + prefix merging for appended non-assistant turns.""" + + max_trim_tokens: int = 0 + trailing_token_ids: frozenset[int] = frozenset() + reasoning_parser: str | None = None + tool_call_parser: str | None = None + + def __init__( + self, + tokenizer: Any, + chat_template_kwargs: dict[str, Any] | None = None, + assistant_start_str: str | None = None, + special_token_ids: set[int] | None = None, + allowed_append_roles: list[str] | None = None, + ) -> None: + self.tokenizer = tokenizer + self.chat_template_kwargs = chat_template_kwargs or {} + self._assistant_start_str = assistant_start_str + self.allowed_append_roles: list[str] = allowed_append_roles if allowed_append_roles is not None else ["tool"] + self.special_token_ids = special_token_ids + + def create_comparator(self) -> TokenSeqComparator: + return TokenSeqComparator( + self.tokenizer, + assistant_start_str=self._assistant_start_str, + special_token_ids=self.special_token_ids, + trim_trailing_ids=self.trailing_token_ids or None, + ) + + def render_messages( + self, + messages: list[dict[str, Any]], + *, + add_generation_prompt: bool, + tools: list[dict[str, Any]] | None = None, + tokenize: bool = False, + ) -> Any: + return apply_chat_template( + messages, + tokenizer=self.tokenizer, + tokenize=tokenize, + add_generation_prompt=add_generation_prompt, + tools=tools, + **self.chat_template_kwargs, + ) + + def _encode_text(self, text: str) -> list[int]: + return self.tokenizer.encode(text, add_special_tokens=False) + + def _split_appended_segments(self, appended_messages: list[dict[str, Any]]) -> list[list[dict[str, Any]]]: + segments: list[list[dict[str, Any]]] = [] + i = 0 + while i < len(appended_messages): + role = appended_messages[i]["role"] + if role == "tool": + j = i + 1 + while j < len(appended_messages) and appended_messages[j]["role"] == "tool": + j += 1 + segments.append(appended_messages[i:j]) + i = j + continue + if role in {"user", "system"}: + segments.append([appended_messages[i]]) + i += 1 + continue + raise ValueError(f"unsupported appended role for TITO segmentation: {role}") + return segments + + def _tokenize_rendered_suffix( + self, + base_messages: list[dict[str, Any]], + appended_messages: list[dict[str, Any]], + *, + tools: list[dict[str, Any]] | None = None, + add_generation_prompt: bool = False, + ) -> list[int]: + text_without = self.render_messages(base_messages, add_generation_prompt=False, tools=tools) + text_with = self.render_messages( + base_messages + appended_messages, add_generation_prompt=add_generation_prompt, tools=tools + ) + if not text_with.startswith(text_without): + roles = [m["role"] for m in appended_messages] if appended_messages else ["generation_prompt"] + raise ValueError(f"rendered suffix diff failed for {roles}") + return self._encode_text(text_with[len(text_without):]) + + def _tokenize_tool_segment( + self, appended_messages: list[dict[str, Any]], tools: list[dict[str, Any]] | None = None + ) -> list[int]: + return self._tokenize_rendered_suffix( + [_DUMMY_SYSTEM, _build_dummy_assistant(appended_messages)], appended_messages, tools=tools + ) + + def _tokenize_user_and_system_segment( + self, appended_message: dict[str, Any], tools: list[dict[str, Any]] | None = None + ) -> list[int]: + return self._tokenize_rendered_suffix([_DUMMY_SYSTEM], [appended_message], tools=tools) + + def tokenize_additional_non_assistant( + self, + old_messages: list[dict[str, Any]], + new_messages: list[dict[str, Any]], + tools: list[dict[str, Any]] | None = None, + ) -> list[int]: + """Incremental token IDs (incl. the next generation prompt) for the + non-assistant messages appended after the pretokenized prefix.""" + assert_messages_append_only_with_allowed_role(old_messages, new_messages, self.allowed_append_roles) + appended_messages = new_messages[len(old_messages):] + incremental: list[int] = [] + for segment in self._split_appended_segments(appended_messages): + role = segment[0]["role"] + if role == "tool": + incremental.extend(self._tokenize_tool_segment(segment, tools)) + elif role in ("user", "system"): + incremental.extend(self._tokenize_user_and_system_segment(segment[0], tools)) + else: + raise ValueError(f"unsupported appended role for TITO tokenization: {role}") + return incremental + self._tokenize_rendered_suffix( + new_messages, [], tools=tools, add_generation_prompt=True + ) + + def merge_tokens( + self, + old_messages: list[dict[str, Any]], + new_messages: list[dict[str, Any]], + pretokenized_token_ids: list[int], + tools: list[dict[str, Any]] | None = None, + ) -> list[int]: + """Default: concatenate the stored prefix with the incremental tokens.""" + incremental = self.tokenize_additional_non_assistant(old_messages, new_messages, tools) + return list(pretokenized_token_ids) + incremental + + +class Qwen3TITOTokenizer(TITOTokenizer): + """Qwen3: the model stops at `<|im_end|>` without the trailing `\\n` the template + emits, so `merge_tokens` re-inserts it so the stored prefix stays canonical.""" + + reasoning_parser = "qwen3" + tool_call_parser = "qwen25" + _default_assistant_start_str = "<|im_start|>assistant" + + def __init__( + self, + tokenizer: Any, + chat_template_kwargs: dict[str, Any] | None = None, + assistant_start_str: str | None = None, + allowed_append_roles: list[str] | None = None, + ) -> None: + super().__init__( + tokenizer, + chat_template_kwargs, + assistant_start_str or self._default_assistant_start_str, + allowed_append_roles=allowed_append_roles, + ) + nl_ids = tokenizer.encode("\n", add_special_tokens=False) + if len(nl_ids) != 1: + raise ValueError(f"expected a single newline token, got {nl_ids}") + self._newline_id: int = nl_ids[0] + self._im_end_id: int = tokenizer.convert_tokens_to_ids("<|im_end|>") + self.trailing_token_ids = frozenset({self._newline_id}) + + def merge_tokens( + self, + old_messages: list[dict[str, Any]], + new_messages: list[dict[str, Any]], + pretokenized_token_ids: list[int], + tools: list[dict[str, Any]] | None = None, + ) -> list[int]: + incremental = self.tokenize_additional_non_assistant(old_messages, new_messages, tools) + prefix = list(pretokenized_token_ids) + if prefix and prefix[-1] == self._im_end_id: + prefix.append(self._newline_id) + return prefix + incremental + + +_QWEN3_FIXED = "qwen3_fixed.jinja" + + +def get_tito_tokenizer( + tokenizer: Any, + tokenizer_type: str = "qwen3", + *, + allowed_append_roles: tuple[str, ...] = ("tool",), +) -> TITOTokenizer: + """Build a TITO tokenizer. `default` uses the tokenizer's own chat template + (model-agnostic); `qwen3` loads the bundled fixed template (and disables thinking + clearing when `user` appends are allowed, so earlier turns keep their reasoning).""" + if tokenizer is None: + raise ValueError("tokenizer must not be None") + roles = frozenset(allowed_append_roles) + invalid = roles - _VALID_ROLES + if invalid: + raise ValueError(f"unknown roles in allowed_append_roles: {sorted(invalid)}; valid: {sorted(_VALID_ROLES)}") + + if tokenizer_type == "default": + return TITOTokenizer(tokenizer, allowed_append_roles=list(allowed_append_roles)) + if tokenizer_type == "qwen3": + kw: dict[str, Any] = {"chat_template": (TEMPLATE_DIR / _QWEN3_FIXED).read_text()} + if "user" in roles: + kw["clear_thinking"] = False + return Qwen3TITOTokenizer(tokenizer, chat_template_kwargs=kw, allowed_append_roles=list(allowed_append_roles)) + raise ValueError(f"unsupported tokenizer_type {tokenizer_type!r}; supported: 'qwen3', 'default'") diff --git a/plugins/tito/agentix/tito/engine/processing.py b/plugins/tito/agentix/tito/engine/processing.py new file mode 100644 index 0000000..038ee1b --- /dev/null +++ b/plugins/tito/agentix/tito/engine/processing.py @@ -0,0 +1,16 @@ +"""Tokenizer loading. Minimal: load an HF tokenizer (tokenizer-only is fine — no +torch needed) and optionally override its chat template from a file.""" + +from __future__ import annotations + +from typing import Any + + +def load_tokenizer(name_or_path: str, chat_template_path: str | None = None, *, trust_remote_code: bool = True) -> Any: + from transformers import AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained(name_or_path, trust_remote_code=trust_remote_code) + if chat_template_path: + with open(chat_template_path) as f: + tokenizer.chat_template = f.read() + return tokenizer diff --git a/plugins/tito/agentix/tito/engine/render.py b/plugins/tito/agentix/tito/engine/render.py new file mode 100644 index 0000000..6d22f83 --- /dev/null +++ b/plugins/tito/agentix/tito/engine/render.py @@ -0,0 +1,94 @@ +"""Chat-template rendering backend. + +`apply_chat_template` renders messages through an HF tokenizer's chat template +(optionally an explicit `chat_template=` string for the fixed template), the same +code path SGLang uses. Tool definitions are canonicalized to the OpenAI +`{type:"function", function:{...}}` shape. No sglang dependency — the one pydantic +`Tool` type the canonicalization needs is defined locally. +""" + +from __future__ import annotations + +import copy +import json +from typing import Any, Literal, Optional + +from jinja2 import TemplateError +from pydantic import BaseModel, TypeAdapter + + +class _Function(BaseModel): + name: str + description: Optional[str] = None + parameters: Optional[dict[str, Any]] = None + + +class Tool(BaseModel): + type: str = "function" + function: _Function + + +def normalize_tool_arguments(messages: list[dict], format: Literal["dict", "json"]) -> list[dict]: + """Deep-copy *messages*, set assistant `content: None` -> "", and coerce tool_call + `arguments` to the form the renderer needs: "dict" (JSON string -> dict, for + HF-Jinja templates) or "json" (dict -> JSON string). Never mutates the input.""" + normalized = copy.deepcopy(messages) + for msg in normalized: + if msg.get("role") == "assistant": + if msg.get("content") is None: + msg["content"] = "" + if isinstance(msg.get("tool_calls"), list): + for item in msg["tool_calls"]: + func = item.get("function") + if not func: + continue + args = func.get("arguments") + if format == "dict" and isinstance(args, str): + func["arguments"] = json.loads(args) + elif format == "json" and isinstance(args, dict): + func["arguments"] = json.dumps(args, ensure_ascii=False) + return normalized + + +def extract_tool_dicts(tools: list[dict] | None) -> list[dict] | None: + """Canonicalize tools to full `{type:"function", function:{...}}` dumps.""" + if not tools: + return None + wrapped = [t if isinstance(t, dict) and "function" in t else {"type": "function", "function": t} for t in tools] + validated = TypeAdapter(list[Tool]).validate_python(wrapped) + return [tool.model_dump() for tool in validated] + + +def apply_chat_template( + messages: list[dict], + *, + tokenizer: Any, + tools: list[dict] | None = None, + add_generation_prompt: bool = True, + tokenize: bool = False, + **kwargs: Any, +) -> str | list[int]: + """Render via the HF tokenizer in SGLang style (`return_dict=False`, so the result + is `str` when tokenize=False or `list[int]` when tokenize=True). `chat_template=` + and other extras pass through `**kwargs`. Falls back to the bare function schema if + the template can't take the wrapped tool dicts.""" + messages = normalize_tool_arguments(messages, "dict") + tool_defs = extract_tool_dicts(tools) + render_kwargs = dict(add_generation_prompt=add_generation_prompt, **kwargs) + try: + return tokenizer.apply_chat_template( + messages, tokenize=tokenize, tools=tool_defs, return_dict=False, **render_kwargs + ) + except TemplateError as e: + if tool_defs is not None: + try: + return tokenizer.apply_chat_template( + messages, + tokenize=tokenize, + tools=[t["function"] if "function" in t else t for t in tool_defs], + return_dict=False, + **render_kwargs, + ) + except TemplateError as te: + raise ValueError(f"Chat template rendering failed (tool format fallback): {te}") from te + raise ValueError(f"Chat template rendering failed: {e}") from e diff --git a/plugins/tito/agentix/tito/engine/session_app.py b/plugins/tito/agentix/tito/engine/session_app.py new file mode 100644 index 0000000..384bbcb --- /dev/null +++ b/plugins/tito/agentix/tito/engine/session_app.py @@ -0,0 +1,180 @@ +"""FastAPI session routes for the TITO gateway. + +The gateway keeps a token-aligned trajectory per session and proxies chat +completions to an OpenAI-compatible backend (sglang). The chat-completions flow: +prepare pretokenized input_ids (lock held briefly) -> force logprobs/meta_info -> +proxy to the backend (no lock) -> validate -> append the trajectory checkpoint +(lock held briefly). The proxy is NOT held under the lock so a slow generation +doesn't block DELETE/other ops. + +`build_session_app` is backend-agnostic: pass any object exposing +``do_proxy(request, path, body=None) -> dict`` and ``build_proxy_response(result)``. +""" + +from __future__ import annotations + +import json +import logging +import time +from typing import Any, Protocol + +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse +from starlette.responses import Response + +from .errors import SessionError, SessionNotFoundError, TokenizationError, UpstreamResponseError +from .processing import load_tokenizer +from .pretokenize import get_tito_tokenizer +from .trajectory import GetSessionResponse, SessionRecord, SessionRegistry + +logger = logging.getLogger(__name__) + + +class Backend(Protocol): + async def do_proxy(self, request: Request, path: str, body: bytes | None = None) -> dict: ... + def build_proxy_response(self, result: dict) -> Response: ... + + +def build_registry(args: Any) -> SessionRegistry | None: + """Construct a SessionRegistry from gateway args, or None if no hf_checkpoint.""" + hf_checkpoint = getattr(args, "hf_checkpoint", None) + if not hf_checkpoint: + logger.info("[session] no hf_checkpoint set — session routes disabled") + return None + tokenizer = load_tokenizer( + hf_checkpoint, chat_template_path=getattr(args, "chat_template_path", None), trust_remote_code=True + ) + roles = getattr(args, "tito_allowed_append_roles", None) or ("tool",) + tito_tokenizer = get_tito_tokenizer( + tokenizer, + tokenizer_type=getattr(args, "tito_model", "default"), + allowed_append_roles=tuple(roles), + ) + return SessionRegistry(args, tokenizer, tito_tokenizer=tito_tokenizer) + + +def setup_session_routes(app: FastAPI, backend: Backend, args: Any) -> None: + registry = build_registry(args) + if registry is None: + return + + instance_id = getattr(args, "session_server_instance_id", None) + + @app.exception_handler(SessionError) + async def _session_error_handler(request: Request, exc: SessionError) -> JSONResponse: + return JSONResponse(status_code=exc.status_code, content={"error": str(exc)}) + + @app.get("/health") + async def health() -> dict[str, Any]: + body: dict[str, Any] = {"status": "ok"} + if instance_id is not None: + body["session_server_instance_id"] = instance_id + return body + + @app.post("/sessions") + async def create_session() -> dict[str, str]: + return {"session_id": registry.create_session()} + + @app.get("/sessions/{session_id}") + async def get_session(session_id: str) -> GetSessionResponse: + session = registry.get_session(session_id) + metadata: dict[str, Any] = {} + try: + mismatch = registry.compute_session_mismatch(session) + except TokenizationError: + logger.exception("failed to compute tito_session_mismatch for %s", session_id) + mismatch = None + if mismatch is not None: + metadata["tito_session_mismatch"] = mismatch + metadata["accumulated_token_ids"] = session.token_ids + metadata["max_trim_tokens"] = registry.tito_tokenizer.max_trim_tokens + return GetSessionResponse(session_id=session_id, records=session.records, metadata=metadata) + + @app.delete("/sessions/{session_id}") + async def delete_session(session_id: str) -> Response: + session = registry.get_session(session_id) + if session.closing: + raise SessionNotFoundError(f"session not found: session_id={session_id}") + session.closing = True + await session.lock.acquire() + try: + registry.remove_session(session_id) + finally: + session.lock.release() + return Response(status_code=204) + + @app.post("/sessions/{session_id}/v1/chat/completions") + async def chat_completions(request: Request, session_id: str) -> Response: + session = registry.get_session(session_id) + if session.closing: + raise SessionNotFoundError(f"session not found: session_id={session_id}") + + # Phase 1: prepare pretokenized input_ids (lock held briefly). + async with session.lock: + if session.closing: + raise SessionNotFoundError(f"session not found: session_id={session_id}") + raw = await request.body() + request_body = json.loads(raw) if raw else {} + # Hardcoded so an agent override can't break token accumulation: + request_body["logprobs"] = True # -> meta_info.output_token_logprobs + request_body["return_meta_info"] = True # -> choice.meta_info + request_body["no_stop_trim"] = False # stop-token text trimmed from content + request_messages = request_body.get("messages", []) + prompt_token_ids = session.prepare_pretokenized( + request_messages, tools=request_body.get("tools"), tito_tokenizer=registry.tito_tokenizer + ) + request_body["input_ids"] = prompt_token_ids + body = json.dumps(request_body).encode() + expected_num_assistant = session.num_assistant + + # Phase 2: proxy to the backend (NO lock). + result = await backend.do_proxy(request, "v1/chat/completions", body=body) + if result["status_code"] != 200: + return backend.build_proxy_response(result) + + response = json.loads(result["response_body"]) + choice = response.get("choices", [{}])[0] + meta_info = choice.get("meta_info") + if not isinstance(meta_info, dict) or "output_token_logprobs" not in meta_info: + raise UpstreamResponseError("meta_info.output_token_logprobs missing (needs logprobs=True)") + assistant_message = choice.get("message", {}) + if assistant_message.get("content") is None: + raise UpstreamResponseError("assistant message content is None") + output_token_logprobs = meta_info["output_token_logprobs"] + completion_tokens = meta_info["completion_tokens"] + if len(output_token_logprobs) != completion_tokens: + raise UpstreamResponseError( + f"len(output_token_logprobs)={len(output_token_logprobs)} != completion_tokens={completion_tokens}" + ) + completion_token_ids = [t[1] for t in output_token_logprobs] + + # Phase 3: append the trajectory checkpoint (lock held briefly). + async with session.lock: + if session.closing: + return backend.build_proxy_response(result) + if session.num_assistant != expected_num_assistant: + logger.warning("session %s changed during proxy; skipping state update", session_id) + return backend.build_proxy_response(result) + session.update_pretokenized_state( + request_messages, + assistant_message, + prompt_token_ids=prompt_token_ids, + completion_token_ids=completion_token_ids, + max_trim_tokens=registry.tito_tokenizer.max_trim_tokens, + ) + session.append_record( + SessionRecord( + timestamp=time.time(), + method=request.method, + path="/v1/chat/completions", + status_code=result["status_code"], + request=request_body, + response=response, + ) + ) + return backend.build_proxy_response(result) + + @app.api_route("/sessions/{session_id}/{path:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"]) + async def session_proxy(request: Request, session_id: str, path: str) -> Response: + result = await backend.do_proxy(request, path) + return backend.build_proxy_response(result) diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/qwen3_fixed.jinja b/plugins/tito/agentix/tito/engine/templates/qwen3_fixed.jinja similarity index 100% rename from sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/qwen3_fixed.jinja rename to plugins/tito/agentix/tito/engine/templates/qwen3_fixed.jinja diff --git a/plugins/tito/agentix/tito/engine/trajectory.py b/plugins/tito/agentix/tito/engine/trajectory.py new file mode 100644 index 0000000..dfc3bd9 --- /dev/null +++ b/plugins/tito/agentix/tito/engine/trajectory.py @@ -0,0 +1,222 @@ +"""Linear trajectory state machine + session registry. + +`LinearTrajectory` holds one session's message history and accumulated token-ID +checkpoints, and is the heart of incremental pretokenization: on each turn it +validates that the request extends the stored history (rolling back at most one +assistant step on agent retries) and reuses the stored token prefix. `SessionRegistry` +maps session IDs to trajectories and computes the from-scratch-vs-accumulated +mismatch report. Mutating methods must be called under `LinearTrajectory.lock`. +""" + +from __future__ import annotations + +import asyncio +import logging +import uuid +from dataclasses import dataclass, field +from typing import Any + +from pydantic import BaseModel, Field + +from .compare import TokenSeqComparator +from .errors import MessageValidationError, SessionNotFoundError, TokenizationError +from .messages import assert_messages_append_only_with_allowed_role, message_matches +from .pretokenize import TITOTokenizer + +logger = logging.getLogger(__name__) + +# Only single-step rollback is supported (an agent retrying one tool call). +MAX_ASSISTANT_ROLLBACK_STEPS = 1 + + +class SessionRecord(BaseModel): + timestamp: float + method: str + path: str + request: dict + response: dict + status_code: int + + +class GetSessionResponse(BaseModel): + session_id: str + records: list[SessionRecord] + metadata: dict = Field(default_factory=dict) + + +@dataclass +class LinearTrajectory: + """Message history + accumulated token-ID checkpoints for one session.""" + + lock: asyncio.Lock = field(default_factory=asyncio.Lock, repr=False, compare=False) + closing: bool = field(default=False, repr=False, compare=False) + messages: list[dict[str, Any]] = field(default_factory=list) + records: list[SessionRecord] = field(default_factory=list) + trajectory_token_ids: list[list[int]] = field(default_factory=list) + num_assistant: int = 0 + + @property + def token_ids(self) -> list[int]: + """The latest assistant checkpoint's token IDs.""" + return self.trajectory_token_ids[-1] if self.trajectory_token_ids else [] + + def append_record(self, record: SessionRecord) -> None: + self.records.append(record) + + def prepare_pretokenized( + self, + request_messages: list[dict[str, Any]], + tools: list[dict[str, Any]] | None = None, + *, + tito_tokenizer: TITOTokenizer, + ) -> list[int]: + """Build the full prompt token IDs for *request_messages*. First turn renders + from scratch; later turns reuse the stored token prefix (rolling back at most + one assistant step on a retry). Must be called under ``self.lock``.""" + if not self.token_ids: + return tito_tokenizer.render_messages( + request_messages, tools=tools, add_generation_prompt=True, tokenize=True + ) + + self._try_detect_and_rollback_to_assistant_checkpoint(request_messages) + try: + assert_messages_append_only_with_allowed_role( + self.messages, request_messages, tito_tokenizer.allowed_append_roles + ) + except ValueError as e: + raise MessageValidationError(f"{e}; to allow more roles use --tito-allowed-append-roles") from e + + return tito_tokenizer.merge_tokens( + old_messages=self.messages, + new_messages=request_messages, + pretokenized_token_ids=self.token_ids, + tools=tools, + ) + + def update_pretokenized_state( + self, + request_messages: list[dict[str, Any]], + assistant_message: dict[str, Any], + prompt_token_ids: list[int], + completion_token_ids: list[int], + max_trim_tokens: int, + ) -> None: + """Append ``prompt+completion`` token IDs as a new checkpoint after a successful + response, validating the previously-stored IDs are a prefix (tolerating up to + ``max_trim_tokens`` trailing differences). Must be called under ``self.lock``.""" + all_token_ids = prompt_token_ids + completion_token_ids + + prev = self.token_ids + if prev: + check_len = len(prev) - max_trim_tokens + if check_len > 0 and all_token_ids[:check_len] != prev[:check_len]: + first_mismatch = next( + ( + i + for i, (a, b) in enumerate(zip(all_token_ids[:check_len], prev[:check_len], strict=True)) + if a != b + ), + min(len(all_token_ids), check_len), + ) + raise TokenizationError( + f"pretokenized prefix mismatch: stored {len(prev)} tokens " + f"(checking first {check_len}, allowing {max_trim_tokens} trailing) are not a prefix of " + f"prompt_token_ids + completion_token_ids ({len(all_token_ids)} tokens), " + f"first mismatch at index {first_mismatch}, matched {first_mismatch}/{check_len} prefix tokens\n" + f"request_messages={request_messages}\nassistant_message={assistant_message}" + ) + + self.messages = list(request_messages) + [assistant_message] + self.trajectory_token_ids.append(all_token_ids) + self.num_assistant += 1 + + def _try_detect_and_rollback_to_assistant_checkpoint(self, request_messages: list[dict[str, Any]]) -> None: + """If *request_messages* diverges from the stored history, truncate state back to + the last assistant checkpoint within the matching prefix (single-step only).""" + stored = self.messages + if not stored or not self.trajectory_token_ids: + return + + match_len = 0 + for i in range(min(len(request_messages), len(stored))): + if message_matches(stored[i], request_messages[i]): + match_len = i + 1 + else: + break + + if match_len >= len(stored): + return + + rollback_msg_end = None + checkpoint_index = -1 + assistant_count = 0 + for i in range(match_len): + if stored[i].get("role") == "assistant": + rollback_msg_end = i + 1 + checkpoint_index = assistant_count + assistant_count += 1 + + if checkpoint_index < 0: + raise MessageValidationError( + f"rollback failed: no assistant message found in the first {match_len} matched messages " + f"(stored has {len(stored)} messages, request has {len(request_messages)} messages)" + ) + + discard_count = self.num_assistant - (checkpoint_index + 1) + if discard_count > MAX_ASSISTANT_ROLLBACK_STEPS: + raise MessageValidationError( + f"rollback failed: discard_count={discard_count} exceeds " + f"max_assistant_rollback_steps={MAX_ASSISTANT_ROLLBACK_STEPS} " + f"(stored has {len(stored)} messages, request has {len(request_messages)} messages)" + ) + + logger.info( + "Rolling back session: stored %d messages / %d checkpoints -> checkpoint %d (messages[:%d]), " + "discarding %d assistant(s)", + len(stored), self.num_assistant, checkpoint_index, rollback_msg_end, discard_count, + ) + self.messages = stored[:rollback_msg_end] + self.trajectory_token_ids = self.trajectory_token_ids[: checkpoint_index + 1] + self.records = self.records[: checkpoint_index + 1] + self.num_assistant = checkpoint_index + 1 + + +class SessionRegistry: + """Session ID -> trajectory map + shared tokenizer/comparator. Pure CRUD plus the + read-only mismatch computation; never mutates trajectory state itself.""" + + def __init__(self, args: Any, tokenizer: Any, *, tito_tokenizer: TITOTokenizer) -> None: + self.sessions: dict[str, LinearTrajectory] = {} + self.args = args + self.tokenizer = tokenizer + self.tito_tokenizer = tito_tokenizer + self.comparator: TokenSeqComparator = tito_tokenizer.create_comparator() + + def create_session(self) -> str: + session_id = uuid.uuid4().hex + self.sessions[session_id] = LinearTrajectory() + return session_id + + def get_session(self, session_id: str) -> LinearTrajectory: + session = self.sessions.get(session_id) + if session is None: + raise SessionNotFoundError(f"session not found: session_id={session_id}") + return session + + def remove_session(self, session_id: str) -> None: + if self.sessions.pop(session_id, None) is None: + raise SessionNotFoundError(f"session not found: session_id={session_id}") + + def compute_session_mismatch(self, session: LinearTrajectory) -> list[dict] | None: + """Compare accumulated token IDs against a from-scratch render. Read-only.""" + if not session.token_ids: + return None + try: + tools = session.records[-1].request.get("tools") if session.records else None + expected_ids = self.tito_tokenizer.render_messages( + session.messages, tools=tools, add_generation_prompt=False, tokenize=True + ) + mismatches = self.comparator.compare_sequences(expected_ids, session.token_ids) + return [m.to_dict() for m in mismatches] + except Exception as e: + raise TokenizationError(f"failed to compute tito_session_mismatch: {e}") from e diff --git a/sidecars/tito/tito_gateway/gateway.py b/plugins/tito/agentix/tito/gateway.py similarity index 80% rename from sidecars/tito/tito_gateway/gateway.py rename to plugins/tito/agentix/tito/gateway.py index b5aec8f..18eb393 100644 --- a/sidecars/tito/tito_gateway/gateway.py +++ b/plugins/tito/agentix/tito/gateway.py @@ -4,10 +4,10 @@ from dataclasses import replace -from tito_gateway.config import TITOGatewayConfig -from tito_gateway.discovery import discover_backend_url, normalize_backend_url -from tito_gateway.pool import BackendPool -from tito_gateway.server import SessionServer +from .config import TITOGatewayConfig +from .discovery import discover_backend_url, normalize_backend_url +from .pool import BackendPool +from .server import SessionServer class TITOGateway: @@ -30,7 +30,7 @@ def __init__(self, config: TITOGatewayConfig): self.config = replace(config, backend_url=backend_url) urls = [backend_url] self.pool = BackendPool(urls, policy=config.routing_policy) - self.server = SessionServer(self.config.as_miles_namespace(), self.pool) + self.server = SessionServer(self.config.as_session_args(), self.pool) self._register_health_alias() @classmethod @@ -38,8 +38,8 @@ def from_server(cls, *, hf_checkpoint: str, backend_url: str | None = None, **kw return cls(TITOGatewayConfig(hf_checkpoint=hf_checkpoint, backend_url=backend_url, **kwargs)) def _register_health_alias(self) -> None: - # abridge's Sidecar probes `/healthz` by default; the vendored session - # server only exposes `/health`. Add a thin alias so a default Sidecar + # abridge's Sidecar probes `/healthz` by default; the engine session + # routes only expose `/health`. Add a thin alias so a default Sidecar # wiring works without overriding `health_path`. async def healthz() -> dict[str, str]: return {"status": "ok"} diff --git a/sidecars/tito/tito_gateway/pool.py b/plugins/tito/agentix/tito/pool.py similarity index 100% rename from sidecars/tito/tito_gateway/pool.py rename to plugins/tito/agentix/tito/pool.py diff --git a/plugins/tito/agentix/tito/server.py b/plugins/tito/agentix/tito/server.py new file mode 100644 index 0000000..a9ec639 --- /dev/null +++ b/plugins/tito/agentix/tito/server.py @@ -0,0 +1,113 @@ +"""Session server — a FastAPI app over the native TITO engine, routing proxied +inference across a multi-backend pool. + +The engine's `session_app` owns the routes (sessions + the token-aligned chat +flow); this module supplies the *backend*: a pooled httpx proxy that picks a +replica per request (sticky by ``session_id`` for prefix-cache locality), reports +a replica down on a transport error, and forgets a session's pin on delete. +""" + +from __future__ import annotations + +import json +import logging +from typing import Any + +import httpx +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse +from starlette.responses import Response + +from .engine.session_app import setup_session_routes +from .pool import BackendPool + +logger = logging.getLogger(__name__) + +_HOP_BY_HOP = ("content-length", "transfer-encoding", "host") +_RESP_STRIP = ("content-length", "transfer-encoding", "content-encoding") + + +def _session_id_from_path(path: str) -> str | None: + """Extract ``{session_id}`` from ``/sessions/{session_id}[/...]``.""" + parts = path.strip("/").split("/") + if len(parts) >= 2 and parts[0] == "sessions": + return parts[1] + return None + + +class _PooledBackend: + """Backend for the session routes: proxy each request to a pool-picked replica.""" + + def __init__(self, args: Any, pool: BackendPool) -> None: + self._pool = pool + timeout = getattr(args, "router_timeout", 600.0) + self.client = httpx.AsyncClient( + limits=httpx.Limits(max_connections=1024), timeout=httpx.Timeout(timeout) + ) + + async def do_proxy(self, request: Request, path: str, body: bytes | None = None) -> dict: + session_id = _session_id_from_path(request.url.path) + backend_url = self._pool.pick(session_id) + url = f"{backend_url}/{path}" + if request.url.query: + url = f"{url}?{request.url.query}" + if body is None: + body = await request.body() + headers = {k: v for k, v in request.headers.items() if k.lower() not in _HOP_BY_HOP} + try: + response = await self.client.request(request.method, url, content=body, headers=headers) + except httpx.TransportError as exc: + self._pool.report_down(backend_url) + logger.warning("pooled proxy transport error %s -> %s: %s", path, backend_url, exc) + error_body = json.dumps({"error": f"backend transport error: {type(exc).__name__}: {exc}"}).encode() + return { + "request_body": body, + "response_body": error_body, + "status_code": 502, + "headers": {"content-type": "application/json"}, + } + content = await response.aread() + return { + "request_body": body, + "response_body": content, + "status_code": response.status_code, + "headers": dict(response.headers), + } + + def build_proxy_response(self, result: dict) -> Response: + content = result["response_body"] + headers = {k: v for k, v in result["headers"].items() if k.lower() not in _RESP_STRIP} + try: + return JSONResponse(content=json.loads(content), status_code=result["status_code"], headers=headers) + except (json.JSONDecodeError, UnicodeDecodeError): + return Response( + content=content, + status_code=result["status_code"], + headers=headers, + media_type=headers.get("content-type", ""), + ) + + async def aclose(self) -> None: + await self.client.aclose() + + +class SessionServer: + """FastAPI session server backed by the native TITO engine + a BackendPool.""" + + def __init__(self, args: Any, pool: BackendPool) -> None: + self.args = args + self.pool = pool + self.backend_url = pool.backends[0] + self.app = FastAPI() + self._backend = _PooledBackend(args, pool) + self.app.router.on_shutdown.append(self._backend.aclose) + setup_session_routes(self.app, self._backend, args) + self.app.middleware("http")(self._forget_on_delete) + + async def _forget_on_delete(self, request: Request, call_next: Any) -> Response: + response = await call_next(request) + if request.method == "DELETE" and response.status_code < 300: + session_id = _session_id_from_path(request.url.path) + if session_id is not None: + self.pool.forget(session_id) + return response diff --git a/plugins/tito/agentix/tito/tokenizer.py b/plugins/tito/agentix/tito/tokenizer.py new file mode 100644 index 0000000..ec26fc9 --- /dev/null +++ b/plugins/tito/agentix/tito/tokenizer.py @@ -0,0 +1,29 @@ +"""Public tokenizer entrypoints — thin re-export of the native TITO engine.""" + +from __future__ import annotations + +from enum import Enum +from typing import Any + +from .engine.pretokenize import get_tito_tokenizer as _engine_get_tito_tokenizer + + +class TITOTokenizerType(str, Enum): + """Tokenizer families the native engine supports. Other models are a small + subclass + a fixed chat template — see agentix.tito.engine.pretokenize.""" + + DEFAULT = "default" + QWEN3 = "qwen3" + + +def get_tito_tokenizer( + tokenizer: Any, + tokenizer_type: TITOTokenizerType | str = TITOTokenizerType.DEFAULT, + *, + allowed_append_roles: tuple[str, ...] | list[str] | None = None, + **_ignored: Any, +) -> Any: + """Build a TITO tokenizer for *tokenizer* (`"qwen3"` or `"default"`).""" + t = tokenizer_type.value if isinstance(tokenizer_type, TITOTokenizerType) else str(tokenizer_type) + roles = tuple(allowed_append_roles) if allowed_append_roles else ("tool",) + return _engine_get_tito_tokenizer(tokenizer, t, allowed_append_roles=roles) diff --git a/plugins/tito/pyproject.toml b/plugins/tito/pyproject.toml new file mode 100644 index 0000000..e1a96c2 --- /dev/null +++ b/plugins/tito/pyproject.toml @@ -0,0 +1,71 @@ +[build-system] +requires = ["uv_build>=0.7,<0.9"] +build-backend = "uv_build" + +[project] +name = "agentix-tito" +version = "0.1.0" +description = "Agentix TITO plugin — token-in-token-out session-recording gateway." +readme = "README.md" +requires-python = ">=3.10" +license = "Apache-2.0" +authors = [ + { name = "Agentix maintainers" }, +] +keywords = ["agentix", "tito", "agentic", "chat-template", "gateway", "rollout"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Environment :: Console", + "Framework :: FastAPI", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Internet :: WWW/HTTP :: HTTP Servers", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] +# The gateway tokenizes prompts itself (the native TITO engine — see +# agentix.tito.engine), so transformers + tokenizers + jinja2 are real runtime +# deps. They are isolated to THIS plugin — agentix core / abridge never pull +# them. No sglang dependency: the engine defines the one pydantic `Tool` type it +# needs itself. +dependencies = [ + # As an `agentix.tito` plugin it lives in the agentix namespace, so importing it + # runs agentix core's __init__ — hence the agentixx dep (pure plumbing: socketio, + # msgpack, fastapi; zero ML/training-framework code). + "agentixx", + "fastapi>=0.110", + "httpx>=0.27", + "pydantic>=2", + "setproctitle>=1.3", + "uvicorn>=0.29", + "transformers>=4.44", + "tokenizers>=0.19", + "jinja2>=3.1", + "huggingface-hub>=0.23", +] + +[project.optional-dependencies] +test = ["pytest>=8", "pytest-asyncio>=0.23"] + +[project.urls] +Homepage = "https://github.com/Agentix-Project/Agentix" + +[project.scripts] +agentix-tito = "agentix.tito.cli:main" + +[tool.uv.sources] +agentixx = { workspace = true } + +# uv_build, like the other plugins: ship under the `agentix.tito` namespace. +[tool.uv.build-backend] +module-name = "agentix.tito" +module-root = "" +namespace = true + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["."] +addopts = "-q" diff --git a/plugins/tito/tests/package/test_cli.py b/plugins/tito/tests/package/test_cli.py new file mode 100644 index 0000000..925e995 --- /dev/null +++ b/plugins/tito/tests/package/test_cli.py @@ -0,0 +1,41 @@ +import pytest + +from agentix.tito.cli import build_parser, main + + +def test_cli_top_level_help(capsys): + with pytest.raises(SystemExit): + main(["--help"]) + assert "serve" in capsys.readouterr().out + + +def test_cli_serve_help(capsys): + with pytest.raises(SystemExit): + main(["serve", "--help"]) + out = capsys.readouterr().out + assert "--hf-checkpoint" in out + assert "--tito-model" in out + assert "--tito-allowed-append-roles" in out + + +def test_cli_serve_parses_args(): + args = build_parser().parse_args( + [ + "serve", + "--hf-checkpoint", "Qwen/Qwen3-0.6B", + "--backend-url", "http://127.0.0.1:8000", + "--tito-model", "qwen3", + "--tito-allowed-append-roles", "tool", "user", + ] + ) + assert args.command == "serve" + assert args.hf_checkpoint == "Qwen/Qwen3-0.6B" + assert args.tito_model == "qwen3" + assert args.tito_allowed_append_roles == ["tool", "user"] + + +def test_cli_tito_model_choices_are_qwen3_and_default(): + args = build_parser().parse_args(["serve", "--hf-checkpoint", "X"]) + assert args.tito_model == "default" + with pytest.raises(SystemExit): + build_parser().parse_args(["serve", "--hf-checkpoint", "X", "--tito-model", "glm47"]) diff --git a/sidecars/tito/tests/package/test_config_discovery.py b/plugins/tito/tests/package/test_config_discovery.py similarity index 93% rename from sidecars/tito/tests/package/test_config_discovery.py rename to plugins/tito/tests/package/test_config_discovery.py index 6b6752b..7bc1a02 100644 --- a/sidecars/tito/tests/package/test_config_discovery.py +++ b/plugins/tito/tests/package/test_config_discovery.py @@ -1,7 +1,7 @@ import pytest -from tito_gateway.config import TITOGatewayConfig -from tito_gateway import discovery +from agentix.tito.config import TITOGatewayConfig +from agentix.tito import discovery def test_explicit_backend_url_wins_over_environment_and_probe(monkeypatch): @@ -113,22 +113,21 @@ def test_no_live_probe_candidate_fails_clearly(monkeypatch): discovery.discover_backend_url(env={}, probe_candidates=("http://dead.example:8000",)) -def test_cli_json_kwargs_parse_to_dict(): +def test_from_cli_values_maps_fields(): config = TITOGatewayConfig.from_cli_values( hf_checkpoint="model", backend_url="http://backend", chat_template_path=None, - apply_chat_template_kwargs='{"enable_thinking": false}', tito_model="qwen3", tito_allowed_append_roles=["tool", "user"], session_server_ip="127.0.0.1", session_server_port=30000, - miles_router_timeout=30, + router_timeout=30, backend_probe_candidates=["http://probe-a:8000", "probe-b:8001"], backend_probe_timeout=2.0, ) - assert config.apply_chat_template_kwargs == {"enable_thinking": False} + assert config.router_timeout == 30 assert config.tito_allowed_append_roles == ("tool", "user") assert config.backend_probe_candidates == ("http://probe-a:8000", "probe-b:8001") assert config.backend_probe_timeout == 2.0 diff --git a/plugins/tito/tests/package/test_engine.py b/plugins/tito/tests/package/test_engine.py new file mode 100644 index 0000000..18404cb --- /dev/null +++ b/plugins/tito/tests/package/test_engine.py @@ -0,0 +1,141 @@ +"""Self-contained tests for the native TITO engine. + +These build a tiny in-memory tokenizer (no model download) and assert the engine's +invariants directly: the incremental tokenization equals a from-scratch render, the +comparator classifies mismatches correctly, message matching collapses falsy +sentinels, and the session state machine rolls back to the last assistant checkpoint. +""" + +from __future__ import annotations + +import pytest +from tokenizers import Tokenizer, models, pre_tokenizers +from transformers import PreTrainedTokenizerFast + +from agentix.tito.engine.compare import MismatchType, TokenSeqComparator +from agentix.tito.engine.messages import assert_messages_append_only_with_allowed_role, message_matches +from agentix.tito.engine.pretokenize import Qwen3TITOTokenizer, get_tito_tokenizer +from agentix.tito.engine.trajectory import LinearTrajectory, SessionRegistry + + +@pytest.fixture(scope="module") +def tok(): + specials = ["", "", "", "<|im_start|>", "<|im_end|>"] + words = ["system", "user", "assistant", "tool", "dummy", "You", "are", "ok", + "done", "compute", "17", "23", "391", "X", "Y", "Hello"] + vocab = {t: i for i, t in enumerate(specials + words)} + tk = Tokenizer(models.WordLevel(vocab=vocab, unk_token="")) + tk.pre_tokenizer = pre_tokenizers.Whitespace() + t = PreTrainedTokenizerFast( + tokenizer_object=tk, unk_token="", bos_token="", eos_token="", + additional_special_tokens=["<|im_start|>", "<|im_end|>"], + ) + t.chat_template = ( + "{%- for m in messages -%}<|im_start|>{{ m['role'] }} {{ m['content'] or '' }}<|im_end|>{%- endfor -%}" + "{%- if add_generation_prompt -%}<|im_start|>assistant {%- endif -%}" + ) + return t + + +def _types(ms): + return [(m.type, m.segment_index) for m in ms] + + +def test_comparator_classifies_mismatches(tok): + cmp = TokenSeqComparator(tok, assistant_start_str="<|im_start|>assistant") + ims, ime = tok.convert_tokens_to_ids("<|im_start|>"), tok.convert_tokens_to_ids("<|im_end|>") + S, U, A = (tok.convert_tokens_to_ids(w) for w in ("system", "user", "assistant")) + Y391, Y23, Yok, YH = (tok.convert_tokens_to_ids(w) for w in ("391", "23", "ok", "Hello")) + + assert cmp.compare_sequences([ims, U, Y391, ime], [ims, U, Y391, ime]) == [] + assert _types(cmp.compare_sequences([ims, U, Y391, ime], [ims, U, Y23, ime])) == [ + (MismatchType.NON_ASSISTANT_TEXT, 1) + ] + assert _types(cmp.compare_sequences([ims, A, Yok, ime], [ims, A, YH, ime])) == [ + (MismatchType.ASSISTANT_TEXT, 1) + ] + assert _types(cmp.compare_sequences([ims, Y391, ime], [ims, Y391, ime, ims])) == [ + (MismatchType.SPECIAL_TOKEN_COUNT, -1) + ] + assert _types(cmp.compare_sequences([ims, Y391, ime], [ime, Y391, ims])) == [ + (MismatchType.SPECIAL_TOKEN_TYPE, 0), + (MismatchType.SPECIAL_TOKEN_TYPE, 2), + ] + # trailing trim removes false structural diffs + assert cmp.compare_sequences([ims, Y391, ime], [ims, Y391, ime, ime], trim_trailing_ids={ime}) == [] + + +def test_message_matches_collapses_falsy_sentinels(): + assert message_matches({"role": "a", "content": ""}, {"role": "a", "content": None}) + assert message_matches({"role": "a", "tool_calls": []}, {"role": "a", "tool_calls": None}) + assert message_matches({"role": "u", "content": "x"}, {"role": "u", "content": "x", "extra": 1}) + # reasoning_content "\n\n" is non-falsy → not collapsed (the bug we hit) + assert not message_matches({"role": "a", "reasoning_content": "\n\n"}, {"role": "a", "reasoning_content": None}) + assert not message_matches({"role": "u", "content": "x"}, {"role": "t", "content": "x"}) + + +def test_append_only_enforced(): + stored = [{"role": "user", "content": "x"}] + assert_messages_append_only_with_allowed_role(stored, stored + [{"role": "tool", "content": "y"}], ["tool"]) + with pytest.raises(ValueError): + assert_messages_append_only_with_allowed_role(stored, stored + [{"role": "user", "content": "z"}], ["tool"]) + with pytest.raises(ValueError): + assert_messages_append_only_with_allowed_role(stored, [{"role": "user", "content": "DIFF"}], ["tool"]) + + +@pytest.mark.parametrize( + "appends", + [ + [{"role": "tool", "content": "391"}], + [{"role": "tool", "content": "391"}, {"role": "tool", "content": "23"}], + [{"role": "user", "content": "Hello"}], + [{"role": "tool", "content": "X"}, {"role": "user", "content": "Y"}], + ], +) +def test_incremental_equals_full_render(tok, appends): + """The core invariant: merge(prefix, incremental) == full from-scratch render.""" + tt = get_tito_tokenizer(tok, "default", allowed_append_roles=("tool", "user")) + old = [{"role": "system", "content": "You are"}, {"role": "user", "content": "compute 17 23"}, + {"role": "assistant", "content": "ok"}] + new = old + appends + prefix = tt.render_messages(old, add_generation_prompt=False, tokenize=True) + merged = tt.merge_tokens(old, new, prefix, None) + full = tt.render_messages(new, add_generation_prompt=True, tokenize=True) + assert merged == full + + +def test_qwen3_newline_fixup(): + class FakeTok: + def encode(self, t, add_special_tokens=False): + return [99] # "\n" -> single id + + def convert_tokens_to_ids(self, t): + return 88 # "<|im_end|>" + + q = Qwen3TITOTokenizer(FakeTok(), chat_template_kwargs={"chat_template": "x"}) + q.tokenize_additional_non_assistant = lambda o, n, t=None: [1, 2, 3] + assert q.merge_tokens([], [], [7, 88], None) == [7, 88, 99, 1, 2, 3] # prefix ends in im_end -> insert \n + assert q.merge_tokens([], [], [7, 5], None) == [7, 5, 1, 2, 3] # otherwise no insert + + +def test_trajectory_rollback_to_assistant_checkpoint(tok): + tt = get_tito_tokenizer(tok, "default", allowed_append_roles=("tool", "user")) + reg = SessionRegistry(None, tok, tito_tokenizer=tt) + tr = LinearTrajectory() + sys = [{"role": "system", "content": "You are"}, {"role": "user", "content": "compute 17 23"}] + a0 = {"role": "assistant", "content": "ok"} + + tr.prepare_pretokenized(sys, None, tito_tokenizer=tt) + tr.update_pretokenized_state(sys, a0, tt.render_messages(sys + [a0], add_generation_prompt=False, tokenize=True), [], tt.max_trim_tokens) + + m1 = sys + [a0, {"role": "tool", "content": "391"}] + a1 = {"role": "assistant", "content": "done"} + tr.prepare_pretokenized(m1, None, tito_tokenizer=tt) + tr.update_pretokenized_state(m1, a1, tt.render_messages(m1 + [a1], add_generation_prompt=False, tokenize=True), [], tt.max_trim_tokens) + assert tr.num_assistant == 2 + assert reg.compute_session_mismatch(tr) == [] # clean chain → no mismatch + + # retry the tool turn with a different result → rollback to a0 checkpoint + tr.prepare_pretokenized(sys + [a0, {"role": "tool", "content": "X"}], None, tito_tokenizer=tt) + assert tr.num_assistant == 1 + assert [m.get("role") for m in tr.messages] == ["system", "user", "assistant"] diff --git a/sidecars/tito/tests/package/test_import_surface.py b/plugins/tito/tests/package/test_import_surface.py similarity index 60% rename from sidecars/tito/tests/package/test_import_surface.py rename to plugins/tito/tests/package/test_import_surface.py index facb732..3bb0fa0 100644 --- a/sidecars/tito/tests/package/test_import_surface.py +++ b/plugins/tito/tests/package/test_import_surface.py @@ -2,31 +2,34 @@ def test_public_import_surface(): - import tito_gateway - from tito_gateway import TITOGateway, TITOGatewayConfig, SessionServer, get_tito_tokenizer + import agentix.tito + from agentix.tito import TITOGateway, TITOGatewayConfig, SessionServer, get_tito_tokenizer - assert tito_gateway.TITOGateway is TITOGateway - assert tito_gateway.TITOGatewayConfig is TITOGatewayConfig - assert tito_gateway.SessionServer is SessionServer + assert agentix.tito.TITOGateway is TITOGateway + assert agentix.tito.TITOGatewayConfig is TITOGatewayConfig + assert agentix.tito.SessionServer is SessionServer assert callable(get_tito_tokenizer) def test_config_requires_hf_checkpoint(): - from tito_gateway import TITOGatewayConfig + from agentix.tito import TITOGatewayConfig with pytest.raises(ValueError, match="hf_checkpoint is required"): TITOGatewayConfig(hf_checkpoint="") def test_gateway_constructs_with_explicit_backend(monkeypatch): - import tito_gateway.gateway as gateway_module - from tito_gateway import TITOGateway + import agentix.tito.gateway as gateway_module + from agentix.tito import TITOGateway class FakeSessionServer: def __init__(self, args, backend_url): + from fastapi import FastAPI + self.args = args self.backend_url = backend_url - self.app = object() + # A real app: the gateway registers a `/healthz` alias on it at construct. + self.app = FastAPI() monkeypatch.setattr(gateway_module, "SessionServer", FakeSessionServer) diff --git a/sidecars/tito/tests/test_pool.py b/plugins/tito/tests/test_pool.py similarity index 97% rename from sidecars/tito/tests/test_pool.py rename to plugins/tito/tests/test_pool.py index da47d77..fabf3e3 100644 --- a/sidecars/tito/tests/test_pool.py +++ b/plugins/tito/tests/test_pool.py @@ -3,7 +3,7 @@ from __future__ import annotations import pytest -from tito_gateway.pool import BackendPool +from agentix.tito.pool import BackendPool A, B, C = "http://h1:8000", "http://h2:8000", "http://h3:8000" diff --git a/sidecars/tito/tests/test_pool_routing.py b/plugins/tito/tests/test_pool_routing.py similarity index 51% rename from sidecars/tito/tests/test_pool_routing.py rename to plugins/tito/tests/test_pool_routing.py index 454ab5d..73dcc08 100644 --- a/sidecars/tito/tests/test_pool_routing.py +++ b/plugins/tito/tests/test_pool_routing.py @@ -1,59 +1,24 @@ """Wiring tests for BackendPool routing in the SessionServer (no model/GPU). -Uses ``hf_checkpoint=None`` so the vendored session server skips tokenizer/route -setup — we drive the pool-aware ``do_proxy`` / forget hook directly. +Uses ``hf_checkpoint=None`` so the session server skips tokenizer/route setup — +we drive the pool-aware ``do_proxy`` / forget hook directly. """ from __future__ import annotations -import sys import types import pytest - -def _install_sglang_stub() -> None: - """Stub `sglang...Tool` so the vendored template module imports without the - sglang runtime (this routing test needs no model). Mirrors tito_experiment.""" - if "sglang" in sys.modules: - return - from typing import Any, Optional - - from pydantic import BaseModel - - class _Function(BaseModel): - name: str - description: Optional[str] = None - parameters: Optional[dict[str, Any]] = None - - class Tool(BaseModel): - type: str = "function" - function: _Function - - names = ["sglang", "sglang.srt", "sglang.srt.entrypoints", "sglang.srt.entrypoints.openai"] - mods = {n: types.ModuleType(n) for n in names} - protocol = types.ModuleType("sglang.srt.entrypoints.openai.protocol") - protocol.Tool = Tool # type: ignore[attr-defined] - mods["sglang.srt.entrypoints.openai"].protocol = protocol # type: ignore[attr-defined] - mods["sglang.srt.entrypoints"].openai = mods["sglang.srt.entrypoints.openai"] # type: ignore[attr-defined] - mods["sglang.srt"].entrypoints = mods["sglang.srt.entrypoints"] # type: ignore[attr-defined] - mods["sglang"].srt = mods["sglang.srt"] # type: ignore[attr-defined] - for n, m in mods.items(): - sys.modules[n] = m - sys.modules["sglang.srt.entrypoints.openai.protocol"] = protocol - - -_install_sglang_stub() - -from tito_gateway.pool import BackendPool # noqa: E402 -from tito_gateway.server import SessionServer, _session_id_from_path # noqa: E402 +from agentix.tito.pool import BackendPool +from agentix.tito.server import SessionServer, _session_id_from_path A = "http://a:8000" B = "http://b:8000" def _args(): - return types.SimpleNamespace(hf_checkpoint=None, miles_router_timeout=600.0) + return types.SimpleNamespace(hf_checkpoint=None, router_timeout=600.0) class _URL: @@ -99,9 +64,9 @@ async def fake_request(method, url, content=None, headers=None): seen.append(url) return _Resp() - monkeypatch.setattr(srv._impl.client, "request", fake_request) + monkeypatch.setattr(srv._backend.client, "request", fake_request) for _ in range(3): - await srv._impl.do_proxy(_Request("/sessions/s1/v1/chat/completions"), "v1/chat/completions") + await srv._backend.do_proxy(_Request("/sessions/s1/v1/chat/completions"), "v1/chat/completions") # all three turns of one session hit the same backend (prefix-cache locality) assert len({u.split("/v1/")[0] for u in seen}) == 1 @@ -116,8 +81,8 @@ async def test_transport_error_reports_backend_down(monkeypatch): async def boom(method, url, content=None, headers=None): raise httpx.ConnectError("refused") - monkeypatch.setattr(srv._impl.client, "request", boom) - result = await srv._impl.do_proxy(_Request("/sessions/s9/v1/chat/completions"), "v1/chat/completions") + monkeypatch.setattr(srv._backend.client, "request", boom) + result = await srv._backend.do_proxy(_Request("/sessions/s9/v1/chat/completions"), "v1/chat/completions") assert result["status_code"] == 502 # the picked backend was marked down assert pool._down # noqa: SLF001 - asserting routing side effect @@ -133,5 +98,5 @@ async def test_forget_on_delete_drops_pin(): async def call_next(_req): return _Resp(status=204) - await srv._impl._forget_on_delete(_Request("/sessions/s2", method="DELETE"), call_next) + await srv._forget_on_delete(_Request("/sessions/s2", method="DELETE"), call_next) assert "s2" not in pool._assigned # noqa: SLF001 diff --git a/pyproject.toml b/pyproject.toml index 1ca6ade..039d2de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,6 +88,7 @@ members = [ "plugins/agents/*", "plugins/datasets/*", "plugins/providers/*", + "plugins/tito", ] # `examples/*` are standalone projects, not members — each has its own # lock and path-depends on the workspace. Excluding them lets `uv lock` @@ -122,6 +123,7 @@ include = [ "plugins/providers/e2b/agentix", "plugins/runtime-basic/agentix", "plugins/trace-otel/agentix", + "plugins/tito/agentix", ] exclude = ["**/__pycache__", "**/.venv", ".venv", "**/build"] # `agentix` is a pkgutil namespace package — the core lives here, the @@ -142,6 +144,7 @@ extraPaths = [ "plugins/providers/e2b", "plugins/runtime-basic", "plugins/trace-otel", + "plugins/tito", ] typeCheckingMode = "basic" # venv lives on local disk (the repo is on a slow FUSE mount); `.venv` diff --git a/sidecars/README.md b/sidecars/README.md index 761f6f4..40ca8c9 100644 --- a/sidecars/README.md +++ b/sidecars/README.md @@ -2,19 +2,20 @@ Host-side gateway sidecars that abridge forwards to. These are **standalone vendored projects** — NOT uv workspace members, NOT part of the `agentix` -package. abridge core stays shape/protocol-blind; all protocol and ML logic -lives here, behind a localhost HTTP process. +package. abridge core stays shape/protocol-blind; all protocol logic lives +here, behind a localhost HTTP process. - `cc_convert/` — Anthropic ↔ OpenAI translation sidecar (Rust core + axum binary + PyO3 wheel). abridge's `agentix.bridge.sidecars.cc_convert_sidecar(...)` preset launches the `cc_convert_sidecar` binary. -- `tito/` — TITO pretokenize + session-recording gateway (FastAPI, wraps - Miles). Sits in front of an sglang / OpenAI-compatible backend and emits - pretokenized RL rollout trajectories. -Each keeps its own build system and dependencies; nothing here is installed -into the core venv. Upstream attributions are preserved in each subtree -(`cc_convert/LICENSE-*`, `tito/VENDORED_MILES_AUDIT.md`). +The TITO pretokenize + session-recording gateway used to live here; it is now a +first-class Agentix plugin at `plugins/tito` (`import agentix.tito`), natively +implemented with no vendored code. + +Each sidecar keeps its own build system and dependencies; nothing here is +installed into the core venv. Upstream attributions are preserved in each +subtree (`cc_convert/LICENSE-*`). ## Status / planned refactor @@ -24,5 +25,3 @@ Vendored as-is to get the sources in-tree; refactor follows. binary requirement and drive translation from code in-process (it already exposes a PyO3 Python package under `cc_convert/python/`), so abridge can call it without launching a separate process. -- **tito** runs as a FastAPI sidecar; its session/trajectory records are - bridged onto the existing `/trace` channel (work in progress). diff --git a/sidecars/tito/.github/workflows/python-package.yml b/sidecars/tito/.github/workflows/python-package.yml deleted file mode 100644 index 662e7f9..0000000 --- a/sidecars/tito/.github/workflows/python-package.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Python package - -on: - push: - branches: [master] - tags: ["v*"] - pull_request: - branches: [master] - -jobs: - test: - name: Test Python ${{ matrix.python-version }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ["3.10", "3.11", "3.12"] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install package - run: python -m pip install -e '.[test]' - - name: Run package tests - run: pytest tests/package -q - - build: - name: Build distributions - runs-on: ubuntu-latest - needs: test - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Install build frontend - run: python -m pip install build - - name: Build wheel and sdist - run: python -m build - - uses: actions/upload-artifact@v4 - with: - name: python-distributions - path: dist/* - - publish: - name: Publish to PyPI - runs-on: ubuntu-latest - needs: build - if: startsWith(github.ref, 'refs/tags/v') - permissions: - id-token: write - environment: - name: pypi - url: https://pypi.org/p/tito-gateway - steps: - - uses: actions/download-artifact@v4 - with: - name: python-distributions - path: dist - - uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/sidecars/tito/README.md b/sidecars/tito/README.md deleted file mode 100644 index 14449ff..0000000 --- a/sidecars/tito/README.md +++ /dev/null @@ -1,83 +0,0 @@ -# TITO Gateway - -[English](README.md) | [中文](README.zh-CN.md) - -TITO Gateway is a standalone Python package and CLI wrapper around the Miles -Agentic Chat Template / TITO session-server work. It is a public-facing -packaging and usage layer for Miles' TITO implementation, not a rewrite of the -underlying algorithms. - -The TITO tokenizer, fixed chat-template approach, session trajectory model, -proxy behavior, and verifier flow are credited to Miles and its contributors. -Vendored source is tracked in `tito_gateway/VENDORED_MILES_AUDIT.md`. - -## Documentation - -- [Docs Home](docs/index.md) -- [Quickstart](docs/quickstart.md) -- [Concepts](docs/concepts.md) -- [Python API](docs/api.md) -- [CLI Reference](docs/cli.md) -- [Verification And Tests](docs/verification.md) -- [Development Notes](docs/development.md) -- [中文文档入口](docs/index.zh-CN.md) - -## Quickstart - -Install from PyPI: - -```bash -pip install tito-gateway -``` - -The copied Miles TITO logic is bundled in this package. Install optional -verifier dependencies only when you want to run the heavy verifier path locally: - -```bash -pip install 'tito-gateway[verify]' -``` - -Start the gateway beside an OpenAI-compatible backend: - -```bash -tito-gateway --hf-checkpoint Qwen/Qwen3-0.6B \ - --tito-model qwen3 \ - --tito-allowed-append-roles tool user \ - --backend-url http://127.0.0.1:8000 \ - --session-server-port 30000 -``` - -Or embed it in Python: - -```python -from tito_gateway import TITOGateway, TITOGatewayConfig - -gateway = TITOGateway( - TITOGatewayConfig( - hf_checkpoint="Qwen/Qwen3-0.6B", - backend_url="http://127.0.0.1:8000", - tito_model="qwen3", - tito_allowed_append_roles=("tool", "user"), - ) -) - -app = gateway.app -``` - -Run the CPU-fast test suite: - -```bash -pip install -e '.[test]' -pytest tests/upstream tests/package -q -``` - -## Acknowledgement - -This package explicitly acknowledges Miles as the source of the underlying TITO -design and implementation: - -- Miles repository: https://github.com/radixark/miles -- Miles documentation: https://www.radixark.com/miles/docs/user-guide/agentic-chat-template - -Future vendoring should preserve upstream notices, keep the source audit -current, and keep copied upstream tests as the compatibility contract. diff --git a/sidecars/tito/README.zh-CN.md b/sidecars/tito/README.zh-CN.md deleted file mode 100644 index f81ce0c..0000000 --- a/sidecars/tito/README.zh-CN.md +++ /dev/null @@ -1,81 +0,0 @@ -# TITO Gateway - -[English](README.md) | [中文](README.zh-CN.md) - -TITO Gateway 是围绕 Miles Agentic Chat Template / TITO session-server 工作做的 -独立 Python package 和 CLI 封装。它是对 Miles TITO 实现的公开封装和使用层，不是 -底层算法的重新实现。 - -TITO tokenizer、fixed chat-template 方案、session trajectory 模型、proxy 行为和验证 -流程都来自 Miles 及其贡献者。vendored source 的审计记录在 -`tito_gateway/VENDORED_MILES_AUDIT.md`。 - -## 文档入口 - -- [中文文档首页](docs/index.zh-CN.md) -- [快速开始](docs/quickstart.zh-CN.md) -- [核心概念](docs/concepts.zh-CN.md) -- [Python API](docs/api.zh-CN.md) -- [CLI 参考](docs/cli.zh-CN.md) -- [验证与测试](docs/verification.zh-CN.md) -- [开发说明](docs/development.zh-CN.md) -- [English Docs Home](docs/index.md) - -## 快速开始 - -从 PyPI 安装： - -```bash -pip install tito-gateway -``` - -Miles TITO 逻辑已经 copy/vendor 在这个 package 里。只有需要在本地跑 heavy verifier -路径时，才安装 verifier 可选依赖： - -```bash -pip install 'tito-gateway[verify]' -``` - -在 OpenAI-compatible backend 旁边启动 gateway： - -```bash -tito-gateway --hf-checkpoint Qwen/Qwen3-0.6B \ - --tito-model qwen3 \ - --tito-allowed-append-roles tool user \ - --backend-url http://127.0.0.1:8000 \ - --session-server-port 30000 -``` - -或在 Python 中嵌入： - -```python -from tito_gateway import TITOGateway, TITOGatewayConfig - -gateway = TITOGateway( - TITOGatewayConfig( - hf_checkpoint="Qwen/Qwen3-0.6B", - backend_url="http://127.0.0.1:8000", - tito_model="qwen3", - tito_allowed_append_roles=("tool", "user"), - ) -) - -app = gateway.app -``` - -运行 CPU-fast 测试： - -```bash -pip install -e '.[test]' -pytest tests/upstream tests/package -q -``` - -## 致谢 - -本 package 明确致谢 Miles：底层 TITO 设计和实现来自 Miles 项目。 - -- Miles repository: https://github.com/radixark/miles -- Miles documentation: https://www.radixark.com/miles/docs/user-guide/agentic-chat-template - -后续 vendor Miles 源码时，需要保留 upstream notices，更新 source audit，并保持 copied -upstream tests 作为兼容性合同。 diff --git a/sidecars/tito/docs/api.md b/sidecars/tito/docs/api.md deleted file mode 100644 index 7051b4f..0000000 --- a/sidecars/tito/docs/api.md +++ /dev/null @@ -1,50 +0,0 @@ -# Python API - -[Docs Home](index.md) | [中文](api.zh-CN.md) - -## Import Surface - -```python -from tito_gateway import TITOGateway, TITOGatewayConfig, SessionServer, get_tito_tokenizer -``` - -## Gateway Configuration - -```python -from tito_gateway import TITOGateway, TITOGatewayConfig - -config = TITOGatewayConfig( - hf_checkpoint="Qwen/Qwen3-0.6B", - backend_url="http://127.0.0.1:8000", - chat_template_path=None, - apply_chat_template_kwargs={"enable_thinking": False}, - tito_model="qwen3", - tito_allowed_append_roles=("tool", "user"), - session_server_ip="127.0.0.1", - session_server_port=30000, - miles_router_timeout=600.0, -) - -gateway = TITOGateway(config) -app = gateway.app -``` - -## Important Fields - -- `hf_checkpoint`: required checkpoint or model ID for tokenizer loading. -- `backend_url`: explicit backend URL. If omitted, discovery is used. -- `chat_template_path`: optional fixed chat template. -- `apply_chat_template_kwargs`: JSON-like dict passed into template rendering. -- `tito_model`: Miles TITO tokenizer family. -- `tito_allowed_append_roles`: roles allowed after an assistant turn. -- `backend_probe_candidates`: optional tuple of URLs to probe. -- `backend_probe_timeout`: per-endpoint probe timeout. - -## Running Directly - -```python -gateway.run() -``` - -For ASGI composition, prefer `gateway.app` and mount or serve it with your own -server process. diff --git a/sidecars/tito/docs/api.zh-CN.md b/sidecars/tito/docs/api.zh-CN.md deleted file mode 100644 index b3fe19d..0000000 --- a/sidecars/tito/docs/api.zh-CN.md +++ /dev/null @@ -1,49 +0,0 @@ -# Python API - -[文档首页](index.zh-CN.md) | [English](api.md) - -## Import Surface - -```python -from tito_gateway import TITOGateway, TITOGatewayConfig, SessionServer, get_tito_tokenizer -``` - -## Gateway 配置 - -```python -from tito_gateway import TITOGateway, TITOGatewayConfig - -config = TITOGatewayConfig( - hf_checkpoint="Qwen/Qwen3-0.6B", - backend_url="http://127.0.0.1:8000", - chat_template_path=None, - apply_chat_template_kwargs={"enable_thinking": False}, - tito_model="qwen3", - tito_allowed_append_roles=("tool", "user"), - session_server_ip="127.0.0.1", - session_server_port=30000, - miles_router_timeout=600.0, -) - -gateway = TITOGateway(config) -app = gateway.app -``` - -## 重要字段 - -- `hf_checkpoint`：必填 checkpoint 或 model ID，用于 tokenizer loading。 -- `backend_url`：显式 backend URL；不填时走 discovery。 -- `chat_template_path`：可选 fixed chat template。 -- `apply_chat_template_kwargs`：传给 template rendering 的 dict。 -- `tito_model`：Miles TITO tokenizer family。 -- `tito_allowed_append_roles`：assistant turn 后允许追加的 roles。 -- `backend_probe_candidates`：可选 probe URL 列表。 -- `backend_probe_timeout`：每个 probe endpoint 的 timeout。 - -## 直接运行 - -```python -gateway.run() -``` - -如果要做 ASGI 组合，建议使用 `gateway.app`，交给自己的 server process mount 或 serve。 diff --git a/sidecars/tito/docs/cli.md b/sidecars/tito/docs/cli.md deleted file mode 100644 index 85f270f..0000000 --- a/sidecars/tito/docs/cli.md +++ /dev/null @@ -1,50 +0,0 @@ -# CLI Reference - -[Docs Home](index.md) | [中文](cli.zh-CN.md) - -## Serve - -```bash -tito-gateway serve --hf-checkpoint Qwen/Qwen3-0.6B \ - --tito-model qwen3 \ - --tito-allowed-append-roles tool user \ - --backend-url http://127.0.0.1:8000 \ - --session-server-port 30000 -``` - -The top-level command aliases to `serve`, so the `serve` word can be omitted. - -## Template Kwargs - -```bash -tito-gateway serve --hf-checkpoint Qwen/Qwen3-0.6B \ - --backend-url http://127.0.0.1:8000 \ - --apply-chat-template-kwargs '{"enable_thinking": false}' -``` - -## Backend Probe Flags - -```bash -tito-gateway serve --hf-checkpoint Qwen/Qwen3-0.6B \ - --backend-probe-candidate http://127.0.0.1:8000 \ - --backend-probe-candidate http://127.0.0.1:30000 \ - --backend-probe-timeout 0.5 -``` - -## Verifiers - -```bash -tito-gateway verify-chat-template --template path/to/template.jinja --thinking off -``` - -```bash -tito-gateway verify-session-tito-tokenizer \ - --hf-checkpoint Qwen/Qwen3-4B \ - --tito-model qwen3 \ - --tito-allowed-append-roles tool user \ - --sglang-reasoning-parser qwen3 \ - --sglang-tool-call-parser qwen25 \ - --rollout-num-gpus-per-engine 1 -``` - -Use `--help` on any subcommand for the full parser surface. diff --git a/sidecars/tito/docs/cli.zh-CN.md b/sidecars/tito/docs/cli.zh-CN.md deleted file mode 100644 index 4d0ca49..0000000 --- a/sidecars/tito/docs/cli.zh-CN.md +++ /dev/null @@ -1,50 +0,0 @@ -# CLI 参考 - -[文档首页](index.zh-CN.md) | [English](cli.md) - -## Serve - -```bash -tito-gateway serve --hf-checkpoint Qwen/Qwen3-0.6B \ - --tito-model qwen3 \ - --tito-allowed-append-roles tool user \ - --backend-url http://127.0.0.1:8000 \ - --session-server-port 30000 -``` - -顶层命令会 alias 到 `serve`，所以可以省略 `serve`。 - -## Template Kwargs - -```bash -tito-gateway serve --hf-checkpoint Qwen/Qwen3-0.6B \ - --backend-url http://127.0.0.1:8000 \ - --apply-chat-template-kwargs '{"enable_thinking": false}' -``` - -## Backend Probe Flags - -```bash -tito-gateway serve --hf-checkpoint Qwen/Qwen3-0.6B \ - --backend-probe-candidate http://127.0.0.1:8000 \ - --backend-probe-candidate http://127.0.0.1:30000 \ - --backend-probe-timeout 0.5 -``` - -## Verifiers - -```bash -tito-gateway verify-chat-template --template path/to/template.jinja --thinking off -``` - -```bash -tito-gateway verify-session-tito-tokenizer \ - --hf-checkpoint Qwen/Qwen3-4B \ - --tito-model qwen3 \ - --tito-allowed-append-roles tool user \ - --sglang-reasoning-parser qwen3 \ - --sglang-tool-call-parser qwen25 \ - --rollout-num-gpus-per-engine 1 -``` - -每个 subcommand 都可以用 `--help` 查看完整 parser surface。 diff --git a/sidecars/tito/docs/concepts.md b/sidecars/tito/docs/concepts.md deleted file mode 100644 index a53f549..0000000 --- a/sidecars/tito/docs/concepts.md +++ /dev/null @@ -1,55 +0,0 @@ -# Concepts - -[Docs Home](index.md) | [中文](concepts.zh-CN.md) - -## Runtime Model - -The gateway has three parts: - -1. An OpenAI-compatible backend. -2. A FastAPI gateway that wraps the Miles session server. -3. Clients that create sessions and send chat completions through session routes. - -Typical flow: - -1. Start or discover a backend. -2. Start the gateway with checkpoint and TITO options. -3. Create a session with `POST /sessions`. -4. Send chat requests through `/sessions/{session_id}/v1/chat/completions`. -5. The gateway injects session/TITO metadata and updates token state from backend response metadata. - -## Session Invariants - -Miles' TITO path relies on append-only message history. The exception is the -Miles-supported rollback around the latest assistant checkpoint. - -`tito_allowed_append_roles` declares which roles may be appended after an -assistant turn. `tool` is the default role surface. - -## Backend Discovery - -Backend selection is deterministic: - -1. Explicit config or `--backend-url`. -2. Environment variables: - - `TITO_BACKEND_URL` - - `OPENAI_BASE_URL` - - `SGLANG_BASE_URL` -3. Configured probe candidates, in order. - -Each probe candidate is checked at `/health` first and `/v1/models` second. The -first live candidate wins. If no backend is found, startup fails before binding -the gateway. - -## Session Routes - -- `GET /health` -- `POST /sessions` -- `GET /sessions/{session_id}` -- `DELETE /sessions/{session_id}` -- `POST /sessions/{session_id}/v1/chat/completions` - -Chat completion requests are proxied to the backend. The proxy path injects the -fields expected by the Miles implementation, including token IDs and metadata -requests. Backend responses must include output token logprob metadata so the -session trajectory can update. diff --git a/sidecars/tito/docs/concepts.zh-CN.md b/sidecars/tito/docs/concepts.zh-CN.md deleted file mode 100644 index c508363..0000000 --- a/sidecars/tito/docs/concepts.zh-CN.md +++ /dev/null @@ -1,53 +0,0 @@ -# 核心概念 - -[文档首页](index.zh-CN.md) | [English](concepts.md) - -## 运行模型 - -Gateway 有三部分： - -1. 一个 OpenAI-compatible backend。 -2. 一个封装 Miles session server 的 FastAPI gateway。 -3. 客户端先创建 session，再通过 session routes 发送 chat completions。 - -典型流程： - -1. 启动或发现 backend。 -2. 用 checkpoint 和 TITO 配置启动 gateway。 -3. 通过 `POST /sessions` 创建 session。 -4. 通过 `/sessions/{session_id}/v1/chat/completions` 发送 chat 请求。 -5. Gateway 注入 session/TITO metadata，并从 backend response metadata 更新 token 状态。 - -## Session 不变量 - -Miles 的 TITO 路径依赖 append-only message history。例外是 Miles 支持的围绕最新 -assistant checkpoint 的 rollback。 - -`tito_allowed_append_roles` 声明 assistant turn 后允许追加的角色。默认 role surface 是 -`tool`。 - -## Backend 自动发现 - -Backend 选择是 deterministic 的： - -1. 显式配置或 `--backend-url`。 -2. 环境变量： - - `TITO_BACKEND_URL` - - `OPENAI_BASE_URL` - - `SGLANG_BASE_URL` -3. 配置的 probe candidates，按顺序。 - -每个 probe candidate 先检查 `/health`，再检查 `/v1/models`。第一个 live candidate -会被选中。如果找不到 backend，gateway 会在绑定前失败。 - -## Session Routes - -- `GET /health` -- `POST /sessions` -- `GET /sessions/{session_id}` -- `DELETE /sessions/{session_id}` -- `POST /sessions/{session_id}/v1/chat/completions` - -Chat completion 请求会被 proxy 到 backend。proxy 路径会注入 Miles 实现所需字段，包括 -token IDs 和 metadata requests。Backend response 必须包含 output token logprob -metadata，session trajectory 才能更新。 diff --git a/sidecars/tito/docs/development.md b/sidecars/tito/docs/development.md deleted file mode 100644 index 680cb52..0000000 --- a/sidecars/tito/docs/development.md +++ /dev/null @@ -1,66 +0,0 @@ -# Development Notes - -[Docs Home](index.md) | [中文](development.zh-CN.md) - -## Source Policy - -This package is a public wrapper around Miles' work. The implementation should -reuse the upstream logic wherever possible. - -Maintenance rules: - -- Preserve Miles attribution and upstream notices. -- Keep the vendored source audit current. -- Prefer wrapper code over changing TITO algorithms. -- Keep copied upstream tests as the compatibility contract. -- Do not weaken negative tests. - -## Public Documentation Standard - -Public docs should make the relationship clear: - -- Miles owns the underlying TITO design and implementation. -- This package makes that work importable and runnable as a standalone gateway. -- Optional heavy verification depends on the Miles/SGLang training stack. - -## Local Setup - -Editable installs are for repository development: - -```bash -pip install -e '.[test]' -pytest tests/package -q -``` - -Public users should install the package distribution instead: - -```bash -pip install tito-gateway -pip install 'tito-gateway[verify]' -``` - -## Build A Distribution - -```bash -python -m pip install build -python -m build -python -m pip install dist/tito_gateway-0.1.0-py3-none-any.whl -tito-gateway --help -``` - -Releases should be tagged as `vX.Y.Z`. The GitHub Actions workflow builds on -every push and publishes to PyPI only from version tags, using PyPI trusted -publishing for the `pypi` environment. - -## Pre-Push Checklist - -1. Run CPU-fast tests. -2. Run CLI help smoke checks. -3. Confirm staged files do not include cache, model weights, credentials, or local env files. -4. Run a secret scan over staged and outgoing changes. -5. Check `git diff --check`. - -## Current Secret Guard - -The `guard-secret` skill can be used before pushing. If installed, run it before -`git push` and only push when it reports `SAFE_TO_PUSH`. diff --git a/sidecars/tito/docs/development.zh-CN.md b/sidecars/tito/docs/development.zh-CN.md deleted file mode 100644 index cd573a6..0000000 --- a/sidecars/tito/docs/development.zh-CN.md +++ /dev/null @@ -1,64 +0,0 @@ -# 开发说明 - -[文档首页](index.zh-CN.md) | [English](development.md) - -## 源码策略 - -这个 package 是对 Miles 工作的公开封装。实现上应尽可能复用 upstream 逻辑。 - -维护规则： - -- 保留 Miles attribution 和 upstream notices。 -- 持续更新 vendored source audit。 -- 优先写 wrapper，不改 TITO 算法。 -- copied upstream tests 是兼容性合同。 -- 不弱化 negative tests。 - -## 公开文档标准 - -公开文档需要讲清楚关系： - -- 底层 TITO 设计和实现来自 Miles。 -- 这个 package 让这套工作可以作为 standalone gateway 被 import 和运行。 -- optional heavy verification 依赖 Miles/SGLang training stack。 - -## 本地设置 - -Editable install 是给仓库开发用的： - -```bash -pip install -e '.[test]' -pytest tests/package -q -``` - -公开用户应该安装 package distribution： - -```bash -pip install tito-gateway -pip install 'tito-gateway[verify]' -``` - -## 构建 Distribution - -```bash -python -m pip install build -python -m build -python -m pip install dist/tito_gateway-0.1.0-py3-none-any.whl -tito-gateway --help -``` - -Release tag 使用 `vX.Y.Z`。GitHub Actions workflow 会在每次 push 时构建，在 version -tag 上通过 PyPI trusted publishing 发布到 `pypi` environment。 - -## Push 前检查 - -1. 跑 CPU-fast tests。 -2. 跑 CLI help smoke checks。 -3. 确认 staged files 不包含 cache、模型权重、credentials 或本地 env 文件。 -4. 对 staged 和 outgoing changes 做 secret scan。 -5. 检查 `git diff --check`。 - -## 当前 Secret Guard - -push 前可以使用 `guard-secret` skill。安装后，在 `git push` 前运行它；只有报告 -`SAFE_TO_PUSH` 时才继续 push。 diff --git a/sidecars/tito/docs/guide.md b/sidecars/tito/docs/guide.md deleted file mode 100644 index 9431391..0000000 --- a/sidecars/tito/docs/guide.md +++ /dev/null @@ -1,13 +0,0 @@ -# User Guide - -The user guide has been split into layered public documentation: - -- [Docs Home](index.md) -- [Quickstart](quickstart.md) -- [Concepts](concepts.md) -- [Python API](api.md) -- [CLI Reference](cli.md) -- [Verification And Tests](verification.md) -- [Development Notes](development.md) - -For Chinese documentation, see [中文文档入口](index.zh-CN.md). diff --git a/sidecars/tito/docs/guide.zh-CN.md b/sidecars/tito/docs/guide.zh-CN.md deleted file mode 100644 index 2530fb3..0000000 --- a/sidecars/tito/docs/guide.zh-CN.md +++ /dev/null @@ -1,13 +0,0 @@ -# 用户指南 - -用户指南已经拆成分层公开文档： - -- [中文文档首页](index.zh-CN.md) -- [快速开始](quickstart.zh-CN.md) -- [核心概念](concepts.zh-CN.md) -- [Python API](api.zh-CN.md) -- [CLI 参考](cli.zh-CN.md) -- [验证与测试](verification.zh-CN.md) -- [开发说明](development.zh-CN.md) - -英文文档见 [Docs Home](index.md)。 diff --git a/sidecars/tito/docs/index.md b/sidecars/tito/docs/index.md deleted file mode 100644 index 9ce76eb..0000000 --- a/sidecars/tito/docs/index.md +++ /dev/null @@ -1,22 +0,0 @@ -# TITO Gateway Documentation - -[English](index.md) | [中文](index.zh-CN.md) - -TITO Gateway is a public package and CLI wrapper around Miles' Agentic Chat -Template / TITO session-server work. It reuses Miles logic and exposes it in a -standalone package. - -## Start Here - -- [Quickstart](quickstart.md): install, run, and test the package. -- [Concepts](concepts.md): runtime model, session flow, and invariants. -- [Python API](api.md): embedding the gateway beside a backend. -- [CLI Reference](cli.md): serve command and backend discovery. -- [Verification And Tests](verification.md): verifier commands and CPU-fast test setup. -- [Development Notes](development.md): source policy, attribution, and release checks. - -## Public Attribution - -This package explicitly credits Miles as the source of the underlying TITO -implementation. It is a wrapper and usage layer, not a reimplementation. The -source audit lives in `tito_gateway/VENDORED_MILES_AUDIT.md`. diff --git a/sidecars/tito/docs/index.zh-CN.md b/sidecars/tito/docs/index.zh-CN.md deleted file mode 100644 index 56d76aa..0000000 --- a/sidecars/tito/docs/index.zh-CN.md +++ /dev/null @@ -1,20 +0,0 @@ -# TITO Gateway 文档 - -[English](index.md) | [中文](index.zh-CN.md) - -TITO Gateway 是围绕 Miles Agentic Chat Template / TITO session-server 工作做的 -公开 package 和 CLI 封装。它复用 Miles 逻辑，并把这条路径整理成独立 package。 - -## 从这里开始 - -- [快速开始](quickstart.zh-CN.md)：安装、运行和测试。 -- [核心概念](concepts.zh-CN.md)：运行模型、session flow 和不变量。 -- [Python API](api.zh-CN.md)：把 gateway 嵌入到 backend 旁边。 -- [CLI 参考](cli.zh-CN.md)：serve 命令和 backend discovery。 -- [验证与测试](verification.zh-CN.md)：verifier 命令和 CPU-fast 测试准备。 -- [开发说明](development.zh-CN.md)：源码策略、致谢和发布检查。 - -## 公开致谢 - -本 package 明确致谢 Miles：底层 TITO 实现来自 Miles。这个项目是封装和使用层，不是 -重新实现。源码审计记录在 `tito_gateway/VENDORED_MILES_AUDIT.md`。 diff --git a/sidecars/tito/docs/quickstart.md b/sidecars/tito/docs/quickstart.md deleted file mode 100644 index 2817cde..0000000 --- a/sidecars/tito/docs/quickstart.md +++ /dev/null @@ -1,70 +0,0 @@ -# Quickstart - -[Docs Home](index.md) | [中文](quickstart.zh-CN.md) - -## Install - -```bash -pip install tito-gateway -``` - -The copied Miles TITO logic is bundled in this package. Install optional -verifier dependencies only when you want to run the heavy verifier path locally: - -```bash -pip install 'tito-gateway[verify]' -``` - -If the console script is not on `PATH`, use: - -```bash -python -m tito_gateway.cli --help -``` - -## Start With An Explicit Backend - -```bash -tito-gateway --hf-checkpoint Qwen/Qwen3-0.6B \ - --tito-model qwen3 \ - --tito-allowed-append-roles tool user \ - --backend-url http://127.0.0.1:8000 \ - --session-server-port 30000 -``` - -The default command is `serve`, so `tito-gateway ...` and `tito-gateway serve ...` -use the same startup path. - -## Start With Backend Probing - -```bash -tito-gateway serve --hf-checkpoint Qwen/Qwen3-0.6B \ - --backend-probe-candidate http://127.0.0.1:8000 \ - --backend-probe-candidate http://127.0.0.1:30000 \ - --backend-probe-timeout 0.5 -``` - -## Embed In Python - -```python -from tito_gateway import TITOGateway, TITOGatewayConfig - -gateway = TITOGateway( - TITOGatewayConfig( - hf_checkpoint="Qwen/Qwen3-0.6B", - backend_url="http://127.0.0.1:8000", - apply_chat_template_kwargs={"enable_thinking": False}, - tito_model="qwen3", - tito_allowed_append_roles=("tool", "user"), - ) -) - -app = gateway.app -``` - -## Run CPU-Fast Tests - -```bash -pip install -e '.[test]' -python scripts/prepare_test_tokenizer_cache.py --endpoint https://huggingface.co -pytest tests/upstream tests/package -q -``` diff --git a/sidecars/tito/docs/quickstart.zh-CN.md b/sidecars/tito/docs/quickstart.zh-CN.md deleted file mode 100644 index 391a1dd..0000000 --- a/sidecars/tito/docs/quickstart.zh-CN.md +++ /dev/null @@ -1,70 +0,0 @@ -# 快速开始 - -[文档首页](index.zh-CN.md) | [English](quickstart.md) - -## 安装 - -```bash -pip install tito-gateway -``` - -Miles TITO 逻辑已经 copy/vendor 在这个 package 里。只有需要在本地跑 heavy verifier -路径时，才安装 verifier 可选依赖： - -```bash -pip install 'tito-gateway[verify]' -``` - -如果 console script 不在 `PATH` 中，可以用： - -```bash -python -m tito_gateway.cli --help -``` - -## 使用显式 backend 启动 - -```bash -tito-gateway --hf-checkpoint Qwen/Qwen3-0.6B \ - --tito-model qwen3 \ - --tito-allowed-append-roles tool user \ - --backend-url http://127.0.0.1:8000 \ - --session-server-port 30000 -``` - -默认命令就是 `serve`，所以 `tito-gateway ...` 和 `tito-gateway serve ...` 走同一套 -启动逻辑。 - -## 使用 backend probing 启动 - -```bash -tito-gateway serve --hf-checkpoint Qwen/Qwen3-0.6B \ - --backend-probe-candidate http://127.0.0.1:8000 \ - --backend-probe-candidate http://127.0.0.1:30000 \ - --backend-probe-timeout 0.5 -``` - -## 在 Python 中嵌入 - -```python -from tito_gateway import TITOGateway, TITOGatewayConfig - -gateway = TITOGateway( - TITOGatewayConfig( - hf_checkpoint="Qwen/Qwen3-0.6B", - backend_url="http://127.0.0.1:8000", - apply_chat_template_kwargs={"enable_thinking": False}, - tito_model="qwen3", - tito_allowed_append_roles=("tool", "user"), - ) -) - -app = gateway.app -``` - -## 运行 CPU-fast 测试 - -```bash -pip install -e '.[test]' -python scripts/prepare_test_tokenizer_cache.py --endpoint https://huggingface.co -pytest tests/upstream tests/package -q -``` diff --git a/sidecars/tito/docs/verification.md b/sidecars/tito/docs/verification.md deleted file mode 100644 index a8f6aff..0000000 --- a/sidecars/tito/docs/verification.md +++ /dev/null @@ -1,46 +0,0 @@ -# Verification And Tests - -[Docs Home](index.md) | [中文](verification.zh-CN.md) - -## Tokenizer Cache - -Copied upstream fast tests need tokenizer assets. Prepare the local cache -without downloading model weights: - -```bash -python scripts/prepare_test_tokenizer_cache.py --endpoint https://huggingface.co -``` - -## CPU-Fast Suite - -```bash -pytest tests/upstream tests/package -q -``` - -Targeted checks: - -```bash -pytest tests/upstream/fast/utils/chat_template_utils/test_tito_tokenizer.py -pytest tests/upstream/fast/utils/chat_template_utils/test_pretokenized_via_tito.py -pytest tests/upstream/fast/router/test_sessions.py -pytest tests/upstream/fast/router/test_session_race_conditions.py -pytest tests/upstream/fast/router/test_session_pretokenized_e2e.py -pytest tests/upstream/fast/utils/test_utils/test_session_verify_runner.py -pytest tests/package -``` - -## CLI Smoke Checks - -```bash -tito-gateway --help -tito-gateway serve --help -tito-gateway verify-chat-template --help -tito-gateway verify-session-tito-tokenizer --help -``` - -## Optional Heavy Verifier - -`verify-session-tito-tokenizer` runs the migrated Miles/SGLang session verifier -when the optional training stack is installed. Without that stack, it exits -with a clear dependency or runtime error. That dependency-gated exit is not a -GPU/e2e pass. diff --git a/sidecars/tito/docs/verification.zh-CN.md b/sidecars/tito/docs/verification.zh-CN.md deleted file mode 100644 index 313faed..0000000 --- a/sidecars/tito/docs/verification.zh-CN.md +++ /dev/null @@ -1,45 +0,0 @@ -# 验证与测试 - -[文档首页](index.zh-CN.md) | [English](verification.md) - -## Tokenizer Cache - -复制过来的 upstream fast tests 需要 tokenizer assets。可以只准备本地 tokenizer cache， -不下载模型权重： - -```bash -python scripts/prepare_test_tokenizer_cache.py --endpoint https://huggingface.co -``` - -## CPU-Fast Suite - -```bash -pytest tests/upstream tests/package -q -``` - -Targeted checks： - -```bash -pytest tests/upstream/fast/utils/chat_template_utils/test_tito_tokenizer.py -pytest tests/upstream/fast/utils/chat_template_utils/test_pretokenized_via_tito.py -pytest tests/upstream/fast/router/test_sessions.py -pytest tests/upstream/fast/router/test_session_race_conditions.py -pytest tests/upstream/fast/router/test_session_pretokenized_e2e.py -pytest tests/upstream/fast/utils/test_utils/test_session_verify_runner.py -pytest tests/package -``` - -## CLI Smoke Checks - -```bash -tito-gateway --help -tito-gateway serve --help -tito-gateway verify-chat-template --help -tito-gateway verify-session-tito-tokenizer --help -``` - -## Optional Heavy Verifier - -`verify-session-tito-tokenizer` 会在安装可选 Miles/SGLang training stack 后运行迁移后的 -session verifier。没有这套依赖时，它会以清晰的 dependency 或 runtime error 退出。 -这个 dependency-gated exit 不能算作 GPU/e2e pass。 diff --git a/sidecars/tito/miles/__init__.py b/sidecars/tito/miles/__init__.py deleted file mode 100644 index 9c61bb1..0000000 --- a/sidecars/tito/miles/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Compatibility namespace for vendored Miles TITO modules.""" - -from pkgutil import extend_path - -__path__ = extend_path(__path__, __name__) diff --git a/sidecars/tito/miles/_upstream_loader.py b/sidecars/tito/miles/_upstream_loader.py deleted file mode 100644 index 49b3e21..0000000 --- a/sidecars/tito/miles/_upstream_loader.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Helpers for delegating compatibility wrappers to an installed Miles tree.""" - -from __future__ import annotations - -import hashlib -import importlib.util -import sys -from pathlib import Path -from types import ModuleType - - -class UpstreamModuleLoadError(ImportError): - """Raised when a present upstream module cannot be imported.""" - - -def _candidate_files(module_name: str, search_root: Path) -> tuple[Path, Path]: - module_parts = module_name.split(".") - module_path = search_root.joinpath(*module_parts) - return module_path.with_suffix(".py"), module_path / "__init__.py" - - -def load_upstream_module(module_name: str, local_file: str) -> ModuleType | None: - """Load an upstream Miles module with the same public name, if available. - - The local compatibility package intentionally occupies `miles.*` import - paths. Exact-name wrappers use this function to look past the current repo - and delegate to a real upstream Miles installation when one is present. - """ - local_path = Path(local_file).resolve() - for entry in sys.path: - search_root = Path(entry or ".").resolve() - for candidate in _candidate_files(module_name, search_root): - try: - candidate = candidate.resolve() - except OSError: - continue - if not candidate.exists() or candidate == local_path: - continue - - digest = hashlib.sha1(str(candidate).encode("utf-8")).hexdigest()[:12] - alias = f"_tito_gateway_upstream_{module_name.replace('.', '_')}_{digest}" - if alias in sys.modules: - return sys.modules[alias] - - is_package = candidate.name == "__init__.py" - spec = importlib.util.spec_from_file_location( - alias, - candidate, - submodule_search_locations=[str(candidate.parent)] if is_package else None, - ) - if spec is None or spec.loader is None: - continue - module = importlib.util.module_from_spec(spec) - sys.modules[alias] = module - try: - spec.loader.exec_module(module) - except Exception as exc: - sys.modules.pop(alias, None) - raise UpstreamModuleLoadError( - f"Found upstream candidate for {module_name} at {candidate}, " - "but importing it failed. Fix the upstream Miles installation " - "or remove it from sys.path." - ) from exc - return module - return None - - -def export_public(module: ModuleType, namespace: dict[str, object]) -> list[str]: - """Copy public symbols from `module` into `namespace`.""" - names = getattr(module, "__all__", None) - if names is None: - names = [name for name in vars(module) if not name.startswith("_")] - for name in names: - namespace[name] = getattr(module, name) - return list(names) diff --git a/sidecars/tito/miles/rollout/__init__.py b/sidecars/tito/miles/rollout/__init__.py deleted file mode 100644 index 09befd7..0000000 --- a/sidecars/tito/miles/rollout/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Compatibility namespace for vendored Miles rollout modules.""" - -from pkgutil import extend_path - -__path__ = extend_path(__path__, __name__) diff --git a/sidecars/tito/miles/rollout/base_types.py b/sidecars/tito/miles/rollout/base_types.py deleted file mode 100644 index 61c9fd7..0000000 --- a/sidecars/tito/miles/rollout/base_types.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for Miles rollout base types.""" - -from miles._upstream_loader import export_public, load_upstream_module - -_upstream = load_upstream_module(__name__, __file__) -if _upstream is not None: - __all__ = export_public(_upstream, globals()) -else: - from tito_gateway.vendor.miles_compat.rollout.base_types import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/rollout/generate_hub/__init__.py b/sidecars/tito/miles/rollout/generate_hub/__init__.py deleted file mode 100644 index cdc6153..0000000 --- a/sidecars/tito/miles/rollout/generate_hub/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Compatibility namespace for Miles generate helpers.""" - -from pkgutil import extend_path - -__path__ = extend_path(__path__, __name__) diff --git a/sidecars/tito/miles/rollout/generate_hub/agentic_tool_call.py b/sidecars/tito/miles/rollout/generate_hub/agentic_tool_call.py deleted file mode 100644 index aab875b..0000000 --- a/sidecars/tito/miles/rollout/generate_hub/agentic_tool_call.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for Miles agentic tool-call generate bridge.""" - -from miles._upstream_loader import export_public, load_upstream_module - -_upstream = load_upstream_module(__name__, __file__) -if _upstream is not None: - __all__ = export_public(_upstream, globals()) -else: - from tito_gateway.vendor.miles_compat.rollout.generate_hub.agentic_tool_call import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/rollout/session/__init__.py b/sidecars/tito/miles/rollout/session/__init__.py deleted file mode 100644 index ebc5c93..0000000 --- a/sidecars/tito/miles/rollout/session/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Compatibility wrapper for Miles session modules.""" diff --git a/sidecars/tito/miles/rollout/session/linear_trajectory.py b/sidecars/tito/miles/rollout/session/linear_trajectory.py deleted file mode 100644 index f45f2a8..0000000 --- a/sidecars/tito/miles/rollout/session/linear_trajectory.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Compatibility wrapper for Miles linear trajectory implementation.""" - -from tito_gateway.vendor.miles_compat.rollout.session.linear_trajectory import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/rollout/session/session_errors.py b/sidecars/tito/miles/rollout/session/session_errors.py deleted file mode 100644 index 0dbaae6..0000000 --- a/sidecars/tito/miles/rollout/session/session_errors.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Compatibility wrapper for Miles session errors.""" - -from tito_gateway.vendor.miles_compat.rollout.session.session_errors import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/rollout/session/session_server.py b/sidecars/tito/miles/rollout/session/session_server.py deleted file mode 100644 index 2d8496f..0000000 --- a/sidecars/tito/miles/rollout/session/session_server.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Compatibility wrapper for Miles session server.""" - -from tito_gateway.vendor.miles_compat.rollout.session.session_server import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/rollout/session/session_types.py b/sidecars/tito/miles/rollout/session/session_types.py deleted file mode 100644 index 8327c0c..0000000 --- a/sidecars/tito/miles/rollout/session/session_types.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Compatibility wrapper for Miles session types.""" - -from tito_gateway.vendor.miles_compat.rollout.session.session_types import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/rollout/session/sessions.py b/sidecars/tito/miles/rollout/session/sessions.py deleted file mode 100644 index 714e037..0000000 --- a/sidecars/tito/miles/rollout/session/sessions.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Compatibility wrapper for Miles session route setup.""" - -from tito_gateway.vendor.miles_compat.rollout.session.sessions import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/__init__.py b/sidecars/tito/miles/utils/__init__.py deleted file mode 100644 index d8cb848..0000000 --- a/sidecars/tito/miles/utils/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Compatibility wrappers for Miles utility modules vendored by TITO Gateway.""" - -from pkgutil import extend_path - -__path__ = extend_path(__path__, __name__) diff --git a/sidecars/tito/miles/utils/chat_template_utils/__init__.py b/sidecars/tito/miles/utils/chat_template_utils/__init__.py deleted file mode 100644 index 14a35f8..0000000 --- a/sidecars/tito/miles/utils/chat_template_utils/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Compatibility wrapper for Miles chat-template utilities.""" - -from tito_gateway.vendor.miles_compat.utils.chat_template_utils import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/chat_template_utils/deepseek_v32.py b/sidecars/tito/miles/utils/chat_template_utils/deepseek_v32.py deleted file mode 100644 index 1c3fe1f..0000000 --- a/sidecars/tito/miles/utils/chat_template_utils/deepseek_v32.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Compatibility wrapper for Miles DeepSeek V3.2 chat-template bridge.""" - -from tito_gateway.vendor.miles_compat.utils.chat_template_utils.deepseek_v32 import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/chat_template_utils/deepseek_v4.py b/sidecars/tito/miles/utils/chat_template_utils/deepseek_v4.py deleted file mode 100644 index 22fa69c..0000000 --- a/sidecars/tito/miles/utils/chat_template_utils/deepseek_v4.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Compatibility wrapper for Miles DeepSeek V4 chat-template bridge.""" - -from tito_gateway.vendor.miles_compat.utils.chat_template_utils.deepseek_v4 import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/chat_template_utils/template.py b/sidecars/tito/miles/utils/chat_template_utils/template.py deleted file mode 100644 index 3d99596..0000000 --- a/sidecars/tito/miles/utils/chat_template_utils/template.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Compatibility wrapper for Miles chat-template rendering helpers.""" - -from tito_gateway.vendor.miles_compat.utils.chat_template_utils.template import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/chat_template_utils/tito_tokenizer.py b/sidecars/tito/miles/utils/chat_template_utils/tito_tokenizer.py deleted file mode 100644 index 5fc2216..0000000 --- a/sidecars/tito/miles/utils/chat_template_utils/tito_tokenizer.py +++ /dev/null @@ -1,4 +0,0 @@ -"""Compatibility wrapper for Miles TITO tokenizer implementations.""" - -from tito_gateway.vendor.miles_compat.utils.chat_template_utils.tito_tokenizer import * # noqa: F401,F403 -from tito_gateway.vendor.miles_compat.utils.chat_template_utils.tito_tokenizer import _build_dummy_assistant diff --git a/sidecars/tito/miles/utils/chat_template_utils/token_seq_comparator.py b/sidecars/tito/miles/utils/chat_template_utils/token_seq_comparator.py deleted file mode 100644 index 432b0e9..0000000 --- a/sidecars/tito/miles/utils/chat_template_utils/token_seq_comparator.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Compatibility wrapper for Miles token sequence comparator.""" - -from tito_gateway.vendor.miles_compat.utils.chat_template_utils.token_seq_comparator import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/external_utils/__init__.py b/sidecars/tito/miles/utils/external_utils/__init__.py deleted file mode 100644 index 736a6d1..0000000 --- a/sidecars/tito/miles/utils/external_utils/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Compatibility namespace for optional Miles external utilities.""" - -from pkgutil import extend_path - -__path__ = extend_path(__path__, __name__) diff --git a/sidecars/tito/miles/utils/external_utils/command_utils.py b/sidecars/tito/miles/utils/external_utils/command_utils.py deleted file mode 100644 index 29f2708..0000000 --- a/sidecars/tito/miles/utils/external_utils/command_utils.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compatibility wrapper for optional Miles command helpers.""" - -from miles._upstream_loader import export_public, load_upstream_module - -_upstream = load_upstream_module(__name__, __file__) -if _upstream is not None: - __all__ = export_public(_upstream, globals()) -else: - from tito_gateway.vendor.miles_compat.utils.external_utils.command_utils import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/hf_config.py b/sidecars/tito/miles/utils/hf_config.py deleted file mode 100644 index 3270150..0000000 --- a/sidecars/tito/miles/utils/hf_config.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Compatibility wrapper for Miles HuggingFace config helpers.""" - -from tito_gateway.vendor.miles_compat.utils.hf_config import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/http_utils.py b/sidecars/tito/miles/utils/http_utils.py deleted file mode 100644 index 0dcdb86..0000000 --- a/sidecars/tito/miles/utils/http_utils.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Compatibility wrapper for Miles HTTP utilities.""" - -from tito_gateway.vendor.miles_compat.utils.http_utils import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/processing_utils.py b/sidecars/tito/miles/utils/processing_utils.py deleted file mode 100644 index eef08c4..0000000 --- a/sidecars/tito/miles/utils/processing_utils.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Compatibility wrapper for Miles tokenizer loading helpers.""" - -from tito_gateway.vendor.miles_compat.utils.processing_utils import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/test_utils/__init__.py b/sidecars/tito/miles/utils/test_utils/__init__.py deleted file mode 100644 index 92f08be..0000000 --- a/sidecars/tito/miles/utils/test_utils/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Compatibility wrappers for vendored Miles test utilities.""" diff --git a/sidecars/tito/miles/utils/test_utils/chat_template_verify.py b/sidecars/tito/miles/utils/test_utils/chat_template_verify.py deleted file mode 100644 index c2fc82a..0000000 --- a/sidecars/tito/miles/utils/test_utils/chat_template_verify.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Compatibility wrapper for Miles chat-template verifier utilities.""" - -from tito_gateway.vendor.miles_compat.utils.test_utils.chat_template_verify import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/test_utils/mock_sglang_server.py b/sidecars/tito/miles/utils/test_utils/mock_sglang_server.py deleted file mode 100644 index 623a6e4..0000000 --- a/sidecars/tito/miles/utils/test_utils/mock_sglang_server.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Compatibility wrapper for Miles mock SGLang process result types.""" - -from tito_gateway.vendor.miles_compat.utils.test_utils.mock_sglang_server import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/test_utils/mock_trajectories.py b/sidecars/tito/miles/utils/test_utils/mock_trajectories.py deleted file mode 100644 index 81aa765..0000000 --- a/sidecars/tito/miles/utils/test_utils/mock_trajectories.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Compatibility wrapper for Miles mock trajectories.""" - -from tito_gateway.vendor.miles_compat.utils.test_utils.mock_trajectories import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/test_utils/session_verify_agent.py b/sidecars/tito/miles/utils/test_utils/session_verify_agent.py deleted file mode 100644 index 98d0127..0000000 --- a/sidecars/tito/miles/utils/test_utils/session_verify_agent.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Compatibility wrapper for Miles session verifier agent.""" - -from tito_gateway.vendor.miles_compat.utils.test_utils.session_verify_agent import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/test_utils/session_verify_runner.py b/sidecars/tito/miles/utils/test_utils/session_verify_runner.py deleted file mode 100644 index 4b2e4a0..0000000 --- a/sidecars/tito/miles/utils/test_utils/session_verify_runner.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Compatibility wrapper for Miles session verifier runner.""" - -from tito_gateway.vendor.miles_compat.utils.test_utils.session_verify_runner import * # noqa: F401,F403 diff --git a/sidecars/tito/miles/utils/test_utils/uvicorn_thread_server.py b/sidecars/tito/miles/utils/test_utils/uvicorn_thread_server.py deleted file mode 100644 index f5b75a3..0000000 --- a/sidecars/tito/miles/utils/test_utils/uvicorn_thread_server.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Compatibility wrapper for Miles uvicorn thread test server.""" - -from tito_gateway.vendor.miles_compat.utils.test_utils.uvicorn_thread_server import * # noqa: F401,F403 diff --git a/sidecars/tito/plan.md b/sidecars/tito/plan.md deleted file mode 100644 index 74f6d39..0000000 --- a/sidecars/tito/plan.md +++ /dev/null @@ -1,228 +0,0 @@ -# TITO Gateway Package Extraction Plan - -## Goal Description - -把 Miles 文档中 Agentic Chat Template / TITO session-server 路径抽成一个独立 Python package。这个项目必须明确 ack Miles 同学/团队的原创工作，定位为“对 Miles TITO 工作的封装、复用和使用层”，不是重新发明或改写 Miles 的算法。核心算法尽可能原样复用 Miles upstream 代码，不重写、不改测试逻辑。新 package 需要同时支持两种使用方式： - -1. Python import 方式：应用在已有 server 旁边包裹一层 gateway，自动发现或接收后端 OpenAI-compatible server 地址，捕捉 `/v1/chat/completions` 调用并维护 TITO session/token 轨迹。 -2. CLI 方式：一行命令启动 gateway，参数与 Miles 对应 TITO/session-server 参数保持兼容，尤其是 `--hf-checkpoint`、`--chat-template-path`、`--apply-chat-template-kwargs`、`--tito-model`、`--tito-allowed-append-roles`、`--session-server-ip`、`--session-server-port`、router/backend URL 相关参数。 - -## Source Exploration Summary - -- 本 package 的技术来源与核心能力来自 Miles 项目；计划、文档和代码注释中需要明确 acknowledgement：TITO tokenizer、fixed chat templates、session trajectory、session-server proxy 和验证体系均基于 Miles 同学/团队已有工作。 -- 文档页 `Agentic Chat Templates (TITO)` 明确了运行不变量：messages 必须 append-only；只允许最新 assistant checkpoint 的单步 rollback；`--tito-allowed-append-roles` 必须准确声明追加角色；`tool` 总是隐含允许。 -- 文档页指向的核心验证脚本是 `scripts/tools/verify_chat_template.py` 和 `scripts/tools/verify_session_tito_tokenizer.py`，它们分别验证固定模板 append-only 与真实 session-server TITO e2e。 -- 代码核心集中在 Miles: - - `miles/utils/chat_template_utils/tito_tokenizer.py` - - `miles/utils/chat_template_utils/template.py` - - `miles/utils/chat_template_utils/token_seq_comparator.py` - - `miles/utils/chat_template_utils/templates/*.jinja` - - `miles/rollout/session/session_server.py` - - `miles/rollout/session/sessions.py` - - `miles/rollout/session/linear_trajectory.py` - - `miles/rollout/session/session_errors.py` - - `miles/rollout/session/session_types.py` -- 现有 session server 是 FastAPI + httpx proxy：创建 `/sessions`，然后通过 `/sessions/{session_id}/v1/chat/completions` 将请求代理到后端，并注入 `logprobs=True`、`return_meta_info=True`、`input_ids`，再从 SGLang/OpenAI-compatible 响应中取 `meta_info.output_token_logprobs` 更新 token checkpoint。 -- 现有测试应作为迁移合同原样保留，优先复制并运行以下测试簇： - - `tests/fast/utils/chat_template_utils/test_tito_tokenizer.py` - - `tests/fast/utils/chat_template_utils/test_pretokenized_via_tito.py` - - `tests/fast/router/test_sessions.py` - - `tests/fast/router/test_session_race_conditions.py` - - `tests/fast/router/test_session_pretokenized_e2e.py` - - `tests/fast/utils/test_utils/test_session_verify_runner.py` - - 可选 GPU/e2e: `tests/e2e/sglang/test_session_server_multi_role/*` - -## Acceptance Criteria - -- AC-1: Package import surface works. - - Positive Tests (expected to PASS): - - `python -c "import tito_gateway; from tito_gateway import TITOGateway, SessionServer, get_tito_tokenizer"` succeeds. - - A test constructs `TITOGateway(...)` with an explicit backend URL and starts the same FastAPI route behavior as Miles `SessionServer`. - - Negative Tests (expected to FAIL): - - Constructing gateway without `hf_checkpoint` raises the same skip/error behavior defined by the wrapper contract, without silently enabling broken TITO tracking. - -- AC-2: Miles TITO core logic is reused with minimal source edits. - - Positive Tests (expected to PASS): - - Upstream copied tests for `TITOTokenizer`, fixed-template resolution, decode-roundtrip verifier, and session routes pass without test body edits. - - A source audit shows `tito_tokenizer.py`, fixed templates, `template.py`, `token_seq_comparator.py`, `linear_trajectory.py`, `sessions.py`, and session error/type models are copied verbatim except import path rewrites required by package namespace. - - Negative Tests (expected to FAIL): - - Any implementation that rewrites TITO merge/tokenize algorithms instead of vendoring upstream code fails review. - -- AC-3: Existing Miles tests are preserved. - - Positive Tests (expected to PASS): - - Migrated tests keep assertions, parametrization, expected failures, and mock trajectory behavior identical to upstream. - - Compatibility shims make original import paths usable where practical, e.g. `miles.utils.chat_template_utils...` can resolve to vendored modules during tests. - - Negative Tests (expected to FAIL): - - Changing upstream test assertions, deleting negative tests like buggy Qwen3 boundary tests, or weakening expected mismatch checks is not allowed. - -- AC-4: CLI starts gateway with Miles-compatible arguments. - - Positive Tests (expected to PASS): - - `tito-gateway --hf-checkpoint Qwen/Qwen3-0.6B --tito-model qwen3 --tito-allowed-append-roles tool user --backend-url http://127.0.0.1:8000 --session-server-port 30000` starts the FastAPI gateway. - - CLI supports JSON parsing for `--apply-chat-template-kwargs` in the same convention as Miles. - - `tito-gateway verify-chat-template ...` delegates to the migrated `verify_chat_template` logic. - - Negative Tests (expected to FAIL): - - Invalid `--tito-model` exits non-zero with argparse/typer validation. - - Unsupported append roles fail before server startup. - -- AC-5: Backend server address can be auto-detected for wrapper usage. - - Positive Tests (expected to PASS): - - Explicit `backend_url` always wins. - - Environment variables are checked in deterministic order, e.g. `TITO_BACKEND_URL`, `OPENAI_BASE_URL`, `SGLANG_BASE_URL`. - - If a common local backend port is configured for probing, `/health` or `/v1/models` detection selects a live backend and logs the selected URL. - - Negative Tests (expected to FAIL): - - If no backend can be found, startup fails with a clear error instead of binding a gateway that cannot proxy calls. - -- AC-6: Session proxy behavior matches Miles. - - Positive Tests (expected to PASS): - - `/health`, `/sessions`, `/sessions/{session_id}`, `DELETE /sessions/{session_id}`, and `/sessions/{session_id}/v1/chat/completions` behave like upstream tests. - - Proxied chat requests inject `input_ids`, `logprobs=True`, `return_meta_info=True`, and `no_stop_trim=False`. - - Concurrent same-session, different-session, and delete-while-inflight race tests pass unchanged. - - Negative Tests (expected to FAIL): - - Missing upstream `meta_info.output_token_logprobs` returns the same upstream-response error behavior. - - Non-append-only messages or forbidden appended roles return 400. - -- AC-7: Verification commands remain available. - - Positive Tests (expected to PASS): - - `tito-gateway verify-chat-template` prints the same PASS/FAIL verdicts as Miles `scripts/tools/verify_chat_template.py`. - - `tito-gateway verify-session-tito-tokenizer` exists as an optional command and either runs the migrated runner when Miles/SGLang training dependencies are installed, or exits with a clear dependency error. - - Negative Tests (expected to FAIL): - - The package must not pretend GPU/e2e verification passed when optional heavy dependencies are unavailable. - -## Path Boundaries - -### Upper Bound (Maximum Scope) - -- New package scaffold with `pyproject.toml`, importable `tito_gateway` package, CLI entrypoints, vendored Miles TITO/session code, compatibility imports, copied tests, and CI commands for CPU-fast tests. -- Optional command namespace for e2e verification that preserves Miles arguments but documents dependency requirements. -- Minimal docs: import usage, CLI usage, backend discovery order, and test commands. - -### Lower Bound (Minimum Scope) - -- Importable package exposing the TITO tokenizer factory and session gateway. -- CLI that starts FastAPI gateway with explicit `--backend-url`. -- Upstream fast tests copied and passing with only import-path compatibility changes outside the test bodies. - -### Allowed Choices - -- Can use FastAPI, httpx, uvicorn, transformers, huggingface_hub, jinja2, pydantic, pytest, requests, typer or argparse. -- Can add a thin namespace compatibility layer so upstream test imports keep working. -- Can add wrapper-only modules such as `tito_gateway.cli`, `tito_gateway.gateway`, `tito_gateway.config`, and `tito_gateway.discovery`. -- Can vendor upstream Miles source with attribution and an upstream commit marker. - -### Disallowed Choices - -- Cannot rewrite TITO tokenization/merge behavior when upstream code can be copied. -- Cannot weaken or edit upstream test assertions. -- Cannot remove negative tests that prove broken templates/subclasses fail. -- Cannot require full Miles training stack for basic package import or gateway startup. -- Cannot silently auto-detect a backend when multiple candidates are alive without deterministic precedence. - -## Proposed Package Layout - -```text -tito_gateway/ - __init__.py - cli.py - config.py - discovery.py - gateway.py - server.py - vendor/ - miles_compat/ - utils/ - chat_template_utils/ - __init__.py - template.py - token_seq_comparator.py - tito_tokenizer.py - deepseek_v32.py - deepseek_v4.py - templates/ - rollout/ - session/ - session_server.py - sessions.py - linear_trajectory.py - session_errors.py - session_types.py - utils/ - processing_utils.py - http_utils.py - test_utils/ - miles/ - __init__.py - ... optional compatibility re-export modules for unchanged tests ... -scripts/ - verify_chat_template.py - verify_session_tito_tokenizer.py -tests/ - upstream/ - ... copied Miles tests, unchanged ... - package/ - test_import_surface.py - test_cli_args.py - test_backend_discovery.py -``` - -## Dependencies and Sequence - -### Milestone 1: Baseline Scaffold - -- Create `pyproject.toml` with package metadata, runtime dependencies, optional test/e2e extras, and console script `tito-gateway`. -- Add `tito_gateway.__init__` export surface. -- Add an upstream metadata file recording Miles repository URL and commit SHA used for extraction. - -### Milestone 2: Vendor Core Miles Logic - -- Copy TITO tokenizer, chat template helpers, fixed jinja templates, token comparator, session types/errors, linear trajectory, sessions route setup, and session server. -- Rewrite only import paths or provide compatibility modules so upstream code remains functionally unchanged. -- Copy required lightweight utility helpers used by tests, especially tokenizer loading, port discovery, mock SGLang server, uvicorn thread server, and mock trajectories. - -### Milestone 3: Python Wrapper API - -- Implement `TITOGatewayConfig` with Miles-compatible names. -- Implement `TITOGateway.from_server(...)` / `TITOGateway(...)` that accepts explicit backend URL or uses discovery. -- Expose `app`, `run()`, and helper methods so users can mount/run beside an existing server. - -### Milestone 4: CLI - -- Implement `tito-gateway serve` and default command alias for one-line startup. -- Preserve Miles-compatible argument names. -- Implement `tito-gateway verify-chat-template` by delegating to copied verifier. -- Implement `tito-gateway verify-session-tito-tokenizer` as optional heavy command with explicit dependency checks. - -### Milestone 5: Test Migration Without Test Edits - -- Copy selected upstream tests into `tests/upstream`. -- Prefer compatibility shims so copied tests import `miles.*` unchanged. -- If import path edits are absolutely unavoidable, perform mechanical path rewrites only and document each changed line in a migration ledger; do not alter assertions, cases, expected exceptions, or parametrization. - -### Milestone 6: New Wrapper Tests - -- Add package-specific tests for import surface, CLI parsing, backend discovery precedence, and explicit backend startup. -- Use Miles mock server utilities to avoid requiring a real SGLang server for CPU-fast tests. - -### Milestone 7: Verification - -- Run CPU-fast subset: - - `pytest tests/upstream/fast/utils/chat_template_utils/test_tito_tokenizer.py` - - `pytest tests/upstream/fast/utils/chat_template_utils/test_pretokenized_via_tito.py` - - `pytest tests/upstream/fast/router/test_sessions.py` - - `pytest tests/upstream/fast/router/test_session_race_conditions.py` - - `pytest tests/upstream/fast/router/test_session_pretokenized_e2e.py` - - `pytest tests/upstream/fast/utils/test_utils/test_session_verify_runner.py` - - `pytest tests/package` -- Run CLI smoke tests: - - `tito-gateway --help` - - `tito-gateway serve --help` - - `tito-gateway verify-chat-template --help` -- Document optional e2e command separately because it requires model/GPU/SGLang dependencies. - -## Implementation Notes - -- The package should treat Miles as the source of truth. Add wrapper code around it; do not “simplify” the TITO algorithm. -- Public docs, README, package metadata, and copied source headers should clearly state that this is a standalone packaging/wrapper effort around Miles TITO work, with attribution to Miles and its contributors. -- Keep upstream test files as a contract. The test migration should be boring and traceable. -- Backend auto-discovery must be deterministic and observable in logs. -- The default import path should be `tito_gateway`, but a `miles` compatibility namespace is acceptable for tests and copied code. -- Preserve Apache-2.0 license notices and upstream attribution when copying Miles code. diff --git a/sidecars/tito/pyproject.toml b/sidecars/tito/pyproject.toml deleted file mode 100644 index 2169583..0000000 --- a/sidecars/tito/pyproject.toml +++ /dev/null @@ -1,91 +0,0 @@ -[build-system] -requires = ["hatchling>=1.25"] -build-backend = "hatchling.build" - -[project] -name = "tito-gateway" -version = "0.1.0" -description = "Standalone wrapper package for Miles TITO session gateway work." -readme = "README.md" -requires-python = ">=3.10" -license = "Apache-2.0" -authors = [ - { name = "TITO Gateway maintainers" }, -] -keywords = ["tito", "agentic", "chat-template", "miles", "gateway"] -classifiers = [ - "Development Status :: 3 - Alpha", - "Environment :: Console", - "Framework :: FastAPI", - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Topic :: Internet :: WWW/HTTP :: HTTP Servers", - "Topic :: Scientific/Engineering :: Artificial Intelligence", -] -dependencies = [ - "fastapi>=0.110", - "httpx>=0.27", - "pydantic>=2", - "setproctitle>=1.3", - "uvicorn>=0.29", -] - -[project.optional-dependencies] -verify = [ - "huggingface-hub>=0.23", - "jinja2>=3.1", - "sglang>=0.4", - "tokenizers>=0.19", - "transformers>=4.44", -] -test = [ - "huggingface-hub>=0.23", - "jinja2>=3.1", - "pytest>=8", - "requests>=2.31", - "sglang>=0.4", - "tokenizers>=0.19", - "transformers>=4.44", -] - -[project.urls] -Homepage = "https://github.com/yitianlian/tito_gateway" -Documentation = "https://github.com/yitianlian/tito_gateway/tree/master/docs" -Repository = "https://github.com/yitianlian/tito_gateway" -"Miles upstream" = "https://github.com/radixark/miles" -"Miles documentation" = "https://www.radixark.com/miles/docs/user-guide/agentic-chat-template" - -[project.scripts] -tito-gateway = "tito_gateway.cli:main" - -[tool.hatch.build.targets.wheel] -packages = ["tito_gateway", "miles"] - -[tool.hatch.build.targets.wheel.force-include] -"LICENSE" = "LICENSE" - -[tool.hatch.build.targets.sdist] -include = [ - "LICENSE", - "README.md", - "README.zh-CN.md", - "docs/**/*.md", - "miles/**/*.py", - "pyproject.toml", - "scripts/**/*.py", - "tests/**/*.py", - "tests/**/*.jinja", - "tito_gateway/**/*.json", - "tito_gateway/**/*.md", - "tito_gateway/**/*.py", - "tito_gateway/**/*.jinja", -] - -[tool.pytest.ini_options] -testpaths = ["tests"] -pythonpath = ["."] -addopts = "-q" diff --git a/sidecars/tito/scripts/prepare_test_tokenizer_cache.py b/sidecars/tito/scripts/prepare_test_tokenizer_cache.py deleted file mode 100644 index 884068f..0000000 --- a/sidecars/tito/scripts/prepare_test_tokenizer_cache.py +++ /dev/null @@ -1,83 +0,0 @@ -"""Prepare tokenizer-only HF cache assets for copied Miles upstream tests.""" - -from __future__ import annotations - -import argparse -import os -from collections.abc import Sequence - -from huggingface_hub import snapshot_download - - -TOKENIZER_REPOS: tuple[str, ...] = ( - "Qwen/Qwen3-0.6B", - "Qwen/Qwen3-4B", - "Qwen/Qwen3-4B-Instruct-2507", - "Qwen/Qwen3-4B-Thinking-2507", - "Qwen/Qwen3-Next-80B-A3B-Thinking", - "Qwen/Qwen3.5-0.8B", - "zai-org/GLM-4.7-Flash", -) - -ALLOW_PATTERNS: tuple[str, ...] = ( - "added_tokens.json", - "chat_template*.jinja", - "config.json", - "configuration*.py", - "generation_config.json", - "merges.txt", - "modeling*.py", - "special_tokens_map.json", - "tokenization*.py", - "tokenizer.json", - "tokenizer.model", - "tokenizer_config.json", - "vocab*.json", -) - -IGNORE_PATTERNS: tuple[str, ...] = ( - "*.bin", - "*.gguf", - "*.h5", - "*.msgpack", - "*.onnx", - "*.pt", - "*.safetensors", - "*.tflite", - "*.th", - "*.weights", -) - - -def prepare_tokenizer_cache(repos: Sequence[str], *, endpoint: str | None = None) -> None: - for repo_id in repos: - print(f"Preparing tokenizer cache for {repo_id}") - snapshot_download( - repo_id=repo_id, - endpoint=endpoint, - allow_patterns=ALLOW_PATTERNS, - ignore_patterns=IGNORE_PATTERNS, - ) - - -def main(argv: Sequence[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--endpoint", - default=os.environ.get("HF_ENDPOINT"), - help="HF endpoint to use. Defaults to HF_ENDPOINT when set.", - ) - parser.add_argument( - "--repo", - action="append", - dest="repos", - help="Override repo list; may be passed multiple times.", - ) - args = parser.parse_args(argv) - - prepare_tokenizer_cache(tuple(args.repos or TOKENIZER_REPOS), endpoint=args.endpoint) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/sidecars/tito/tests/ci/__init__.py b/sidecars/tito/tests/ci/__init__.py deleted file mode 100644 index 14e7098..0000000 --- a/sidecars/tito/tests/ci/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Compatibility package for copied Miles CI marker helpers.""" diff --git a/sidecars/tito/tests/ci/ci_register.py b/sidecars/tito/tests/ci/ci_register.py deleted file mode 100644 index 5c72d0a..0000000 --- a/sidecars/tito/tests/ci/ci_register.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Runtime no-op compatibility for copied Miles CPU CI markers.""" - - -def register_cpu_ci( - est_time: float, - suite: str, - *, - labels: list[str] | None = None, - nightly: bool = False, - disabled: str | None = None, -): - return None diff --git a/sidecars/tito/tests/fast/__init__.py b/sidecars/tito/tests/fast/__init__.py deleted file mode 100644 index 5e0f4b4..0000000 --- a/sidecars/tito/tests/fast/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Compatibility namespace for copied Miles fast-test helpers.""" diff --git a/sidecars/tito/tests/fast/router/__init__.py b/sidecars/tito/tests/fast/router/__init__.py deleted file mode 100644 index 136f6f4..0000000 --- a/sidecars/tito/tests/fast/router/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Compatibility namespace for copied Miles router test helpers.""" diff --git a/sidecars/tito/tests/fast/router/session_pretokenized_test_utils.py b/sidecars/tito/tests/fast/router/session_pretokenized_test_utils.py deleted file mode 100644 index ca8f7bb..0000000 --- a/sidecars/tito/tests/fast/router/session_pretokenized_test_utils.py +++ /dev/null @@ -1,202 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from types import SimpleNamespace -from typing import Any - -import requests -from fastapi import FastAPI, Request -from fastapi.responses import JSONResponse - -from miles.rollout.session.session_server import SessionServer -from miles.utils.chat_template_utils import MismatchType, apply_chat_template, get_tito_tokenizer -from miles.utils.http_utils import find_available_port -from miles.utils.processing_utils import load_tokenizer -from miles.utils.test_utils.uvicorn_thread_server import UvicornThreadServer - -FORBIDDEN_MISMATCH_TYPES: frozenset[str] = frozenset( - { - MismatchType.SPECIAL_TOKEN_COUNT.value, - MismatchType.SPECIAL_TOKEN_TYPE.value, - MismatchType.NON_ASSISTANT_TEXT.value, - } -) - - -@dataclass(frozen=True) -class ScriptedBackendTurn: - response_message: dict[str, Any] - render_message: dict[str, Any] - - -def load_test_tokenizer(hf_checkpoint: str, chat_template_path: str | None): - return load_tokenizer( - hf_checkpoint, - chat_template_path=chat_template_path, - trust_remote_code=True, - ) - - -def make_router_env( - backend, - *, - hf_checkpoint: str, - chat_template_path: str | None, - tito_model: str, - allowed_append_roles: list[str], -): - args = SimpleNamespace( - miles_router_timeout=30, - hf_checkpoint=hf_checkpoint, - chat_template_path=chat_template_path, - tito_model=tito_model, - tito_allowed_append_roles=allowed_append_roles, - use_rollout_routing_replay=False, - ) - session_server = SessionServer(args, backend_url=backend.url) - - port = find_available_port(31000) - server = UvicornThreadServer(session_server.app, host="127.0.0.1", port=port) - server.start() - - return SimpleNamespace( - url=f"http://127.0.0.1:{port}", - backend=backend, - server=server, - ) - - -def teardown_router_env(env) -> None: - env.server.stop() - env.backend.stop() - - -def fetch_session_payload(base_url: str, session_id: str) -> dict[str, Any]: - response = requests.get(f"{base_url}/sessions/{session_id}", timeout=5.0) - response.raise_for_status() - return response.json() - - -def compute_local_session_mismatch( - tokenizer, - *, - tito_model: str, - allowed_append_roles: list[str], - messages: list[dict[str, Any]], - accumulated_token_ids: list[int], - tools: list[dict[str, Any]] | None, -) -> list[dict[str, Any]]: - comparator = get_tito_tokenizer( - tokenizer, - tokenizer_type=tito_model, - allowed_append_roles=allowed_append_roles, - ).create_comparator() - expected_ids = apply_chat_template( - messages, - tokenizer=tokenizer, - tools=tools, - add_generation_prompt=False, - tokenize=True, - ) - return [m.to_dict() for m in comparator.compare_sequences(expected_ids, accumulated_token_ids)] - - -def forbidden_mismatches(mismatch: list[dict[str, Any]]) -> list[dict[str, Any]]: - return [m for m in mismatch if m.get("type") in FORBIDDEN_MISMATCH_TYPES] - - -class ScriptedChatBackend: - def __init__(self, tokenizer, scripted_turns: list[ScriptedBackendTurn]): - self.tokenizer = tokenizer - self._scripted_turns = scripted_turns - self._call_count = 0 - self.request_log: list[dict[str, Any]] = [] - self.host = "127.0.0.1" - self.port = find_available_port(32000) - self.app = FastAPI() - self._server = UvicornThreadServer(self.app, host=self.host, port=self.port) - self._setup_routes() - - @property - def url(self) -> str: - return f"http://{self.host}:{self.port}" - - def start(self): - self._server.start() - - def stop(self): - self._server.stop() - - def reset_stats(self): - self.request_log.clear() - self._call_count = 0 - - def _setup_routes(self): - @self.app.get("/health") - async def health(): - return JSONResponse(content={"status": "ok"}) - - @self.app.post("/v1/chat/completions") - async def chat_completions(request: Request): - payload = await request.json() - self.request_log.append(payload) - - idx = self._call_count - assert idx < len(self._scripted_turns), f"Unexpected extra request #{idx + 1}" - self._call_count += 1 - - turn = self._scripted_turns[idx] - messages = payload["messages"] - tools = payload.get("tools") - - prompt_text = apply_chat_template( - messages, - tokenizer=self.tokenizer, - tools=tools, - add_generation_prompt=True, - tokenize=False, - ) - with_assistant_text = apply_chat_template( - messages + [turn.render_message], - tokenizer=self.tokenizer, - tools=tools, - add_generation_prompt=False, - tokenize=False, - ) - assert with_assistant_text.startswith(prompt_text), "Scripted assistant must extend prompt text" - - response_text = with_assistant_text[len(prompt_text) :] - output_ids = self.tokenizer.encode(response_text, add_special_tokens=False) - input_ids = payload.get("input_ids") - prompt_ids = ( - list(input_ids) - if input_ids is not None - else apply_chat_template( - messages, - tokenizer=self.tokenizer, - tools=tools, - add_generation_prompt=True, - tokenize=True, - ) - ) - - return JSONResponse( - content={ - "id": f"scripted-{idx}", - "object": "chat.completion", - "created": 0, - "model": "scripted-model", - "choices": [ - { - "index": 0, - "message": turn.response_message, - "prompt_token_ids": prompt_ids, - "finish_reason": "tool_calls" if turn.response_message.get("tool_calls") else "stop", - "meta_info": { - "completion_tokens": len(output_ids), - "output_token_logprobs": [[-i / 128, tid] for i, tid in enumerate(output_ids)], - }, - } - ], - } - ) diff --git a/sidecars/tito/tests/miles/utils/chat_template_utils/templates/qwen3_thinking_2507_and_next_fixed.jinja b/sidecars/tito/tests/miles/utils/chat_template_utils/templates/qwen3_thinking_2507_and_next_fixed.jinja deleted file mode 100644 index 1588dfb..0000000 --- a/sidecars/tito/tests/miles/utils/chat_template_utils/templates/qwen3_thinking_2507_and_next_fixed.jinja +++ /dev/null @@ -1,82 +0,0 @@ -{%- if tools %} - {{- '<|im_start|>system\n' }} - {%- if messages[0].role == 'system' %} - {{- messages[0].content + '\n\n' }} - {%- endif %} - {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} - {%- for tool in tools %} - {{- "\n" }} - {{- tool | tojson }} - {%- endfor %} - {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} -{%- else %} - {%- if messages[0].role == 'system' %} - {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} - {%- endif %} -{%- endif %} -{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} -{%- for message in messages[::-1] %} - {%- set index = (messages|length - 1) - loop.index0 %} - {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} - {%- set ns.multi_step_tool = false %} - {%- set ns.last_query_index = index %} - {%- endif %} -{%- endfor %} -{%- for message in messages %} - {%- if message.content is string %} - {%- set content = message.content %} - {%- else %} - {%- set content = '' %} - {%- endif %} - {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} - {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} - {%- elif message.role == "assistant" %} - {%- set reasoning_content = '' %} - {%- if message.reasoning_content is string %} - {%- set reasoning_content = message.reasoning_content %} - {%- else %} - {%- if '' in content %} - {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} - {%- set content = content.split('')[-1].lstrip('\n') %} - {%- endif %} - {%- endif %} - {%- if (not (clear_thinking | default(true))) or loop.index0 > ns.last_query_index %} - {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} - {%- else %} - {{- '<|im_start|>' + message.role + '\n' + content }} - {%- endif %} - {%- if message.tool_calls %} - {%- for tool_call in message.tool_calls %} - {%- if (loop.first and content) or (not loop.first) %} - {{- '\n' }} - {%- endif %} - {%- if tool_call.function %} - {%- set tool_call = tool_call.function %} - {%- endif %} - {{- '\n{"name": "' }} - {{- tool_call.name }} - {{- '", "arguments": ' }} - {%- if tool_call.arguments is string %} - {{- tool_call.arguments }} - {%- else %} - {{- tool_call.arguments | tojson }} - {%- endif %} - {{- '}\n' }} - {%- endfor %} - {%- endif %} - {{- '<|im_end|>\n' }} - {%- elif message.role == "tool" %} - {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} - {{- '<|im_start|>user' }} - {%- endif %} - {{- '\n\n' }} - {{- content }} - {{- '\n' }} - {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} - {{- '<|im_end|>\n' }} - {%- endif %} - {%- endif %} -{%- endfor %} -{%- if add_generation_prompt %} - {{- '<|im_start|>assistant\n\n' }} -{%- endif %} diff --git a/sidecars/tito/tests/package/test_cli.py b/sidecars/tito/tests/package/test_cli.py deleted file mode 100644 index cd8147c..0000000 --- a/sidecars/tito/tests/package/test_cli.py +++ /dev/null @@ -1,212 +0,0 @@ -import os - -from miles.utils.test_utils import session_verify_runner as runner -from tito_gateway.cli import build_parser, main - - -def test_cli_top_level_help(capsys): - try: - main(["--help"]) - except SystemExit as exc: - assert exc.code == 0 - - assert "Standalone wrapper around Miles TITO" in capsys.readouterr().out - - -def test_cli_serve_help(capsys): - try: - main(["serve", "--help"]) - except SystemExit as exc: - assert exc.code == 0 - - assert "--hf-checkpoint" in capsys.readouterr().out - - -def test_cli_verify_session_returns_clear_dependency_error(capsys, monkeypatch): - monkeypatch.setenv("http_proxy", "http://proxy.example:8888") - monkeypatch.setenv("HTTPS_PROXY", "http://secure-proxy.example:8888") - - code = main( - [ - "verify-session-tito-tokenizer", - "--hf-checkpoint", - "Qwen/Qwen3-4B", - "--tito-model", - "qwen3", - "--tito-allowed-append-roles", - "tool", - "user", - "--sglang-reasoning-parser", - "qwen3", - "--sglang-tool-call-parser", - "qwen25", - "--rollout-num-gpus-per-engine", - "1", - ] - ) - - assert code == 2 - assert "requires the optional Miles/SGLang training stack" in capsys.readouterr().out - assert os.environ.get("http_proxy") == "http://proxy.example:8888" - assert os.environ.get("HTTPS_PROXY") == "http://secure-proxy.example:8888" - - -def test_cli_verify_session_help(capsys): - try: - main(["verify-session-tito-tokenizer", "--help"]) - except SystemExit as exc: - assert exc.code == 0 - - out = capsys.readouterr().out - assert "--hf-checkpoint" in out - assert "--rollout-num-gpus-per-engine" in out - assert "--assistant-text-threshold" in out - assert f"Default {runner.ASSISTANT_TEXT_MISMATCH_RATIO_THRESHOLD}" in out - - -def test_cli_verify_session_default_threshold_matches_runner_constant(): - args = build_parser().parse_args( - [ - "verify-session-tito-tokenizer", - "--hf-checkpoint", - "Qwen/Qwen3-4B", - "--tito-model", - "qwen3", - ] - ) - - assert args.assistant_text_threshold == runner.ASSISTANT_TEXT_MISMATCH_RATIO_THRESHOLD - - -def test_cli_verify_session_parses_representative_miles_style_invocation(tmp_path): - prompt_data = tmp_path / "session.jsonl" - template_path = tmp_path / "template.jinja" - args = build_parser().parse_args( - [ - "verify-session-tito-tokenizer", - "--hf-checkpoint", - "Qwen/Qwen3-4B", - "--chat-template-path", - str(template_path), - "--apply-chat-template", - "--apply-chat-template-kwargs", - '{"enable_thinking": false}', - "--tito-model", - "qwen3", - "--tito-allowed-append-roles", - "tool", - "user", - "--prompt-data", - str(prompt_data), - "--input-key", - "messages", - "--backend-url", - "http://127.0.0.1:8000", - "--session-server-ip", - "127.0.0.1", - "--session-server-port", - "31000", - "--miles-router-timeout", - "12.5", - "--sglang-reasoning-parser", - "qwen3", - "--sglang-tool-call-parser", - "qwen25", - "--rollout-num-gpus-per-engine", - "2", - "--sglang-expert-parallel-size", - "4", - "--num-rollout", - "2", - "--rollout-batch-size", - "8", - "--rollout-max-response-len", - "4096", - "--rollout-temperature", - "0.2", - "--global-batch-size", - "32", - "--actor-num-nodes", - "2", - "--actor-num-gpus-per-node", - "4", - "--n-samples-per-prompt", - "6", - "--session-verify-cycles", - "5", - "--tool-call-failure-mode", - "skip", - "--assistant-text-threshold", - "0.4", - "--train-backend", - "fsdp", - "--custom-generate-function-path", - "pkg.verify.generate", - "--custom-agent-function-path", - "pkg.verify.run_agent", - "--rm-type", - "random", - "--use-session-server", - "--debug-rollout-only", - "--ci-test", - "--colocate", - ] - ) - - assert args.verify_command == "session-tito-tokenizer" - assert args.chat_template_path == str(template_path) - assert args.apply_chat_template is True - assert args.apply_chat_template_kwargs == {"enable_thinking": False} - assert args.backend_url == "http://127.0.0.1:8000" - assert args.session_server_port == 31000 - assert args.miles_router_timeout == 12.5 - assert args.assistant_text_threshold == 0.4 - - train_args = runner.namespace_to_train_args(args) - assert f"--prompt-data {prompt_data}" in train_args - assert "--input-key messages" in train_args - assert "--rollout-batch-size 8" in train_args - assert "--n-samples-per-prompt 6" in train_args - assert "--rollout-max-response-len 4096" in train_args - assert "--rollout-temperature 0.2" in train_args - assert "--global-batch-size 32" in train_args - assert "--custom-generate-function-path pkg.verify.generate" in train_args - assert "--custom-agent-function-path pkg.verify.run_agent" in train_args - assert "--session-verify-cycles 5" in train_args - assert "--tool-call-failure-mode skip" in train_args - assert "--rollout-num-gpus-per-engine 2" in train_args - assert "--sglang-expert-parallel-size 4" in train_args - assert "--actor-num-nodes 2" in train_args - assert "--actor-num-gpus-per-node 4" in train_args - assert "--train-backend fsdp" in train_args - assert "--use-session-server" in train_args - assert "--debug-rollout-only" in train_args - assert "--ci-test" in train_args - assert "--colocate" in train_args - - -def test_cli_verify_chat_template_help(capsys): - try: - main(["verify-chat-template", "--help"]) - except SystemExit as exc: - assert exc.code == 0 - - out = capsys.readouterr().out - assert "--template" in out - assert "--tito-allowed-append-roles" in out - - -def test_cli_verify_chat_template_runs_real_verifier(tmp_path, capsys): - template = tmp_path / "simple.jinja" - template.write_text( - "{%- for message in messages -%}" - "{{ '<|' + message['role'] + '|>' + (message.get('content') or '') }}" - "{%- endfor -%}" - "{%- if add_generation_prompt -%}{{ '<|assistant|>' }}{%- endif -%}" - ) - - code = main(["verify-chat-template", "--template", str(template), "--thinking", "off"]) - - captured = capsys.readouterr() - assert code == 0 - assert "Verdict: PASS - template IS append-only" in captured.out diff --git a/sidecars/tito/tests/package/test_gateway_integration.py b/sidecars/tito/tests/package/test_gateway_integration.py deleted file mode 100644 index 429d56f..0000000 --- a/sidecars/tito/tests/package/test_gateway_integration.py +++ /dev/null @@ -1,82 +0,0 @@ -from unittest.mock import patch - -import requests - -from miles.utils.http_utils import find_available_port -from miles.utils.test_utils.mock_sglang_server import MockSGLangServer, ProcessResult, with_mock_server -from miles.utils.test_utils.uvicorn_thread_server import UvicornThreadServer -from tito_gateway import TITOGateway, TITOGatewayConfig - - -def test_tito_gateway_serves_real_session_routes_with_explicit_backend(): - def process_fn(prompt: str) -> ProcessResult: - return ProcessResult(text=f"echo: {prompt}", finish_reason="stop") - - original_chat_response = MockSGLangServer._compute_chat_completions_response - - def patched_chat_response(self, payload: dict) -> dict: - response = original_chat_response(self, payload) - choice = response["choices"][0] - logprobs_content = choice["logprobs"]["content"] - output_token_logprobs = [ - (item["logprob"], self.tokenizer.convert_tokens_to_ids(item["token"])) for item in logprobs_content - ] - choice["meta_info"] = { - "output_token_logprobs": output_token_logprobs, - "completion_tokens": len(output_token_logprobs), - } - return response - - with ( - patch.object(MockSGLangServer, "_compute_chat_completions_response", new=patched_chat_response), - with_mock_server(process_fn=process_fn) as backend, - ): - gateway = TITOGateway( - TITOGatewayConfig( - hf_checkpoint="Qwen/Qwen3-0.6B", - backend_url=backend.url, - apply_chat_template_kwargs={"enable_thinking": False}, - tito_model="default", - tito_allowed_append_roles=("tool",), - miles_router_timeout=30, - ) - ) - - port = find_available_port(33000) - server = UvicornThreadServer(gateway.app, host="127.0.0.1", port=port) - server.start() - url = f"http://127.0.0.1:{port}" - - try: - health = requests.get(f"{url}/health", timeout=5.0) - assert health.status_code == 200 - assert health.json()["status"] == "ok" - - session_id = requests.post(f"{url}/sessions", timeout=5.0).json()["session_id"] - payload = { - "messages": [{"role": "user", "content": "What is 1+2?"}], - "return_logprob": True, - } - response = requests.post( - f"{url}/sessions/{session_id}/v1/chat/completions", - json=payload, - timeout=10.0, - ) - - assert response.status_code == 200 - assert response.json()["choices"] - assert len(backend.request_log) == 1 - proxied_payload = backend.request_log[0] - assert proxied_payload["messages"] == payload["messages"] - assert proxied_payload["logprobs"] is True - assert proxied_payload["return_meta_info"] is True - assert proxied_payload["no_stop_trim"] is False - assert isinstance(proxied_payload["input_ids"], list) - assert proxied_payload["input_ids"] - - session = requests.get(f"{url}/sessions/{session_id}", timeout=5.0).json() - assert len(session["records"]) == 1 - assert session["records"][0]["path"] == "/v1/chat/completions" - assert session["records"][0]["status_code"] == 200 - finally: - server.stop() diff --git a/sidecars/tito/tests/package/test_session_verifier_plumbing.py b/sidecars/tito/tests/package/test_session_verifier_plumbing.py deleted file mode 100644 index 69f7a65..0000000 --- a/sidecars/tito/tests/package/test_session_verifier_plumbing.py +++ /dev/null @@ -1,74 +0,0 @@ -import argparse -import importlib -import json - -from miles.utils.test_utils import session_verify_runner as runner - - -def _load_dotted(path): - module_name, attr_name = path.rsplit(".", 1) - return getattr(importlib.import_module(module_name), attr_name) - - -def _build_args(tmp_path): - values = { - **runner.SESSION_VERIFY_INVARIANT_ARGS, - "hf_checkpoint": str(tmp_path / "local-model"), - "tito_model": "qwen3", - "tito_allowed_append_roles": ["tool", "user"], - "rollout_num_gpus_per_engine": 1, - "actor_num_nodes": 1, - "actor_num_gpus_per_node": 1, - "n_samples_per_prompt": 4, - "session_verify_cycles": 3, - "tool_call_failure_mode": "rollback", - "sglang_reasoning_parser": "qwen3", - "sglang_tool_call_parser": "qwen25", - "assistant_text_threshold": 0.1, - "sglang_expert_parallel_size": 1, - } - model_dir = tmp_path / "local-model" - model_dir.mkdir() - return argparse.Namespace(**values) - - -def test_session_verify_agent_function_paths_are_importable(): - generate = _load_dotted(runner.SESSION_VERIFY_INVARIANT_ARGS["custom_generate_function_path"]) - run_agent = _load_dotted(runner.SESSION_VERIFY_INVARIANT_ARGS["custom_agent_function_path"]) - - assert callable(generate) - assert callable(run_agent) - - -def test_run_session_verify_cpu_fast_positive_path(tmp_path, monkeypatch): - import miles.utils.external_utils.command_utils as command_utils - - calls = [] - - def fake_execute_train(**kwargs): - calls.append(kwargs) - metrics_path = kwargs["extra_env_vars"]["MILES_SESSION_VERIFY_METRICS_PATH"] - with open(metrics_path, "w") as f: - f.write( - json.dumps( - { - "driver_events": ["initial", "append_tool", "rollback"], - "had_assistant_mismatch": False, - } - ) - + "\n" - ) - - monkeypatch.setattr(command_utils, "execute_train", fake_execute_train) - monkeypatch.setattr(runner, "PROMPT_DATA_PATH", str(tmp_path / "session_multi_role_verify.jsonl")) - - args = _build_args(tmp_path) - runner.run_session_verify(args) - - assert len(calls) == 1 - assert calls[0]["num_gpus_per_node"] == 1 - assert calls[0]["megatron_model_type"] is None - train_args = calls[0]["train_args"] - assert f"--hf-checkpoint {tmp_path / 'local-model'}" in train_args - assert "--custom-generate-function-path miles.utils.test_utils.session_verify_agent.generate" in train_args - assert "--custom-agent-function-path miles.utils.test_utils.session_verify_agent.run_agent" in train_args diff --git a/sidecars/tito/tests/package/test_upstream_delegation.py b/sidecars/tito/tests/package/test_upstream_delegation.py deleted file mode 100644 index 8e7cdd5..0000000 --- a/sidecars/tito/tests/package/test_upstream_delegation.py +++ /dev/null @@ -1,121 +0,0 @@ -import importlib -import sys - -import pytest - -from miles._upstream_loader import UpstreamModuleLoadError, load_upstream_module - - -TARGET_MODULES = { - "miles.utils.external_utils.command_utils", - "miles.rollout.generate_hub.agentic_tool_call", - "miles.rollout.base_types", -} - - -def _clear_target_modules(): - for name in list(sys.modules): - if name in TARGET_MODULES or name.startswith("_tito_gateway_upstream_"): - sys.modules.pop(name, None) - - -@pytest.fixture(autouse=True) -def clear_target_modules(): - _clear_target_modules() - yield - _clear_target_modules() - - -def _write_fake_upstream(root): - command_utils = root / "miles" / "utils" / "external_utils" - command_utils.mkdir(parents=True) - (command_utils / "command_utils.py").write_text( - "SOURCE = 'fake-upstream-command-utils'\n" - "def exec_command(*args, **kwargs):\n" - " return ('upstream-exec', args, kwargs)\n" - "def execute_train(*args, **kwargs):\n" - " return ('upstream-train', args, kwargs)\n" - ) - - generate_hub = root / "miles" / "rollout" / "generate_hub" - generate_hub.mkdir(parents=True) - (generate_hub / "agentic_tool_call.py").write_text( - "SOURCE = 'fake-upstream-agentic-tool-call'\n" - "async def generate(input):\n" - " return ('upstream-generate', input)\n" - "def _add_arguments(parser):\n" - " parser.add_argument('--fake-upstream-agentic-flag')\n" - "generate.add_arguments = _add_arguments\n" - ) - - rollout = root / "miles" / "rollout" - (rollout / "base_types.py").write_text( - "SOURCE = 'fake-upstream-base-types'\n" - "class GenerateFnInput:\n" - " ORIGIN = 'upstream'\n" - "class GenerateFnOutput:\n" - " ORIGIN = 'upstream'\n" - ) - - -def test_exact_name_wrappers_delegate_to_later_upstream_sys_path(tmp_path, monkeypatch): - fake_root = tmp_path / "fake_upstream" - _write_fake_upstream(fake_root) - monkeypatch.setattr(sys, "path", [*sys.path, str(fake_root)]) - importlib.invalidate_caches() - - command_utils = importlib.import_module("miles.utils.external_utils.command_utils") - agentic_tool_call = importlib.import_module("miles.rollout.generate_hub.agentic_tool_call") - base_types = importlib.import_module("miles.rollout.base_types") - - assert command_utils.SOURCE == "fake-upstream-command-utils" - assert command_utils.exec_command("x")[0] == "upstream-exec" - assert command_utils.execute_train(train_args="--debug")[0] == "upstream-train" - - assert agentic_tool_call.SOURCE == "fake-upstream-agentic-tool-call" - assert callable(agentic_tool_call.generate) - assert callable(agentic_tool_call.generate.add_arguments) - - assert base_types.SOURCE == "fake-upstream-base-types" - assert base_types.GenerateFnInput.ORIGIN == "upstream" - assert base_types.GenerateFnOutput.ORIGIN == "upstream" - - -def test_loader_considers_upstream_candidate_under_shared_install_root(tmp_path, monkeypatch): - shared_root = tmp_path / "site-packages" - upstream = shared_root / "miles" / "utils" / "external_utils" - upstream.mkdir(parents=True) - (upstream / "command_utils.py").write_text("SOURCE = 'shared-root-upstream'\n") - - local_file = shared_root / "tito_gateway_wrapper" / "miles" / "utils" / "external_utils" / "command_utils.py" - local_file.parent.mkdir(parents=True) - local_file.write_text("SOURCE = 'local-wrapper'\n") - - monkeypatch.setattr(sys, "path", [str(shared_root)]) - - module = load_upstream_module("miles.utils.external_utils.command_utils", str(local_file)) - - assert module is not None - assert module.SOURCE == "shared-root-upstream" - - -def test_present_upstream_import_failure_is_not_masked(tmp_path, monkeypatch): - fake_root = tmp_path / "broken_upstream" - command_utils = fake_root / "miles" / "utils" / "external_utils" - command_utils.mkdir(parents=True) - (command_utils / "command_utils.py").write_text("raise RuntimeError('upstream exploded')\n") - monkeypatch.setattr(sys, "path", [*sys.path, str(fake_root)]) - importlib.invalidate_caches() - - with pytest.raises(UpstreamModuleLoadError, match="Found upstream candidate") as exc_info: - importlib.import_module("miles.utils.external_utils.command_utils") - - assert isinstance(exc_info.value.__cause__, RuntimeError) - assert "upstream exploded" in str(exc_info.value.__cause__) - - -def test_command_utils_fallback_remains_clear_without_upstream(): - command_utils = importlib.import_module("miles.utils.external_utils.command_utils") - - with pytest.raises(command_utils.MissingMilesTrainingStackError, match="not bundled with tito-gateway"): - command_utils.execute_train(train_args="--debug", num_gpus_per_node=1, megatron_model_type=None) diff --git a/sidecars/tito/tests/package/test_vendored_miles.py b/sidecars/tito/tests/package/test_vendored_miles.py deleted file mode 100644 index 175590a..0000000 --- a/sidecars/tito/tests/package/test_vendored_miles.py +++ /dev/null @@ -1,31 +0,0 @@ -from pathlib import Path - - -def test_miles_chat_template_compat_import_resolves_fixed_template(): - from miles.utils.chat_template_utils import TITOTokenizerType, resolve_fixed_chat_template - - template_path, kwargs = resolve_fixed_chat_template(TITOTokenizerType.QWEN3, ["tool"]) - - assert template_path is not None - assert Path(template_path).name == "qwen3_fixed.jinja" - assert Path(template_path).is_file() - assert kwargs == {} - - -def test_miles_session_compat_import_resolves_errors(): - from miles.rollout.session.session_errors import SessionError, SessionNotFoundError - - assert SessionError.status_code == 500 - assert SessionNotFoundError.status_code == 404 - - -def test_public_get_tito_tokenizer_delegates_to_vendored_default(): - from tito_gateway import get_tito_tokenizer - from tito_gateway.vendor.miles_compat.utils.chat_template_utils.tito_tokenizer import TITOTokenizer - - fake_tokenizer = object() - - result = get_tito_tokenizer(fake_tokenizer, tokenizer_type="default") - - assert isinstance(result, TITOTokenizer) - assert result.tokenizer is fake_tokenizer diff --git a/sidecars/tito/tests/upstream/fast/router/test_session_pretokenized_e2e.py b/sidecars/tito/tests/upstream/fast/router/test_session_pretokenized_e2e.py deleted file mode 100644 index d7a9dfb..0000000 --- a/sidecars/tito/tests/upstream/fast/router/test_session_pretokenized_e2e.py +++ /dev/null @@ -1,174 +0,0 @@ -"""Representative session-layer smoke tests for miles-maintained fixed templates. - -These tests intentionally stay narrow: - -- only bundled fixed templates maintained by miles -- only tool-only multi-turn session flow -- only session/TITO plumbing + mismatch taxonomy checks - -Detailed template correctness remains covered by the lower-level chat-template -tests in ``tests/fast/utils/chat_template_utils/``. -""" - -from __future__ import annotations - -from dataclasses import dataclass - -import pytest -import requests -from tests.fast.router.session_pretokenized_test_utils import ( - ScriptedBackendTurn, - ScriptedChatBackend, - compute_local_session_mismatch, - fetch_session_payload, - forbidden_mismatches, - load_test_tokenizer, - make_router_env, - teardown_router_env, -) - -from miles.utils.chat_template_utils import TITOTokenizerType, resolve_fixed_chat_template -from miles.utils.test_utils.mock_trajectories import LongChainTrajectory, build_trajectory - - -@dataclass(frozen=True) -class FixedTemplateSmokeConfig: - name: str - hf_checkpoint: str - chat_template_path: str - tito_model: str - - -FIXED_TEMPLATE_SMOKE_CONFIGS: tuple[FixedTemplateSmokeConfig, ...] = ( - FixedTemplateSmokeConfig( - name="qwen3-fixed", - hf_checkpoint="Qwen/Qwen3-0.6B", - chat_template_path=resolve_fixed_chat_template(TITOTokenizerType.QWEN3, ["tool"])[0], - tito_model=TITOTokenizerType.QWEN3.value, - ), - FixedTemplateSmokeConfig( - name="qwen3.5-fixed", - hf_checkpoint="Qwen/Qwen3.5-0.8B", - chat_template_path=resolve_fixed_chat_template(TITOTokenizerType.QWEN35, ["tool"])[0], - tito_model=TITOTokenizerType.QWEN35.value, - ), - FixedTemplateSmokeConfig( - name="qwen3-thinking2507-fixed", - hf_checkpoint="Qwen/Qwen3-4B-Thinking-2507", - chat_template_path=resolve_fixed_chat_template(TITOTokenizerType.QWENNEXT, ["tool"])[0], - tito_model=TITOTokenizerType.QWENNEXT.value, - ), - FixedTemplateSmokeConfig( - name="qwen3-next-thinking-fixed", - hf_checkpoint="Qwen/Qwen3-Next-80B-A3B-Thinking", - chat_template_path=resolve_fixed_chat_template(TITOTokenizerType.QWENNEXT, ["tool"])[0], - tito_model=TITOTokenizerType.QWENNEXT.value, - ), -) - - -def _get_followup_messages_after_assistant(full_messages: list[dict], assistant_idx: int) -> list[dict]: - followup = [] - i = assistant_idx + 1 - while i < len(full_messages) and full_messages[i]["role"] != "assistant": - followup.append(full_messages[i]) - i += 1 - return followup - - -def _remap_followup_messages(followup_msgs: list[dict], response_tool_calls: list[dict]) -> list[dict]: - remapped = [] - tool_idx = 0 - for msg in followup_msgs: - new_msg = dict(msg) - if msg["role"] == "tool": - if tool_idx < len(response_tool_calls): - new_msg["tool_call_id"] = response_tool_calls[tool_idx]["id"] - tool_idx += 1 - remapped.append(new_msg) - return remapped - - -@pytest.mark.parametrize("config", FIXED_TEMPLATE_SMOKE_CONFIGS, ids=[c.name for c in FIXED_TEMPLATE_SMOKE_CONFIGS]) -def test_bundled_fixed_template_session_smoke(config: FixedTemplateSmokeConfig): - assert config.chat_template_path is not None, f"{config.name} should resolve to a bundled fixed template" - - try: - tokenizer = load_test_tokenizer(config.hf_checkpoint, config.chat_template_path) - except (ValueError, OSError) as exc: - pytest.skip(f"Cannot load tokenizer for {config.hf_checkpoint}: {exc}") - - trajectory = build_trajectory(tokenizer, LongChainTrajectory) - scripted_turns = [ - ScriptedBackendTurn( - response_message={**turn.assistant_message, "content": turn.assistant_message.get("content") or ""}, - render_message=turn.assistant_message, - ) - for turn in trajectory.turns - ] - backend = ScriptedChatBackend(tokenizer, scripted_turns) - backend.start() - env = make_router_env( - backend, - hf_checkpoint=config.hf_checkpoint, - chat_template_path=config.chat_template_path, - tito_model=config.tito_model, - allowed_append_roles=["tool"], - ) - - try: - backend.reset_stats() - session_id = requests.post(f"{env.url}/sessions", timeout=5.0).json()["session_id"] - assistant_indices = [i for i, m in enumerate(trajectory.full_messages) if m["role"] == "assistant"] - - accumulated_messages: list[dict] = [] - for turn_idx, turn in enumerate(trajectory.turns): - messages = list(accumulated_messages) if turn_idx > 0 else list(turn.request_messages) - if turn_idx == 0: - accumulated_messages = list(messages) - - payload = {"messages": messages, "tools": trajectory.tools} - response = requests.post( - f"{env.url}/sessions/{session_id}/v1/chat/completions", - json=payload, - timeout=10.0, - ) - assert response.status_code == 200, f"{config.name} turn {turn_idx} failed: {response.text}" - - body = response.json() - assert len(body["choices"]) == 1 - if turn_idx > 0: - assert "input_ids" in backend.request_log[turn_idx], f"{config.name} turn {turn_idx} missing input_ids" - - assistant_msg = body["choices"][0]["message"] - session_messages = list(messages) + [assistant_msg] - - session_payload = fetch_session_payload(env.url, session_id) - metadata = session_payload["metadata"] - remote_mismatch = metadata.get("tito_session_mismatch", []) - local_mismatch = compute_local_session_mismatch( - tokenizer, - tito_model=config.tito_model, - allowed_append_roles=["tool"], - messages=session_messages, - accumulated_token_ids=metadata["accumulated_token_ids"], - tools=trajectory.tools, - ) - assert remote_mismatch == local_mismatch - assert ( - forbidden_mismatches(remote_mismatch) == [] - ), f"{config.name} turn {turn_idx} has forbidden mismatch types: {remote_mismatch}" - - accumulated_messages.append(assistant_msg) - ass_idx = assistant_indices[turn_idx] - followup_msgs = _get_followup_messages_after_assistant(trajectory.full_messages, ass_idx) - response_tool_calls = assistant_msg.get("tool_calls") or [] - accumulated_messages.extend(_remap_followup_messages(followup_msgs, response_tool_calls)) - - final_session_payload = fetch_session_payload(env.url, session_id) - records = final_session_payload["records"] - assert len(records) == len(trajectory.turns) - assert all(r["status_code"] == 200 for r in records) - assert all(r["path"] == "/v1/chat/completions" for r in records) - finally: - teardown_router_env(env) diff --git a/sidecars/tito/tests/upstream/fast/router/test_session_race_conditions.py b/sidecars/tito/tests/upstream/fast/router/test_session_race_conditions.py deleted file mode 100644 index 95c6a69..0000000 --- a/sidecars/tito/tests/upstream/fast/router/test_session_race_conditions.py +++ /dev/null @@ -1,422 +0,0 @@ -"""E2E session stress tests. - -Contract under test (with split-lock / session.closing): -- Phase 1 (prepare) and Phase 3 (state update) hold session.lock briefly; - Phase 2 (proxy to SGLang) does NOT hold the lock. -- Concurrent same-session requests can overlap at the backend (Phase 2), - but state updates (Phase 3) are serialized; stale-update guard - (expected_num_assistant check) ensures only one concurrent writer wins. -- Different sessions can run in parallel (no global lock). -- Per-session clients can run turn-by-turn without idle gaps while global load stays parallel. -- Delete marks session.closing=True, acquires session.lock, then removes. - Because the lock is not held during Phase 2, delete can proceed while a - chat request is mid-proxy; the chat's Phase 3 will see closing=True and - skip the state update gracefully. -- Chat requests to a closing session get 404 immediately (pre-lock check). -- Chat requests arriving while delete waits for lock get 404 (double-check after lock). -- Concurrent deletes on the same session: second delete gets 404. -""" - -from __future__ import annotations - -import time -from concurrent.futures import ThreadPoolExecutor -from contextlib import contextmanager -from types import SimpleNamespace -from unittest.mock import patch - -import requests - -from miles.rollout.session.session_server import SessionServer -from miles.utils.http_utils import find_available_port -from miles.utils.test_utils.mock_sglang_server import MockSGLangServer, ProcessResult, with_mock_server -from miles.utils.test_utils.uvicorn_thread_server import UvicornThreadServer - -HF_CHECKPOINT = "Qwen/Qwen3-0.6B" - - -def _patch_mock_chat_response(): - original_chat_response = MockSGLangServer._compute_chat_completions_response - - def patched_chat_response(self, payload: dict) -> dict: - response = original_chat_response(self, payload) - # Session server expects output_token_logprobs as (logprob, token_id). - choice = response["choices"][0] - logprobs_content = choice["logprobs"]["content"] - output_token_logprobs = [ - (item["logprob"], self.tokenizer.convert_tokens_to_ids(item["token"])) for item in logprobs_content - ] - choice["meta_info"] = { - "output_token_logprobs": output_token_logprobs, - "completion_tokens": len(output_token_logprobs), - } - return response - - return patch.object(MockSGLangServer, "_compute_chat_completions_response", new=patched_chat_response) - - -@contextmanager -def _router_env(process_fn, *, latency: float = 0.0): - with _patch_mock_chat_response(): - with with_mock_server(model_name=HF_CHECKPOINT, process_fn=process_fn, latency=latency) as backend: - args = SimpleNamespace( - miles_router_timeout=30, - hf_checkpoint=HF_CHECKPOINT, - chat_template_path=None, - trajectory_manager="linear_trajectory", - tito_allowed_append_roles=["tool", "system"], - ) - server_obj = SessionServer(args, backend_url=backend.url) - - port = find_available_port(31000) - server = UvicornThreadServer(server_obj.app, host="127.0.0.1", port=port) - server.start() - url = f"http://127.0.0.1:{port}" - - try: - yield SimpleNamespace(url=url, backend=backend, server=server) - finally: - server.stop() - - -def _create_session(url: str) -> str: - response = requests.post(f"{url}/sessions", timeout=5.0) - assert response.status_code == 200 - return response.json()["session_id"] - - -def _chat(url: str, session_id: str, payload: dict, timeout: float = 20.0) -> requests.Response: - return requests.post( - f"{url}/sessions/{session_id}/v1/chat/completions", - json=payload, - timeout=timeout, - ) - - -class TestSessionConcurrencyContracts: - def test_same_session_concurrent_requests_reach_backend(self): - """With the split-lock, same-session requests CAN overlap at the backend. - - Phase 2 (proxy) runs without the lock, so concurrent requests are not - serialized at the backend level. Phase 3 state updates are still - serialized; the stale-update guard ensures only one writer wins per - generation, so no state corruption occurs. - """ - - def process_fn(prompt: str) -> ProcessResult: - return ProcessResult(text="concurrent-ok", finish_reason="stop") - - with _router_env(process_fn, latency=0.2) as env: - session_id = _create_session(env.url) - - # Warm up one assistant checkpoint so repeated identical retry payloads are valid. - warmup_payload = {"messages": [{"role": "user", "content": "warmup"}]} - warmup_resp = _chat(env.url, session_id, warmup_payload) - assert warmup_resp.status_code == 200 - assistant = warmup_resp.json()["choices"][0]["message"] - - retry_payload = { - "messages": [ - {"role": "user", "content": "warmup"}, - assistant, - {"role": "system", "content": "retry-from-assistant-checkpoint"}, - ] - } - - env.backend.reset_stats() - with ThreadPoolExecutor(max_workers=4) as pool: - futures = [pool.submit(_chat, env.url, session_id, retry_payload) for _ in range(4)] - responses = [f.result(timeout=30.0) for f in futures] - - # All requests should succeed (200) — no 500s. - assert all(resp.status_code == 200 for resp in responses) - assert len(env.backend.request_log) == 4 - # With split-lock, concurrent backend access is expected (not == 1). - assert env.backend.max_concurrent >= 1 - - def test_different_sessions_can_run_in_parallel(self): - def process_fn(prompt: str) -> ProcessResult: - return ProcessResult(text="parallel-ok", finish_reason="stop") - - with _router_env(process_fn, latency=0.2) as env: - session_ids = [_create_session(env.url) for _ in range(6)] - - env.backend.reset_stats() - with ThreadPoolExecutor(max_workers=6) as pool: - futures = [ - pool.submit( - _chat, - env.url, - sid, - {"messages": [{"role": "user", "content": f"parallel-{i}"}]}, - ) - for i, sid in enumerate(session_ids) - ] - responses = [f.result(timeout=30.0) for f in futures] - - assert all(resp.status_code == 200 for resp in responses) - assert len(env.backend.request_log) == 6 - assert env.backend.max_concurrent >= 3 - - def test_e2e_pressure_serial_per_session_parallel_globally(self): - num_sessions = 8 - turns_per_session = 3 - - def process_fn(prompt: str) -> ProcessResult: - return ProcessResult(text="turn-ok", finish_reason="stop") - - with _router_env(process_fn, latency=0.08) as env: - session_ids = [_create_session(env.url) for _ in range(num_sessions)] - - def run_session_worker(session_id: str, idx: int) -> bool: - messages: list[dict] = [{"role": "user", "content": f"session-{idx}-turn-0"}] - for turn in range(turns_per_session): - resp = _chat(env.url, session_id, {"messages": messages}, timeout=30.0) - assert resp.status_code == 200 - assistant = resp.json()["choices"][0]["message"] - if turn < turns_per_session - 1: - messages = [ - *messages, - assistant, - {"role": "system", "content": f"session-{idx}-continue-{turn}"}, - ] - return True - - env.backend.reset_stats() - with ThreadPoolExecutor(max_workers=num_sessions) as pool: - futures = [pool.submit(run_session_worker, sid, idx) for idx, sid in enumerate(session_ids)] - results = [f.result(timeout=120.0) for f in futures] - - assert all(results) - assert len(env.backend.request_log) == num_sessions * turns_per_session - assert env.backend.max_concurrent >= 4 - - def test_delete_can_proceed_while_chat_is_mid_proxy(self): - """With split-lock, delete can acquire the lock while chat is in Phase 2. - - The inflight chat's Phase 3 sees session.closing=True and skips - state update gracefully. Both chat and delete complete without error. - """ - - def process_fn(prompt: str) -> ProcessResult: - return ProcessResult(text="slow-turn", finish_reason="stop") - - with _router_env(process_fn, latency=0.35) as env: - session_id = _create_session(env.url) - payload = {"messages": [{"role": "user", "content": "slow-turn-0"}]} - - with ThreadPoolExecutor(max_workers=2) as pool: - inflight = pool.submit(_chat, env.url, session_id, payload, 30.0) - - # Wait until the first request has reached backend before deleting. - deadline = time.time() + 5.0 - while time.time() < deadline: - if env.backend.request_log: - break - time.sleep(0.01) - else: - raise AssertionError("in-flight request did not reach backend in time") - - delete_resp = requests.delete(f"{env.url}/sessions/{session_id}", timeout=30.0) - inflight_resp = inflight.result(timeout=30.0) - - # Chat returns 200 (backend responded); delete returns 204. - assert inflight_resp.status_code == 200 - assert delete_resp.status_code == 204 - # Session is gone after delete. - post_delete = _chat(env.url, session_id, payload, timeout=10.0) - assert post_delete.status_code == 404 - - -class TestClosingRaceConditions: - """Tests for race conditions around session.closing flag.""" - - def test_chat_during_delete_returns_404(self): - """Chat requests arriving after delete sets closing=True get 404. - - Timeline: - 1. Chat A starts, acquires lock (Phase 1), releases it, proxying (Phase 2) - 2. Delete arrives, sets session.closing=True, acquires lock, removes session - 3. Chat B arrives, sees session.closing=True, returns 404 immediately - 4. Chat A's Phase 3 sees closing=True, skips state update, returns 200 - """ - - def process_fn(prompt: str) -> ProcessResult: - return ProcessResult(text="slow", finish_reason="stop") - - with _router_env(process_fn, latency=0.5) as env: - session_id = _create_session(env.url) - payload = {"messages": [{"role": "user", "content": "slow-chat"}]} - - with ThreadPoolExecutor(max_workers=3) as pool: - # 1. Start slow chat A - chat_a = pool.submit(_chat, env.url, session_id, payload, 30.0) - - # Wait for chat A to reach backend - deadline = time.time() + 5.0 - while time.time() < deadline: - if env.backend.request_log: - break - time.sleep(0.01) - - # 2. Start delete (will block waiting for lock) - delete_future = pool.submit( - requests.delete, - f"{env.url}/sessions/{session_id}", - timeout=30.0, - ) - # Small delay to ensure delete has set closing=True - time.sleep(0.05) - - # 3. Chat B should get 404 because session.closing=True - chat_b = _chat(env.url, session_id, payload, timeout=10.0) - assert chat_b.status_code == 404, f"Chat during closing should return 404, got {chat_b.status_code}" - - # Wait for remaining futures - chat_a_resp = chat_a.result(timeout=30.0) - delete_resp = delete_future.result(timeout=30.0) - - assert chat_a_resp.status_code == 200 - assert delete_resp.status_code == 204 - - def test_double_delete_second_returns_404(self): - """Concurrent delete on the same session: second delete gets 404. - - With session.closing flag, the first delete sets closing=True. - The second delete sees closing=True and returns 404. - """ - - def process_fn(prompt: str) -> ProcessResult: - return ProcessResult(text="ok", finish_reason="stop") - - with _router_env(process_fn, latency=0.3) as env: - session_id = _create_session(env.url) - - # Start a slow chat to hold the lock - payload = {"messages": [{"role": "user", "content": "hold-lock"}]} - with ThreadPoolExecutor(max_workers=3) as pool: - chat_future = pool.submit(_chat, env.url, session_id, payload, 30.0) - - # Wait for chat to reach backend - deadline = time.time() + 5.0 - while time.time() < deadline: - if env.backend.request_log: - break - time.sleep(0.01) - - # Fire two deletes concurrently - delete_1 = pool.submit( - requests.delete, - f"{env.url}/sessions/{session_id}", - timeout=30.0, - ) - time.sleep(0.02) # tiny delay to let first delete set closing - delete_2 = pool.submit( - requests.delete, - f"{env.url}/sessions/{session_id}", - timeout=30.0, - ) - - chat_resp = chat_future.result(timeout=30.0) - d1 = delete_1.result(timeout=30.0) - d2 = delete_2.result(timeout=30.0) - - assert chat_resp.status_code == 200 - # One delete succeeds, the other gets 404 - codes = sorted([d1.status_code, d2.status_code]) - assert codes == [204, 404], f"Expected [204, 404], got {codes}" - - def test_chat_after_delete_returns_404(self): - """Chat request after session is fully deleted returns 404.""" - - def process_fn(prompt: str) -> ProcessResult: - return ProcessResult(text="ok", finish_reason="stop") - - with _router_env(process_fn) as env: - session_id = _create_session(env.url) - - # Delete the session - delete_resp = requests.delete(f"{env.url}/sessions/{session_id}", timeout=5.0) - assert delete_resp.status_code == 204 - - # Chat should get 404 - payload = {"messages": [{"role": "user", "content": "hello"}]} - chat_resp = _chat(env.url, session_id, payload, timeout=5.0) - assert chat_resp.status_code == 404 - - # GET should also get 404 - get_resp = requests.get(f"{env.url}/sessions/{session_id}", timeout=5.0) - assert get_resp.status_code == 404 - - def test_multiple_chats_queued_then_delete(self): - """Multiple chat requests queued behind session.lock, then delete. - - After delete marks closing=True, queued chats that acquire the lock - should check closing and return 404. - """ - - def process_fn(prompt: str) -> ProcessResult: - return ProcessResult(text="queued-ok", finish_reason="stop") - - with _router_env(process_fn, latency=0.3) as env: - session_id = _create_session(env.url) - payload = {"messages": [{"role": "user", "content": "queued"}]} - - with ThreadPoolExecutor(max_workers=6) as pool: - # Fire 3 chats (first holds lock, others queue) - chat_futures = [pool.submit(_chat, env.url, session_id, payload, 30.0) for _ in range(3)] - - # Wait for first to reach backend - deadline = time.time() + 5.0 - while time.time() < deadline: - if env.backend.request_log: - break - time.sleep(0.01) - - # Now delete - sets closing, waits for first chat to finish - delete_future = pool.submit( - requests.delete, - f"{env.url}/sessions/{session_id}", - timeout=30.0, - ) - - results = [f.result(timeout=30.0) for f in chat_futures] - delete_resp = delete_future.result(timeout=30.0) - - assert delete_resp.status_code == 204 - - # At least one chat must succeed (the one holding the lock when - # delete arrived). Others may get 200 (acquired lock before - # closing) or 404 (saw closing=True). No 500s allowed. - status_codes = [r.status_code for r in results] - assert all(c in (200, 404) for c in status_codes), f"Unexpected status codes: {status_codes}" - assert 200 in status_codes, f"Expected at least one 200, got {status_codes}" - - def test_rapid_create_chat_delete_cycles(self): - """Rapidly create, chat, and delete sessions to stress the lifecycle. - - Ensures no deadlocks or crashes from rapid session lifecycle operations. - """ - - def process_fn(prompt: str) -> ProcessResult: - return ProcessResult(text="cycle-ok", finish_reason="stop") - - with _router_env(process_fn) as env: - - def lifecycle_cycle(idx: int) -> bool: - session_id = _create_session(env.url) - payload = {"messages": [{"role": "user", "content": f"cycle-{idx}"}]} - chat_resp = _chat(env.url, session_id, payload, timeout=10.0) - assert chat_resp.status_code == 200 - delete_resp = requests.delete(f"{env.url}/sessions/{session_id}", timeout=5.0) - assert delete_resp.status_code == 204 - # Verify gone - get_resp = requests.get(f"{env.url}/sessions/{session_id}", timeout=5.0) - assert get_resp.status_code == 404 - return True - - with ThreadPoolExecutor(max_workers=8) as pool: - futures = [pool.submit(lifecycle_cycle, i) for i in range(20)] - results = [f.result(timeout=60.0) for f in futures] - - assert all(results) diff --git a/sidecars/tito/tests/upstream/fast/router/test_sessions.py b/sidecars/tito/tests/upstream/fast/router/test_sessions.py deleted file mode 100644 index 86f2e2d..0000000 --- a/sidecars/tito/tests/upstream/fast/router/test_sessions.py +++ /dev/null @@ -1,140 +0,0 @@ -"""Integration tests for session HTTP routes (create / get / delete / proxy).""" - -import re -import uuid -from types import SimpleNamespace -from unittest.mock import patch - -import pytest -import requests - -from miles.rollout.session.session_server import SessionServer -from miles.utils.http_utils import find_available_port -from miles.utils.test_utils.mock_sglang_server import MockSGLangServer, ProcessResult, with_mock_server -from miles.utils.test_utils.uvicorn_thread_server import UvicornThreadServer - - -@pytest.fixture(scope="class") -def router_env(): - """Create a standalone SessionServer with session routes and a mock backend.""" - - def process_fn(prompt: str) -> ProcessResult: - return ProcessResult(text=f"echo: {prompt}", finish_reason="stop") - - original_chat_response = MockSGLangServer._compute_chat_completions_response - - def patched_chat_response(self, payload: dict) -> dict: - response = original_chat_response(self, payload) - choice = response["choices"][0] - logprobs_content = choice["logprobs"]["content"] - output_token_logprobs = [ - (item["logprob"], self.tokenizer.convert_tokens_to_ids(item["token"])) for item in logprobs_content - ] - choice["meta_info"] = { - "output_token_logprobs": output_token_logprobs, - "completion_tokens": len(output_token_logprobs), - } - return response - - with patch.object(MockSGLangServer, "_compute_chat_completions_response", new=patched_chat_response): - with with_mock_server(process_fn=process_fn) as backend: - args = SimpleNamespace( - miles_router_timeout=30, - hf_checkpoint="Qwen/Qwen3-0.6B", - chat_template_path=None, - apply_chat_template_kwargs={"enable_thinking": False}, - tito_model="default", - tito_allowed_append_roles=["tool"], - trajectory_manager="linear_trajectory", - session_server_instance_id=uuid.uuid4().hex, - ) - server_obj = SessionServer(args, backend_url=backend.url) - - port = find_available_port(31000) - server = UvicornThreadServer(server_obj.app, host="127.0.0.1", port=port) - server.start() - - url = f"http://127.0.0.1:{port}" - - try: - yield SimpleNamespace(url=url, backend=backend) - finally: - server.stop() - - -class TestSessionRoutes: - def test_health_reports_stable_instance_id(self, router_env): - first = requests.get(f"{router_env.url}/health", timeout=5.0) - second = requests.get(f"{router_env.url}/health", timeout=5.0) - - assert first.status_code == 200 - assert second.status_code == 200 - first_body = first.json() - second_body = second.json() - assert first_body["status"] == "ok" - assert second_body["status"] == "ok" - assert re.fullmatch(r"[0-9a-f]{32}", first_body["session_server_instance_id"]) - assert second_body["session_server_instance_id"] == first_body["session_server_instance_id"] - - def test_create_session(self, router_env): - response = requests.post(f"{router_env.url}/sessions", timeout=5.0) - assert response.status_code == 200 - data = response.json() - assert "session_id" in data - assert len(data["session_id"]) == 32 - - def test_get_session_initial_state(self, router_env): - session_id = requests.post(f"{router_env.url}/sessions", timeout=5.0).json()["session_id"] - - get_resp = requests.get(f"{router_env.url}/sessions/{session_id}", timeout=5.0) - assert get_resp.status_code == 200 - data = get_resp.json() - assert data["session_id"] == session_id - assert data["records"] == [] - - def test_get_session_not_found(self, router_env): - response = requests.get(f"{router_env.url}/sessions/nonexistent", timeout=5.0) - assert response.status_code == 404 - assert response.json()["error"] == "session not found: session_id=nonexistent" - - def test_delete_session(self, router_env): - session_id = requests.post(f"{router_env.url}/sessions", timeout=5.0).json()["session_id"] - - delete_resp = requests.delete(f"{router_env.url}/sessions/{session_id}", timeout=5.0) - assert delete_resp.status_code == 204 - assert delete_resp.text == "" - - assert requests.delete(f"{router_env.url}/sessions/{session_id}", timeout=5.0).status_code == 404 - - def test_delete_session_not_found(self, router_env): - response = requests.delete(f"{router_env.url}/sessions/nonexistent", timeout=5.0) - assert response.status_code == 404 - assert response.json()["error"] == "session not found: session_id=nonexistent" - - -class TestSessionProxy: - def test_proxy_chat_appends_record(self, router_env): - session_id = requests.post(f"{router_env.url}/sessions", timeout=5.0).json()["session_id"] - - payload = { - "messages": [{"role": "user", "content": "What is 1+2?"}], - "return_logprob": True, - } - resp = requests.post( - f"{router_env.url}/sessions/{session_id}/v1/chat/completions", - json=payload, - timeout=10.0, - ) - assert resp.status_code == 200 - body = resp.json() - assert "choices" in body - assert body["choices"] - - get_resp = requests.get(f"{router_env.url}/sessions/{session_id}", timeout=5.0) - records = get_resp.json()["records"] - - assert isinstance(records, list) - assert len(records) == 1 - record = records[0] - assert record["path"] == "/v1/chat/completions" - assert record["status_code"] == 200 diff --git a/sidecars/tito/tests/upstream/fast/utils/chat_template_utils/test_pretokenized_via_tito.py b/sidecars/tito/tests/upstream/fast/utils/chat_template_utils/test_pretokenized_via_tito.py deleted file mode 100644 index 68b392c..0000000 --- a/sidecars/tito/tests/upstream/fast/utils/chat_template_utils/test_pretokenized_via_tito.py +++ /dev/null @@ -1,161 +0,0 @@ -"""Unit tests for ``verify_append_only_via_tito_instance`` / -``run_all_checks_via_tito``: PASS on registered TITO families, FAIL on the -unfixed Qwen3 chat template, FAIL on a test-local ``_BuggyQwen3TITOTokenizer`` -that omits the ``\\n`` insertion at the ``<|im_end|>`` boundary. -""" - -from copy import deepcopy - -import pytest -from tests.ci.ci_register import register_cpu_ci -from transformers import AutoTokenizer - -register_cpu_ci(est_time=120, suite="stage-b-cpu", labels=[]) - - -from miles.utils.chat_template_utils import TITOTokenizerType, resolve_fixed_chat_template -from miles.utils.chat_template_utils.tito_tokenizer import Qwen3TITOTokenizer -from miles.utils.test_utils.chat_template_verify import run_all_checks_via_tito, verify_append_only_via_tito_instance -from miles.utils.test_utils.mock_trajectories import SingleToolTrajectory - -# --------------------------------------------------------------------------- -# Test helpers -# --------------------------------------------------------------------------- - - -def _setup_tokenizer_with_registered_template( - model_id: str, - family: TITOTokenizerType, - roles: list[str], -): - """Mirror what production wiring does at startup. - - Loads tokenizer, looks up the registered ``SUPPORTED_TEMPLATES`` row for - ``(family, roles)``, and applies the resolved fixed template (if any) onto - ``tokenizer.chat_template``. Returns ``(tokenizer, extra_kwargs)``. - - A fresh tokenizer instance per call avoids state-mutation hazards from - overwriting ``chat_template``. - """ - tokenizer = AutoTokenizer.from_pretrained(model_id) - fixed_path, extra_kwargs = resolve_fixed_chat_template(family, roles) - if fixed_path is not None: - with open(fixed_path) as f: - tokenizer.chat_template = f.read() - return tokenizer, dict(extra_kwargs) - - -# --------------------------------------------------------------------------- -# (1) PASS on registered families × role surfaces -# --------------------------------------------------------------------------- - - -_PASS_PARAMS = [ - pytest.param(TITOTokenizerType.QWEN3, "Qwen/Qwen3-0.6B", frozenset({"tool"}), id="qwen3-tool"), - pytest.param(TITOTokenizerType.QWEN3, "Qwen/Qwen3-0.6B", frozenset({"tool", "user"}), id="qwen3-tool_user"), - pytest.param(TITOTokenizerType.QWEN35, "Qwen/Qwen3.5-0.8B", frozenset({"tool"}), id="qwen35-tool"), - pytest.param(TITOTokenizerType.QWEN35, "Qwen/Qwen3.5-0.8B", frozenset({"tool", "user"}), id="qwen35-tool_user"), - pytest.param(TITOTokenizerType.QWENNEXT, "Qwen/Qwen3-4B-Thinking-2507", frozenset({"tool"}), id="qwennext-tool"), - pytest.param( - TITOTokenizerType.QWENNEXT, - "Qwen/Qwen3-4B-Thinking-2507", - frozenset({"tool", "user"}), - id="qwennext-tool_user", - ), - pytest.param(TITOTokenizerType.GLM47, "zai-org/GLM-4.7-Flash", frozenset({"tool"}), id="glm47-tool"), - pytest.param(TITOTokenizerType.GLM47, "zai-org/GLM-4.7-Flash", frozenset({"tool", "user"}), id="glm47-tool_user"), - pytest.param( - TITOTokenizerType.GLM47, - "zai-org/GLM-4.7-Flash", - frozenset({"tool", "user", "system"}), - id="glm47-tool_user_system", - ), -] - - -@pytest.mark.parametrize("family,model_id,roles", _PASS_PARAMS) -def test_via_tito_pass_on_registered_families(family, model_id, roles): - """All 4 registered TITO families round-trip cleanly via decode-roundtrip.""" - tokenizer, extra_kwargs = _setup_tokenizer_with_registered_template(model_id, family, sorted(roles)) - results = run_all_checks_via_tito( - tokenizer, - family, - allowed_append_roles=set(roles), - thinking="both", - extra_template_kwargs=extra_kwargs, - ) - failures = [r for r in results if not r.passed] - assert not failures, ( - f"Expected all PASS for {family.value} × {sorted(roles)} via TITO primitive; " - f"got {len(failures)} FAIL(s) out of {len(results)}:\n" - + "\n".join(f" [{r.case_name}] {r.error}" for r in failures[:5]) - ) - - -# --------------------------------------------------------------------------- -# (2) FAIL on the original unfixed Qwen3 chat template -# --------------------------------------------------------------------------- - - -def test_via_tito_fail_on_original_qwen3_template(): - """The original Qwen3 chat template uses ``loop.last`` and breaks append-only. - - Bypass ``resolve_fixed_chat_template`` entirely — keep the HF default - ``tokenizer.chat_template`` and assert the primitive surfaces a FAIL. - """ - tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B") - # Do NOT overwrite tokenizer.chat_template — keep the broken HF default. - - # Cast wide: thinking=both + multi-user-turn surface so trajectories that - # actually advance ``last_query_index`` between prefix and full are exercised. - # Those are the ones where ``loop.index0 > ns.last_query_index`` truncation - # in the original Qwen3 template renders the same assistant turn differently - # depending on the boundary position. - results = run_all_checks_via_tito( - tokenizer, - TITOTokenizerType.QWEN3, - allowed_append_roles={"tool", "user"}, - thinking="both", - ) - failures = [r for r in results if not r.passed] - assert failures, "Expected ≥1 FAIL on the original (unfixed) Qwen3 chat template; got all PASS." - - -# --------------------------------------------------------------------------- -# (3) FAIL on a test-local buggy subclass -# --------------------------------------------------------------------------- - - -class _BuggyQwen3TITOTokenizer(Qwen3TITOTokenizer): - """Test-only Qwen3 variant with the ``\\n`` boundary insertion deleted. - - Real ``Qwen3TITOTokenizer.merge_tokens`` appends ``self._newline_id`` after - a trailing ``<|im_end|>`` because the model stops without emitting the - newline the chat template would otherwise produce. This variant skips - that fixup; the decode-roundtrip primitive is expected to surface it as a - single-character diff at the prefix-suffix junction. - """ - - def merge_tokens(self, old_messages, new_messages, pretokenized_token_ids, tools=None): - incremental = self.tokenize_additional_non_assistant(old_messages, new_messages, tools) - # Intentionally omit the `+\n` insertion — that's the bug we're catching. - return list(pretokenized_token_ids) + incremental - - -def test_via_tito_fail_on_buggy_qwen3_subclass(): - """A buggy ``merge_tokens`` produces a junction-level diff that the verifier surfaces.""" - tokenizer, _ = _setup_tokenizer_with_registered_template("Qwen/Qwen3-0.6B", TITOTokenizerType.QWEN3, ["tool"]) - buggy = _BuggyQwen3TITOTokenizer(tokenizer, allowed_append_roles=["tool"]) - - result = verify_append_only_via_tito_instance( - buggy, - tokenizer, - deepcopy(SingleToolTrajectory.MESSAGES), - pretokenized_num_message=3, - tools=SingleToolTrajectory.TOOLS, - case_name="buggy_qwen3-single_tool-N3", - ) - assert not result.passed, "Expected FAIL on _BuggyQwen3TITOTokenizer (omits the `+\\n` boundary patch); got PASS." - assert "Decode-roundtrip mismatch" in ( - result.error or "" - ), f"Expected decode-roundtrip diff in error message; got: {result.error}" diff --git a/sidecars/tito/tests/upstream/fast/utils/chat_template_utils/test_tito_tokenizer.py b/sidecars/tito/tests/upstream/fast/utils/chat_template_utils/test_tito_tokenizer.py deleted file mode 100644 index 31b2aff..0000000 --- a/sidecars/tito/tests/upstream/fast/utils/chat_template_utils/test_tito_tokenizer.py +++ /dev/null @@ -1,582 +0,0 @@ -"""Tests for TITOTokenizer: merge_tokens boundary logic, incremental tokenization, and factory. - -## Test structure - -TestConfig - Smoke-checks that each subclass stores the correct model-specific config - (assistant_start_str, trailing_token_ids, max_trim_tokens) and propagates - them to the comparator. These are NOT behavioral tests — they guard - against accidental config regressions when modifying __init__. - -TestMergeTokensBoundary - Unit tests for the core merge_tokens boundary logic, using *synthetic* - prefix IDs ([100, 200, ...]) so the assertions are purely about prefix - manipulation — not about template rendering. - - Why synthetic IDs? merge_tokens is: ``prefix + [boundary fix] + incremental``. - The incremental part comes from tokenize_additional_non_assistant (tested - separately); boundary logic depends only on the last token of the prefix. - Synthetic IDs isolate this and make failures trivially diagnosable. - - Covers three subclass behaviors: - - Qwen3: inserts ``\\n`` when prefix ends with ``<|im_end|>`` (model stops - at im_end without the trailing newline the template expects). - - GLM47: strips trailing ``<|observation|>`` or ``<|user|>`` (model emits - the stop token, but the template also emits it as the next turn's opener). - - Default: plain concatenation (no boundary handling). - -TestTokenizeAdditional - Behavioral tests for tokenize_additional_non_assistant — the role-segmented - synthetic-prefix diff that computes incremental token IDs for appended - non-assistant messages. - - ``test_produces_nonempty_incremental`` is parametrized over: - _TOOL_TRAJECTORIES (trajectory classes) × _TITO_MODELS (qwen3, glm47) - Split points are auto-detected by _find_tito_splits from message structure, - so adding a trajectory to _TOOL_TRAJECTORIES automatically extends coverage. - - Remaining tests cover segmentation logic, generation-prompt timing, - reasoning-content shape, merge structure preservation, and append-only - validation (reject prefix mutation, fewer messages, or forbidden roles). - -TestFactory - get_tito_tokenizer factory: string/enum dispatch, invalid input handling. -""" - -from __future__ import annotations - -from pathlib import Path - -import pytest -from transformers import AutoTokenizer - -from miles.utils.chat_template_utils import MismatchType, apply_chat_template, resolve_fixed_chat_template -from miles.utils.chat_template_utils.tito_tokenizer import ( - GLM47TITOTokenizer, - Qwen3TITOTokenizer, - Qwen35TITOTokenizer, - QwenNextTITOTokenizer, - TITOTokenizer, - TITOTokenizerType, - _build_dummy_assistant, - get_tito_tokenizer, -) -from miles.utils.processing_utils import load_tokenizer -from miles.utils.test_utils.mock_trajectories import ( - IntermediateSystemTrajectory, - LongChainTrajectory, - MultiToolSingleTurnTrajectory, - MultiTurnTrajectory, - ParallelToolsTrajectory, - RetrySystemTrajectory, - SingleToolThinkingTrajectory, - SingleToolTrajectory, -) - -# --------------------------------------------------------------------------- -# Tokenizer cache -# --------------------------------------------------------------------------- - -_TOK_CACHE: dict[tuple[str, str | None], AutoTokenizer] = {} - - -def _get_tokenizer(model_id: str, tito_type: TITOTokenizerType | None = None) -> AutoTokenizer: - chat_template_path = resolve_fixed_chat_template(tito_type, ["tool"])[0] if tito_type is not None else None - cache_key = (model_id, chat_template_path) - if cache_key not in _TOK_CACHE: - _TOK_CACHE[cache_key] = load_tokenizer( - model_id, - chat_template_path=chat_template_path, - trust_remote_code=True, - ) - return _TOK_CACHE[cache_key] - - -# --------------------------------------------------------------------------- -# Fixtures — model-specific TITO tokenizers -# -# `tito` is parametrized over all supported models; use it for tests that -# should run against every model. Named fixtures (qwen3_tito, etc.) are -# for tests specific to one subclass's boundary logic. -# --------------------------------------------------------------------------- - -_TITO_MODELS: dict[str, tuple[str, type[TITOTokenizer], TITOTokenizerType]] = { - "qwen3": ("Qwen/Qwen3-4B", Qwen3TITOTokenizer, TITOTokenizerType.QWEN3), - "glm47": ("zai-org/GLM-4.7-Flash", GLM47TITOTokenizer, TITOTokenizerType.GLM47), -} - -_ALLOWED_APPEND_ROLES = ["tool", "user", "system"] - - -@pytest.fixture(params=list(_TITO_MODELS.keys())) -def tito(request) -> TITOTokenizer: - model_id, cls, tito_type = _TITO_MODELS[request.param] - return cls(_get_tokenizer(model_id, tito_type), allowed_append_roles=_ALLOWED_APPEND_ROLES) - - -@pytest.fixture -def qwen3_tito() -> Qwen3TITOTokenizer: - return Qwen3TITOTokenizer( - _get_tokenizer("Qwen/Qwen3-4B", TITOTokenizerType.QWEN3), - allowed_append_roles=_ALLOWED_APPEND_ROLES, - ) - - -@pytest.fixture -def glm47_tito() -> GLM47TITOTokenizer: - return GLM47TITOTokenizer( - _get_tokenizer("zai-org/GLM-4.7-Flash", TITOTokenizerType.GLM47), - allowed_append_roles=_ALLOWED_APPEND_ROLES, - ) - - -@pytest.fixture -def default_tito() -> TITOTokenizer: - return TITOTokenizer(_get_tokenizer("Qwen/Qwen3-4B"), allowed_append_roles=_ALLOWED_APPEND_ROLES) - - -# --------------------------------------------------------------------------- -# Trajectory parametrization -# -# Instead of relying on PRETOKENIZE_POSITIONS (which serves the pretokenized -# *chat* tests), we derive TITO split points directly from message structure: -# every assistant(tool_calls) followed by a tool/system message is a valid -# split. This way new trajectories get coverage automatically. -# -# To extend: add a trajectory class to _TOOL_TRAJECTORIES. -# To add a model: add an entry to _TITO_MODELS above. -# --------------------------------------------------------------------------- - - -def _find_tito_splits(traj_cls) -> list[int]: - """Find TITO split positions from message structure. - - A valid split is at index ``i+1`` whenever ``messages[i]`` is an assistant - message with tool_calls and ``messages[i+1]`` is a tool or system message. - Returns a list of such positions (the index of the first appended message). - """ - msgs = traj_cls.MESSAGES - splits = [] - for i, msg in enumerate(msgs): - if ( - msg.get("role") == "assistant" - and msg.get("tool_calls") - and i + 1 < len(msgs) - and msgs[i + 1].get("role") in ("tool", "system") - ): - splits.append(i + 1) - return splits - - -def _split_at(traj_cls, pos: int): - """Split trajectory at *pos* into ``(old_msgs, new_msgs, tools)``. - - ``old_msgs = messages[:pos]`` — the pretokenized prefix (ends with assistant turn). - ``new_msgs`` extends through all subsequent non-assistant messages - (tool/user/system), stopping before the next assistant turn. - """ - msgs = traj_cls.MESSAGES - end = pos - while end < len(msgs) and msgs[end].get("role") != "assistant": - end += 1 - return msgs[:pos], msgs[:end], traj_cls.TOOLS - - -_TOOL_TRAJECTORIES = [ - SingleToolTrajectory, # 1 tool call, 1 response - MultiTurnTrajectory, # 2 sequential tool turns - MultiToolSingleTurnTrajectory, # 2 parallel tool calls (weather + date) - ParallelToolsTrajectory, # 3 parallel tool calls - LongChainTrajectory, # 3 sequential turns (weather → date → weather) - RetrySystemTrajectory, # tool + system retry injection mid-conversation - IntermediateSystemTrajectory, # system messages interleaved with tool turns - SingleToolThinkingTrajectory, # tool call with reasoning_content -] - -_TRAJ_CASES = [ - pytest.param(traj_cls, pos, id=f"{traj_cls.__name__}-N{pos}") - for traj_cls in _TOOL_TRAJECTORIES - for pos in _find_tito_splits(traj_cls) -] - -# --------------------------------------------------------------------------- -# TestConfig — subclass configuration smoke-checks -# --------------------------------------------------------------------------- - - -class TestConfig: - """Each subclass stores the correct model-specific configuration at init.""" - - def test_qwen3(self, qwen3_tito: Qwen3TITOTokenizer): - assert qwen3_tito._assistant_start_str == "<|im_start|>assistant" - assert qwen3_tito._newline_id in qwen3_tito.trailing_token_ids - - def test_glm47(self, glm47_tito: GLM47TITOTokenizer): - assert glm47_tito._assistant_start_str == "<|assistant|>" - assert glm47_tito._observation_id in glm47_tito.trailing_token_ids - assert glm47_tito._user_id in glm47_tito.trailing_token_ids - assert glm47_tito.max_trim_tokens == 1 - - def test_default(self, default_tito: TITOTokenizer): - assert default_tito._assistant_start_str is None - assert default_tito.trailing_token_ids == frozenset() - - def test_comparator_inherits_trailing_ids(self, qwen3_tito: Qwen3TITOTokenizer): - """create_comparator propagates trailing_token_ids to the comparator's trim set.""" - comp = qwen3_tito.create_comparator() - assert comp._trim_trailing_ids == set(qwen3_tito.trailing_token_ids) - - -# --------------------------------------------------------------------------- -# TestMergeTokensBoundary — prefix manipulation with synthetic IDs -# -# All tests use the same trajectory (SingleToolTrajectory split at pos=3) -# to compute incremental tokens, then verify prefix manipulation with -# synthetic IDs like [100, 200, ]. -# --------------------------------------------------------------------------- - -_BND_OLD, _BND_NEW, _BND_TOOLS = _split_at(SingleToolTrajectory, 3) - - -class TestMergeTokensBoundary: - """merge_tokens correctly manipulates the prefix before concatenating incremental tokens.""" - - # -- Qwen3: insert \n after <|im_end|> -- - - def test_qwen3_inserts_newline_after_im_end(self, qwen3_tito: Qwen3TITOTokenizer): - """Model stops at <|im_end|> without trailing \\n; merge_tokens inserts it.""" - incremental = qwen3_tito.tokenize_additional_non_assistant(_BND_OLD, _BND_NEW, _BND_TOOLS) - im_end = qwen3_tito._im_end_id - nl = qwen3_tito._newline_id - - result = qwen3_tito.merge_tokens(_BND_OLD, _BND_NEW, [100, 200, im_end], _BND_TOOLS) - assert result == [100, 200, im_end, nl] + incremental - - def test_qwen3_no_newline_otherwise(self, qwen3_tito: Qwen3TITOTokenizer): - """No insertion when prefix does not end with <|im_end|>.""" - incremental = qwen3_tito.tokenize_additional_non_assistant(_BND_OLD, _BND_NEW, _BND_TOOLS) - result = qwen3_tito.merge_tokens(_BND_OLD, _BND_NEW, [100, 200, 300], _BND_TOOLS) - assert result == [100, 200, 300] + incremental - - # -- GLM47: strip ambiguous boundary tokens -- - - def test_glm47_strips_observation(self, glm47_tito: GLM47TITOTokenizer): - """Model emits <|observation|> as stop token; merge_tokens strips the duplicate.""" - incremental = glm47_tito.tokenize_additional_non_assistant(_BND_OLD, _BND_NEW, _BND_TOOLS) - result = glm47_tito.merge_tokens(_BND_OLD, _BND_NEW, [100, 200, glm47_tito._observation_id], _BND_TOOLS) - assert result == [100, 200] + incremental - - def test_glm47_strips_user(self, glm47_tito: GLM47TITOTokenizer): - """<|user|> is also an ambiguous boundary — stripped the same way.""" - incremental = glm47_tito.tokenize_additional_non_assistant(_BND_OLD, _BND_NEW, _BND_TOOLS) - result = glm47_tito.merge_tokens(_BND_OLD, _BND_NEW, [100, 200, glm47_tito._user_id], _BND_TOOLS) - assert result == [100, 200] + incremental - - def test_glm47_no_strip_otherwise(self, glm47_tito: GLM47TITOTokenizer): - """Non-boundary trailing token is preserved.""" - incremental = glm47_tito.tokenize_additional_non_assistant(_BND_OLD, _BND_NEW, _BND_TOOLS) - result = glm47_tito.merge_tokens(_BND_OLD, _BND_NEW, [100, 200, 300], _BND_TOOLS) - assert result == [100, 200, 300] + incremental - - # -- Default: no boundary handling -- - - def test_default_concatenates(self, default_tito: TITOTokenizer): - """Base class does plain concatenation without any prefix modification.""" - incremental = default_tito.tokenize_additional_non_assistant(_BND_OLD, _BND_NEW, _BND_TOOLS) - result = default_tito.merge_tokens(_BND_OLD, _BND_NEW, [100, 200, 300], _BND_TOOLS) - assert result == [100, 200, 300] + incremental - - # -- Edge case -- - - def test_empty_prefix(self, qwen3_tito: Qwen3TITOTokenizer): - """Empty prefix → no boundary handling, result is just incremental.""" - incremental = qwen3_tito.tokenize_additional_non_assistant(_BND_OLD, _BND_NEW, _BND_TOOLS) - result = qwen3_tito.merge_tokens(_BND_OLD, _BND_NEW, [], _BND_TOOLS) - assert result == incremental - - -# --------------------------------------------------------------------------- -# TestTokenizeAdditional — incremental tokenization via role-segmented synthetic diff -# -# test_produces_nonempty_incremental is the scalable core: parametrized over -# _TRAJ_CASES (trajectories × split points) × tito fixture (models). -# 8 trajectories × ~14 splits × 2 models = 28 test cases currently. -# -# Validation tests use a single trajectory since the validation logic -# (assert_messages_append_only_with_allowed_role) is model/trajectory-independent. -# --------------------------------------------------------------------------- - - -class TestTokenizeAdditional: - """tokenize_additional_non_assistant produces valid incremental tokens.""" - - @pytest.mark.parametrize("traj_cls, pos", _TRAJ_CASES) - def test_produces_nonempty_incremental(self, tito: TITOTokenizer, traj_cls, pos): - """Every valid TITO split yields non-empty incremental tokens. - - This is the primary scalability test — it runs every trajectory's - TITO splits against every model tokenizer. - """ - old_msgs, new_msgs, tools = _split_at(traj_cls, pos) - incremental = tito.tokenize_additional_non_assistant(old_msgs, new_msgs, tools) - assert len(incremental) > 0 - - def test_contiguous_tool_segment_is_tokenized_together(self, qwen3_tito: Qwen3TITOTokenizer): - old_msgs, new_msgs, tools = _split_at(MultiToolSingleTurnTrajectory, 3) - appended = new_msgs[len(old_msgs) :] - - segments = qwen3_tito._split_appended_segments(appended) - assert len(segments) == 1 - assert [msg["role"] for msg in segments[0]] == ["tool", "tool"] - - incremental = qwen3_tito.tokenize_additional_non_assistant(old_msgs, new_msgs, tools) - decoded = qwen3_tito.tokenizer.decode(incremental) - assert MultiToolSingleTurnTrajectory.MESSAGES[3]["content"] in decoded - assert MultiToolSingleTurnTrajectory.MESSAGES[4]["content"] in decoded - - def test_user_and_system_segments_are_singletons(self, default_tito: TITOTokenizer): - appended = [ - {"role": "system", "content": "Use JSON."}, - {"role": "user", "content": "Hello"}, - {"role": "tool", "tool_call_id": "call_1", "content": '{"ok": true}'}, - {"role": "tool", "tool_call_id": "call_2", "content": '{"ok": false}'}, - {"role": "user", "content": "Try again"}, - ] - - segments = default_tito._split_appended_segments(appended) - assert [[msg["role"] for msg in segment] for segment in segments] == [ - ["system"], - ["user"], - ["tool", "tool"], - ["user"], - ] - - def test_generation_prompt_is_appended_once_for_full_suffix(self, qwen3_tito: Qwen3TITOTokenizer): - old_msgs = list(SingleToolThinkingTrajectory.MESSAGES[:3]) - new_msgs = old_msgs + [ - SingleToolThinkingTrajectory.MESSAGES[3], - {"role": "user", "content": "Now check Shanghai too."}, - ] - tools = SingleToolThinkingTrajectory.TOOLS - - incremental = qwen3_tito.tokenize_additional_non_assistant(old_msgs, new_msgs, tools) - decoded = qwen3_tito.tokenizer.decode(incremental) - assert decoded.count(qwen3_tito._assistant_start_str) == 1 - assert decoded.endswith( - qwen3_tito.tokenizer.decode( - qwen3_tito._tokenize_rendered_suffix(new_msgs, [], tools=tools, add_generation_prompt=True) - ) - ) - - def test_qwen3_tool_dummy_assistant_preserves_reasoning_shape(self): - thinking_template_path = ( - Path(__file__).resolve().parents[4] - / "miles/utils/chat_template_utils/templates/qwen3_thinking_2507_and_next_fixed.jinja" - ) - thinking_tito = Qwen3TITOTokenizer( - load_tokenizer( - "Qwen/Qwen3-4B-Instruct-2507", - chat_template_path=str(thinking_template_path), - trust_remote_code=True, - ), - allowed_append_roles=_ALLOWED_APPEND_ROLES, - ) - tool_messages = [SingleToolThinkingTrajectory.MESSAGES[3]] - dummy_assistant = _build_dummy_assistant(tool_messages) - rendered = thinking_tito.render_messages( - [{"role": "system", "content": "dummy system"}, dummy_assistant], - add_generation_prompt=False, - tools=SingleToolThinkingTrajectory.TOOLS, - ) - - assert dummy_assistant["reasoning_content"] == " " - assert rendered.endswith( - '<|im_start|>assistant\n\n{"name": "dummy_func", "arguments": {}}\n<|im_end|>\n' - ) - - @pytest.mark.parametrize( - "traj_cls, pos", - [ - pytest.param(SingleToolTrajectory, 3, id="single-tool"), - pytest.param(RetrySystemTrajectory, 3, id="tool-plus-system"), - pytest.param(IntermediateSystemTrajectory, 3, id="intermediate-system"), - ], - ) - def test_qwen3_merge_preserves_non_assistant_structure(self, qwen3_tito: Qwen3TITOTokenizer, traj_cls, pos): - """Merged tokens may differ in assistant text, but not in tool/system structure.""" - old_msgs, new_msgs, tools = _split_at(traj_cls, pos) - pretokenized = apply_chat_template( - old_msgs, - tokenizer=qwen3_tito.tokenizer, - tokenize=True, - add_generation_prompt=False, - tools=tools, - ) - merged = qwen3_tito.merge_tokens(old_msgs, new_msgs, pretokenized, tools) - expected = apply_chat_template( - new_msgs, - tokenizer=qwen3_tito.tokenizer, - tokenize=True, - add_generation_prompt=True, - tools=tools, - ) - mismatches = qwen3_tito.create_comparator().compare_sequences(expected, merged) - assert all(m.type == MismatchType.ASSISTANT_TEXT for m in mismatches) - - # -- Append-only validation (assert_messages_append_only_with_allowed_role is called internally) -- - - def test_rejects_prefix_mutation(self, qwen3_tito: Qwen3TITOTokenizer): - """Modifying an existing message in new_messages raises ValueError.""" - old_msgs, new_msgs, _ = _split_at(SingleToolTrajectory, 3) - mutated_old = [{"role": "user", "content": "CHANGED"}] + list(old_msgs[1:]) - mutated_new = mutated_old + list(new_msgs[len(old_msgs) :]) - with pytest.raises(ValueError, match="mismatch"): - qwen3_tito.tokenize_additional_non_assistant(old_msgs, mutated_new) - - def test_rejects_fewer_messages(self, qwen3_tito: Qwen3TITOTokenizer): - """new_messages shorter than old_messages raises ValueError.""" - old_msgs = SingleToolTrajectory.MESSAGES[:3] - with pytest.raises(ValueError, match="fewer"): - qwen3_tito.tokenize_additional_non_assistant(old_msgs, old_msgs[:1]) - - def test_rejects_assistant_append(self, qwen3_tito: Qwen3TITOTokenizer): - """Appending an assistant message (not tool/system) raises ValueError.""" - old_msgs = SingleToolTrajectory.MESSAGES[:3] - bad_new = list(old_msgs) + [{"role": "assistant", "content": "hi"}] - with pytest.raises(ValueError, match="role"): - qwen3_tito.tokenize_additional_non_assistant(old_msgs, bad_new) - - -# --------------------------------------------------------------------------- -# TestFactory — get_tito_tokenizer dispatch -# --------------------------------------------------------------------------- - - -class TestFactory: - """get_tito_tokenizer creates the correct subclass from string or enum type.""" - - @pytest.mark.parametrize( - "type_str, model_id, cls", - [ - ("qwen3", "Qwen/Qwen3-4B", Qwen3TITOTokenizer), - ("qwen35", "Qwen/Qwen3-4B", Qwen35TITOTokenizer), - ("qwennext", "Qwen/Qwen3-4B", QwenNextTITOTokenizer), - ("glm47", "zai-org/GLM-4.7-Flash", GLM47TITOTokenizer), - ("default", "Qwen/Qwen3-4B", TITOTokenizer), - ], - ) - def test_creates_correct_type(self, type_str, model_id, cls): - tito = get_tito_tokenizer(_get_tokenizer(model_id), tokenizer_type=type_str) - assert isinstance(tito, cls) - - def test_enum_input(self): - """Enum values work the same as string values.""" - tito = get_tito_tokenizer(_get_tokenizer("Qwen/Qwen3-4B"), tokenizer_type=TITOTokenizerType.QWEN3) - assert isinstance(tito, Qwen3TITOTokenizer) - - @pytest.mark.parametrize( - "type_str, cls", - [("qwen35", Qwen35TITOTokenizer), ("qwennext", QwenNextTITOTokenizer)], - ) - def test_qwen_variant_inherits_qwen3_boundary_logic(self, type_str, cls): - """Qwen3.5 / Qwen3-Next reuse Qwen3's boundary handling via inheritance. - The named subclass exists so fixed_templates can key on (tito_model, - surface) — but token-level merge behavior is identical to Qwen3.""" - tito = get_tito_tokenizer(_get_tokenizer("Qwen/Qwen3-4B"), tokenizer_type=type_str) - assert isinstance(tito, cls) - assert isinstance(tito, Qwen3TITOTokenizer) - - def test_invalid_type_raises(self): - with pytest.raises(ValueError): - get_tito_tokenizer(_get_tokenizer("Qwen/Qwen3-4B"), tokenizer_type="nonexistent") - - def test_none_tokenizer_raises(self): - with pytest.raises(ValueError, match="must not be None"): - get_tito_tokenizer(None) - - -class TestParserBinding: - """Each TITO subclass binds sglang ``--reasoning-parser`` and - ``--tool-call-parser`` values; ``resolve_reasoning_and_tool_call_parser`` - enforces user-supplied values agree with the bindings (or returns the - bound values when the user didn't pass one). The two parsers are - resolved independently — a missing binding on one doesn't suppress the - assert on the other.""" - - @pytest.mark.parametrize( - "tito_model, expected_reasoning, expected_tool_call", - [ - (TITOTokenizerType.QWEN3, "qwen3", "qwen25"), - (TITOTokenizerType.QWEN35, "qwen3", "qwen3_coder"), - (TITOTokenizerType.QWENNEXT, "qwen3", "qwen25"), - (TITOTokenizerType.GLM47, "glm45", "glm47"), - (TITOTokenizerType.NEMOTRON3, "nemotron_3", "qwen3_coder"), - (TITOTokenizerType.KIMI25, None, None), - (TITOTokenizerType.KIMI26, "kimi_k2", "kimi_k2_raw_id"), - (TITOTokenizerType.MINIMAX_M25, "minimax-append-think", "minimax-m2"), - (TITOTokenizerType.MINIMAX_M27, "minimax-append-think", "minimax-m2"), - (TITOTokenizerType.DEEPSEEKV32, "deepseek-v3", "deepseekv32"), - (TITOTokenizerType.DEEPSEEKV4, "deepseek-v4", "deepseekv4"), - (TITOTokenizerType.DEFAULT, None, None), - ], - ) - def test_subclass_binding(self, tito_model, expected_reasoning, expected_tool_call): - cls = TITOTokenizerType.get_tokenizer_class(tito_model) - assert cls.reasoning_parser == expected_reasoning - assert cls.tool_call_parser == expected_tool_call - - def test_resolve_returns_binding_when_user_omits(self): - from miles.utils.chat_template_utils import resolve_reasoning_and_tool_call_parser - - assert resolve_reasoning_and_tool_call_parser(TITOTokenizerType.QWEN3) == ("qwen3", "qwen25") - assert resolve_reasoning_and_tool_call_parser(TITOTokenizerType.QWEN35) == ("qwen3", "qwen3_coder") - assert resolve_reasoning_and_tool_call_parser(TITOTokenizerType.GLM47) == ("glm45", "glm47") - assert resolve_reasoning_and_tool_call_parser(TITOTokenizerType.DEEPSEEKV4) == ("deepseek-v4", "deepseekv4") - # DEFAULT family has no binding for either parser; both come back None. - assert resolve_reasoning_and_tool_call_parser(TITOTokenizerType.DEFAULT) == (None, None) - - def test_resolve_accepts_matching_user_value(self): - from miles.utils.chat_template_utils import resolve_reasoning_and_tool_call_parser - - assert resolve_reasoning_and_tool_call_parser("qwen3", "qwen3", "qwen25") == ("qwen3", "qwen25") - assert resolve_reasoning_and_tool_call_parser(TITOTokenizerType.QWEN35, "qwen3", "qwen3_coder") == ( - "qwen3", - "qwen3_coder", - ) - - def test_resolve_raises_on_reasoning_mismatch(self): - from miles.utils.chat_template_utils import resolve_reasoning_and_tool_call_parser - - with pytest.raises(ValueError, match="--reasoning-parser='glm45' disagrees"): - resolve_reasoning_and_tool_call_parser(TITOTokenizerType.QWEN3, user_reasoning_parser="glm45") - - def test_resolve_raises_on_tool_call_mismatch(self): - from miles.utils.chat_template_utils import resolve_reasoning_and_tool_call_parser - - with pytest.raises(ValueError, match="--tool-call-parser='glm47' disagrees"): - resolve_reasoning_and_tool_call_parser(TITOTokenizerType.QWEN3, user_tool_call_parser="glm47") - - def test_resolve_accepts_user_value_when_family_unbound(self): - # DEFAULT family has no binding for either parser; user-provided wins - # (for families that haven't been wired up to a sglang parser yet). - from miles.utils.chat_template_utils import resolve_reasoning_and_tool_call_parser - - assert resolve_reasoning_and_tool_call_parser( - TITOTokenizerType.DEFAULT, "custom_reasoning", "custom_tool_call" - ) == ("custom_reasoning", "custom_tool_call") - - def test_resolve_partial_user_input(self): - # User can pass only one of the two; the other auto-resolves from - # the family binding independently. - from miles.utils.chat_template_utils import resolve_reasoning_and_tool_call_parser - - # User passes reasoning only — tool_call comes from binding. - assert resolve_reasoning_and_tool_call_parser(TITOTokenizerType.QWEN3, user_reasoning_parser="qwen3") == ( - "qwen3", - "qwen25", - ) - # User passes tool_call only — reasoning comes from binding. - assert resolve_reasoning_and_tool_call_parser(TITOTokenizerType.GLM47, user_tool_call_parser="glm47") == ( - "glm45", - "glm47", - ) diff --git a/sidecars/tito/tests/upstream/fast/utils/test_utils/test_session_verify_runner.py b/sidecars/tito/tests/upstream/fast/utils/test_utils/test_session_verify_runner.py deleted file mode 100644 index 770f7f5..0000000 --- a/sidecars/tito/tests/upstream/fast/utils/test_utils/test_session_verify_runner.py +++ /dev/null @@ -1,85 +0,0 @@ -import argparse -import json - -import pytest - -from miles.utils.test_utils.session_verify_runner import ( - SESSION_VERIFY_INVARIANT_ARGS, - assert_session_verify_metrics, - namespace_to_train_args, -) - - -def _build_args(**overrides) -> str: - values = { - **SESSION_VERIFY_INVARIANT_ARGS, - "hf_checkpoint": "/root/models/test-model", - "tito_model": "qwen3", - "tito_allowed_append_roles": ["tool", "user"], - "rollout_num_gpus_per_engine": 2, - "actor_num_nodes": 1, - "actor_num_gpus_per_node": 8, - "n_samples_per_prompt": 4, - "session_verify_cycles": 3, - "tool_call_failure_mode": "rollback", - "sglang_reasoning_parser": "qwen3", - "sglang_tool_call_parser": "qwen25", - } - values.update(overrides) - return namespace_to_train_args(argparse.Namespace(**values)) - - -def test_namespace_to_train_args_uses_default_rollout_max_response_len(): - train_args = _build_args() - - assert "--rollout-max-response-len 8192" in train_args - - -def test_namespace_to_train_args_allows_model_specific_rollout_max_response_len(): - train_args = _build_args(rollout_max_response_len=16384) - - assert "--rollout-max-response-len 16384" in train_args - - -def test_namespace_to_train_args_keeps_ci_test_enabled_for_fsdp_debug_rollout(): - train_args = _build_args() - - assert "--train-backend fsdp" in train_args - assert "--ci-test" in train_args - - -def test_namespace_to_train_args_omits_expert_parallel_for_single_expert(): - train_args = _build_args() - - assert "--sglang-expert-parallel-size" not in train_args - - -def test_namespace_to_train_args_emits_expert_parallel_for_moe(): - train_args = _build_args(sglang_expert_parallel_size=8) - - assert "--sglang-expert-parallel-size 8" in train_args - - -def _write_metrics(path, entries: list[dict]) -> None: - path.write_text("\n".join(json.dumps(entry) for entry in entries) + "\n") - - -def test_session_verify_metrics_accepts_cross_sample_append_tool(tmp_path): - metrics_path = tmp_path / "metrics.jsonl" - _write_metrics( - metrics_path, - [ - {"driver_events": ["initial", "append_user"], "had_assistant_mismatch": False}, - {"driver_events": ["initial", "append_tool"], "had_assistant_mismatch": False}, - ], - ) - - assert_session_verify_metrics(str(metrics_path), assistant_text_threshold=0.1) - - -def test_session_verify_metrics_requires_at_least_one_append_tool(tmp_path): - metrics_path = tmp_path / "metrics.jsonl" - _write_metrics(metrics_path, [{"driver_events": ["initial", "append_user"], "had_assistant_mismatch": False}]) - - with pytest.raises(AssertionError, match="no sample produced an append_tool action"): - assert_session_verify_metrics(str(metrics_path), assistant_text_threshold=0.1) diff --git a/sidecars/tito/tito_gateway/VENDORED_MILES_AUDIT.md b/sidecars/tito/tito_gateway/VENDORED_MILES_AUDIT.md deleted file mode 100644 index 6f053f4..0000000 --- a/sidecars/tito/tito_gateway/VENDORED_MILES_AUDIT.md +++ /dev/null @@ -1,29 +0,0 @@ -# Vendored Miles Source Audit - -This package is a standalone wrapper/packaging layer around Miles TITO work. It is not a rewrite of Miles TITO algorithms. Core Miles files below were audited against: - -- Repository: `https://github.com/radixark/miles` -- Commit: `9437366e0aa3a25294720f70d18b081067595f85` -- Local upstream checkout used for audit: `/tmp/miles-explore` - -## Audit Result - -| Upstream file | Vendored file | Status | Allowed differences | -|---|---|---|---| -| `miles/utils/chat_template_utils/tito_tokenizer.py` | `tito_gateway/vendor/miles_compat/utils/chat_template_utils/tito_tokenizer.py` | Import-path rewrite plus Python 3.10 enum compatibility | `miles.*` imports rewritten to `tito_gateway.vendor.miles_compat.*`; upstream `StrEnum` replaced by `str, Enum` because this package supports Python 3.10. TITO tokenization, merge, fixed-template resolution, and factory logic are otherwise preserved. | -| `miles/utils/chat_template_utils/template.py` | `tito_gateway/vendor/miles_compat/utils/chat_template_utils/template.py` | Import-path rewrite only | `miles.utils.chat_template_utils` import rewritten to `tito_gateway.vendor.miles_compat.utils.chat_template_utils`. | -| `miles/utils/chat_template_utils/token_seq_comparator.py` | `tito_gateway/vendor/miles_compat/utils/chat_template_utils/token_seq_comparator.py` | Byte-identical | None. | -| `miles/utils/chat_template_utils/templates/*.jinja` | `tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/*.jinja` | Byte-identical | None. Audited templates: `kimi_k25_fixed.jinja`, `minimax_m25_fixed.jinja`, `minimax_m27_fixed.jinja`, `qwen3.5_fixed.jinja`, `qwen3_fixed.jinja`, `qwen3_thinking_2507_and_next_fixed.jinja`. | -| `miles/rollout/session/linear_trajectory.py` | `tito_gateway/vendor/miles_compat/rollout/session/linear_trajectory.py` | Import-path rewrite only | `miles.*` imports rewritten to `tito_gateway.vendor.miles_compat.*`. | -| `miles/rollout/session/sessions.py` | `tito_gateway/vendor/miles_compat/rollout/session/sessions.py` | Import-path rewrite only | `miles.*` imports rewritten to `tito_gateway.vendor.miles_compat.*`. | -| `miles/rollout/session/session_errors.py` | `tito_gateway/vendor/miles_compat/rollout/session/session_errors.py` | Byte-identical | None. | -| `miles/rollout/session/session_types.py` | `tito_gateway/vendor/miles_compat/rollout/session/session_types.py` | Byte-identical | None. | -| `miles/rollout/session/session_server.py` | `tito_gateway/vendor/miles_compat/rollout/session/session_server.py` | Import-path rewrite only | `miles.rollout.session.sessions` import rewritten to `tito_gateway.vendor.miles_compat.rollout.session.sessions`. | - -## Compatibility Test Path - -The copied upstream tokenizer test computes a repository-root-relative template path from its own `tests/upstream/...` location. To preserve the unchanged test body, this package provides: - -- `tests/miles/utils/chat_template_utils/templates/qwen3_thinking_2507_and_next_fixed.jinja` - -That file is byte-identical to the vendored/upstream template. diff --git a/sidecars/tito/tito_gateway/__init__.py b/sidecars/tito/tito_gateway/__init__.py deleted file mode 100644 index 0b531b2..0000000 --- a/sidecars/tito/tito_gateway/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Standalone wrapper package around Miles TITO session gateway work.""" - -from tito_gateway.config import TITOGatewayConfig -from tito_gateway.discovery import discover_backend_url -from tito_gateway.gateway import TITOGateway -from tito_gateway.server import SessionServer -from tito_gateway.tokenizer import TITOTokenizerType, get_tito_tokenizer - -__all__ = [ - "TITOGateway", - "TITOGatewayConfig", - "SessionServer", - "TITOTokenizerType", - "discover_backend_url", - "get_tito_tokenizer", -] diff --git a/sidecars/tito/tito_gateway/cli.py b/sidecars/tito/tito_gateway/cli.py deleted file mode 100644 index f348e8f..0000000 --- a/sidecars/tito/tito_gateway/cli.py +++ /dev/null @@ -1,241 +0,0 @@ -"""Command-line entrypoint for TITO Gateway.""" - -from __future__ import annotations - -import argparse -import json -import sys - -from tito_gateway.config import TITOGatewayConfig -from tito_gateway.gateway import TITOGateway -from tito_gateway.tokenizer import TITOTokenizerType - - -def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - prog="tito-gateway", - description="Standalone wrapper around Miles TITO session gateway work.", - ) - subparsers = parser.add_subparsers(dest="command") - - _add_serve_parser(subparsers) - _add_verify_chat_template_parser(subparsers) - _add_verify_session_parser(subparsers) - return parser - - -def _add_serve_parser(subparsers: argparse._SubParsersAction) -> argparse.ArgumentParser: - serve = subparsers.add_parser("serve", help="Start the TITO gateway server.") - _add_serve_arguments(serve) - return serve - - -def _add_serve_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument("--hf-checkpoint", required=True, help="HuggingFace model ID or local checkpoint path.") - parser.add_argument("--backend-url", default=None, help="OpenAI-compatible backend URL to proxy to.") - parser.add_argument("--chat-template-path", default=None, help="Optional fixed chat template path.") - parser.add_argument( - "--apply-chat-template-kwargs", - default=None, - help="JSON object forwarded as chat-template kwargs, matching Miles convention.", - ) - parser.add_argument( - "--tito-model", - choices=[item.value for item in TITOTokenizerType], - default=TITOTokenizerType.DEFAULT.value, - help="Miles TITO tokenizer family.", - ) - parser.add_argument( - "--tito-allowed-append-roles", - nargs="+", - choices=["tool", "user", "system"], - default=["tool"], - help="Roles allowed after an assistant turn; tool is the default.", - ) - parser.add_argument("--session-server-ip", default="127.0.0.1", help="Gateway bind host.") - parser.add_argument("--session-server-port", type=int, default=30000, help="Gateway bind port.") - parser.add_argument("--miles-router-timeout", type=float, default=600.0, help="Proxy timeout in seconds.") - parser.add_argument( - "--backend-probe-candidate", - action="append", - default=None, - metavar="URL", - help="Local backend URL candidate to probe after explicit and environment URLs; repeatable.", - ) - parser.add_argument( - "--backend-probe-timeout", - type=float, - default=0.25, - help="Per-endpoint backend probe timeout in seconds.", - ) - - -def _add_verify_chat_template_parser(subparsers: argparse._SubParsersAction) -> None: - parser = subparsers.add_parser( - "verify-chat-template", - help="Verify that a chat template is append-only after last user message.", - ) - parser.add_argument("--template", metavar="PATH") - parser.add_argument("--model", metavar="MODEL_ID") - parser.add_argument( - "--tito-model", - choices=[item.value for item in TITOTokenizerType], - default=None, - ) - parser.add_argument( - "--tito-allowed-append-roles", - nargs="+", - default=["tool"], - choices=["tool", "user", "system"], - metavar="ROLE", - ) - parser.add_argument("--thinking", choices=["off", "on", "both"], default="on") - parser.add_argument("--chat-template-kwargs", type=json.loads, default=None, metavar="JSON") - parser.set_defaults(verify_command="chat-template") - - -def _add_verify_session_parser(subparsers: argparse._SubParsersAction) -> None: - from miles.utils.test_utils.session_verify_runner import ( - ASSISTANT_TEXT_MISMATCH_RATIO_THRESHOLD, - SESSION_VERIFY_INVARIANT_ARGS, - ) - - parser = subparsers.add_parser( - "verify-session-tito-tokenizer", - help="Run the optional Miles/SGLang session-server TITO verifier.", - ) - parser.add_argument("--hf-checkpoint", required=True, help="HuggingFace model ID or local checkpoint path.") - parser.add_argument("--chat-template-path", default=None, help="Optional fixed chat template path.") - parser.add_argument( - "--apply-chat-template", - action="store_true", - default=False, - help="Miles-compatible flag for applying the chat template in data preprocessing.", - ) - parser.add_argument( - "--apply-chat-template-kwargs", - type=json.loads, - default={}, - metavar="JSON", - help="JSON object forwarded as Miles chat-template kwargs.", - ) - parser.add_argument( - "--tito-model", - choices=[item.value for item in TITOTokenizerType], - required=True, - help="Miles TITO tokenizer family.", - ) - parser.add_argument( - "--tito-allowed-append-roles", - nargs="+", - default=["tool"], - choices=["tool", "user", "system"], - metavar="ROLE", - ) - parser.add_argument("--prompt-data", default=SESSION_VERIFY_INVARIANT_ARGS["prompt_data"]) - parser.add_argument("--input-key", default=SESSION_VERIFY_INVARIANT_ARGS["input_key"]) - parser.add_argument( - "--custom-generate-function-path", - default=SESSION_VERIFY_INVARIANT_ARGS["custom_generate_function_path"], - ) - parser.add_argument( - "--custom-agent-function-path", - default=SESSION_VERIFY_INVARIANT_ARGS["custom_agent_function_path"], - ) - parser.add_argument("--backend-url", default=None, help="Optional OpenAI-compatible backend URL for session tests.") - parser.add_argument("--session-server-ip", default="127.0.0.1") - parser.add_argument("--session-server-port", type=int, default=30000) - parser.add_argument("--miles-router-timeout", type=float, default=600.0) - parser.add_argument("--sglang-reasoning-parser", default=None) - parser.add_argument("--sglang-tool-call-parser", default=None) - parser.add_argument("--rollout-num-gpus-per-engine", type=int, default=1) - parser.add_argument("--sglang-expert-parallel-size", type=int, default=1) - parser.add_argument("--num-rollout", type=int, default=SESSION_VERIFY_INVARIANT_ARGS["num_rollout"]) - parser.add_argument("--rollout-batch-size", type=int, default=SESSION_VERIFY_INVARIANT_ARGS["rollout_batch_size"]) - parser.add_argument( - "--rollout-max-response-len", - type=int, - default=SESSION_VERIFY_INVARIANT_ARGS["rollout_max_response_len"], - ) - parser.add_argument( - "--rollout-temperature", - type=float, - default=SESSION_VERIFY_INVARIANT_ARGS["rollout_temperature"], - ) - parser.add_argument("--global-batch-size", type=int, default=SESSION_VERIFY_INVARIANT_ARGS["global_batch_size"]) - parser.add_argument("--rm-type", default=SESSION_VERIFY_INVARIANT_ARGS["rm_type"]) - parser.add_argument("--actor-num-nodes", type=int, default=1) - parser.add_argument("--actor-num-gpus-per-node", type=int, default=1) - parser.add_argument("--n-samples-per-prompt", type=int, default=4) - parser.add_argument("--session-verify-cycles", type=int, default=3) - parser.add_argument("--tool-call-failure-mode", default="rollback") - parser.add_argument( - "--assistant-text-threshold", - type=float, - default=ASSISTANT_TEXT_MISMATCH_RATIO_THRESHOLD, - help=( - "Soft threshold for assistant_text mismatch ratio. " - f"Default {ASSISTANT_TEXT_MISMATCH_RATIO_THRESHOLD}." - ), - ) - parser.add_argument( - "--train-backend", - choices=["megatron", "fsdp"], - default=SESSION_VERIFY_INVARIANT_ARGS["train_backend"], - ) - parser.add_argument("--use-session-server", action=argparse.BooleanOptionalAction, default=None) - parser.add_argument("--debug-rollout-only", action=argparse.BooleanOptionalAction, default=None) - parser.add_argument("--ci-test", action=argparse.BooleanOptionalAction, default=None) - parser.add_argument("--colocate", action=argparse.BooleanOptionalAction, default=None) - parser.set_defaults(**SESSION_VERIFY_INVARIANT_ARGS) - parser.set_defaults(verify_command="session-tito-tokenizer") - - -def _serve(args: argparse.Namespace) -> int: - config = TITOGatewayConfig.from_cli_values( - hf_checkpoint=args.hf_checkpoint, - backend_url=args.backend_url, - chat_template_path=args.chat_template_path, - apply_chat_template_kwargs=args.apply_chat_template_kwargs, - tito_model=args.tito_model, - tito_allowed_append_roles=args.tito_allowed_append_roles, - session_server_ip=args.session_server_ip, - session_server_port=args.session_server_port, - miles_router_timeout=args.miles_router_timeout, - backend_probe_candidates=args.backend_probe_candidate, - backend_probe_timeout=args.backend_probe_timeout, - ) - TITOGateway(config).run() - return 0 - - -def main(argv: list[str] | None = None) -> int: - parser = build_parser() - raw_args = list(sys.argv[1:] if argv is None else argv) - commands = {"serve", "verify-chat-template", "verify-session-tito-tokenizer", "-h", "--help"} - if not raw_args or raw_args[0] not in commands: - raw_args.insert(0, "serve") - args = parser.parse_args(raw_args) - - if getattr(args, "verify_command", None): - if args.verify_command == "chat-template": - from tito_gateway.verify_chat_template import run_from_args - - try: - return run_from_args(args) - except Exception as exc: - print(f"tito-gateway verify-chat-template: error: {exc}", file=sys.stderr) - return 1 - from tito_gateway.verify_session_tito_tokenizer import run_from_args - - return run_from_args(args) - - try: - return _serve(args) - except Exception as exc: - print(f"tito-gateway: error: {exc}", file=sys.stderr) - return 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/sidecars/tito/tito_gateway/server.py b/sidecars/tito/tito_gateway/server.py deleted file mode 100644 index 9ec08e3..0000000 --- a/sidecars/tito/tito_gateway/server.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Session server wrapper around the vendored Miles implementation. - -Adds multi-backend routing on top of the vendored single-backend -``SessionServer`` WITHOUT modifying any vendored file: a thin subclass -overrides ``do_proxy`` to pick a backend from a :class:`BackendPool` per -request (sticky by ``session_id`` for prefix-cache locality), reports a -backend down on a transport error, and forgets a session's pin when the -session is deleted. -""" - -from __future__ import annotations - -import json -import logging -from typing import Any - -import httpx - -from tito_gateway.pool import BackendPool - -logger = logging.getLogger(__name__) - -_HOP_BY_HOP = ("content-length", "transfer-encoding", "host") - - -def _session_id_from_path(path: str) -> str | None: - """Extract ``{session_id}`` from ``/sessions/{session_id}[/...]``.""" - parts = path.strip("/").split("/") - if len(parts) >= 2 and parts[0] == "sessions": - return parts[1] - return None - - -class SessionServer: - """Wrapper for Miles' standalone FastAPI session server, routing proxied - inference across a :class:`BackendPool`.""" - - def __init__(self, args: Any, pool: BackendPool): - from tito_gateway.vendor.miles_compat.rollout.session.session_server import ( - SessionServer as MilesSessionServer, - ) - - class _PooledSessionServer(MilesSessionServer): - def __init__(self, args: Any, pool: BackendPool) -> None: - self._pool = pool - # Nominal backend_url for any vendored code that reads it; the - # per-request route is chosen in `do_proxy` below. - super().__init__(args, pool.backends[0]) - self.app.middleware("http")(self._forget_on_delete) - - async def do_proxy(self, request, path, body=None, headers=None) -> dict: # type: ignore[override] - session_id = _session_id_from_path(request.url.path) - backend_url = self._pool.pick(session_id) - url = f"{backend_url}/{path}" - if request.url.query: - url = f"{url}?{request.url.query}" - if body is None: - body = await request.body() - if headers is None: - headers = dict(request.headers) - headers = {k: v for k, v in headers.items() if k.lower() not in _HOP_BY_HOP} - try: - response = await self.client.request(request.method, url, content=body, headers=headers) - except httpx.TransportError as exc: - # Mark this replica down so the session re-pins on its next - # request; surface the error to the agent unchanged. - self._pool.report_down(backend_url) - logger.warning("pooled proxy transport error %s -> %s: %s", path, backend_url, exc) - error_body = json.dumps( - {"error": f"backend transport error: {type(exc).__name__}: {exc}"} - ).encode() - return { - "request_body": body, - "response_body": error_body, - "status_code": 502, - "headers": {"content-type": "application/json"}, - } - content = await response.aread() - return { - "request_body": body, - "response_body": content, - "status_code": response.status_code, - "headers": dict(response.headers), - } - - async def _forget_on_delete(self, request, call_next): - response = await call_next(request) - if request.method == "DELETE" and response.status_code < 300: - session_id = _session_id_from_path(request.url.path) - if session_id is not None: - # Drop the sticky pin so `_assigned` doesn't grow without - # bound across a long-lived gateway. - self._pool.forget(session_id) - return response - - self._impl = _PooledSessionServer(args, pool) - self.args = args - self.pool = pool - self.backend_url = pool.backends[0] - self.app = self._impl.app diff --git a/sidecars/tito/tito_gateway/tokenizer.py b/sidecars/tito/tito_gateway/tokenizer.py deleted file mode 100644 index bab7c9e..0000000 --- a/sidecars/tito/tito_gateway/tokenizer.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Public tokenizer entrypoints for the vendored Miles TITO implementation.""" - -from __future__ import annotations - -from enum import Enum -from typing import Any - - -class TITOTokenizerType(str, Enum): - DEFAULT = "default" - QWEN3 = "qwen3" - QWEN35 = "qwen35" - QWENNEXT = "qwennext" - GLM47 = "glm47" - NEMOTRON3 = "nemotron3" - KIMI25 = "kimi25" - KIMI26 = "kimi26" - MINIMAX_M25 = "minimax_m25" - MINIMAX_M27 = "minimax_m27" - DEEPSEEKV32 = "deepseekv32" - DEEPSEEKV4 = "deepseekv4" - - -def get_tito_tokenizer(*args: Any, **kwargs: Any) -> Any: - """Return a vendored Miles TITO tokenizer instance.""" - from tito_gateway.vendor.miles_compat.utils.chat_template_utils import ( - get_tito_tokenizer as _get_tito_tokenizer, - ) - - return _get_tito_tokenizer(*args, **kwargs) diff --git a/sidecars/tito/tito_gateway/upstream.json b/sidecars/tito/tito_gateway/upstream.json deleted file mode 100644 index ef8ca13..0000000 --- a/sidecars/tito/tito_gateway/upstream.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "project": "Miles", - "repository": "https://github.com/radixark/miles", - "documentation": "https://www.radixark.com/miles/docs/user-guide/agentic-chat-template", - "source_commit": "9437366e0aa3a25294720f70d18b081067595f85", - "acknowledgement": "TITO Gateway is a standalone wrapper and packaging layer around Miles TITO work, not a rewrite of the Miles TITO algorithms." -} diff --git a/sidecars/tito/tito_gateway/vendor/__init__.py b/sidecars/tito/tito_gateway/vendor/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/__init__.py b/sidecars/tito/tito_gateway/vendor/miles_compat/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/__init__.py b/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/base_types.py b/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/base_types.py deleted file mode 100644 index 1041a6f..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/base_types.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Lightweight Miles rollout type shapes used by verifier imports. - -The full Miles training stack owns the production rollout implementation. This -module preserves the small dataclass surface that `session_verify_agent` needs -for import and CPU-fast wrapper tests. -""" - -from __future__ import annotations - -from argparse import Namespace -from dataclasses import dataclass -from typing import Any - - -@dataclass(frozen=True) -class GenerateFnInput: - state: Any - sample: Any - sampling_params: dict[str, Any] - evaluation: bool - - @property - def args(self) -> Namespace: - return self.state.args - - -@dataclass(frozen=True) -class GenerateFnOutput: - samples: Any diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/generate_hub/__init__.py b/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/generate_hub/__init__.py deleted file mode 100644 index d5a9064..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/generate_hub/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Lightweight compatibility namespace for Miles generate helpers.""" diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/generate_hub/agentic_tool_call.py b/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/generate_hub/agentic_tool_call.py deleted file mode 100644 index 1bfdd8b..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/generate_hub/agentic_tool_call.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Optional Miles agentic tool-call generate bridge. - -TITO Gateway vendors the session verifier wrapper, but the full Miles rollout -engine remains an optional dependency. The callable exists so verifier helper -paths are importable and testable; real e2e execution must install/provide the -Miles rollout stack or monkeypatch this bridge in CPU-fast tests. -""" - -from __future__ import annotations - -import argparse -from typing import Any - - -async def generate(input: Any) -> Any: - raise RuntimeError( - "Miles agentic tool-call rollout generation is not bundled with " - "tito-gateway. Install the optional Miles/SGLang training stack before " - "running full session verifier e2e jobs." - ) - - -def _add_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument("--custom-agent-function-path", type=str) - parser.add_argument("--generate-multi-samples", action="store_true", default=False) - parser.add_argument("--max-seq-len", type=int, default=None, dest="max_seq_len") - - -generate.add_arguments = _add_arguments diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/__init__.py b/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/linear_trajectory.py b/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/linear_trajectory.py deleted file mode 100644 index 491bd64..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/linear_trajectory.py +++ /dev/null @@ -1,285 +0,0 @@ -import asyncio -import logging -import uuid -from dataclasses import dataclass, field -from typing import Any - -from tito_gateway.vendor.miles_compat.rollout.session.session_errors import MessageValidationError, SessionNotFoundError, TokenizationError -from tito_gateway.vendor.miles_compat.rollout.session.session_types import SessionRecord -from tito_gateway.vendor.miles_compat.utils.chat_template_utils import assert_messages_append_only_with_allowed_role, message_matches -from tito_gateway.vendor.miles_compat.utils.chat_template_utils.tito_tokenizer import TITOTokenizer - -logger = logging.getLogger(__name__) - - -# TODO: hardcoded to 1 for now; if multi-step rollback is actually needed, -# raise this limit or make it configurable and remove the restriction. -MAX_ASSISTANT_ROLLBACK_STEPS = 1 - - -@dataclass -class LinearTrajectory: - """State for a linear trajectory. - - Tracks the full message history and accumulated token IDs for one session. - The typical message sequence is: [system?, user, assistant, tool, assistant, tool, …], - but the agent may retry from an earlier point (e.g. re-running a tool call), - in which case the session is rolled back at most one assistant step. - - Concurrency contract: all mutating methods must be called under ``self.lock``. - """ - - lock: asyncio.Lock = field(default_factory=asyncio.Lock, repr=False, compare=False) - closing: bool = field(default=False, repr=False, compare=False) - messages: list[dict[str, Any]] = field(default_factory=list) - records: list[SessionRecord] = field(default_factory=list) - trajectory_token_ids: list[list[int]] = field(default_factory=list) - num_assistant: int = 0 - - @property - def token_ids(self) -> list[int]: - """Current token IDs — the latest assistant checkpoint.""" - return self.trajectory_token_ids[-1] if self.trajectory_token_ids else [] - - def append_record(self, record: SessionRecord) -> None: - self.records.append(record) - - def prepare_pretokenized( - self, - request_messages: list[dict[str, Any]], - tools: list[dict[str, Any]] | None = None, - *, - tito_tokenizer: TITOTokenizer, - ) -> list[int]: - """Build the full prompt input_ids for *request_messages*. - - On the first turn (no stored token_ids), renders *request_messages* - from scratch via the chat template. On subsequent turns, validates - that *request_messages* extends the stored history (rolling back at - most one assistant step on agent retries) and reuses the stored - token_ids as the pretokenized prefix. - - Must be called under ``self.lock``. - """ - if not self.token_ids: - return tito_tokenizer.render_messages( - request_messages, - tools=tools, - add_generation_prompt=True, - tokenize=True, - ) - - # 1. Detect agent retries and roll back (at most one assistant step). - self._try_detect_and_rollback_to_assistant_checkpoint(request_messages) - # 2. Confirm the (possibly rolled-back) stored messages are a prefix of request, - # and that each appended message role is in tito_tokenizer.allowed_append_roles. - try: - assert_messages_append_only_with_allowed_role( - self.messages, request_messages, tito_tokenizer.allowed_append_roles - ) - except ValueError as e: - raise MessageValidationError(f"{e}; to allow more roles use --tito-allowed-append-roles") from e - - return tito_tokenizer.merge_tokens( - old_messages=self.messages, - new_messages=request_messages, - pretokenized_token_ids=self.token_ids, - tools=tools, - ) - - def update_pretokenized_state( - self, - request_messages: list[dict[str, Any]], - assistant_message: dict[str, Any], - prompt_token_ids: list[int], - completion_token_ids: list[int], - max_trim_tokens: int, - ) -> None: - """Store raw token IDs after a successful response. - - Appends ``prompt_token_ids + completion_token_ids`` as a new checkpoint. - Validates that the previously stored token_ids are a prefix of the new - checkpoint (tolerating up to ``max_trim_tokens`` trailing differences). - Must be called under ``self.lock``. - """ - all_token_ids = prompt_token_ids + completion_token_ids - - prev = self.token_ids - if prev: - check_len = len(prev) - max_trim_tokens - if check_len > 0 and all_token_ids[:check_len] != prev[:check_len]: - first_mismatch = next( - ( - i - for i, (a, b) in enumerate(zip(all_token_ids[:check_len], prev[:check_len], strict=True)) - if a != b - ), - min(len(all_token_ids), check_len), - ) - raise TokenizationError( - f"pretokenized prefix mismatch: " - f"stored {len(prev)} tokens (checking first {check_len}, " - f"allowing {max_trim_tokens} trailing) are not a prefix of " - f"prompt_token_ids + completion_token_ids " - f"({len(all_token_ids)} tokens), " - f"first mismatch at index {first_mismatch}, " - f"matched {first_mismatch}/{check_len} prefix tokens\n" - f"request_messages={request_messages}\n" - f"assistant_message={assistant_message}" - ) - - self.messages = list(request_messages) + [assistant_message] - self.trajectory_token_ids.append(all_token_ids) - self.num_assistant += 1 - - def _try_detect_and_rollback_to_assistant_checkpoint( - self, - request_messages: list[dict[str, Any]], - ) -> None: - """Detect if *request_messages* diverges from stored history and roll back. - - In agentic workflows the agent may retry from an earlier point — for - example, re-running a tool call with different arguments. When that - happens the new request shares a common prefix with the stored messages - but diverges before the end. This method truncates session state back - to the last assistant checkpoint within the matching prefix. - - Only a single-step rollback is allowed (controlled by - ``MAX_ASSISTANT_ROLLBACK_STEPS``). Discarding exactly one assistant - message means the agent is retrying from the preceding checkpoint — - the request shares the stored prefix up to that assistant and then - continues with whatever the agent chooses (same or different tool - result, additional messages, etc.). Any request that would need to - discard more than one assistant (i.e. jump back across multiple - turns) is rejected with ``MessageValidationError`` and no state is - modified. - - Example — agent retries after the first tool call:: - - stored: [sys, user, assistant₁, tool₁, assistant₂] - ───────────────────── ▲ - checkpoint 0 (assistant₁) checkpoint 1 (assistant₂) - - request: [sys, user, assistant₁, tool₁_different, ...] - ↑ diverges here (index 3) - - match_len = 3 (sys, user, assistant₁ all match) - Last assistant in matched prefix → assistant₁ (checkpoint 0) - discard_count = 2 - 1 = 1 (≤ MAX_ASSISTANT_ROLLBACK_STEPS) - - After rollback: - messages = [sys, user, assistant₁] - trajectory_token_ids = [checkpoint_0_ids] - records = [record_0] - num_assistant = 1 - - No rollback occurs when: - - The stored history is empty. - - *request_messages* is a strict extension of stored messages - (``match_len >= len(stored)``). - """ - stored = self.messages - if not stored or not self.trajectory_token_ids: - return - - match_len = 0 - for i in range(min(len(request_messages), len(stored))): - if message_matches(stored[i], request_messages[i]): - match_len = i + 1 - else: - break - - if match_len >= len(stored): - return - - # Find the last assistant message within the matched prefix. - rollback_msg_end = None - checkpoint_index = -1 - assistant_count = 0 - for i in range(match_len): - if stored[i].get("role") == "assistant": - rollback_msg_end = i + 1 - checkpoint_index = assistant_count - assistant_count += 1 - - if checkpoint_index < 0: - raise MessageValidationError( - f"rollback failed: no assistant message found in the first " - f"{match_len} matched messages (stored has {len(stored)} messages, " - f"request has {len(request_messages)} messages)" - ) - - discard_count = self.num_assistant - (checkpoint_index + 1) - if discard_count > MAX_ASSISTANT_ROLLBACK_STEPS: - raise MessageValidationError( - f"rollback failed: discard_count={discard_count} exceeds " - f"max_assistant_rollback_steps={MAX_ASSISTANT_ROLLBACK_STEPS} " - f"(stored has {len(stored)} messages, " - f"request has {len(request_messages)} messages)" - ) - - logger.info( - "Rolling back session: stored %d messages / %d checkpoints -> " - "checkpoint %d (messages[:%d]), discarding %d assistant(s)", - len(stored), - self.num_assistant, - checkpoint_index, - rollback_msg_end, - discard_count, - ) - - self.messages = stored[:rollback_msg_end] - self.trajectory_token_ids = self.trajectory_token_ids[: checkpoint_index + 1] - self.records = self.records[: checkpoint_index + 1] - self.num_assistant = checkpoint_index + 1 - - -class SessionRegistry: - """Session ID -> trajectory mapping with shared tokenizer resources. - - Pure CRUD plus read-only computation (compute_session_mismatch). - Does NOT mutate session state - all mutations are methods on - LinearTrajectory; called by the route handler under session.lock. - """ - - def __init__(self, args, tokenizer: Any, *, tito_tokenizer: TITOTokenizer): - self.sessions: dict[str, LinearTrajectory] = {} - self.args = args - self.tokenizer = tokenizer - self.tito_tokenizer = tito_tokenizer - self.comparator = tito_tokenizer.create_comparator() - - def create_session(self) -> str: - session_id = uuid.uuid4().hex - self.sessions[session_id] = LinearTrajectory() - return session_id - - def get_session(self, session_id: str) -> LinearTrajectory: - session = self.sessions.get(session_id) - if session is None: - raise SessionNotFoundError(f"session not found: session_id={session_id}") - return session - - def remove_session(self, session_id: str) -> None: - if self.sessions.pop(session_id, None) is None: - raise SessionNotFoundError(f"session not found: session_id={session_id}") - - def compute_session_mismatch(self, session: LinearTrajectory) -> list[dict] | None: - """Compare accumulated token IDs against canonical chat template output. - - Read-only: does not mutate session state. - """ - if not session.token_ids: - return None - try: - tools = session.records[-1].request.get("tools") if session.records else None - expected_ids = self.tito_tokenizer.render_messages( - session.messages, - tools=tools, - add_generation_prompt=False, - tokenize=True, - ) - mismatches = self.comparator.compare_sequences(expected_ids, session.token_ids) - return [m.to_dict() for m in mismatches] - except Exception as e: - raise TokenizationError(f"failed to compute tito_session_mismatch: {e}") from e diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/session_errors.py b/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/session_errors.py deleted file mode 100644 index 30e6784..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/session_errors.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Error types for the session module. - -Hierarchy ---------- -SessionError (base) -├── SessionNotFoundError → 404 session does not exist -├── MessageValidationError → 400 messages structure/content invalid -├── TokenizationError → 500 TITO tokenizer / prefix mismatch -└── UpstreamResponseError → 502 SGLang response invalid or unexpected -""" - - -class SessionError(Exception): - """Base class for all session-related errors.""" - - status_code: int = 500 - - -class SessionNotFoundError(SessionError): - """Raised when the requested session ID does not exist.""" - - status_code: int = 404 - - -class MessageValidationError(SessionError): - """Raised when request messages fail structural validation. - - Examples: user message after assistant, messages not append-only, - rollback failed (no assistant checkpoint in matched prefix). - """ - - status_code: int = 400 - - -class TokenizationError(SessionError): - """Raised when TITO tokenization invariants are violated. - - Examples: pretokenized prefix mismatch between stored and new token IDs. - """ - - status_code: int = 500 - - -class UpstreamResponseError(SessionError): - """Raised when the upstream SGLang response is invalid or unexpected. - - Examples: missing meta_info, assistant content is None, - output_token_logprobs length mismatch. - """ - - status_code: int = 502 diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/session_server.py b/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/session_server.py deleted file mode 100644 index b787e16..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/session_server.py +++ /dev/null @@ -1,111 +0,0 @@ -"""Standalone Session Server that proxies through the inference router. - -This decouples session/TITO logic from the Miles Router, allowing sessions -to work with the SGLang Rust Router or any other backend. Inference -requests are proxied through the router (sglang or miles), which handles -load balancing and forwarding to worker engines. -""" - -import json -import logging - -import httpx -import setproctitle -import uvicorn -from fastapi import FastAPI, Request -from fastapi.responses import JSONResponse -from starlette.responses import Response - -from tito_gateway.vendor.miles_compat.rollout.session.sessions import setup_session_routes - -logger = logging.getLogger(__name__) - - -class SessionServer: - """Lightweight FastAPI server that manages sessions and proxies inference - requests through the inference router (sglang or miles).""" - - def __init__(self, args, backend_url: str): - self.backend_url = backend_url - self.app = FastAPI() - - timeout = getattr(args, "miles_router_timeout", 600.0) - self.client = httpx.AsyncClient( - limits=httpx.Limits(max_connections=1024), - timeout=httpx.Timeout(timeout), - ) - - # Close the httpx connection pool when uvicorn shuts down to avoid FD leaks. - self.app.router.on_shutdown.append(self.client.aclose) - - setup_session_routes(self.app, self, args) - - async def do_proxy( - self, - request: Request, - path: str, - body: bytes | None = None, - headers: dict | None = None, - ) -> dict: - url = f"{self.backend_url}/{path}" - if request.url.query: - url = f"{url}?{request.url.query}" - - if body is None: - body = await request.body() - if headers is None: - headers = dict(request.headers) - headers = { - k: v for k, v in headers.items() if k.lower() not in ("content-length", "transfer-encoding", "host") - } - - try: - response = await self.client.request(request.method, url, content=body, headers=headers) - except httpx.TransportError as exc: - logger.warning("Proxy transport error for %s %s: %s", request.method, path, exc) - error_body = json.dumps({"error": f"backend transport error: {type(exc).__name__}: {exc}"}).encode() - return { - "request_body": body, - "response_body": error_body, - "status_code": 502, - "headers": {"content-type": "application/json"}, - } - content = await response.aread() - return { - "request_body": body, - "response_body": content, - "status_code": response.status_code, - "headers": dict(response.headers), - } - - def build_proxy_response(self, result: dict) -> Response: - content = result["response_body"] - status_code = result["status_code"] - # Drop wire-level framing headers from upstream so Starlette rebuilds them - # from the body we actually send: transfer-encoding is hop-by-hop - headers = { - k: v - for k, v in result["headers"].items() - if k.lower() not in ("content-length", "transfer-encoding", "content-encoding") - } - content_type = headers.get("content-type", "") - try: - data = json.loads(content) - return JSONResponse(content=data, status_code=status_code, headers=headers) - except (json.JSONDecodeError, UnicodeDecodeError): - return Response(content=content, status_code=status_code, headers=headers, media_type=content_type) - - -def run_session_server(args, backend_url: str): - """Entry point to start the standalone session server as a subprocess.""" - # Visible to `pkill -9 miles`; without this the daemon inherits "python". - setproctitle.setproctitle("miles-session-server") - - server = SessionServer(args, backend_url) - logger.info( - "[session-server] Starting on %s:%s, proxying to %s", - args.session_server_ip, - args.session_server_port, - backend_url, - ) - uvicorn.run(server.app, host=args.session_server_ip, port=args.session_server_port, log_level="info") diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/session_types.py b/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/session_types.py deleted file mode 100644 index 6548902..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/session_types.py +++ /dev/null @@ -1,16 +0,0 @@ -from pydantic import BaseModel, Field - - -class SessionRecord(BaseModel): - timestamp: float - method: str - path: str - request: dict - response: dict - status_code: int - - -class GetSessionResponse(BaseModel): - session_id: str - records: list[SessionRecord] - metadata: dict = Field(default_factory=dict) diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/sessions.py b/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/sessions.py deleted file mode 100644 index 17e722b..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/rollout/session/sessions.py +++ /dev/null @@ -1,251 +0,0 @@ -import json -import logging -import time - -from fastapi import Request -from fastapi.responses import JSONResponse -from starlette.responses import Response - -from tito_gateway.vendor.miles_compat.rollout.session.linear_trajectory import SessionRegistry -from tito_gateway.vendor.miles_compat.rollout.session.session_errors import ( - SessionError, - SessionNotFoundError, - TokenizationError, - UpstreamResponseError, -) -from tito_gateway.vendor.miles_compat.rollout.session.session_types import GetSessionResponse, SessionRecord -from tito_gateway.vendor.miles_compat.utils.chat_template_utils import get_tito_tokenizer -from tito_gateway.vendor.miles_compat.utils.processing_utils import load_tokenizer - -logger = logging.getLogger(__name__) - - -def setup_session_routes(app, backend, args): - hf_checkpoint = getattr(args, "hf_checkpoint", None) - if not hf_checkpoint: - logger.info("[session] Skipping session routes (hf_checkpoint not set).") - return - - session_server_instance_id = getattr(args, "session_server_instance_id", None) - - tokenizer = load_tokenizer( - hf_checkpoint, chat_template_path=getattr(args, "chat_template_path", None), trust_remote_code=True - ) - - tito_tokenizer = get_tito_tokenizer( - tokenizer, - tokenizer_type=getattr(args, "tito_model", "default"), - chat_template_kwargs=getattr(args, "apply_chat_template_kwargs", None), - allowed_append_roles=getattr(args, "tito_allowed_append_roles", None), - ) - - registry = SessionRegistry(args, tokenizer, tito_tokenizer=tito_tokenizer) - - @app.get("/health") - async def health(): - body = {"status": "ok"} - if session_server_instance_id is not None: - body["session_server_instance_id"] = session_server_instance_id - return body - - # --- DEBUG: track in-flight chat_completions --- - _inflight_chat = {"count": 0} - - @app.middleware("http") - async def debug_request_logger(request: Request, call_next): - client = request.client - client_info = f"{client.host}:{client.port}" if client else "unknown" - logger.info( - f"[session-server] REQUEST ARRIVED: {request.method} {request.url.path} from={client_info} inflight_chat={_inflight_chat['count']}" - ) - t0 = time.time() - response = await call_next(request) - elapsed = time.time() - t0 - logger.info( - f"[session-server] REQUEST DONE: {request.method} {request.url.path} status={response.status_code} elapsed={elapsed:.3f}s from={client_info}" - ) - return response - - @app.exception_handler(SessionError) - async def session_error_handler(request: Request, exc: SessionError): - return JSONResponse(status_code=exc.status_code, content={"error": str(exc)}) - - @app.post("/sessions") - async def create_session(): - session_id = registry.create_session() - return {"session_id": session_id} - - @app.get("/sessions/{session_id}") - async def get_session(session_id: str): - session = registry.get_session(session_id) - metadata = {} - try: - mismatch = registry.compute_session_mismatch(session) - except TokenizationError: - logger.exception("Failed to compute tito_session_mismatch for session %s", session_id) - mismatch = None - if mismatch is not None: - metadata["tito_session_mismatch"] = mismatch - metadata["accumulated_token_ids"] = session.token_ids - metadata["max_trim_tokens"] = registry.tito_tokenizer.max_trim_tokens - return GetSessionResponse( - session_id=session_id, - records=session.records, - metadata=metadata, - ) - - @app.delete("/sessions/{session_id}") - async def delete_session(session_id: str): - session = registry.get_session(session_id) - if session.closing: - raise SessionNotFoundError(f"session not found: session_id={session_id}") - session.closing = True - logger.debug( - f"[session-server] DELETE waiting for lock: session={session_id} lock_locked={session.lock.locked()}" - ) - await session.lock.acquire() - logger.debug(f"[session-server] DELETE acquired lock: session={session_id}") - try: - registry.remove_session(session_id) - finally: - session.lock.release() - return Response(status_code=204) - - @app.post("/sessions/{session_id}/v1/chat/completions") - async def chat_completions(request: Request, session_id: str): - """Proxy a chat completion through SGLang with TITO token tracking. - - Flow: prepare pretokenized input_ids (lock held briefly) → inject - SGLang flags → proxy to backend (NO lock) → validate response → - update trajectory checkpoint (lock held briefly) → append session record. - - The lock is NOT held during the slow proxy call to avoid blocking - DELETE/other operations when the agent disconnects mid-request. - """ - _inflight_chat["count"] += 1 - try: - session = registry.get_session(session_id) - if session.closing: - raise SessionNotFoundError(f"session not found: session_id={session_id}") - - # --- Phase 1: prepare request (lock held briefly) --- - async with session.lock: - # Double-check: session may have been marked closing while waiting for lock. - if session.closing: - raise SessionNotFoundError(f"session not found: session_id={session_id}") - - body = await request.body() - request_body = json.loads(body) if body else {} - - # TITO token tracking requires Miles-owned input_ids plus SGLang - # output-token metadata: - # logprobs=True → populates meta_info.output_token_logprobs - # return_meta_info → wraps the above in choice.meta_info - # Both flags are hardcoded (not set default) to prevent agent-side - # overrides from breaking the token accumulation invariants. - request_body["logprobs"] = True - request_body["return_meta_info"] = True - if getattr(args, "use_rollout_routing_replay", False): - request_body["return_routed_experts"] = True - if getattr(args, "use_rollout_indexer_replay", False): - request_body["return_indexer_topk"] = True - # Must be False so stop-token text is trimmed from assistant - # message content; token IDs are still taken from logprobs below. - request_body["no_stop_trim"] = False - - request_messages = request_body.get("messages", []) - prompt_token_ids = session.prepare_pretokenized( - request_messages, - tools=request_body.get("tools"), - tito_tokenizer=registry.tito_tokenizer, - ) - request_body["input_ids"] = prompt_token_ids - logger.debug( - "Using TITO input_ids: %d tokens", - len(prompt_token_ids), - ) - - body = json.dumps(request_body).encode() - expected_num_assistant = session.num_assistant - # --- lock released here --- - - # --- Phase 2: proxy to SGLang (NO lock held) --- - result = await backend.do_proxy(request, "v1/chat/completions", body=body) - - # If SGLang returned a non-200 error (e.g. 400 for context too long), - # pass it through to the agent without recording — the agent can retry - # or handle the error. - if result["status_code"] != 200: - return backend.build_proxy_response(result) - - response = json.loads(result["response_body"]) - - choice = response.get("choices", [{}])[0] - - meta_info = choice.get("meta_info") - if not isinstance(meta_info, dict) or "output_token_logprobs" not in meta_info: - raise UpstreamResponseError( - "meta_info and output_token_logprobs must be in choice (requires logprobs=True)" - ) - assistant_message = choice.get("message", {}) - if assistant_message.get("content") is None: - raise UpstreamResponseError( - "assistant message content is None, when tool call parser failed SGLang should still return " - "an empty content rather than None. Please check your modified SGLang version." - ) - - output_token_logprobs = meta_info["output_token_logprobs"] - completion_tokens = meta_info["completion_tokens"] - - actual_output_logprobs_len = len(output_token_logprobs) - if actual_output_logprobs_len != completion_tokens: - raise UpstreamResponseError( - "invalid chat completion response: " - f"len(output_token_logprobs)={actual_output_logprobs_len} " - f"!= completion_tokens={completion_tokens}. " - f"Please check whether you use the correct SGLang branch which has fix the tokenizer batch decode issue." - ) - - completion_token_ids = [t[1] for t in output_token_logprobs] - - # --- Phase 3: update state (lock held briefly) --- - async with session.lock: - if session.closing: - logger.warning(f"Session {session_id} closed during proxy, skipping state update") - return backend.build_proxy_response(result) - - if session.num_assistant != expected_num_assistant: - logger.warning( - f"Session {session_id} state changed during proxy " - f"(expected num_assistant={expected_num_assistant}, " - f"got {session.num_assistant}), skipping state update" - ) - return backend.build_proxy_response(result) - - session.update_pretokenized_state( - request_messages, - assistant_message, - prompt_token_ids=prompt_token_ids, - completion_token_ids=completion_token_ids, - max_trim_tokens=registry.tito_tokenizer.max_trim_tokens, - ) - - record = SessionRecord( - timestamp=time.time(), - method=request.method, - path="/v1/chat/completions", - status_code=result["status_code"], - request=request_body, - response=response, - ) - session.append_record(record) - # --- lock released here --- - - return backend.build_proxy_response(result) - finally: - _inflight_chat["count"] -= 1 - - @app.api_route("/sessions/{session_id}/{path:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"]) - async def session_proxy(request: Request, session_id: str, path: str): - result = await backend.do_proxy(request, path) - return backend.build_proxy_response(result) diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/__init__.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/__init__.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/__init__.py deleted file mode 100644 index 24976a4..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/__init__.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Chat template utilities for agentic-workflow token consistency.""" - -from tito_gateway.vendor.miles_compat.utils.chat_template_utils.template import ( - apply_chat_template, - apply_chat_template_from_str, - assert_messages_append_only_with_allowed_role, - extract_tool_dicts, - load_hf_chat_template, - message_matches, - normalize_tool_arguments, -) -from tito_gateway.vendor.miles_compat.utils.chat_template_utils.tito_tokenizer import ( - TEMPLATE_DIR, - TITOTokenizer, - TITOTokenizerType, - get_tito_tokenizer, - resolve_fixed_chat_template, - resolve_reasoning_and_tool_call_parser, -) -from tito_gateway.vendor.miles_compat.utils.chat_template_utils.token_seq_comparator import Mismatch, MismatchType, TokenSeqComparator - -__all__ = [ - "TITOTokenizer", - "TITOTokenizerType", - "get_tito_tokenizer", - "TEMPLATE_DIR", - "resolve_fixed_chat_template", - "resolve_reasoning_and_tool_call_parser", - "load_hf_chat_template", - "apply_chat_template", - "apply_chat_template_from_str", - "assert_messages_append_only_with_allowed_role", - "message_matches", - "extract_tool_dicts", - "normalize_tool_arguments", - "Mismatch", - "TokenSeqComparator", - "MismatchType", -] diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/deepseek_v32.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/deepseek_v32.py deleted file mode 100644 index bf29b22..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/deepseek_v32.py +++ /dev/null @@ -1,97 +0,0 @@ -from __future__ import annotations - -import copy -import functools -import json -import logging -import os -from typing import Any - -from sglang.srt.entrypoints.openai.protocol import Tool - -try: - from sglang.srt.entrypoints.openai import encoding_dsv32 -except ImportError: # pragma: no cover - depends on the installed sglang build. - encoding_dsv32 = None - -logger = logging.getLogger(__name__) - -_MODEL_TYPE = "deepseek_v32" - -_KNOWN_KWARGS = frozenset( - { - "thinking_mode", - "drop_thinking", - "add_default_bos_token", - "context", - } -) - - -@functools.cache -def _read_model_type(name_or_path: str) -> str: - """Read ``model_type`` from a checkpoint's ``config.json`` (cached per path).""" - if not name_or_path: - return "" - config_path = os.path.join(name_or_path, "config.json") - if not os.path.isfile(config_path): - return "" - try: - with open(config_path, encoding="utf-8") as f: - config = json.load(f) - except (OSError, UnicodeDecodeError, json.JSONDecodeError): - return "" - if not isinstance(config, dict): - return "" - return config.get("model_type", "") or "" - - -def is_deepseek_v32(tokenizer: Any) -> bool: - """Return True when *tokenizer* is a DeepSeek V3.2 checkpoint.""" - return _read_model_type(tokenizer.name_or_path) == _MODEL_TYPE - - -def _build_deepseek_encode_config(kwargs: dict) -> dict: - kwargs = dict(kwargs) - if (enable_thinking := kwargs.pop("enable_thinking", None)) is not None: - kwargs.setdefault("thinking_mode", "thinking" if enable_thinking else "chat") - # reject unknown kwargs to avoid silent config drop - unknown = set(kwargs) - _KNOWN_KWARGS - if unknown: - raise ValueError( - f"apply_chat_template_kwargs has unsupported kwargs {sorted(unknown)} " - f"for the DeepSeek encoder. Known keys: {sorted(_KNOWN_KWARGS)}" - ) - cfg = {"thinking_mode": "thinking", "drop_thinking": True, "add_default_bos_token": True} - for key in _KNOWN_KWARGS: - if key in kwargs: - cfg[key] = kwargs[key] - return cfg - - -def _inject_tools_into_system(messages: list[dict[str, Any]], tools: list[dict[str, Any]]) -> list[dict[str, Any]]: - """Put *tools* in the system message, where ``encode_messages`` reads them. - - The encoder serializes each tool dict verbatim into ````, so they - must round-trip through ``Tool.model_dump()`` (fills defaults / orders fields) - or the token ids drift from what sglang serves. - """ - out = copy.deepcopy(messages) - if not out or out[0].get("role") != "system": - out.insert(0, {"role": "system", "content": ""}) - out[0]["tools"] = [Tool.model_validate(t).model_dump() for t in tools] - return out - - -def render_messages(messages: list[dict[str, Any]], *, tools: list[dict] | None = None, **kwargs: Any) -> str: - """Render *messages* into a DeepSeek V3.2 prompt via sglang ``encode_messages``. - - Tool_call ``arguments`` must already be JSON strings; *tools*, if given, are - injected into the system message (see ``_inject_tools_into_system``). - """ - encode_config = _build_deepseek_encode_config(kwargs) - if tools: - messages = _inject_tools_into_system(messages, tools) - if encoding_dsv32 is None: - raise ImportError("sglang encoding_dsv32 is required for DeepSeek V3.2 chat-template rendering") - return encoding_dsv32.encode_messages(messages, **encode_config) diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/deepseek_v4.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/deepseek_v4.py deleted file mode 100644 index e8bf2a0..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/deepseek_v4.py +++ /dev/null @@ -1,100 +0,0 @@ -from __future__ import annotations - -import copy -import functools -import json -import logging -import os -from typing import Any - -from sglang.srt.entrypoints.openai.protocol import Tool - -try: - from sglang.srt.entrypoints.openai import encoding_dsv4 -except ImportError: # pragma: no cover - depends on the installed sglang build. - encoding_dsv4 = None - -logger = logging.getLogger(__name__) - -_MODEL_TYPE = "deepseek_v4" - -_KNOWN_KWARGS = frozenset( - { - "thinking_mode", - "drop_thinking", - "add_default_bos_token", - "context", - "reasoning_effort", - } -) - - -@functools.cache -def _read_model_type(name_or_path: str) -> str: - """Read ``model_type`` from a checkpoint's ``config.json`` (cached per path).""" - if not name_or_path: - return "" - config_path = os.path.join(name_or_path, "config.json") - if not os.path.isfile(config_path): - return "" - try: - with open(config_path, encoding="utf-8") as f: - config = json.load(f) - except (OSError, UnicodeDecodeError, json.JSONDecodeError): - return "" - if not isinstance(config, dict): - return "" - return config.get("model_type", "") or "" - - -def is_deepseek_v4(tokenizer: Any) -> bool: - """Return True when *tokenizer* is a DeepSeek V4 checkpoint.""" - return _read_model_type(tokenizer.name_or_path) == _MODEL_TYPE - - -def _build_deepseek_encode_config(kwargs: dict) -> dict: - kwargs = dict(kwargs) - if (enable_thinking := kwargs.pop("enable_thinking", None)) is not None: - kwargs.setdefault("thinking_mode", "thinking" if enable_thinking else "chat") - # reject unknown kwargs to avoid silent config drop - unknown = set(kwargs) - _KNOWN_KWARGS - if unknown: - raise ValueError( - f"apply_chat_template_kwargs has unsupported kwargs {sorted(unknown)} " - f"for the DeepSeek encoder. Known keys: {sorted(_KNOWN_KWARGS)}" - ) - # reasoning_effort has no default: like context, it is only forwarded when the - # caller supplies it, and its value is validated by encoding_dsv4 (not here). - cfg = {"thinking_mode": "thinking", "drop_thinking": True, "add_default_bos_token": True} - for key in _KNOWN_KWARGS: - if key in kwargs: - cfg[key] = kwargs[key] - return cfg - - -def _inject_tools_into_system(messages: list[dict[str, Any]], tools: list[dict[str, Any]]) -> list[dict[str, Any]]: - """Put *tools* in the system message, where ``encode_messages`` reads them. - - The encoder serializes each tool dict verbatim into ````, so they - must round-trip through ``Tool.model_dump()`` (fills defaults / orders fields) - or the token ids drift from what sglang serves. - """ - out = copy.deepcopy(messages) - if not out or out[0].get("role") != "system": - out.insert(0, {"role": "system", "content": ""}) - out[0]["tools"] = [Tool.model_validate(t).model_dump() for t in tools] - return out - - -def render_messages(messages: list[dict[str, Any]], *, tools: list[dict] | None = None, **kwargs: Any) -> str: - """Render *messages* into a DeepSeek V4 prompt via sglang ``encode_messages``. - - Tool_call ``arguments`` must already be JSON strings; *tools*, if given, are - injected into the system message (see ``_inject_tools_into_system``). - """ - encode_config = _build_deepseek_encode_config(kwargs) - if tools: - messages = _inject_tools_into_system(messages, tools) - if encoding_dsv4 is None: - raise ImportError("sglang encoding_dsv4 is required for DeepSeek V4 chat-template rendering") - return encoding_dsv4.encode_messages(messages, **encode_config) diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/template.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/template.py deleted file mode 100644 index 45464c4..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/template.py +++ /dev/null @@ -1,256 +0,0 @@ -"""Core chat template operations: load from HuggingFace and render from string. - -``load_hf_chat_template`` fetches original (unmodified) chat templates via -``hf_hub_download``. Files are cached locally after the first download — -subsequent calls read from disk without network access. - -``apply_chat_template_from_str`` renders a Jinja2 chat template string -without depending on a HuggingFace tokenizer, equivalent to -``tokenizer.apply_chat_template(..., tokenize=False)``. - -``apply_chat_template`` applies via an HF tokenizer object (returns -``str`` or ``list[int]``). Both functions normalize tool arguments, -canonicalize tool definitions, and fall back between tool dict formats. -""" - -from __future__ import annotations - -import copy -import json -from typing import Any, Literal - -from huggingface_hub import hf_hub_download -from jinja2 import TemplateError -from pydantic import TypeAdapter -from sglang.srt.entrypoints.openai.protocol import Tool -from transformers.utils.chat_template_utils import render_jinja_template - -from tito_gateway.vendor.miles_compat.utils.chat_template_utils import deepseek_v4, deepseek_v32 - - -def load_hf_chat_template(model_id: str) -> str: - """Load an original chat template from HuggingFace (cached locally). - - Handles two layouts: - - ``chat_template`` field in ``tokenizer_config.json`` (most models) - - Separate ``chat_template.jinja`` file (e.g. GLM-5) - """ - config_path = hf_hub_download(model_id, "tokenizer_config.json") - with open(config_path) as f: - config = json.load(f) - template = config.get("chat_template", "") - if template: - if isinstance(template, list): - for t in template: - if t.get("name") == "default" or not t.get("name"): - return t["template"] - return template[0]["template"] - return template - - jinja_path = hf_hub_download(model_id, "chat_template.jinja") - with open(jinja_path) as f: - return f.read() - - -def normalize_tool_arguments(messages: list[dict], format: Literal["dict", "json"]) -> list[dict]: - """Deep-copy *messages*, normalize assistant ``content: None`` -> "", and coerce - tool_call ``arguments`` to the form the downstream renderer needs (``format`` picks - the direction; never mutates the input): - - ``"dict"``: JSON string -> dict, for HF-Jinja templates (they index args as objects). - - ``"json"``: dict -> JSON string, for the DeepSeek DSML encoders (they ``json.loads`` them). - """ - normalized = copy.deepcopy(messages) - for msg in normalized: - if msg.get("role") == "assistant": - if msg.get("content") is None: - msg["content"] = "" - if isinstance(msg.get("tool_calls"), list): - for item in msg["tool_calls"]: - func = item.get("function") - if not func: - continue - args = func.get("arguments") - if format == "dict" and isinstance(args, str): - func["arguments"] = json.loads(args) - elif format == "json" and isinstance(args, dict): - func["arguments"] = json.dumps(args, ensure_ascii=False) - return normalized - - -def extract_tool_dicts(tools: list[dict] | None) -> list[dict] | None: - """Canonicalize tools via Pydantic, returning full Tool model dumps. - - Matches SGLang's ``_process_messages`` (``serving_chat.py`` lines 343-344): - ``tools = [item.model_dump() for item in request.tools]`` — each tool is - a full ``Tool`` model dump (``{"type": "function", "function": {...}}``). - """ - if not tools: - return None - - wrapped = [t if isinstance(t, dict) and "function" in t else {"type": "function", "function": t} for t in tools] - validated = TypeAdapter(list[Tool]).validate_python(wrapped) - return [tool.model_dump() for tool in validated] - - -def apply_chat_template_from_str( - chat_template: str, - messages: list[dict], - add_generation_prompt: bool = True, - tools: list[dict] | None = None, - **kwargs, -) -> str: - """Render a Jinja2 chat template string (tokenize=False, no tokenizer needed). - - Calls HF transformers' ``render_jinja_template`` directly — the same - function that ``tokenizer.apply_chat_template`` uses internally. Both - SGLang and our ``apply_chat_template`` go through that same HF code path. - - Applies SGLang-style normalizations (tool argument parsing, tool dict - canonicalization, tool format fallback). - """ - - def _render(tool_defs): - rendered, _ = render_jinja_template( - conversations=[messages], - chat_template=chat_template, - add_generation_prompt=add_generation_prompt, - tools=tool_defs, - **kwargs, - ) - return rendered[0] - - messages = normalize_tool_arguments(messages, "dict") - tool_defs = extract_tool_dicts(tools) - try: - return _render(tool_defs) - except TemplateError as e: - if tool_defs is not None: - try: - return _render([t["function"] if "function" in t else t for t in tool_defs]) - except TemplateError as te: - raise ValueError(f"Chat template rendering failed (tool format fallback): {te}") from te - raise ValueError(f"Chat template rendering failed: {e}") from e - - -_TEMPLATE_RELEVANT_KEYS = ("role", "content", "reasoning_content", "tool_calls") - - -def _normalize_value(value: Any) -> Any: - """Normalize falsy sentinels that produce identical Jinja2 output. - - None, "" and [] are all falsy in Jinja2 and render the same way, - but client libraries may interchange them (e.g. content: null vs "" - for tool-call-only responses, or tool_calls: null vs []). - - Only collapses falsy values — non-falsy content (including whitespace - like trailing newlines) is returned as-is. Message boundary characters - must be preserved exactly so they tokenize identically across turns. - """ - if value is None or value == "" or value == []: - return None - return value - - -def message_matches(stored: dict[str, Any], new: dict[str, Any]) -> bool: - """Compare only the fields that affect chat-template tokenization. - - External client libraries (e.g. litellm) may inject extra keys like - ``provider_specific_fields`` into messages. These have no effect on - the Jinja2 chat template output, so we only compare the keys that - templates actually read: role, content, reasoning_content, tool_calls. - """ - for key in _TEMPLATE_RELEVANT_KEYS: - if _normalize_value(stored.get(key)) != _normalize_value(new.get(key)): - return False - return True - - -_DEFAULT_APPEND_ROLES: list[str] = ["tool"] - - -def assert_messages_append_only_with_allowed_role( - stored_messages: list[dict[str, Any]], - new_messages: list[dict[str, Any]], - allowed_append_roles: list[str] = _DEFAULT_APPEND_ROLES, -) -> None: - """Assert *new_messages* is an append-only extension of *stored_messages*. - - The stored prefix must match exactly (compared by template-relevant keys), - and any appended messages must have a role in *allowed_append_roles* - (default: ``{'tool'}``). - """ - if not stored_messages: - return - - if len(new_messages) < len(stored_messages): - raise ValueError( - f"new messages ({len(new_messages)}) are fewer than stored messages ({len(stored_messages)})", - new_messages, - stored_messages, - ) - - for i, stored_msg in enumerate(stored_messages): - if not message_matches(stored_msg, new_messages[i]): - diffs = { - key: {"stored": repr(stored_msg.get(key))[:200], "new": repr(new_messages[i].get(key))[:200]} - for key in _TEMPLATE_RELEVANT_KEYS - if stored_msg.get(key) != new_messages[i].get(key) - } - raise ValueError( - f"message mismatch at index {i} " - f"(role: stored={stored_msg.get('role')}, new={new_messages[i].get('role')}). " - f"Diffs: {diffs}" - ) - - for j, msg in enumerate(new_messages[len(stored_messages) :]): - if msg.get("role") not in allowed_append_roles: - raise ValueError( - f"appended message at index {len(stored_messages) + j} " - f"has role={msg.get('role')!r}, allowed={allowed_append_roles}" - ) - - -def apply_chat_template( - messages: list[dict], - *, - tokenizer, - tools: list[dict] | None = None, - add_generation_prompt: bool = True, - tokenize: bool = False, - **kwargs, -) -> str | list[int]: - """Apply chat template via HF tokenizer in SGLang style. - - Passes ``return_dict=False`` to match SGLang's ``serving_chat.py``, - ensuring the result is ``str`` (tokenize=False) or ``list[int]`` - (tokenize=True), not a ``BatchEncoding`` or ``dict``. - """ - if deepseek_v32.is_deepseek_v32(tokenizer): - rendered = deepseek_v32.render_messages(normalize_tool_arguments(messages, "json"), tools=tools, **kwargs) - return tokenizer.encode(rendered, add_special_tokens=False) if tokenize else rendered - - if deepseek_v4.is_deepseek_v4(tokenizer): - rendered = deepseek_v4.render_messages(normalize_tool_arguments(messages, "json"), tools=tools, **kwargs) - return tokenizer.encode(rendered, add_special_tokens=False) if tokenize else rendered - - messages = normalize_tool_arguments(messages, "dict") - tool_defs = extract_tool_dicts(tools) - render_kwargs = dict(add_generation_prompt=add_generation_prompt, **kwargs) - - try: - return tokenizer.apply_chat_template( - messages, tokenize=tokenize, tools=tool_defs, return_dict=False, **render_kwargs - ) - except TemplateError as e: - if tool_defs is not None: - try: - return tokenizer.apply_chat_template( - messages, - tokenize=tokenize, - tools=[t["function"] if "function" in t else t for t in tool_defs], - return_dict=False, - **render_kwargs, - ) - except TemplateError as te: - raise ValueError(f"Chat template rendering failed (tool format fallback): {te}") from te - raise ValueError(f"Chat template rendering failed: {e}") from e diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/kimi_k25_fixed.jinja b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/kimi_k25_fixed.jinja deleted file mode 100644 index 0f05753..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/kimi_k25_fixed.jinja +++ /dev/null @@ -1,111 +0,0 @@ -{%- macro render_content(msg) -%} - {%- set c = msg.get('content') -%} - {%- if c is string -%} - {{ c }} - {%- elif c is not none -%} - {% for content in c -%} - {% if content['type'] == 'image' or content['type'] == 'image_url' -%} - <|media_begin|>image<|media_content|><|media_pad|><|media_end|> - {% elif content['type'] == 'video' or content['type']== 'video_url'-%} - <|kimi_k25_video_placeholder|> - {% else -%} - {{ content['text'] }} - {%- endif -%} - {%- endfor -%} - {%- endif -%} -{%- endmacro -%} - -{% macro set_roles(message) -%} - {%- set role_name = message.get('name') or message['role'] -%} - {%- if message['role'] == 'user' -%} - <|im_user|>{{role_name}}<|im_middle|> - {%- elif message['role'] == 'assistant' -%} - <|im_assistant|>{{role_name}}<|im_middle|> - {%- else -%} - <|im_system|>{{role_name}}<|im_middle|> - {%- endif -%} -{%- endmacro -%} - - -{%- macro render_toolcalls(message) -%} - <|tool_calls_section_begin|> - {%- for tool_call in message['tool_calls'] -%} - {%- set formatted_id = tool_call['id'] -%} - <|tool_call_begin|>{{ formatted_id }}<|tool_call_argument_begin|>{% if tool_call['function']['arguments'] is string %}{{ tool_call['function']['arguments'] }}{% else %}{{ tool_call['function']['arguments'] | tojson }}{% endif %}<|tool_call_end|> - {%- endfor -%} - <|tool_calls_section_end|> -{%- endmacro -%} - - -{%- set preserve_thinking = preserve_thinking | default(false) -%} -{# Find last non-tool-call assistant message. If preserve_thinking, keep -1 so hist is empty and all msgs use suffix (retain reasoning). #} -{%- set ns = namespace(last_non_tool_call_assistant_msg=-1) -%} -{%- if not preserve_thinking -%} -{%- for idx in range(messages|length-1, -1, -1) -%} - {%- if messages[idx]['role'] == 'assistant' and not messages[idx].get('tool_calls') -%} - {%- set ns.last_non_tool_call_assistant_msg = idx -%} - {%- break -%} - {%- endif -%} -{%- endfor -%} -{%- endif -%} - -{# split all messages into history & suffix, reasoning_content in suffix should be reserved.#} -{%- set hist_msgs = messages[:ns.last_non_tool_call_assistant_msg+1] -%} -{%- set suffix_msgs = messages[ns.last_non_tool_call_assistant_msg+1:] -%} - -{%- if tools -%} - {%- if tools_ts_str -%} - <|im_system|>tool_declare<|im_middle|>{{ tools_ts_str }}<|im_end|> - {%- else -%} - <|im_system|>tool_declare<|im_middle|>{{ tools | tojson(separators=(',', ':')) }}<|im_end|> - {%- endif -%} -{%- endif -%} - -{%- for message in hist_msgs -%} - {{set_roles(message)}} - {%- if message['role'] == 'assistant' -%} - {{render_content(message)}} - {%- if message.get('tool_calls') -%} - {{render_toolcalls(message)}} - {%- endif -%} - {%- elif message['role'] == 'tool' -%} - {%- set tool_call_id = message.tool_call_id -%} - ## Return of {{ tool_call_id }} -{{render_content(message)}} - {%- elif message['content'] is not none -%} - {{render_content(message)}} - {%- endif -%} - <|im_end|> -{%- endfor -%} - -{%- for message in suffix_msgs -%} - {{set_roles(message)}} - {%- if message['role'] == 'assistant' -%} - {%- if thinking is defined and thinking is false -%} - {{render_content(message)}} - {%- else -%} - {%- set rc = message.get('reasoning_content', '') -%} - {{rc}}{{render_content(message)}} - {%- endif -%} - {%- if message.get('tool_calls') -%} - {{render_toolcalls(message)}} - {%- endif -%} - {%- elif message['role'] == 'tool' -%} - {%- set tool_call_id = message.tool_call_id -%} - ## Return of {{ tool_call_id }} -{{render_content(message)}} - {%- elif message['content'] is not none -%} - {{render_content(message)}} - {%- endif -%} - <|im_end|> -{%- endfor -%} - - -{%- if add_generation_prompt -%} - <|im_assistant|>assistant<|im_middle|> - {%- if thinking is defined and thinking is false -%} - - {%- else -%} - - {%- endif -%} -{%- endif -%} diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/minimax_m25_fixed.jinja b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/minimax_m25_fixed.jinja deleted file mode 100644 index c121f2d..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/minimax_m25_fixed.jinja +++ /dev/null @@ -1,159 +0,0 @@ -{# ----------‑‑‑ special token variables ‑‑‑---------- #} -{%- set toolcall_begin_token = '' -%} -{%- set toolcall_end_token = '' -%} -{#- Tool Rendering Functions ============================================== -#} -{%- macro render_tool_namespace(namespace_name, tool_list) -%} -{%- for tool in tool_list -%} -{{ tool.function | tojson(ensure_ascii=False) }} -{% endfor -%} -{%- endmacro -%} -{%- macro visible_text(content) -%} - {%- if content is string -%} - {{ content }} - {%- elif content is iterable and content is not mapping -%} - {%- for item in content -%} - {%- if item is mapping and item.type == 'text' -%} - {{- item.text }} - {%- elif item is string -%} - {{- item }} - {%- endif -%} - {%- endfor -%} - {%- else -%} - {{- content }} - {%- endif -%} -{%- endmacro -%} -{#- System Message Construction ============================================ -#} -{%- macro build_system_message(system_message) -%} - {%- if system_message and system_message.content -%} - {{- visible_text(system_message.content) }} - {%- else -%} - {%- if model_identity is not defined -%} - {%- set model_identity = "You are a helpful assistant. Your name is MiniMax-M2.5 and is built by MiniMax." -%} - {%- endif -%} - {{- model_identity }} - {%- endif -%} - - {#- Handle current_date -#} - {%- if system_message and system_message.current_date -%} - {{- '\n' ~ 'Current date: ' + system_message.current_date }} - {%- endif -%} - {#- Handle current_location -#} - {%- if system_message and system_message.current_location -%} - {{- '\n' ~ 'Current location: ' + system_message.current_location }} - {%- endif -%} -{%- endmacro -%} -{#- Main Template Logic ================================================= -#} -{#- Extract system message (only first message if it's system) -#} -{%- set system_message = none -%} -{%- set conversation_messages = messages -%} -{%- if messages and messages[0].role == "system" -%} - {%- set system_message = messages[0] -%} - {%- set conversation_messages = messages[1:] -%} -{%- endif -%} -{#- Get the last user message turn, for interleved thinking -#} -{%- set ns = namespace(last_user_index=-1) %} -{% for m in conversation_messages %} - {%- if m.role == 'user' %} - {% set ns.last_user_index = loop.index0 -%} - {%- endif %} -{%- endfor %} -{#- Render system message -#} -{{- ']~!b[' ~ ']~b]system' ~ '\n' }} -{{- build_system_message(system_message) }} -{#- Render tools if available -#} -{%- if tools -%} - {{- '\n\n' ~ '# Tools' ~ '\n' ~ 'You may call one or more tools to assist with the user query.\nHere are the tools available in JSONSchema format:' ~ '\n' }} - {{- '\n' ~ '' ~ '\n' }} - {{- render_tool_namespace("functions", tools) }} - {{- '' ~ '\n\n' }} -{{- 'When making tool calls, use XML format to invoke tools and pass parameters:' ~ '\n' }} -{{- '\n' ~ toolcall_begin_token }} - -param-value-1 -param-value-2 -... - -{{- '\n' ~ toolcall_end_token }} -{%- endif -%} -{{- '[e~[\n' }} - -{#- Render messages -#} -{%- set last_tool_call = namespace(name=none) -%} -{%- for message in conversation_messages -%} - {%- if message.role == 'assistant' -%} - {#- Only render reasoning_content if no user message follows (or clear_thinking disabled) -#} - {{- ']~b]ai' ~ '\n' }} - - {%- set reasoning_content = '' %} - {%- set content = visible_text(message.content) %} - {%- if message.reasoning_content is string %} - {%- set reasoning_content = message.reasoning_content %} - {%- else %} - {%- if '' in content %} - {%- set reasoning_content = content.split('')[0].strip('\n').split('')[-1].strip('\n') %} - {%- set content = content.split('')[-1].strip('\n') %} - {%- endif %} - {%- endif %} - {%- if reasoning_content and (not (clear_thinking | default(true)) or loop.index0 > ns.last_user_index) -%} - {{- '' ~ '\n' ~ reasoning_content ~ '\n' ~ '' ~ '\n\n' }} - {%- endif -%} - {%- if content -%} - {{- content }} - {%- endif -%} - {%- if message.tool_calls -%} - {{- '\n' ~ toolcall_begin_token ~ '\n' }} - - {%- for tool_call in message.tool_calls -%} - {%- if tool_call.function %} - {%- set tool_call = tool_call.function %} - {%- endif %} - {{- '' }} - {% set _args = tool_call.arguments %} - {%- for k, v in _args.items() %} - {{- '' }} - {{- v | tojson(ensure_ascii=False) if v is not string else v }} - {{- '' }} - {% endfor %} - {{- '' ~ '\n' }} - {%- endfor -%} - - {{- toolcall_end_token}} - {%- set last_tool_call.name = message.tool_calls[-1].name -%} - {%- else -%} - {%- set last_tool_call.name = none -%} - {%- endif -%} - {{- '[e~[' ~ '\n' }} - - {%- elif message.role == 'tool' -%} - {%- if last_tool_call.name is none -%} - {{- raise_exception("Message has tool role, but there was no previous assistant message with a tool call!") }} - {%- endif -%} - {%- if loop.first or (conversation_messages[loop.index0 - 1].role != 'tool') -%} - {{- ']~b]tool' }} - {%- endif -%} - {%- if message.content is string -%} - {{- '\n' }} - {{- message.content }} - {{- '' }} - {%- else -%} - {%- for tr in message.content -%} - {{- '\n' }} - {{- tr.output if tr.output is defined else (tr.text if tr.type == 'text' and tr.text is defined else tr) }} - {{- '\n' }} - {%- endfor -%} - {%- endif -%} - {%- if loop.last or (conversation_messages[loop.index0 + 1].role != 'tool') -%} - {{- '[e~[\n' -}} - {%- endif -%} - - {%- elif message.role == 'user' -%} - {{- ']~b]user' ~ '\n' }} - {{- visible_text(message.content) }} - {{- '[e~[' ~ '\n' }} - {%- endif -%} -{%- endfor -%} - -{#- Generation prompt -#} -{%- if add_generation_prompt -%} -{{- ']~b]ai' ~ '\n' ~ '' ~ '\n' }} -{%- endif -%} diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/minimax_m27_fixed.jinja b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/minimax_m27_fixed.jinja deleted file mode 100644 index b49b815..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/minimax_m27_fixed.jinja +++ /dev/null @@ -1,159 +0,0 @@ -{# ----------‑‑‑ special token variables ‑‑‑---------- #} -{%- set toolcall_begin_token = '' -%} -{%- set toolcall_end_token = '' -%} -{#- Tool Rendering Functions ============================================== -#} -{%- macro render_tool_namespace(namespace_name, tool_list) -%} -{%- for tool in tool_list -%} -{{ tool.function | tojson(ensure_ascii=False) }} -{% endfor -%} -{%- endmacro -%} -{%- macro visible_text(content) -%} - {%- if content is string -%} - {{ content }} - {%- elif content is iterable and content is not mapping -%} - {%- for item in content -%} - {%- if item is mapping and item.type == 'text' -%} - {{- item.text }} - {%- elif item is string -%} - {{- item }} - {%- endif -%} - {%- endfor -%} - {%- else -%} - {{- content }} - {%- endif -%} -{%- endmacro -%} -{#- System Message Construction ============================================ -#} -{%- macro build_system_message(system_message) -%} - {%- if system_message and system_message.content -%} - {{- visible_text(system_message.content) }} - {%- else -%} - {%- if model_identity is not defined -%} - {%- set model_identity = "You are a helpful assistant. Your name is MiniMax-M2.7 and is built by MiniMax." -%} - {%- endif -%} - {{- model_identity }} - {%- endif -%} - - {#- Handle current_date -#} - {%- if system_message and system_message.current_date -%} - {{- '\n' ~ 'Current date: ' + system_message.current_date }} - {%- endif -%} - {#- Handle current_location -#} - {%- if system_message and system_message.current_location -%} - {{- '\n' ~ 'Current location: ' + system_message.current_location }} - {%- endif -%} -{%- endmacro -%} -{#- Main Template Logic ================================================= -#} -{#- Extract system message (only first message if it's system) -#} -{%- set system_message = none -%} -{%- set conversation_messages = messages -%} -{%- if messages and messages[0].role == "system" -%} - {%- set system_message = messages[0] -%} - {%- set conversation_messages = messages[1:] -%} -{%- endif -%} -{#- Get the last user message turn, for interleved thinking -#} -{%- set ns = namespace(last_user_index=-1) %} -{% for m in conversation_messages %} - {%- if m.role == 'user' %} - {% set ns.last_user_index = loop.index0 -%} - {%- endif %} -{%- endfor %} -{#- Render system message -#} -{{- ']~!b[' ~ ']~b]system' ~ '\n' }} -{{- build_system_message(system_message) }} -{#- Render tools if available -#} -{%- if tools -%} - {{- '\n\n' ~ '# Tools' ~ '\n' ~ 'You may call one or more tools to assist with the user query.\nHere are the tools available in JSONSchema format:' ~ '\n' }} - {{- '\n' ~ '' ~ '\n' }} - {{- render_tool_namespace("functions", tools) }} - {{- '' ~ '\n\n' }} -{{- 'When making tool calls, use XML format to invoke tools and pass parameters:' ~ '\n' }} -{{- '\n' ~ toolcall_begin_token }} - -param-value-1 -param-value-2 -... - -{{- '\n' ~ toolcall_end_token }} -{%- endif -%} -{{- '[e~[\n' }} - -{#- Render messages -#} -{%- set last_tool_call = namespace(name=none) -%} -{%- for message in conversation_messages -%} - {%- if message.role == 'assistant' -%} - {#- Only render reasoning_content if no user message follows (or clear_thinking disabled) -#} - {{- ']~b]ai' ~ '\n' }} - - {%- set reasoning_content = '' %} - {%- set content = visible_text(message.content) %} - {%- if message.reasoning_content is string %} - {%- set reasoning_content = message.reasoning_content %} - {%- else %} - {%- if '' in content %} - {%- set reasoning_content = content.split('')[0].strip('\n').split('')[-1].strip('\n') %} - {%- set content = content.split('')[-1].strip('\n') %} - {%- endif %} - {%- endif %} - {%- if reasoning_content and (not (clear_thinking | default(true)) or loop.index0 > ns.last_user_index) -%} - {{- '' ~ '\n' ~ reasoning_content ~ '\n' ~ '' ~ '\n\n' }} - {%- endif -%} - {%- if content -%} - {{- content }} - {%- endif -%} - {%- if message.tool_calls -%} - {{- '\n' ~ toolcall_begin_token ~ '\n' }} - - {%- for tool_call in message.tool_calls -%} - {%- if tool_call.function %} - {%- set tool_call = tool_call.function %} - {%- endif %} - {{- '' }} - {% set _args = tool_call.arguments %} - {%- for k, v in _args.items() %} - {{- '' }} - {{- v | tojson(ensure_ascii=False) if v is not string else v }} - {{- '' }} - {% endfor %} - {{- '' ~ '\n' }} - {%- endfor -%} - - {{- toolcall_end_token}} - {%- set last_tool_call.name = message.tool_calls[-1].name -%} - {%- else -%} - {%- set last_tool_call.name = none -%} - {%- endif -%} - {{- '[e~[' ~ '\n' }} - - {%- elif message.role == 'tool' -%} - {%- if last_tool_call.name is none -%} - {{- raise_exception("Message has tool role, but there was no previous assistant message with a tool call!") }} - {%- endif -%} - {%- if loop.first or (conversation_messages[loop.index0 - 1].role != 'tool') -%} - {{- ']~b]tool' }} - {%- endif -%} - {%- if message.content is string -%} - {{- '\n' }} - {{- message.content }} - {{- '' }} - {%- else -%} - {%- for tr in message.content -%} - {{- '\n' }} - {{- tr.output if tr.output is defined else (tr.text if tr.type == 'text' and tr.text is defined else tr) }} - {{- '\n' }} - {%- endfor -%} - {%- endif -%} - {%- if loop.last or (conversation_messages[loop.index0 + 1].role != 'tool') -%} - {{- '[e~[\n' -}} - {%- endif -%} - - {%- elif message.role == 'user' -%} - {{- ']~b]user' ~ '\n' }} - {{- visible_text(message.content) }} - {{- '[e~[' ~ '\n' }} - {%- endif -%} -{%- endfor -%} - -{#- Generation prompt -#} -{%- if add_generation_prompt -%} -{{- ']~b]ai' ~ '\n' ~ '' ~ '\n' }} -{%- endif -%} diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/qwen3.5_fixed.jinja b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/qwen3.5_fixed.jinja deleted file mode 100644 index b003bcb..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/qwen3.5_fixed.jinja +++ /dev/null @@ -1,151 +0,0 @@ -{%- set image_count = namespace(value=0) %} -{%- set video_count = namespace(value=0) %} -{%- macro render_content(content, do_vision_count, is_system_content=false) %} - {%- if content is string %} - {{- content }} - {%- elif content is iterable and content is not mapping %} - {%- for item in content %} - {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} - {%- if is_system_content %} - {{- raise_exception('System message cannot contain images.') }} - {%- endif %} - {%- if do_vision_count %} - {%- set image_count.value = image_count.value + 1 %} - {%- endif %} - {%- if add_vision_id %} - {{- 'Picture ' ~ image_count.value ~ ': ' }} - {%- endif %} - {{- '<|vision_start|><|image_pad|><|vision_end|>' }} - {%- elif 'video' in item or item.type == 'video' %} - {%- if is_system_content %} - {{- raise_exception('System message cannot contain videos.') }} - {%- endif %} - {%- if do_vision_count %} - {%- set video_count.value = video_count.value + 1 %} - {%- endif %} - {%- if add_vision_id %} - {{- 'Video ' ~ video_count.value ~ ': ' }} - {%- endif %} - {{- '<|vision_start|><|video_pad|><|vision_end|>' }} - {%- elif 'text' in item %} - {{- item.text }} - {%- else %} - {{- raise_exception('Unexpected item type in content.') }} - {%- endif %} - {%- endfor %} - {%- elif content is none or content is undefined %} - {{- '' }} - {%- else %} - {{- raise_exception('Unexpected content type.') }} - {%- endif %} -{%- endmacro %} -{%- if not messages %} - {{- raise_exception('No messages provided.') }} -{%- endif %} -{%- if tools and tools is iterable and tools is not mapping %} - {{- '<|im_start|>system\n' }} - {{- "# Tools\n\nYou have access to the following functions:\n\n" }} - {%- for tool in tools %} - {{- "\n" }} - {{- tool | tojson }} - {%- endfor %} - {{- "\n" }} - {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} - {%- if messages[0].role == 'system' %} - {%- set content = render_content(messages[0].content, false, true)|trim %} - {%- if content %} - {{- '\n\n' + content }} - {%- endif %} - {%- endif %} - {{- '<|im_end|>\n' }} -{%- else %} - {%- if messages[0].role == 'system' %} - {%- set content = render_content(messages[0].content, false, true)|trim %} - {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} - {%- endif %} -{%- endif %} -{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} -{%- for message in messages[::-1] %} - {%- set index = (messages|length - 1) - loop.index0 %} - {%- if ns.multi_step_tool and message.role == "user" %} - {%- set content = render_content(message.content, false)|trim %} - {%- if not(content.startswith('') and content.endswith('')) %} - {%- set ns.multi_step_tool = false %} - {%- set ns.last_query_index = index %} - {%- endif %} - {%- endif %} -{%- endfor %} -{%- for message in messages %} - {%- set content = render_content(message.content, true)|trim %} - {%- if message.role == "system" %} - {%- if not loop.first %} - {{- raise_exception('System message must be at the beginning.') }} - {%- endif %} - {%- elif message.role == "user" %} - {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} - {%- elif message.role == "assistant" %} - {%- set reasoning_content = '' %} - {%- if message.reasoning_content is string %} - {%- set reasoning_content = message.reasoning_content %} - {%- else %} - {%- if '' in content %} - {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} - {%- set content = content.split('')[-1].lstrip('\n') %} - {%- endif %} - {%- endif %} - {%- set reasoning_content = reasoning_content|trim %} - {%- if (not (clear_thinking | default(true))) or loop.index0 > ns.last_query_index %} - {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} - {%- else %} - {{- '<|im_start|>' + message.role + '\n' + content }} - {%- endif %} - {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} - {%- for tool_call in message.tool_calls %} - {%- if tool_call.function is defined %} - {%- set tool_call = tool_call.function %} - {%- endif %} - {%- if loop.first %} - {%- if content|trim %} - {{- '\n\n\n\n' }} - {%- else %} - {{- '\n\n' }} - {%- endif %} - {%- else %} - {{- '\n\n\n' }} - {%- endif %} - {%- if tool_call.arguments is defined %} - {%- for args_name, args_value in tool_call.arguments|items %} - {{- '\n' }} - {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} - {{- args_value }} - {{- '\n\n' }} - {%- endfor %} - {%- endif %} - {{- '\n' }} - {%- endfor %} - {%- endif %} - {{- '<|im_end|>\n' }} - {%- elif message.role == "tool" %} - {%- if loop.previtem and loop.previtem.role != "tool" %} - {{- '<|im_start|>user' }} - {%- endif %} - {{- '\n\n' }} - {{- content }} - {{- '\n' }} - {%- if not loop.last and loop.nextitem.role != "tool" %} - {{- '<|im_end|>\n' }} - {%- elif loop.last %} - {{- '<|im_end|>\n' }} - {%- endif %} - {%- else %} - {{- raise_exception('Unexpected message role.') }} - {%- endif %} -{%- endfor %} -{%- if add_generation_prompt %} - {{- '<|im_start|>assistant\n' }} - {%- if enable_thinking is defined and enable_thinking is false %} - {{- '\n\n\n\n' }} - {%- else %} - {{- '\n' }} - {%- endif %} -{%- endif %} diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/qwen3_thinking_2507_and_next_fixed.jinja b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/qwen3_thinking_2507_and_next_fixed.jinja deleted file mode 100644 index 1588dfb..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/templates/qwen3_thinking_2507_and_next_fixed.jinja +++ /dev/null @@ -1,82 +0,0 @@ -{%- if tools %} - {{- '<|im_start|>system\n' }} - {%- if messages[0].role == 'system' %} - {{- messages[0].content + '\n\n' }} - {%- endif %} - {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} - {%- for tool in tools %} - {{- "\n" }} - {{- tool | tojson }} - {%- endfor %} - {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} -{%- else %} - {%- if messages[0].role == 'system' %} - {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} - {%- endif %} -{%- endif %} -{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} -{%- for message in messages[::-1] %} - {%- set index = (messages|length - 1) - loop.index0 %} - {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} - {%- set ns.multi_step_tool = false %} - {%- set ns.last_query_index = index %} - {%- endif %} -{%- endfor %} -{%- for message in messages %} - {%- if message.content is string %} - {%- set content = message.content %} - {%- else %} - {%- set content = '' %} - {%- endif %} - {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} - {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} - {%- elif message.role == "assistant" %} - {%- set reasoning_content = '' %} - {%- if message.reasoning_content is string %} - {%- set reasoning_content = message.reasoning_content %} - {%- else %} - {%- if '' in content %} - {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} - {%- set content = content.split('')[-1].lstrip('\n') %} - {%- endif %} - {%- endif %} - {%- if (not (clear_thinking | default(true))) or loop.index0 > ns.last_query_index %} - {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} - {%- else %} - {{- '<|im_start|>' + message.role + '\n' + content }} - {%- endif %} - {%- if message.tool_calls %} - {%- for tool_call in message.tool_calls %} - {%- if (loop.first and content) or (not loop.first) %} - {{- '\n' }} - {%- endif %} - {%- if tool_call.function %} - {%- set tool_call = tool_call.function %} - {%- endif %} - {{- '\n{"name": "' }} - {{- tool_call.name }} - {{- '", "arguments": ' }} - {%- if tool_call.arguments is string %} - {{- tool_call.arguments }} - {%- else %} - {{- tool_call.arguments | tojson }} - {%- endif %} - {{- '}\n' }} - {%- endfor %} - {%- endif %} - {{- '<|im_end|>\n' }} - {%- elif message.role == "tool" %} - {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} - {{- '<|im_start|>user' }} - {%- endif %} - {{- '\n\n' }} - {{- content }} - {{- '\n' }} - {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} - {{- '<|im_end|>\n' }} - {%- endif %} - {%- endif %} -{%- endfor %} -{%- if add_generation_prompt %} - {{- '<|im_start|>assistant\n\n' }} -{%- endif %} diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/tito_tokenizer.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/tito_tokenizer.py deleted file mode 100644 index df8df5a..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/tito_tokenizer.py +++ /dev/null @@ -1,1014 +0,0 @@ -"""TITO tokenizer — incremental tokenization for pretokenized prefix reuse. - -``TITOTokenizer`` computes incremental token IDs for non-assistant messages -(tool responses, user follow-ups, system injections) that follow the -assistant's generated token sequence, then merges them with the pretokenized -prefix — handling model-specific boundary tokens at the junction. - -The default implementation incrementally tokenizes appended non-assistant turns -with role-specific synthetic prefixes: - -- contiguous ``tool`` runs use ``[dummy_system, dummy_assistant]`` -- each ``user`` or ``system`` message uses ``[dummy_system]`` - -The appended suffix is processed left-to-right, then the generation prompt for -the next assistant turn is appended once at the end. Model-specific -subclasses only override ``merge_tokens`` for boundary quirks at the prefix -junction. -""" - -from __future__ import annotations - -import logging -from collections.abc import Iterable -from dataclasses import dataclass, field -from enum import Enum -from pathlib import Path -from typing import Any - -from tito_gateway.vendor.miles_compat.utils.chat_template_utils.template import apply_chat_template, assert_messages_append_only_with_allowed_role -from tito_gateway.vendor.miles_compat.utils.chat_template_utils.token_seq_comparator import TokenSeqComparator - -logger = logging.getLogger(__name__) - -# Bundled fixed-template files live under this directory; ``FixedTemplateRow.template`` -# values are filenames relative to it. -TEMPLATE_DIR = Path(__file__).parent / "templates" - -# Roles the TITO merge logic understands; passing anything else is a typo. -_VALID_ROLES = frozenset({"tool", "user", "system"}) - -_DUMMY_SYSTEM: dict[str, Any] = {"role": "system", "content": "dummy system"} - - -@dataclass(frozen=True) -class FixedTemplateRow: - """A ``(roles, template, extra_kwargs)`` row owned by a TITO tokenizer family. - - Each row says: when the session is configured for ``allowed_roles``, this - family expects the given chat template plus the given extra kwargs. - ``template`` is a path relative to ``TEMPLATE_DIR`` for a bundled fixed - template, or ``None`` to keep the HF-native template (kwargs-only fix). - """ - - allowed_roles: frozenset[str] - template: str | None = None - extra_kwargs: dict[str, Any] = field(default_factory=dict) - - -def _build_dummy_assistant(tool_responses: list[dict[str, Any]]) -> dict[str, Any]: - """Build a dummy assistant message with tool_calls matching *tool_responses*, - so the template correctly renders the subsequent tool-response turn boundaries.""" - return { - "role": "assistant", - "content": "", - "reasoning_content": " ", - "tool_calls": [ - { - "id": resp.get("tool_call_id") or f"call0000{i}", - "type": "function", - "function": { - "name": resp.get("name") or "dummy_func", - "arguments": {}, - }, - } - for i, resp in enumerate(tool_responses) - ], - } - - -# --------------------------------------------------------------------------- -# Base / default tokenizer -# --------------------------------------------------------------------------- -# TODO: split different model's TITO tokenizer into different files - - -class TITOTokenizer: - """Incremental tokenization and prefix merging for appended non-assistant turns.""" - - max_trim_tokens: int = 0 - trailing_token_ids: frozenset[int] = frozenset() - - # ``(roles, template, extra_kwargs)`` rows this family supports. Resolved - # by ``resolve_fixed_chat_template`` via smallest-superset match against - # the caller's ``allowed_append_roles``. - SUPPORTED_TEMPLATES: tuple[FixedTemplateRow, ...] = () - - # sglang ``--reasoning-parser`` and ``--tool-call-parser`` values bound to - # this family. - reasoning_parser: str | None = None - tool_call_parser: str | None = None - - def __init__( - self, - tokenizer: Any, - chat_template_kwargs: dict[str, Any] | None = None, - assistant_start_str: str | None = None, - special_token_ids: set[int] | None = None, - allowed_append_roles: list[str] | None = None, - ): - self.tokenizer = tokenizer - self.chat_template_kwargs = chat_template_kwargs or {} - self._assistant_start_str = assistant_start_str - self.allowed_append_roles: list[str] = allowed_append_roles if allowed_append_roles is not None else ["tool"] - self.special_token_ids: set[int] = special_token_ids - - def create_comparator(self) -> TokenSeqComparator: - """Create a :class:`TokenSeqComparator` configured with this - tokenizer's model-specific settings.""" - return TokenSeqComparator( - self.tokenizer, - assistant_start_str=self._assistant_start_str, - special_token_ids=self.special_token_ids, - trim_trailing_ids=self.trailing_token_ids or None, - ) - - def render_messages( - self, - messages: list[dict[str, Any]], - *, - add_generation_prompt: bool, - tools: list[dict[str, Any]] | None = None, - tokenize: bool = False, - ) -> str | list[int]: - return apply_chat_template( - messages, - tokenizer=self.tokenizer, - tokenize=tokenize, - add_generation_prompt=add_generation_prompt, - tools=tools, - **self.chat_template_kwargs, - ) - - def _encode_text(self, text: str) -> list[int]: - return self.tokenizer.encode(text, add_special_tokens=False) - - def _split_appended_segments(self, appended_messages: list[dict[str, Any]]) -> list[list[dict[str, Any]]]: - segments: list[list[dict[str, Any]]] = [] - i = 0 - while i < len(appended_messages): - role = appended_messages[i]["role"] - # Many templates wrap a contiguous tool-response run as one logical - # block, so tool messages are diffed together instead of one-by-one. - if role == "tool": - j = i + 1 - while j < len(appended_messages) and appended_messages[j]["role"] == "tool": - j += 1 - segments.append(appended_messages[i:j]) - i = j - continue - if role in {"user", "system"}: - segments.append([appended_messages[i]]) - i += 1 - continue - raise ValueError(f"unsupported appended role for TITO segmentation: {role}") - - return segments - - def _tokenize_rendered_suffix( - self, - base_messages: list[dict[str, Any]], - appended_messages: list[dict[str, Any]], - *, - tools: list[dict[str, Any]] | None = None, - add_generation_prompt: bool = False, - ) -> list[int]: - """Render *base_messages* and *base_messages + appended_messages*, return - tokens for the suffix. - - When *add_generation_prompt* is True and *appended_messages* is empty, - this computes the generation-prompt suffix (the assistant opener tokens). - """ - text_without = self.render_messages(base_messages, add_generation_prompt=False, tools=tools) - text_with = self.render_messages( - base_messages + appended_messages, - add_generation_prompt=add_generation_prompt, - tools=tools, - ) - if not text_with.startswith(text_without): - roles = [msg["role"] for msg in appended_messages] if appended_messages else ["generation_prompt"] - raise ValueError(f"rendered suffix diff failed for {roles}") - return self._encode_text(text_with[len(text_without) :]) - - def _tokenize_tool_segment( - self, - appended_messages: list[dict[str, Any]], - tools: list[dict[str, Any]] | None = None, - ) -> list[int]: - # No dummy user to avoid cut think issues. - return self._tokenize_rendered_suffix( - [_DUMMY_SYSTEM, _build_dummy_assistant(appended_messages)], - appended_messages, - tools=tools, - ) - - def _tokenize_user_and_system_segment( - self, - appended_message: dict[str, Any], - tools: list[dict[str, Any]] | None = None, - ) -> list[int]: - # User/system single-message appends share one synthetic context. - return self._tokenize_rendered_suffix( - [_DUMMY_SYSTEM], - [appended_message], - tools=tools, - ) - - def tokenize_additional_non_assistant( - self, - old_messages: list[dict[str, Any]], - new_messages: list[dict[str, Any]], - tools: list[dict[str, Any]] | None = None, - ) -> list[int]: - """Compute incremental token IDs for non-assistant messages appended - after the pretokenized prefix. - - Handles tool responses, user, and system messages — - never an assistant message. Validates that *new_messages* is an - append-only extension of *old_messages* via - ``assert_messages_append_only_with_allowed_role``. - - Args: - old_messages: Previously stored messages (prefix). - new_messages: Full new message list (must be a superset of - *old_messages* with only allowed-role messages appended). - tools: Tool definitions in OpenAI format (may vary per call). - - Returns: - Incremental token IDs (including the generation prompt) that, - when merged with pretokenized prefix via ``merge_tokens``, - form the full prompt token IDs. - """ - assert_messages_append_only_with_allowed_role(old_messages, new_messages, self.allowed_append_roles) - appended_messages = new_messages[len(old_messages) :] - incremental: list[int] = [] - - # Incremental non-assistant content is assembled segment-by-segment - # using the smallest synthetic context that preserves each role's - # boundary tokens. - for segment in self._split_appended_segments(appended_messages): - role = segment[0]["role"] - if role == "tool": - incremental.extend(self._tokenize_tool_segment(segment, tools)) - elif role == "user" or role == "system": - incremental.extend(self._tokenize_user_and_system_segment(segment[0], tools)) - else: - raise ValueError(f"unsupported appended role for TITO tokenization: {role}") - - # The next assistant opener depends on the full post-append history, so - # it is derived from the real messages once and appended only at the end. - return incremental + self._tokenize_rendered_suffix( - new_messages, - [], - tools=tools, - add_generation_prompt=True, - ) - - def merge_tokens( - self, - old_messages: list[dict[str, Any]], - new_messages: list[dict[str, Any]], - pretokenized_token_ids: list[int], - tools: list[dict[str, Any]] | None = None, - ) -> list[int]: - """Merge *pretokenized_token_ids* with incremental tokens to produce - the complete prompt token IDs (including generation prompt). - - The default implementation is simple concatenation. Subclasses - override this to handle model-specific boundary token logic. - """ - incremental = self.tokenize_additional_non_assistant(old_messages, new_messages, tools) - return list(pretokenized_token_ids) + incremental - - -# --------------------------------------------------------------------------- -# Qwen3 implementation -# --------------------------------------------------------------------------- - - -class Qwen3TITOTokenizer(TITOTokenizer): - """Qwen3 variant: handles missing newline at the boundary. - - The Qwen3 chat template emits ``<|im_end|>\\n`` after every message, but - the model stops at ``<|im_end|>`` without generating the trailing ``\\n``. - ``merge_tokens`` inserts the missing newline so that the pretokenized - prefix matches the canonical template output. - """ - - reasoning_parser = "qwen3" - tool_call_parser = "qwen25" - - SUPPORTED_TEMPLATES = ( - FixedTemplateRow( - allowed_roles=frozenset({"tool"}), - template="qwen3_fixed.jinja", - ), - FixedTemplateRow( - allowed_roles=frozenset({"tool", "user"}), - template="qwen3_fixed.jinja", - extra_kwargs={"clear_thinking": False}, - ), - ) - - _default_assistant_start_str: str = "<|im_start|>assistant" - - def __init__( - self, - tokenizer: Any, - chat_template_kwargs: dict[str, Any] | None = None, - assistant_start_str: str | None = None, - allowed_append_roles: list[str] | None = None, - ): - super().__init__( - tokenizer, - chat_template_kwargs, - assistant_start_str or self._default_assistant_start_str, - allowed_append_roles=allowed_append_roles, - ) - nl_ids = tokenizer.encode("\n", add_special_tokens=False) - assert len(nl_ids) == 1, f"Expected single newline token, got {nl_ids}" - self._newline_id: int = nl_ids[0] - self._im_end_id: int = tokenizer.convert_tokens_to_ids("<|im_end|>") - self.trailing_token_ids = frozenset({self._newline_id}) - - def merge_tokens( - self, - old_messages: list[dict[str, Any]], - new_messages: list[dict[str, Any]], - pretokenized_token_ids: list[int], - tools: list[dict[str, Any]] | None = None, - ) -> list[int]: - incremental = self.tokenize_additional_non_assistant(old_messages, new_messages, tools) - prefix = list(pretokenized_token_ids) - if prefix and prefix[-1] == self._im_end_id: - prefix.append(self._newline_id) - return prefix + incremental - - -# Qwen3.5 and Qwen3-Next-Thinking share the ``<|im_end|>`` boundary handling -# with Qwen3, so they reuse Qwen3TITOTokenizer's token-level logic via plain -# inheritance. They are still split into named subclasses because each owns -# its own ``SUPPORTED_TEMPLATES`` row pointing to a distinct fixed jinja, even -# though their boundary behavior is identical. - - -class Qwen35TITOTokenizer(Qwen3TITOTokenizer): - """Qwen3.5 — same boundary behavior as Qwen3, distinct fixed template.""" - - tool_call_parser = "qwen3_coder" - - SUPPORTED_TEMPLATES = ( - FixedTemplateRow( - allowed_roles=frozenset({"tool"}), - template="qwen3.5_fixed.jinja", - ), - FixedTemplateRow( - allowed_roles=frozenset({"tool", "user"}), - template="qwen3.5_fixed.jinja", - extra_kwargs={"clear_thinking": False}, - ), - ) - - -class QwenNextTITOTokenizer(Qwen3TITOTokenizer): - """Qwen3-Thinking-2507 / Qwen3-Next-Thinking — same boundary behavior as - Qwen3, distinct (shared) fixed template.""" - - SUPPORTED_TEMPLATES = ( - FixedTemplateRow( - allowed_roles=frozenset({"tool"}), - template="qwen3_thinking_2507_and_next_fixed.jinja", - ), - FixedTemplateRow( - allowed_roles=frozenset({"tool", "user"}), - template="qwen3_thinking_2507_and_next_fixed.jinja", - extra_kwargs={"clear_thinking": False}, - ), - ) - - -# --------------------------------------------------------------------------- -# GLM 4.7 implementation -# --------------------------------------------------------------------------- - - -class GLM47TITOTokenizer(TITOTokenizer): - """GLM 4.7 variant: handles ambiguous boundary tokens in ``merge_tokens``. - - ``<|user|>`` and ``<|observation|>`` are both assistant stop tokens *and* - next-message start tokens in the chat template. In ``merge_tokens``, - the last token of the pretokenized prefix is always stripped when it is - one of these boundary tokens — whether it matches the first incremental - token (overlap) or differs (e.g. model stopped with ``<|observation|>`` but - next turn is ``<|user|>`` because the tool call failed and a system message - is injected instead). - """ - - reasoning_parser = "glm45" - tool_call_parser = "glm47" - - # GLM's HF-native chat template already exposes a ``clear_thinking`` kwarg, - # so no fixed-jinja patch is needed for either append surface. - SUPPORTED_TEMPLATES = ( - FixedTemplateRow( - allowed_roles=frozenset({"tool"}), - template=None, - ), - FixedTemplateRow( - allowed_roles=frozenset({"tool", "user"}), - template=None, - extra_kwargs={"clear_thinking": False}, - ), - FixedTemplateRow( - allowed_roles=frozenset({"tool", "user", "system"}), - template=None, - extra_kwargs={"clear_thinking": False}, - ), - ) - - max_trim_tokens: int = 1 - _default_assistant_start_str: str = "<|assistant|>" - - def __init__( - self, - tokenizer: Any, - chat_template_kwargs: dict[str, Any] | None = None, - assistant_start_str: str | None = None, - allowed_append_roles: list[str] | None = None, - ): - super().__init__( - tokenizer, - chat_template_kwargs, - assistant_start_str or self._default_assistant_start_str, - allowed_append_roles=allowed_append_roles, - ) - self._observation_id: int = tokenizer.convert_tokens_to_ids("<|observation|>") - self._user_id: int = tokenizer.convert_tokens_to_ids("<|user|>") - self._ambiguous_boundary_ids: set[int] = {self._observation_id, self._user_id} - self.trailing_token_ids = frozenset(self._ambiguous_boundary_ids) - - def merge_tokens( - self, - old_messages: list[dict[str, Any]], - new_messages: list[dict[str, Any]], - pretokenized_token_ids: list[int], - tools: list[dict[str, Any]] | None = None, - ) -> list[int]: - incremental = self.tokenize_additional_non_assistant(old_messages, new_messages, tools) - prefix = list(pretokenized_token_ids) - if prefix and prefix[-1] in self._ambiguous_boundary_ids: - prefix = prefix[:-1] - return prefix + incremental - - -# --------------------------------------------------------------------------- -# Nemotron 3 implementation -# --------------------------------------------------------------------------- - - -class Nemotron3TITOTokenizer(Qwen3TITOTokenizer): - """NVIDIA Nemotron 3 family: ``<|im_end|>\\n`` message boundaries. - - Inherits Qwen3's boundary handling — Nemotron 3 emits the same - ``<|im_end|>\\n`` after every message and the model stops at - ``<|im_end|>`` without the trailing newline. - - No fixed jinja is shipped — HF native template is append-only when - ``truncate_history_thinking=False``. Multi-user-turn surfaces - auto-merge that kwarg via ``extra_kwargs`` below; ``{tool}``-only does - not need it (no user-turn boundary to truncate across). - - The plain-text assistant turn does not roundtrip cleanly under - sglang's upstream ``nemotron_3`` reasoning parser (it keeps a trailing - ``\\n`` in ``reasoning_content``), so step-4 ``assistant_text`` soft - assertion is expected to fail until the parser is patched upstream — - out of scope for this family registration. - """ - - reasoning_parser = "nemotron_3" - tool_call_parser = "qwen3_coder" - - SUPPORTED_TEMPLATES = ( - FixedTemplateRow( - allowed_roles=frozenset({"tool"}), - template=None, - ), - FixedTemplateRow( - allowed_roles=frozenset({"tool", "user"}), - template=None, - extra_kwargs={"truncate_history_thinking": False}, - ), - FixedTemplateRow( - allowed_roles=frozenset({"tool", "user", "system"}), - template=None, - extra_kwargs={"truncate_history_thinking": False}, - ), - ) - - _default_assistant_start_str: str = "<|im_start|>assistant\n" - - def __init__( - self, - tokenizer: Any, - chat_template_kwargs: dict[str, Any] | None = None, - assistant_start_str: str | None = None, - allowed_append_roles: list[str] | None = None, - ): - super().__init__( - tokenizer, - chat_template_kwargs, - assistant_start_str or self._default_assistant_start_str, - allowed_append_roles=allowed_append_roles, - ) - - -# --------------------------------------------------------------------------- -# Kimi K2 implementation -# --------------------------------------------------------------------------- - - -def _kimi_segment_special_token_ids(tokenizer: Any) -> set[int]: - """Kimi specials minus ``<|im_middle|>`` (intra-turn role-name/body - separator, not a role boundary; must not be a segment boundary).""" - return TokenSeqComparator.collect_special_ids(tokenizer) - {tokenizer.convert_tokens_to_ids("<|im_middle|>")} - - -class Kimi25TITOTokenizer(TITOTokenizer): - """Moonshot Kimi K2.5: ``<|im_end|>`` boundary (no trailing newline). - - K2.5 has no kwarg escape hatch for the "drop reasoning of prior assistants - once a new non-tool-call assistant arrives" behavior. Ships a - bundled fixed jinja that wraps the ``last_non_tool_call_assistant_msg`` - loop in ``{%- if not preserve_thinking -%}`` so multi-user-turn rollout - can pass ``preserve_thinking=True`` to keep history append-only. Only the - ``{tool, user}`` surface is registered (per current onboarding scope). - """ - - SUPPORTED_TEMPLATES = ( - FixedTemplateRow( - allowed_roles=frozenset({"tool", "user"}), - template="kimi_k25_fixed.jinja", - extra_kwargs={"preserve_thinking": True}, - ), - ) - - _default_assistant_start_str: str = "<|im_assistant|>" - - def __init__( - self, - tokenizer: Any, - chat_template_kwargs: dict[str, Any] | None = None, - assistant_start_str: str | None = None, - allowed_append_roles: list[str] | None = None, - ): - super().__init__( - tokenizer, - chat_template_kwargs, - assistant_start_str or self._default_assistant_start_str, - special_token_ids=_kimi_segment_special_token_ids(tokenizer), - allowed_append_roles=allowed_append_roles, - ) - - -class Kimi26TITOTokenizer(TITOTokenizer): - """Moonshot Kimi K2.6: same boundary as K2.5 + native ``preserve_thinking`` kwarg. - - K2.6's HF-native template already carries the ``preserve_thinking`` gate - that K2.5 needs patched in. No bundled fixed - template required; ``{tool, user}`` row registers ``template=None`` and - auto-merges ``preserve_thinking=True`` for multi-user-turn rollout. - - Tool-call parser is bound to ``kimi_k2_raw_id`` rather than ``kimi_k2``: - RL trajectories need the model-emitted ``tool_call_id`` to round-trip - verbatim across turns (no ``history_tool_calls_cnt`` renumbering), and - miles is the primary consumer of this TITO family. - """ - - reasoning_parser = "kimi_k2" - tool_call_parser = "kimi_k2_raw_id" - - SUPPORTED_TEMPLATES = ( - FixedTemplateRow( - allowed_roles=frozenset({"tool", "user"}), - template=None, - extra_kwargs={"preserve_thinking": True}, - ), - ) - - _default_assistant_start_str: str = "<|im_assistant|>" - - def __init__( - self, - tokenizer: Any, - chat_template_kwargs: dict[str, Any] | None = None, - assistant_start_str: str | None = None, - allowed_append_roles: list[str] | None = None, - ): - super().__init__( - tokenizer, - chat_template_kwargs, - assistant_start_str or self._default_assistant_start_str, - special_token_ids=_kimi_segment_special_token_ids(tokenizer), - allowed_append_roles=allowed_append_roles, - ) - - -# --------------------------------------------------------------------------- -# MiniMax M2 family implementation (M2.5 and M2.7 share tokenizer/arch and -# stop-token semantics; only their default system identity strings differ). -# --------------------------------------------------------------------------- - - -class MinimaxM25TITOTokenizer(TITOTokenizer): - """MiniMax-M2.5 family: bespoke ``]~!b[`` / ``[e~[`` / ``]~b]`` tag set. - - Shares tokenizer.json (sha256) and architecture (MiniMaxM2ForCausalLM) - with M2.7 — only the chat template's default system identity string - differs (``MiniMax-M2.5`` vs ``MiniMax-M2.7``). Stop-token handling - (``[e~[`` / trailing newline) is identical to M2.7. - - Reasoning is gated by a per-message ``last_user_index`` check: - ``reasoning_content`` is only rendered for assistant turns *after* the - last ``user`` — appending a new ``user`` therefore strips prior assistant - ```` blocks and breaks append-only. Only ``{tool}`` surface is - registered on HF-native template for that reason; multi-user-turn - requires the fixed jinja with ``clear_thinking=False`` to always - preserve history reasoning. - """ - - reasoning_parser = "minimax-append-think" - tool_call_parser = "minimax-m2" - - SUPPORTED_TEMPLATES = ( - FixedTemplateRow( - allowed_roles=frozenset({"tool"}), - template=None, - ), - FixedTemplateRow( - allowed_roles=frozenset({"tool", "user"}), - template="minimax_m25_fixed.jinja", - extra_kwargs={"clear_thinking": False}, - ), - ) - - _default_assistant_start_str: str = "]~b]ai" - - def __init__( - self, - tokenizer: Any, - chat_template_kwargs: dict[str, Any] | None = None, - assistant_start_str: str | None = None, - allowed_append_roles: list[str] | None = None, - ): - super().__init__( - tokenizer, - chat_template_kwargs, - assistant_start_str or self._default_assistant_start_str, - allowed_append_roles=allowed_append_roles, - ) - nl_ids = tokenizer.encode("\n", add_special_tokens=False) - assert len(nl_ids) == 1, f"Expected single newline token, got {nl_ids}" - self._newline_id: int = nl_ids[0] - self._eos_id: int = tokenizer.convert_tokens_to_ids("[e~[") - self.trailing_token_ids = frozenset({self._newline_id}) - - def merge_tokens( - self, - old_messages: list[dict[str, Any]], - new_messages: list[dict[str, Any]], - pretokenized_token_ids: list[int], - tools: list[dict[str, Any]] | None = None, - ) -> list[int]: - incremental = self.tokenize_additional_non_assistant(old_messages, new_messages, tools) - prefix = list(pretokenized_token_ids) - if prefix and prefix[-1] == self._eos_id: - prefix.append(self._newline_id) - return prefix + incremental - - -class MinimaxM27TITOTokenizer(MinimaxM25TITOTokenizer): - """MiniMax-M2.7 family: tokenizer / arch / stop-token semantics identical - to M2.5; the chat template only differs by default system identity string. - - Inherits parsers, ``__init__``, ``merge_tokens``, and - ``_default_assistant_start_str`` from M2.5; only ``SUPPORTED_TEMPLATES`` - is rebound to ``minimax_m27_fixed.jinja`` so the fixed-template lookup - points at the M2.7-derived jinja. - """ - - SUPPORTED_TEMPLATES = ( - FixedTemplateRow( - allowed_roles=frozenset({"tool"}), - template=None, - ), - FixedTemplateRow( - allowed_roles=frozenset({"tool", "user"}), - template="minimax_m27_fixed.jinja", - extra_kwargs={"clear_thinking": False}, - ), - ) - - -# --------------------------------------------------------------------------- -# DeepSeek V3.2 implementation -# --------------------------------------------------------------------------- - - -class DeepSeekV32TITOTokenizer(TITOTokenizer): - """DeepSeek V3.2 — official encoder via sglang's ``encoding_dsv32``. - - V3.2 ships no jinja chat_template; sglang renders prompts through - ``encoding_dsv32.encode_messages``, and miles' ``apply_chat_template`` routes - any V3.2 tokenizer to the thin ``chat_template_utils.deepseek_v32`` bridge. - TITO incremental tokenization rides that same bridge so it stays - byte-aligned with what the runtime serves. - - Only the ``{tool}`` surface is registered. DeepSeek's official - ``encoding_dsv32`` gates an assistant's thinking block on - ``index > last_user_idx``: appending a *user* turn re-classifies every prior - assistant as "before last user" and strips its thinking block, which is not - append-only. Tool-only append is safe because ``find_last_user_index`` - ignores tool roles, so the last-user position never moves. - """ - - reasoning_parser = "deepseek-v3" - tool_call_parser = "deepseekv32" - - SUPPORTED_TEMPLATES = ( - FixedTemplateRow( - allowed_roles=frozenset({"tool"}), - template=None, - ), - ) - - _DEFAULT_ASSISTANT_START = "<｜Assistant｜>" - - def __init__( - self, - tokenizer: Any, - chat_template_kwargs: dict[str, Any] | None = None, - assistant_start_str: str | None = None, - allowed_append_roles: list[str] | None = None, - ): - # V3.2 has no jinja template, so assistant_start_str can't be sniffed - # from one; pin it explicitly. The comparator keys off the User / - # Assistant sentinels to find assistant-content boundaries. - super().__init__( - tokenizer, - chat_template_kwargs=chat_template_kwargs, - assistant_start_str=assistant_start_str or self._DEFAULT_ASSISTANT_START, - special_token_ids={ - tokenizer.convert_tokens_to_ids("<｜User｜>"), - tokenizer.convert_tokens_to_ids("<｜Assistant｜>"), - }, - allowed_append_roles=allowed_append_roles, - ) - - -# --------------------------------------------------------------------------- -# DeepSeek V4 implementation -# --------------------------------------------------------------------------- - - -class DeepSeekV4TITOTokenizer(TITOTokenizer): - """DeepSeek V4 — official encoder via sglang's ``encoding_dsv4``. - - Like V3.2, V4 ships no jinja chat_template; miles' ``apply_chat_template`` - routes any V4 tokenizer to the ``chat_template_utils.deepseek_v4`` bridge, and - TITO incremental tokenization rides that same bridge to stay byte-aligned - with what the runtime serves. Only the ``{tool}`` surface is registered, so - the base ``_split_appended_segments`` (contiguous tool runs) covers it - without a custom override. - """ - - reasoning_parser = "deepseek-v4" - tool_call_parser = "deepseekv4" - - SUPPORTED_TEMPLATES = ( - FixedTemplateRow( - allowed_roles=frozenset({"tool"}), - template=None, - ), - ) - - _DEFAULT_ASSISTANT_START = "<｜Assistant｜>" - - def __init__( - self, - tokenizer: Any, - chat_template_kwargs: dict[str, Any] | None = None, - assistant_start_str: str | None = None, - allowed_append_roles: list[str] | None = None, - ): - super().__init__( - tokenizer, - chat_template_kwargs=chat_template_kwargs, - assistant_start_str=assistant_start_str or self._DEFAULT_ASSISTANT_START, - special_token_ids={ - tokenizer.convert_tokens_to_ids("<｜User｜>"), - tokenizer.convert_tokens_to_ids("<｜Assistant｜>"), - }, - allowed_append_roles=allowed_append_roles, - ) - - -# --------------------------------------------------------------------------- -# Enum + Factory -# --------------------------------------------------------------------------- - - -class TITOTokenizerType(str, Enum): - DEFAULT = "default" - QWEN3 = "qwen3" - QWEN35 = "qwen35" - QWENNEXT = "qwennext" - GLM47 = "glm47" - NEMOTRON3 = "nemotron3" - KIMI25 = "kimi25" - KIMI26 = "kimi26" - MINIMAX_M25 = "minimax_m25" - MINIMAX_M27 = "minimax_m27" - DEEPSEEKV32 = "deepseekv32" - DEEPSEEKV4 = "deepseekv4" - - @classmethod - def get_tokenizer_class(cls, t: TITOTokenizerType) -> type[TITOTokenizer]: - """Resolve the concrete ``TITOTokenizer`` subclass for *t*.""" - match t: - case cls.DEFAULT: - return TITOTokenizer - case cls.QWEN3: - return Qwen3TITOTokenizer - case cls.QWEN35: - return Qwen35TITOTokenizer - case cls.QWENNEXT: - return QwenNextTITOTokenizer - case cls.GLM47: - return GLM47TITOTokenizer - case cls.NEMOTRON3: - return Nemotron3TITOTokenizer - case cls.KIMI25: - return Kimi25TITOTokenizer - case cls.KIMI26: - return Kimi26TITOTokenizer - case cls.MINIMAX_M25: - return MinimaxM25TITOTokenizer - case cls.MINIMAX_M27: - return MinimaxM27TITOTokenizer - case cls.DEEPSEEKV32: - return DeepSeekV32TITOTokenizer - case cls.DEEPSEEKV4: - return DeepSeekV4TITOTokenizer - case _: - raise ValueError(f"Unknown TITOTokenizerType: {t!r}") - - -def get_tito_tokenizer( - tokenizer: Any, - tokenizer_type: TITOTokenizerType | str = TITOTokenizerType.DEFAULT, - chat_template_kwargs: dict[str, Any] | None = None, - assistant_start_str: str | None = None, - allowed_append_roles: list[str] | None = None, -) -> TITOTokenizer: - """Create a ``TITOTokenizer`` instance. - - Args: - tokenizer: HuggingFace tokenizer object. - tokenizer_type: Explicit type (string or enum). Corresponds to the - ``--tito-model`` CLI argument. - chat_template_kwargs: Extra kwargs forwarded to ``apply_chat_template``. - assistant_start_str: Decoded text prefix identifying assistant content - segments (e.g. ``"<|im_start|>assistant"``). Auto-detected from - the chat template by default; pass explicitly to override. - allowed_append_roles: Roles allowed in appended messages. Defaults to - ``["tool"]``. Passed to - ``assert_messages_append_only_with_allowed_role``. - """ - if tokenizer is None: - raise ValueError("tokenizer must not be None") - if isinstance(tokenizer_type, str): - tokenizer_type = TITOTokenizerType(tokenizer_type) - cls = TITOTokenizerType.get_tokenizer_class(tokenizer_type) - kwargs: dict[str, Any] = {"chat_template_kwargs": chat_template_kwargs} - if assistant_start_str is not None: - kwargs["assistant_start_str"] = assistant_start_str - if allowed_append_roles is not None: - kwargs["allowed_append_roles"] = allowed_append_roles - return cls(tokenizer, **kwargs) - - -# --------------------------------------------------------------------------- -# Fixed-template resolution (smallest-superset over SUPPORTED_TEMPLATES) -# --------------------------------------------------------------------------- - - -def resolve_fixed_chat_template( - tito_model: TITOTokenizerType | str, - allowed_append_roles: Iterable[str], -) -> tuple[str | None, dict[str, Any]]: - """Smallest-superset lookup over the requested family's ``SUPPORTED_TEMPLATES``. - - Returns ``(template_path, extra_kwargs)``: - - - ``template_path``: absolute path to a bundled ``.jinja`` file, or ``None`` - when the matched row registers HF-native (kwargs-only fix) or when no - row matches at all. - - ``extra_kwargs``: kwargs the caller should merge into - ``apply_chat_template`` (caller's explicit user kwargs win on conflict). - Empty when no row matches or the matched row needs none. - - Raises ``ValueError`` on equally-minimal supersets — register a stricter - row to disambiguate. - """ - if isinstance(tito_model, str): - tito_model = TITOTokenizerType(tito_model) - - requested = frozenset(allowed_append_roles) - invalid = requested - _VALID_ROLES - if invalid: - raise ValueError( - f"Unknown roles in allowed_append_roles: {sorted(invalid)}. " f"Supported: {sorted(_VALID_ROLES)}." - ) - - cls = TITOTokenizerType.get_tokenizer_class(tito_model) - candidates = [row for row in cls.SUPPORTED_TEMPLATES if requested.issubset(row.allowed_roles)] - if not candidates: - raise ValueError( - f"No SUPPORTED_TEMPLATES row registered for tito_model={tito_model.value} " - f"with allowed_append_roles={sorted(requested)}. Register a row in " - f"{cls.__name__}.SUPPORTED_TEMPLATES (template=None for HF-native models)." - ) - - # Pick the most specific superset. Ties surface registration mistakes - # immediately rather than depending on iteration order. - min_size = min(len(row.allowed_roles) for row in candidates) - minimal = [row for row in candidates if len(row.allowed_roles) == min_size] - if len(minimal) > 1: - raise ValueError( - f"Ambiguous fixed-template registration for tito_model={tito_model.value}, " - f"requested_roles={sorted(requested)}: multiple equally-minimal supersets " - f"{[sorted(row.allowed_roles) for row in minimal]}. Register a stricter row to disambiguate." - ) - row = minimal[0] - - path = str(TEMPLATE_DIR / row.template) if row.template else None - logger.info( - "tito_model=%s requested_roles=%s -> matched registered_roles=%s -> template=%s kwargs=%s", - tito_model.value, - sorted(requested), - sorted(row.allowed_roles), - path, - row.extra_kwargs, - ) - return path, dict(row.extra_kwargs) - - -# --------------------------------------------------------------------------- -# sglang parser resolution (per-family binding + assert-equal on user input) -# --------------------------------------------------------------------------- - - -def resolve_reasoning_and_tool_call_parser( - tito_model: TITOTokenizerType | str, - user_reasoning_parser: str | None = None, - user_tool_call_parser: str | None = None, -) -> tuple[str | None, str | None]: - """Resolve sglang ``--reasoning-parser`` and ``--tool-call-parser`` for the - given TITO family. - - Both parsers are bound on the TITO subclass as class attributes because - the model's reasoning / tool-call emission shapes are per-family facts. - For each parser independently: - - * If the user didn't pass a value, return the family's bound value - (which may itself be ``None`` for ``DEFAULT`` or unbound subclasses - — the caller is then responsible for supplying one downstream). - * If the user passed a value and the family is bound, assert equality; - a mismatch is a configuration bug and raises ``ValueError`` rather - than silently overriding. - * If the user passed a value and the family is unbound, accept it. - - Returns ``(reasoning_parser, tool_call_parser)``. - """ - if isinstance(tito_model, str): - tito_model = TITOTokenizerType(tito_model) - cls = TITOTokenizerType.get_tokenizer_class(tito_model) - - def _resolve_one(field: str, bound: str | None, user: str | None) -> str | None: - if user is None: - return bound - if bound is None: - return user - if user != bound: - raise ValueError( - f"--{field.replace('_', '-')}={user!r} disagrees with the parser " - f"registered for tito_model={tito_model.value!r}: {bound!r}. The " - f"parser is bound on the TITO subclass; either pass {bound!r} or " - f"omit the flag to auto-resolve." - ) - return user - - return ( - _resolve_one("reasoning_parser", cls.reasoning_parser, user_reasoning_parser), - _resolve_one("tool_call_parser", cls.tool_call_parser, user_tool_call_parser), - ) diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/token_seq_comparator.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/token_seq_comparator.py deleted file mode 100644 index 2f4b93d..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/chat_template_utils/token_seq_comparator.py +++ /dev/null @@ -1,289 +0,0 @@ -"""TokenSeqComparator: segment token IDs by special-token boundaries and compare sequences.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from enum import Enum - - -@dataclass -class Segment: - """A contiguous run of token IDs — either a special token or a content segment.""" - - token_ids: list[int] = field(default_factory=list) - is_special: bool = False - - -class MismatchType(Enum): - # Segment count or structure (special/content pattern) differs between - # expected and actual. When this happens, segments can't be aligned so - # no per-segment comparison is possible. - SPECIAL_TOKEN_COUNT = "special_token_count" - - # A special-token segment has the same position in both sequences but - # contains a different token ID. - SPECIAL_TOKEN_TYPE = "special_token_type" - - # Non-assistant content (user, system, tool, etc.) differs. This indicates - # a bug in the TITO algorithm — these regions should match exactly. - NON_ASSISTANT_TEXT = "non_assistant_text" - - # Assistant content differs. Expected and non-severe: assistant tokens - # are inherited directly from the pretokenized prefix across turns, - # so they may not match the chat template's canonical tokenization. - ASSISTANT_TEXT = "assistant_text" - - -@dataclass -class Mismatch: - """A single difference found between two token sequences.""" - - type: MismatchType - segment_index: int - expected_text: str - actual_text: str - detail: str = "" - - def to_dict(self) -> dict: - return { - "type": self.type.value, - "segment_index": self.segment_index, - "expected_text": self.expected_text, - "actual_text": self.actual_text, - "detail": self.detail, - } - - -class TokenSeqComparator: - """Segment token sequences by special tokens and compare them. - - Parameters - ---------- - tokenizer : PreTrainedTokenizerBase - special_token_ids : set[int] | None - Token IDs that mark segment boundaries. Default None auto-detects - them (see :meth:`collect_special_ids`). Pass an explicit set to - override — needed when the tokenizer flags a token ``special=True`` - even though it lives inside a role's turn (e.g. Kimi's - ``<|im_middle|>`` between role name and body) and so must not - split the segment. - assistant_start_str : str - Decoded text prefix identifying assistant content segments, e.g. - ``"<|im_start|>assistant"`` (Qwen3) or ``"<|assistant|>"`` (GLM). - Used to classify content mismatches as assistant vs non-assistant. - trim_trailing_ids : set[int] | None - Token IDs to strip from both sequence tails before comparison - (see :func:`_trim_trailing`). Stored as a default; callers of - :meth:`compare_sequences` may supply additional IDs that are - **unioned** with this set. - """ - - def __init__( - self, - tokenizer, - assistant_start_str: str, - special_token_ids: set[int] | None = None, - trim_trailing_ids: set[int] | None = None, - ): - self.tokenizer = tokenizer - if special_token_ids is not None: - self._special_ids = set(special_token_ids) - else: - self._special_ids = self.collect_special_ids(tokenizer) - self._assistant_start_str = assistant_start_str - self._trim_trailing_ids: set[int] | None = set(trim_trailing_ids) if trim_trailing_ids else None - - @staticmethod - def collect_special_ids(tokenizer) -> set[int]: - """Collect token IDs with ``special=True`` from the tokenizer. - - Special tokens are structural markers added by the chat template to - delimit messages, roles, and control flow — for example - ``<|im_start|>``, ``<|im_end|>``, ``<|endoftext|>``, ````, - ````, and ``<|assistant|>``. - - Tokens that encode *content* produced by a role are **not** special, - even if they look "special" to a human. For instance, ```` and - ```` in reasoning models are regular content tokens generated - by the assistant — the tokenizer does not flag them as - ``special=True``, so they are not collected here. - """ - ids = set(tokenizer.all_special_ids) - decoder = getattr(tokenizer, "added_tokens_decoder", None) - if decoder: - ids |= {k for k, v in decoder.items() if v.special} - return ids - - def segment_by_special_tokens(self, token_ids: list[int]) -> list[Segment]: - """Split *token_ids* into segments at special-token boundaries. - - Each special token becomes its own single-ID segment with - ``is_special=True``. Consecutive non-special tokens are grouped - into content segments. Example for Qwen3:: - - [<|im_start|>, "assistant", "\\n", "Hi", <|im_end|>, "\\n"] - → [special(<|im_start|>), content("assistant\\nHi"), special(<|im_end|>), content("\\n")] - """ - if not token_ids: - return [] - - segments: list[Segment] = [] - current: list[int] = [] - for tid in token_ids: - if tid in self._special_ids: - if current: - segments.append(Segment(token_ids=current)) - current = [] - segments.append(Segment(token_ids=[tid], is_special=True)) - else: - current.append(tid) - if current: - segments.append(Segment(token_ids=current)) - return segments - - def compare_sequences( - self, - expected_ids: list[int], - actual_ids: list[int], - trim_trailing_ids: set[int] | None = None, - ) -> list[Mismatch]: - """Compare two token-ID sequences and return mismatches. - - Parameters - ---------- - trim_trailing_ids : set[int] | None - Additional token IDs to strip from both sequence tails before - comparison. **Unioned** with the IDs passed at construction time. - """ - trim = self._trim_trailing_ids or set() - if trim_trailing_ids: - trim = trim | trim_trailing_ids - if trim: - expected_ids = _trim_trailing(expected_ids, trim) - actual_ids = _trim_trailing(actual_ids, trim) - - exp_segs = self.segment_by_special_tokens(expected_ids) - act_segs = self.segment_by_special_tokens(actual_ids) - - structural = self._check_segment_structure(exp_segs, act_segs) - if structural: - return [structural] - - mismatches: list[Mismatch] = [] - for idx, (exp, act) in enumerate(zip(exp_segs, act_segs, strict=True)): - is_assistant_content = self._is_assistant_content(exp_segs, idx) and self._is_assistant_content( - act_segs, idx - ) - m = self._compare_single_segment(idx, exp, act, is_assistant_content=is_assistant_content) - if m is not None: - mismatches.append(m) - return mismatches - - def _check_segment_structure( - self, - exp_segs: list[Segment], - act_segs: list[Segment], - ) -> Mismatch | None: - """Pre-check that expected and actual segment lists have the same count - and the same special/content pattern before per-segment comparison.""" - if len(exp_segs) != len(act_segs): - detail = f"segment count differs: expected {len(exp_segs)}, got {len(act_segs)}" - elif [s.is_special for s in exp_segs] != [s.is_special for s in act_segs]: - detail = "segment structure (special/content pattern) differs" - else: - return None - return Mismatch( - type=MismatchType.SPECIAL_TOKEN_COUNT, - segment_index=-1, - expected_text=self._describe_structure(exp_segs), - actual_text=self._describe_structure(act_segs), - detail=detail, - ) - - def _compare_single_segment( - self, - idx: int, - exp: Segment, - act: Segment, - *, - is_assistant_content: bool, - ) -> Mismatch | None: - """Compare a single aligned segment pair and return a mismatch if they differ. - - Special segments are compared by token ID. Content segments are decoded - and compared as stripped text — leading/trailing whitespace (``\\n``, - spaces) is ignored because chat templates may insert boundary newlines - that differ from the TITO prefix. This whitespace-only difference does - not cause meaningful misalignment with the chat template, so we strip - to avoid noisy false positives. - """ - if exp.is_special: - if exp.token_ids != act.token_ids: - return Mismatch( - type=MismatchType.SPECIAL_TOKEN_TYPE, - segment_index=idx, - expected_text=self._decode(exp.token_ids), - actual_text=self._decode(act.token_ids), - ) - return None - - # After ignoring assistant text diff, there is no need to keep the strip operator, - # as other text should be exact match. - exp_text = self._decode(exp.token_ids) - act_text = self._decode(act.token_ids) - if exp_text == act_text: - return None - - return Mismatch( - type=MismatchType.ASSISTANT_TEXT if is_assistant_content else MismatchType.NON_ASSISTANT_TEXT, - segment_index=idx, - expected_text=exp_text, - actual_text=act_text, - ) - - def _is_assistant_content(self, segments: list[Segment], idx: int) -> bool: - """Check if the content segment at *idx* belongs to an assistant message. - - Decodes the preceding special-token segment and the first few tokens of - the current segment *separately*, then concatenates the decoded strings. - If the result starts with ``assistant_start_str`` (e.g. - ``"<|im_start|>assistant"``), this segment is classified as assistant - content — mismatches there are expected and non-severe. - - """ - if self._assistant_start_str is None: - return False - if segments[idx].is_special: - return False - if idx == 0: - return False - prev = segments[idx - 1] - if not prev.is_special: - return False - special_text = self._decode(prev.token_ids) - # Decode enough prefix tokens to capture the role label (e.g. "assistant\n"). - content_prefix = self._decode(segments[idx].token_ids[:20]) - return (special_text + content_prefix).startswith(self._assistant_start_str) - - def _decode(self, token_ids: list[int]) -> str: - return self.tokenizer.decode(token_ids, skip_special_tokens=False) - - def _describe_structure(self, segments: list[Segment]) -> str: - return " ".join( - f"[{self._decode(s.token_ids)}]" if s.is_special else f"({len(s.token_ids)} tokens)" for s in segments - ) - - -def _trim_trailing(ids: list[int], to_remove: set[int]) -> list[int]: - """Strip trailing token IDs that belong to *to_remove*. - - The model's generated output typically ends with a stop token (e.g. - ``<|observation|>`` for GLM, ``<|im_end|>`` for Qwen) that won't appear - at the same position in the template-rendered expected sequence. Stripping - these trailing tokens from both sides before comparison avoids false - structural mismatches. - """ - end = len(ids) - while end > 0 and ids[end - 1] in to_remove: - end -= 1 - return ids[:end] diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/external_utils/__init__.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/external_utils/__init__.py deleted file mode 100644 index 101568c..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/external_utils/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Optional Miles external utility compatibility namespace.""" diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/external_utils/command_utils.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/external_utils/command_utils.py deleted file mode 100644 index db771f9..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/external_utils/command_utils.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Optional Miles command helpers used by session verifier e2e jobs.""" - -from __future__ import annotations - -from dataclasses import dataclass - - -class MissingMilesTrainingStackError(RuntimeError): - """Raised when full Miles training helpers are required but unavailable.""" - - -@dataclass -class ExecuteTrainConfig: - cuda_core_dump: bool = False - num_nodes: int = 1 - extra_env_vars: str = "" - output_dir: str = "/root/shared_data" - - -def exec_command(*args, **kwargs): - raise MissingMilesTrainingStackError( - "Miles command execution helpers are not bundled with tito-gateway. " - "Install/provide the optional Miles training stack before running full " - "session verifier e2e jobs." - ) - - -def execute_train(*args, **kwargs): - raise MissingMilesTrainingStackError( - "Miles execute_train helper is not bundled with tito-gateway. " - "Install/provide the optional Miles training stack before running full " - "session verifier e2e jobs." - ) diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/hf_config.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/hf_config.py deleted file mode 100644 index f949b02..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/hf_config.py +++ /dev/null @@ -1,108 +0,0 @@ -"""HuggingFace config loader with model-type alias registration and overrides. - -`load_hf_config` is the single entry point miles uses to load an HF config from a -local checkpoint. It supports 2 customizations: - -- Registers model_type aliases before calling AutoConfig, in case the model is - not recognized in huggingface. -- Accepts an `overrides` dict applied via setattr after loading, so callers can - adjust fields without touching the checkpoint. - -The default behavior is exactly the same as `AutoConfig.from_pretrained`. -""" - -import importlib -from dataclasses import dataclass - -from transformers import AutoConfig, AutoModelForCausalLM -from transformers.models.auto.configuration_auto import CONFIG_MAPPING_NAMES - - -@dataclass(frozen=True) -class _HFConfigAlias: - model_type: str - base_module: str - base_class: str - compat_class_name: str - auto_model_classes: tuple = (AutoModelForCausalLM,) - # Set True to override transformers' native config. - override_hf_native: bool = False - - -_CONFIG_ALIASES: tuple[_HFConfigAlias, ...] = ( - _HFConfigAlias( - model_type="deepseek_v32", - base_module="transformers.models.deepseek_v3.configuration_deepseek_v3", - base_class="DeepseekV3Config", - compat_class_name="DeepseekV32Config", - ), - _HFConfigAlias( - model_type="deepseek_v4", - base_module="transformers.models.deepseek_v3.configuration_deepseek_v3", - base_class="DeepseekV3Config", - compat_class_name="DeepseekV4Config", - auto_model_classes=(), - override_hf_native=True, - ), -) - -_REGISTERED_ALIASES: set[str] = set() - - -def register_hf_config_aliases() -> None: - """Register miles model_type aliases with transformers. Idempotent. - - Already called inside `load_hf_config` and `load_tokenizer`. Only call - directly before a third-party entry point that won't go through either - (e.g. megatron's `_build_tokenizer`). - """ - for alias in _CONFIG_ALIASES: - if alias.model_type in _REGISTERED_ALIASES: - continue - if alias.model_type in CONFIG_MAPPING_NAMES and not alias.override_hf_native: - raise RuntimeError( - f"transformers now natively supports model_type={alias.model_type!r}; " - f"set override_hf_native=True to override." - ) - module = importlib.import_module(alias.base_module) - base_config = getattr(module, alias.base_class) - compat_config = type( - alias.compat_class_name, - (base_config,), - {"model_type": alias.model_type, "__module__": __name__}, - ) - AutoConfig.register(alias.model_type, compat_config, exist_ok=alias.override_hf_native) - for auto_cls in alias.auto_model_classes: - base_model_cls = auto_cls._model_mapping[base_config] - compat_model_cls = type( - base_model_cls.__name__, (base_model_cls,), {"config_class": compat_config, "__module__": __name__} - ) - auto_cls.register(compat_config, compat_model_cls, exist_ok=alias.override_hf_native) - _REGISTERED_ALIASES.add(alias.model_type) - - -def load_hf_config( - checkpoint_path: str, - *, - overrides: dict | None = None, - trust_remote_code: bool = True, - **autoconfig_kwargs, -): - """Load an HF config from a local checkpoint. - - Registers model aliases first for pre-set aliases. - - overrides: optional dict of attributes to setattr on the returned config - after loading. Lets callers patch fields without mutating the checkpoint. - """ - register_hf_config_aliases() - config = AutoConfig.from_pretrained(checkpoint_path, trust_remote_code=trust_remote_code, **autoconfig_kwargs) - - if overrides: - for key, value in overrides.items(): - setattr(config, key, value) - return config - - -def is_dsa(hf_config) -> bool: - return getattr(hf_config, "model_type", None) in ("deepseek_v32", "glm_moe_dsa") diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/http_utils.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/http_utils.py deleted file mode 100644 index 0aaf792..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/http_utils.py +++ /dev/null @@ -1,315 +0,0 @@ -import asyncio -import ipaddress -import json -import logging -import multiprocessing -import os -import random -import socket -import time - -import httpx - -logger = logging.getLogger(__name__) - -MILES_HOST_IP_ENV = "MILES_HOST_IP" - - -def find_available_port(base_port: int): - port = base_port + random.randint(100, 1000) - while True: - if is_port_available(port): - return port - if port < 60000: - port += 42 - else: - port -= 43 - - -def is_port_available(port): - """Return whether a port is available.""" - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - try: - s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - s.bind(("", port)) - s.listen(1) - return True - except OSError: - return False - except OverflowError: - return False - - -def wait_for_server_ready( - host: str, - port: int, - process: "multiprocessing.Process | None" = None, - timeout: float = 30, -) -> None: - """Poll until a TCP port is accepting connections. - - Raises ``RuntimeError`` if the process dies or the timeout is exceeded. - """ - deadline = time.time() + timeout - while time.time() < deadline: - if process is not None and not process.is_alive(): - raise RuntimeError(f"Server process died before port {port} became ready") - try: - with socket.create_connection((host, port), timeout=1): - return - except OSError: - time.sleep(0.5) - raise RuntimeError(f"Server at {host}:{port} not ready after {timeout}s") - - -def get_host_info(): - hostname = socket.gethostname() - - if env_overwrite_local_ip := os.getenv(MILES_HOST_IP_ENV, None): - return hostname, env_overwrite_local_ip - - def _is_loopback(ip): - return ip.startswith("127.") or ip == "::1" - - def _resolve_ip(family, test_target_ip): - """ - Attempt to get the local LAN IP for the specific family (IPv4/IPv6). - Strategy: UDP Probe (Preferred) -> Hostname Resolution (Fallback) -> None - """ - - # Strategy 1: UDP Connect Probe (Most accurate, relies on routing table) - # Useful when the machine has a default gateway or internet access. - try: - with socket.socket(family, socket.SOCK_DGRAM) as s: - # The IP doesn't need to be reachable, but the routing table must exist. - s.connect((test_target_ip, 80)) - ip = s.getsockname()[0] - if not _is_loopback(ip): - return ip - except Exception: - pass # Route unreachable or network error, move to next strategy. - - # Strategy 2: Hostname Resolution (Fallback for offline clusters) - # Useful for offline environments where UDP connect fails but /etc/hosts is configured. - try: - # getaddrinfo allows specifying the family (AF_INET or AF_INET6) - # Result format: [(family, type, proto, canonname, sockaddr), ...] - infos = socket.getaddrinfo(hostname, None, family=family, type=socket.SOCK_STREAM) - - for info in infos: - ip = info[4][0] # The first element of sockaddr is the IP - # Must filter out loopback addresses to avoid "127.0.0.1" issues - if not _is_loopback(ip): - return ip - except Exception: - pass - - return None - - prefer_ipv6 = os.getenv("MILES_PREFER_IPV6", "0").lower() in ("1", "true", "yes", "on") - local_ip = None - final_fallback = "127.0.0.1" - - if prefer_ipv6: - # [Strict Mode] IPv6 Only - # 1. Try UDP V6 Probe - # 2. Try Hostname Resolution (V6) - # If failed, fallback to V6 loopback. Never mix with V4. - local_ip = _resolve_ip(socket.AF_INET6, "2001:4860:4860::8888") - final_fallback = "::1" - else: - # [Strict Mode] IPv4 Only (Default) - # 1. Try UDP V4 Probe - # 2. Try Hostname Resolution (V4) - # If failed, fallback to V4 loopback. Never mix with V6. - local_ip = _resolve_ip(socket.AF_INET, "8.8.8.8") - final_fallback = "127.0.0.1" - - return hostname, local_ip or final_fallback - - -def _wrap_ipv6(host): - """Wrap IPv6 address in [] if needed.""" - try: - ipaddress.IPv6Address(host.strip("[]")) - return f"[{host.strip('[]')}]" - except ipaddress.AddressValueError: - return host - - -def run_router(args): - try: - from sglang_router.launch_router import launch_router - - router = launch_router(args) - if router is None: - return 1 - return 0 - except Exception as e: - logger.info(e) - return 1 - - -def terminate_process(process: multiprocessing.Process, timeout: float = 1.0) -> None: - """Terminate a process gracefully, with forced kill as fallback. - - Args: - process: The process to terminate - timeout: Seconds to wait for graceful termination before forcing kill - """ - if not process.is_alive(): - return - - process.terminate() - process.join(timeout=timeout) - if process.is_alive(): - process.kill() - process.join() - - -_http_client: httpx.AsyncClient | None = None -_client_concurrency: int = 0 - -# Optional Ray-based distributed POST dispatch -_distributed_post_enabled: bool = False -_post_actors: list[object] = [] -_post_actor_idx: int = 0 - - -def _next_actor(): - global _post_actor_idx - if not _post_actors: - return None - actor = _post_actors[_post_actor_idx % len(_post_actors)] - _post_actor_idx = (_post_actor_idx + 1) % len(_post_actors) - return actor - - -async def _post(client, url, payload, max_retries=60, action="post", headers=None): - retry_count = 0 - while retry_count < max_retries: - try: - if action in ("delete", "get"): - assert not payload - response = await getattr(client, action)(url, headers=headers) - else: - response = await getattr(client, action)(url, json=payload or {}, headers=headers) - response.raise_for_status() - try: - output = response.json() - except json.JSONDecodeError: - output = response.text - except Exception as e: - retry_count += 1 - - if isinstance(e, httpx.HTTPStatusError): - response_text = e.response.text - else: - response_text = None - - logger.info( - f"Error: {e}, retrying... (attempt {retry_count}/{max_retries}, url={url}, response={response_text})" - ) - if retry_count >= max_retries: - logger.info(f"Max retries ({max_retries}) reached, failing... (url={url})") - raise e - await asyncio.sleep(1) - continue - break - - return output - - -def init_http_client(args): - """Initialize HTTP client and optionally enable distributed POST via Ray.""" - global _http_client, _client_concurrency, _distributed_post_enabled - if not args.rollout_num_gpus: - return - - _client_concurrency = args.sglang_server_concurrency * args.rollout_num_gpus // args.rollout_num_gpus_per_engine - if _http_client is None: - _http_client = httpx.AsyncClient( - limits=httpx.Limits(max_connections=_client_concurrency), - timeout=httpx.Timeout(None), - ) - - # Optionally initialize distributed POST via Ray without changing interfaces - if args.use_distributed_post: - _init_ray_distributed_post(args) - _distributed_post_enabled = True - - -def _init_ray_distributed_post(args): - """Initialize one or more Ray async actors per node for HTTP POST. - - Uses NodeAffinitySchedulingStrategy to place actors on distinct nodes. - Controlled by MILES_HTTP_POST_ACTORS_PER_NODE. - """ - global _post_actors - if _post_actors: - return # Already initialized - - import ray - from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy - - # Discover alive nodes - nodes = [n for n in ray.nodes() if n.get("Alive")] - if not nodes: - raise RuntimeError("No alive Ray nodes to place HTTP POST actors.") - - # Define the async actor - @ray.remote - class _HttpPosterActor: - def __init__(self, concurrency: int): - # Lazy creation to this actor's event loop - self._client = httpx.AsyncClient( - limits=httpx.Limits(max_connections=max(1, concurrency)), - timeout=httpx.Timeout(None), - ) - - async def do_post(self, url, payload, max_retries=60, action="post", headers=None): - return await _post(self._client, url, payload, max_retries, action=action, headers=headers) - - # Create actors per node - created = [] - # Distribute client concurrency across actors (at least 1 per actor) - per_actor_conc = (_client_concurrency + len(nodes)) // len(nodes) - - for node in nodes: - node_id = node["NodeID"] - scheduling = NodeAffinitySchedulingStrategy(node_id=node_id, soft=False) - for _ in range(args.num_gpus_per_node): - actor = _HttpPosterActor.options( - name=None, - lifetime="detached", - scheduling_strategy=scheduling, - max_concurrency=per_actor_conc, - # Use tiny CPU to schedule - num_cpus=0.001, - ).remote(per_actor_conc) - created.append(actor) - - _post_actors = created - - -# TODO may generalize the name since it now contains http DELETE/GET etc (with retries and remote-execution) -async def post(url, payload, max_retries=60, action="post", headers=None): - # If distributed mode is enabled and actors exist, dispatch via Ray. - if _distributed_post_enabled and _post_actors: - try: - actor = _next_actor() - if actor is not None: - return await actor.do_post.remote(url, payload, max_retries, action=action, headers=headers) - except Exception as e: - logger.info(f"[http_utils] Distributed POST failed, falling back to local: {e} (url={url})") - # fall through to local - - return await _post(_http_client, url, payload, max_retries, action=action, headers=headers) - - -# TODO unify w/ `post` to add retries and remote-execution -async def get(url): - response = await _http_client.get(url) - response.raise_for_status() - output = response.json() - return output diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/processing_utils.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/processing_utils.py deleted file mode 100644 index ac6e122..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/processing_utils.py +++ /dev/null @@ -1,175 +0,0 @@ -import base64 -import inspect -import io -import logging -import os -from pathlib import Path - -from huggingface_hub import hf_hub_download -from tokenizers import Tokenizer as RawTokenizer -from transformers import AutoProcessor, AutoTokenizer, PreTrainedTokenizerBase, ProcessorMixin - -from tito_gateway.vendor.miles_compat.utils.hf_config import register_hf_config_aliases - -logger = logging.getLogger(__name__) - - -def _fix_v5_tokenizer_components(tokenizer: PreTrainedTokenizerBase, model_name_or_path: str) -> None: - # transformers v5's LlamaTokenizerFast rebuilds pre_tokenizer/decoder in - # __init__, discarding the originals from tokenizer.json. DeepSeek-V3.2 - # declares LlamaTokenizerFast but actually uses ByteLevel, so without this - # fix the loaded tokenizer decodes Metaspace ▁ instead of ByteLevel Ġ/Ċ - # and diverges from the sglang-served tokenizer. Mirrors sglang's - # _fix_v5_tokenizer_components (hf_transformers_utils.py). - backend = getattr(tokenizer, "_tokenizer", None) - if backend is None: - return - - try: - local_path = Path(model_name_or_path) / "tokenizer.json" - if local_path.is_file(): - tok_file = str(local_path) - else: - tok_file = hf_hub_download(model_name_or_path, "tokenizer.json", local_files_only=True) - raw = RawTokenizer.from_file(tok_file) - except Exception as e: - logger.warning("Could not load tokenizer.json for %s: %s", model_name_or_path, e) - return - - raw_pre = type(raw.pre_tokenizer).__name__ if raw.pre_tokenizer else None - loaded_pre = type(backend.pre_tokenizer).__name__ if backend.pre_tokenizer else None - - if raw_pre and loaded_pre and raw_pre != loaded_pre: - logger.info( - "Fixing v5 tokenizer component mismatch for %s: pre_tokenizer %s -> %s, decoder %s -> %s", - model_name_or_path, - loaded_pre, - raw_pre, - type(backend.decoder).__name__ if backend.decoder else None, - type(raw.decoder).__name__ if raw.decoder else None, - ) - backend.pre_tokenizer = raw.pre_tokenizer - backend.decoder = raw.decoder - - -# Default image patch size for vision-language models -# Note: Qwen3-VL uses 16, Qwen2.5-VL uses 14 -# Reference: https://github.com/QwenLM/Qwen3-VL/blob/main/qwen-vl-utils/README.md -DEFAULT_PATCH_SIZE = 14 - - -_TOKENIZER_CACHE: dict[tuple, PreTrainedTokenizerBase] = {} - - -def _make_cache_key(name_or_path: str, chat_template_path: str | None, kwargs: dict) -> tuple | None: - try: - kwargs_items = tuple(sorted(kwargs.items())) - hash(kwargs_items) - except TypeError: - return None - return (name_or_path, chat_template_path, kwargs_items) - - -def load_tokenizer(name_or_path: str, chat_template_path: str | None = None, **kwargs) -> PreTrainedTokenizerBase: - # Cache keyed by (name, chat_template_path, kwargs) — the fast suite creates - # hundreds of SessionServer / MockSGLangServer fixtures and each previously - # triggered a fresh AutoTokenizer.from_pretrained, tripping HF Hub rate limits. - cache_key = _make_cache_key(name_or_path, chat_template_path, kwargs) - if cache_key is not None and cache_key in _TOKENIZER_CACHE: - return _TOKENIZER_CACHE[cache_key] - - register_hf_config_aliases() - tokenizer = AutoTokenizer.from_pretrained(name_or_path, **kwargs) - _fix_v5_tokenizer_components(tokenizer, name_or_path) - if chat_template_path: - assert os.path.isfile(chat_template_path), ( - f"chat_template_path not found: {chat_template_path}. " - f"Ensure the path is accessible on this node (e.g. inside the miles repo or on a shared filesystem)." - ) - with open(chat_template_path) as f: - tokenizer.chat_template = f.read() - logger.info("Loaded custom chat template from %s", chat_template_path) - - if cache_key is not None: - _TOKENIZER_CACHE[cache_key] = tokenizer - return tokenizer - - -def build_processor_kwargs(multimodal_inputs: dict | None = None) -> dict: - - modality_forced = {"return_tensors": "pt"} - - result = dict(multimodal_inputs) if multimodal_inputs else {} - - # return_tensors=None for text (input_ids), "pt" for modality-specific outputs. - # Use per-modality dicts to avoid transformers >=5.0 duplicate kwarg error. - result["text_kwargs"] = {**result.get("text_kwargs", {}), "return_tensors": None} - for key in ("audio_kwargs", "images_kwargs", "videos_kwargs"): - if key in result: - result[key] = {**result[key], **modality_forced} - else: - result[key] = modality_forced.copy() - - return result - - -def processor_requires_medias(processor) -> bool: - try: - params = inspect.signature(processor).parameters - return "medias" in params and "text" in params - except (TypeError, ValueError): - return hasattr(processor, "media_processor") - - -def call_processor(processor, text, multimodal_inputs: dict | None = None): - multimodal_inputs = multimodal_inputs or {} - - # for kimi-vl & kimi-2.5 - if processor_requires_medias(processor): - medias = [] - if images := multimodal_inputs.get("images"): - medias.extend({"type": "image", "image": image} for image in images) - if videos := multimodal_inputs.get("videos"): - medias.extend({"type": "video", "video": video} for video in videos) - return processor(text=text, medias=medias) - - kwargs = build_processor_kwargs(multimodal_inputs) - return processor(text=text, **kwargs) - - -def load_processor(name_or_path: str, **kwargs): - try: - proc = AutoProcessor.from_pretrained(name_or_path, **kwargs) - except (OSError, ValueError) as e: - logger.warning(f"Failed to load processor from {name_or_path}: {e}") - proc = None - - # If HF returned a tokenizer, discard it. - if isinstance(proc, PreTrainedTokenizerBase) or not isinstance(proc, ProcessorMixin): - proc = None - - return proc - - -def process_vision_info(prompt, processor): - # TODO: temporary solution, will write image utils for miles later - from qwen_vl_utils import process_vision_info as qwen_process_vision_info - - if hasattr(processor.image_processor, "patch_size"): - image_patch_size = processor.image_processor.patch_size - else: - logger.info(f"Using default patch size: {DEFAULT_PATCH_SIZE}") - image_patch_size = DEFAULT_PATCH_SIZE - images, videos = qwen_process_vision_info(prompt, image_patch_size=image_patch_size) - multimodal_inputs = {"images": images, "videos": videos} - return multimodal_inputs - - -def encode_image_for_rollout_engine(image) -> str: - """Load an image from path, ensure RGB, encode as PNG base64 string.""" - buffer = io.BytesIO() - if image.mode != "RGB": - image = image.convert("RGB") - image.save(buffer, format="PNG") - image_base64 = base64.b64encode(buffer.getvalue()).decode("utf-8") - return f"data:image/png;base64,{image_base64}" diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/__init__.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/chat_template_verify.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/chat_template_verify.py deleted file mode 100644 index 79e2533..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/chat_template_verify.py +++ /dev/null @@ -1,602 +0,0 @@ -"""Verify that a chat template satisfies the append-only invariant. - -The append-only invariant means: rendering the first N messages (without -generation prompt) produces a string that is an exact prefix of rendering -all messages (with generation prompt). This is required by sglang's -pretokenized prefix mechanism for agentic workflows. - -Core functions are used by both the CLI script -(``scripts/tools/verify_chat_template.py``) and the test suite -(``tests/fast/utils/chat_template_utils/test_pretokenized_chat.py``). -""" - -from __future__ import annotations - -from collections.abc import Iterable -from copy import deepcopy -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any - -from tito_gateway.vendor.miles_compat.utils.chat_template_utils.template import apply_chat_template_from_str - -if TYPE_CHECKING: - from tito_gateway.vendor.miles_compat.utils.chat_template_utils.tito_tokenizer import TITOTokenizer, TITOTokenizerType - - -def simulate_pretokenized_path( - chat_template: str, - messages: list[dict], - pretokenized_num_message: int, - tools: list[dict] | None = None, - **template_kwargs, -) -> str: - """Simulate the pretokenized incremental path at text level. - - 1. Render first N messages (no generation prompt) -> prefix_text - 2. Render ALL messages (with generation prompt) -> full_text - 3. Verify prefix_text is a prefix of full_text - - Raises ``ValueError`` on prefix mismatch. - """ - prefix_text = apply_chat_template_from_str( - chat_template, - messages[:pretokenized_num_message], - add_generation_prompt=False, - tools=tools, - **template_kwargs, - ) - - full_text = apply_chat_template_from_str( - chat_template, - messages, - add_generation_prompt=True, - tools=tools, - **template_kwargs, - ) - - if not full_text.startswith(prefix_text): - raise ValueError( - f"Prefix mismatch!\n" - f"prefix_text ({len(prefix_text)} chars):\n{repr(prefix_text[-200:])}\n\n" - f"full_text at same position:\n{repr(full_text[:len(prefix_text)][-200:])}" - ) - - return full_text - - -def get_standard_result( - chat_template: str, - messages: list[dict], - tools: list[dict] | None = None, - **template_kwargs, -) -> str: - """Standard path: render all messages with generation prompt.""" - return apply_chat_template_from_str( - chat_template, - messages, - add_generation_prompt=True, - tools=tools, - **template_kwargs, - ) - - -def assert_pretokenized_equals_standard(chat_template, messages, pretokenized_num_message, tools=None, **kwargs): - """Assert pretokenized incremental path produces same text as standard full render.""" - standard = get_standard_result(chat_template, messages, tools=tools, **kwargs) - pretokenized = simulate_pretokenized_path(chat_template, messages, pretokenized_num_message, tools=tools, **kwargs) - assert pretokenized == standard, f"Pretokenized (N={pretokenized_num_message}) != standard" - - -# --------------------------------------------------------------------------- -# Non-raising verification API for CLI / programmatic use -# --------------------------------------------------------------------------- - - -@dataclass -class VerifyResult: - """Result of a single append-only verification case.""" - - case_name: str - passed: bool - error: str | None = None - - -def verify_append_only( - chat_template: str, - messages: list[dict], - pretokenized_num_message: int, - tools: list[dict] | None = None, - case_name: str = "", - **template_kwargs, -) -> VerifyResult: - """Check that the template satisfies the append-only invariant. - - Returns a ``VerifyResult`` instead of raising, making it suitable for - batch verification in CLI scripts. - """ - try: - standard = get_standard_result(chat_template, deepcopy(messages), tools=tools, **template_kwargs) - pretokenized = simulate_pretokenized_path( - chat_template, deepcopy(messages), pretokenized_num_message, tools=tools, **template_kwargs - ) - if pretokenized != standard: - return VerifyResult( - case_name=case_name, passed=False, error=f"Pretokenized (N={pretokenized_num_message}) != standard" - ) - return VerifyResult(case_name=case_name, passed=True) - except ValueError as e: - return VerifyResult(case_name=case_name, passed=False, error=str(e)) - except Exception as e: - return VerifyResult(case_name=case_name, passed=False, error=f"{type(e).__name__}: {e}") - - -# --------------------------------------------------------------------------- -# Built-in test cases (shared between CLI and test suite) -# --------------------------------------------------------------------------- -# -# Trajectories expose two class attributes used for verify-layer filtering: -# -# * ``APPEND_ROLES: frozenset[str]`` — non-assistant roles that appear after -# the first assistant message. Drives ``--tito-allowed-append-roles``. -# * ``IS_THINKING: bool`` — any assistant carries ``reasoning_content``. -# Drives ``--thinking`` and whether ``enable_thinking`` kwarg is passed. -# -# Both are declared on the trajectory class (mock_trajectories.py), alongside -# ``TOOLS`` / ``PRETOKENIZE_POSITIONS`` / ``MESSAGES``. This file only lists -# which trajectories to exercise and expands them into concrete cases. - -import re # noqa: E402 - -from tito_gateway.vendor.miles_compat.utils.test_utils.mock_trajectories import ( # noqa: E402 - IntermediateSystemThinkingTrajectory, - IntermediateSystemTrajectory, - LongChainThinkingTrajectory, - LongChainTrajectory, - MultiRoleSequenceTrajectory, - MultiToolSingleTurnTrajectory, - MultiTurnNoToolThinkingTrajectory, - MultiTurnNoToolTrajectory, - MultiTurnThinkingTrajectory, - MultiTurnTrajectory, - MultiUserToolChainTrajectory, - MultiUserTurnThinkingTrajectory, - ParallelToolsTrajectory, - RetrySystemTrajectory, - SimpleNoToolTrajectory, - SingleToolThinkingTrajectory, - SingleToolTrajectory, -) - - -def _short_name(cls: type) -> str: - name = cls.__name__.replace("Trajectory", "") - return re.sub(r"(? list[CaseSpec]: - """Expand one trajectory into one CaseSpec per PRETOKENIZE_POSITIONS value.""" - short = _short_name(traj_cls) - return [ - CaseSpec( - case_name=f"{short}-N{n}", - traj_cls=traj_cls, - pretokenize_n=n, - tools=traj_cls.TOOLS, - append_roles=traj_cls.APPEND_ROLES, - is_thinking=traj_cls.IS_THINKING, - ) - for n in traj_cls.PRETOKENIZE_POSITIONS - ] - - -ALL_CASES: list[CaseSpec] = [c for t in _TRAJECTORIES for c in _expand(t)] - -THINKING_MODES: tuple[str, ...] = ("off", "on", "both") - - -def select_cases( - *, - allowed_append_roles: Iterable[str], - is_thinking: bool | None = None, -) -> list[CaseSpec]: - """Select trajectory cases by append-role surface and (optionally) thinking flag. - - A case is included iff ``case.append_roles`` is a subset of - *allowed_append_roles*, and (when *is_thinking* is not ``None``) - ``case.is_thinking`` matches. - - The caller is responsible for including ``"tool"`` in *allowed_append_roles* - when the session is tool-capable; this function does not silently union it. - """ - allowed = frozenset(allowed_append_roles) - out: list[CaseSpec] = [] - for c in ALL_CASES: - if not c.append_roles.issubset(allowed): - continue - if is_thinking is not None and c.is_thinking != is_thinking: - continue - out.append(c) - return out - - -def enable_thinking_variants(thinking: str) -> list[dict]: - """Return the list of ``enable_thinking`` kwarg variants to apply per case. - - * ``"off"`` → ``[{}]`` (no ``enable_thinking`` kwarg). - * ``"on"`` → ``[{"enable_thinking": True}]``. - * ``"both"`` → ``[{"enable_thinking": True}, {"enable_thinking": False}]``. - - Both CLI (:func:`run_all_checks`) and pytest parametrize callers use this - to avoid drifting in how the ``enable_thinking`` knob is exercised. - """ - if thinking == "off": - return [{}] - if thinking == "on": - return [{"enable_thinking": True}] - if thinking == "both": - return [{"enable_thinking": True}, {"enable_thinking": False}] - raise ValueError(f"thinking must be one of {THINKING_MODES}; got {thinking!r}") - - -def format_case_id(case: CaseSpec, kwargs: dict) -> str: - """Human-readable label for a ``(case, template_kwargs)`` tuple. - - Used for both CLI ``VerifyResult.case_name`` and pytest test ids so the - same tuple is identified the same way in both surfaces. Format: - - * empty kwargs → ``case.case_name``. - * otherwise → ``-_on/off-=val`` (keys sorted; - bool values emit ``key_on`` / ``key_off``; other values ``key=val``). - """ - if not kwargs: - return case.case_name - parts: list[str] = [] - for k, v in sorted(kwargs.items()): - if isinstance(v, bool): - parts.append(f"{k}_{'on' if v else 'off'}") - else: - parts.append(f"{k}={v}") - return f"{case.case_name}-{'-'.join(parts)}" - - -@dataclass -class CoverageReport: - """Coverage of cases across ``(is_thinking, append_roles \\ {tool})``. - - ``covered`` maps each combination to the case names that fall in it; - ``missing`` lists combinations with no case. ``tool`` is excluded from - the role axis because it is implicitly always allowed. - """ - - covered: dict[tuple[bool, tuple[str, ...]], list[str]] - missing: list[tuple[bool, tuple[str, ...]]] - - -def check_coverage( - cases: list[CaseSpec] | None = None, - *, - role_universe: set[str] | None = None, -) -> CoverageReport: - """Enumerate ``thinking × append-role-subset`` combinations and report gaps. - - Used as a sanity check that every meaningful combination of - ``--tito-allowed-append-roles`` and ``--thinking`` is backed by at least - one trajectory — otherwise certain CLI settings would be no-ops. - """ - if cases is None: - cases = ALL_CASES - if role_universe is None: - role_universe = {"user", "system"} - - from itertools import chain, combinations - - ordered_universe = sorted(role_universe) - all_subsets: list[tuple[str, ...]] = [ - tuple(sub) - for sub in chain.from_iterable(combinations(ordered_universe, r) for r in range(len(ordered_universe) + 1)) - ] - - covered: dict[tuple[bool, tuple[str, ...]], list[str]] = { - (is_thinking, sub): [] for is_thinking in (False, True) for sub in all_subsets - } - for c in cases: - roles_key = tuple(sorted(c.append_roles - {"tool"})) - key = (c.is_thinking, roles_key) - if key in covered: - covered[key].append(c.case_name) - - missing = [k for k, v in covered.items() if not v] - return CoverageReport(covered=covered, missing=missing) - - -def run_all_checks( - chat_template: str, - *, - allowed_append_roles: set[str] | frozenset[str] | None = None, - thinking: str = "off", - extra_template_kwargs: dict | None = None, -) -> list[VerifyResult]: - """Run verification cases filtered by *allowed_append_roles* and *thinking*. - - ``allowed_append_roles`` is the role surface the session may append after - an assistant turn; defaults to ``{"tool"}`` for the agentic baseline. - Trajectories whose ``append_roles`` are not a subset are skipped. Caller - must include ``"tool"`` explicitly when relevant — there is no implicit - union. - - ``thinking`` selects which ``enable_thinking`` variants are exercised — - see :func:`enable_thinking_variants`. When ``"both"``, **every** selected - trajectory (thinking or not) is rerun with ``enable_thinking=True`` and - ``enable_thinking=False``, so templates that branch on the flag are - validated against non-reasoning input too. - - ``extra_template_kwargs`` is merged into every invocation — use it to - thread template-specific kwargs (e.g. GLM's ``clear_thinking=False``) - through the CLI. - """ - if allowed_append_roles is None: - allowed_append_roles = {"tool"} - if thinking not in THINKING_MODES: - raise ValueError(f"thinking must be one of {THINKING_MODES}; got {thinking!r}") - extra = extra_template_kwargs or {} - - is_thinking_filter = {"off": False, "on": True, "both": None}[thinking] - selected = select_cases(allowed_append_roles=allowed_append_roles, is_thinking=is_thinking_filter) - variants = enable_thinking_variants(thinking) - - results: list[VerifyResult] = [] - for case in selected: - for variant in variants: - kwargs = {**variant, **extra} - results.append( - verify_append_only( - chat_template, - deepcopy(case.traj_cls.MESSAGES), - case.pretokenize_n, - tools=case.tools, - case_name=format_case_id(case, kwargs), - **kwargs, - ) - ) - - return results - - -# --------------------------------------------------------------------------- -# TITO-instance verification: decode-roundtrip equality -# --------------------------------------------------------------------------- -# -# The string-based primitive above asserts text-prefix at the chat-template -# layer. This is necessary but not sufficient for production correctness — -# production runs ``get_tito_tokenizer(...)`` and exercises ``merge_tokens`` -# (model-specific token-level boundary patches) plus -# ``tokenize_additional_non_assistant`` (renders appended segments under a -# synthetic ``[_DUMMY_SYSTEM, ...]`` context, not the real history). -# -# The primitive below mirrors the production path: it instantiates the actual -# TITO subclass + HF tokenizer, runs ``merge_tokens`` against the encoded -# prefix, decodes, and asserts text equality with the canonical full render. - - -def verify_append_only_via_tito_instance( - tito: TITOTokenizer, - tokenizer: Any, - messages: list[dict], - pretokenized_num_message: int, - tools: list[dict] | None = None, - case_name: str = "", - **template_kwargs, -) -> VerifyResult: - """Decode-roundtrip verify with a pre-built TITO instance. - - Asserts ``decode(tito.merge_tokens(prefix_msgs, full_msgs, encode(prefix_text))) - == full_text`` where ``prefix_text`` and ``full_text`` come from running the - chat template through ``tokenizer`` with the same kwargs ``tito`` was built - with. The test-only path (e.g. ``BuggyQwen3TITOTokenizer``) uses this - instance form directly; production-shape callers go through - :func:`verify_append_only_via_tito`. - """ - try: - # TITO's incremental path requires the appendix to be all non-assistant. - # From the pretokenized boundary N, greedily extend M forward over the - # maximal non-assistant run — that's the chunk production would call - # merge_tokens for (between two assistant generations, or up to end). - n = pretokenized_num_message - m = n - while m < len(messages) and messages[m].get("role") != "assistant": - m += 1 - if m == n: - return VerifyResult( - case_name=case_name, - passed=False, - error=( - f"Empty appendix at N={n}: messages[{n}] is assistant. " - "PRETOKENIZE_POSITIONS must land at a post-assistant boundary " - "where messages[N:] starts with a non-assistant turn." - ), - ) - - prefix_msgs = deepcopy(messages[:n]) - full_msgs = deepcopy(messages[:m]) - - prefix_text = tito.render_messages( - prefix_msgs, - tools=tools, - add_generation_prompt=False, - ) - full_text = tito.render_messages( - full_msgs, - tools=tools, - add_generation_prompt=True, - ) - - prefix_ids = tokenizer.encode(prefix_text, add_special_tokens=False) - # Simulate production's model-stop: in production, ``pretokenized_token_ids`` - # ends where the model actually stopped — typically before the trailing - # tokens the chat template would otherwise emit (Qwen's ``\n`` after - # ``<|im_end|>``, GLM's ambiguous ``<|user|>``/``<|observation|>`` boundary). - # The TITO subclass declares those as ``trailing_token_ids``. Trim them - # here so ``merge_tokens``'s boundary patches see the prefix in its - # production shape so the verifier sees the same prefix the - # subclass merge_tokens / trailing trim path operates on. - trailing = tito.trailing_token_ids - while prefix_ids and prefix_ids[-1] in trailing: - prefix_ids = prefix_ids[:-1] - merged_ids = tito.merge_tokens(prefix_msgs, full_msgs, prefix_ids, tools=tools) - merged_text = tokenizer.decode(merged_ids) - - if merged_text == full_text: - return VerifyResult(case_name=case_name, passed=True) - - # Find first divergence and quote ~60 chars of context on each side. - common_len = min(len(merged_text), len(full_text)) - diff_idx = next( - (i for i in range(common_len) if merged_text[i] != full_text[i]), - common_len, - ) - ctx_start = max(0, diff_idx - 60) - ctx_end = diff_idx + 60 - return VerifyResult( - case_name=case_name, - passed=False, - error=( - f"Decode-roundtrip mismatch (N={pretokenized_num_message}) at char {diff_idx}\n" - f" expected: ...{full_text[ctx_start:ctx_end]!r}...\n" - f" actual: ...{merged_text[ctx_start:ctx_end]!r}..." - ), - ) - except Exception as e: - return VerifyResult(case_name=case_name, passed=False, error=f"{type(e).__name__}: {e}") - - -def verify_append_only_via_tito( - tokenizer: Any, - tito_model: TITOTokenizerType | str, - allowed_append_roles: list[str], - messages: list[dict], - pretokenized_num_message: int, - tools: list[dict] | None = None, - case_name: str = "", - **template_kwargs, -) -> VerifyResult: - """Decode-roundtrip verify, building TITO from the registered family. - - Matches the production wiring at ``miles/rollout/session/sessions.py:35`` — - the same ``get_tito_tokenizer`` factory call, with ``chat_template_kwargs`` - threaded through so ``merge_tokens`` and the dummy-context segment renders - use the same kwargs as the reference full render. - """ - from tito_gateway.vendor.miles_compat.utils.chat_template_utils import get_tito_tokenizer - - tito = get_tito_tokenizer( - tokenizer, - tokenizer_type=tito_model, - chat_template_kwargs=dict(template_kwargs), - allowed_append_roles=list(allowed_append_roles), - ) - return verify_append_only_via_tito_instance( - tito, - tokenizer, - messages, - pretokenized_num_message, - tools=tools, - case_name=case_name, - **template_kwargs, - ) - - -def run_all_checks_via_tito( - tokenizer: Any, - tito_model: TITOTokenizerType | str, - *, - allowed_append_roles: set[str] | frozenset[str] | None = None, - thinking: str = "off", - extra_template_kwargs: dict | None = None, -) -> list[VerifyResult]: - """Same shape as :func:`run_all_checks` but routes through TITO + tokenizer. - - Per-case TITO rebuild: each (case, ``enable_thinking`` variant) gets a fresh - TITO instance constructed with the merged kwargs, so the dummy-context - segment renders inside ``tokenize_additional_non_assistant`` see the same - ``enable_thinking`` value as the reference render. Construction is - millisecond-level and runs ~50 times per CLI invocation; cheap. - - The caller is responsible for setting ``tokenizer.chat_template`` (e.g. via - ``resolve_fixed_chat_template`` lookup or ``--template`` override) before - calling this — this function does not consult ``SUPPORTED_TEMPLATES``. - """ - if allowed_append_roles is None: - allowed_append_roles = {"tool"} - if thinking not in THINKING_MODES: - raise ValueError(f"thinking must be one of {THINKING_MODES}; got {thinking!r}") - extra = extra_template_kwargs or {} - - is_thinking_filter = {"off": False, "on": True, "both": None}[thinking] - selected = select_cases(allowed_append_roles=allowed_append_roles, is_thinking=is_thinking_filter) - variants = enable_thinking_variants(thinking) - roles_list = sorted(allowed_append_roles) - - results: list[VerifyResult] = [] - for case in selected: - # TITO incremental requires a non-empty non-assistant appendix at the - # boundary. Trajectories that end at the assistant turn (e.g. plain - # ``[sys, user, assistant]``) have no appendix to verify and are - # silently skipped here — the string-based primitive still covers - # them at the text-prefix layer. - msgs = case.traj_cls.MESSAGES - n = case.pretokenize_n - if n >= len(msgs) or msgs[n].get("role") == "assistant": - continue - for variant in variants: - kwargs = {**variant, **extra} - results.append( - verify_append_only_via_tito( - tokenizer, - tito_model, - roles_list, - deepcopy(case.traj_cls.MESSAGES), - case.pretokenize_n, - tools=case.tools, - case_name=format_case_id(case, kwargs), - **kwargs, - ) - ) - - return results diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/mock_sglang_server.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/mock_sglang_server.py deleted file mode 100644 index 294aa41..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/mock_sglang_server.py +++ /dev/null @@ -1,270 +0,0 @@ -import asyncio -import re -import time -import uuid -from collections.abc import Callable -from contextlib import contextmanager -from dataclasses import asdict, dataclass - -from fastapi import FastAPI, Request -from fastapi.responses import JSONResponse -from pydantic import TypeAdapter -from sglang.srt.entrypoints.openai.protocol import Tool -from sglang.srt.function_call.function_call_parser import FunctionCallParser - -from miles.utils.http_utils import find_available_port -from miles.utils.processing_utils import load_tokenizer -from miles.utils.test_utils.uvicorn_thread_server import UvicornThreadServer - - -@dataclass(frozen=True) -class ProcessResultMetaInfo: - weight_version: str | None = None - routed_experts: str | None = None - spec_accept_token_num: int | None = None - spec_draft_token_num: int | None = None - spec_verify_ct: int | None = None - - def to_dict(self) -> dict: - return {k: v for k, v in asdict(self).items() if v is not None} - - -@dataclass(frozen=True) -class ProcessResult: - text: str - finish_reason: str = "stop" - cached_tokens: int = 0 - meta_info: ProcessResultMetaInfo = ProcessResultMetaInfo() - - -ProcessFn = Callable[[str], ProcessResult] - - -class MockSGLangServer: - def __init__( - self, - model_name: str, - process_fn: ProcessFn, - host: str, - port: int, - latency: float = 0.0, - chat_template_path: str | None = None, - ): - self.tokenizer = load_tokenizer(model_name, chat_template_path=chat_template_path, trust_remote_code=True) - self.process_fn = process_fn - self.host = host - self.port = port or find_available_port(30000) - self.latency = latency - - self.app = FastAPI() - self._server: UvicornThreadServer | None = None - - self.request_log: list[dict] = [] - self._concurrency = Counter() - - self._setup_routes() - - @property - def max_concurrent(self) -> int: - return self._concurrency.max_value - - def reset_stats(self): - self.request_log.clear() - self._concurrency.reset() - - def start(self): - self._server = UvicornThreadServer(self.app, host=self.host, port=self.port) - self._server.start() - - def stop(self): - if self._server is not None: - self._server.stop() - - @property - def url(self) -> str: - return f"http://{self.host}:{self.port}" - - def _setup_routes(self): - @self.app.post("/generate") - async def generate(request: Request): - return await self._handle_generate_like_request(request, self._compute_generate_response) - - @self.app.post("/v1/chat/completions") - async def chat_completions(request: Request): - return await self._handle_generate_like_request(request, self._compute_chat_completions_response) - - @self.app.get("/health") - async def health(): - return JSONResponse(content={"status": "ok"}) - - @self.app.post("/abort_request") - async def abort_request(_request: Request): - return JSONResponse(content={"status": "ok"}) - - async def _handle_generate_like_request(self, request: Request, compute_fn: Callable[[dict], dict]): - payload = await request.json() - self.request_log.append(payload) - with self._concurrency.track(): - if self.latency > 0: - await asyncio.sleep(self.latency) - response = compute_fn(payload) - return JSONResponse(content=response) - - def _compute_generate_response(self, payload: dict) -> dict: - assert payload.get("return_logprob", True) is True, "MockSGLangServer requires return_logprob=True" - input_ids = payload.get("input_ids", []) - - prompt_str = self.tokenizer.decode(input_ids, skip_special_tokens=False) - process_result = self.process_fn(prompt_str) - output_ids = self.tokenizer.encode(process_result.text, add_special_tokens=False) - - prompt_tokens = len(input_ids) - completion_tokens = len(output_ids) - - finish_reason_dict = {"type": process_result.finish_reason} - if process_result.finish_reason == "length": - finish_reason_dict["length"] = completion_tokens - - output_token_logprobs = [(-1 / 128 * i, token_id) for i, token_id in enumerate(output_ids)] - - meta_info = { - "finish_reason": finish_reason_dict, - "prompt_tokens": prompt_tokens, - "cached_tokens": process_result.cached_tokens, - "completion_tokens": completion_tokens, - "output_token_logprobs": output_token_logprobs, - **process_result.meta_info.to_dict(), - } - - return {"text": process_result.text, "meta_info": meta_info} - - def _compute_chat_completions_response(self, payload: dict) -> dict: - messages = payload.get("messages", []) - tools = payload.get("tools") - - prompt_str = self.tokenizer.apply_chat_template( - messages, tokenize=False, add_generation_prompt=True, tools=tools - ) - - prompt_ids = None - if payload.get("return_prompt_token_ids"): - input_ids = payload.get("input_ids") - if input_ids is not None: - prompt_ids = list(input_ids) - else: - prompt_ids = self.tokenizer.encode(prompt_str, add_special_tokens=False) - - process_result = self.process_fn(prompt_str) - output_ids = self.tokenizer.encode(process_result.text, add_special_tokens=False) - - logprobs_content = [ - { - "token": self.tokenizer.convert_ids_to_tokens(tid), - "token_id": tid, - "logprob": -1 / 128 * i, - } - for i, tid in enumerate(output_ids) - ] - - finish_reason = process_result.finish_reason - tool_calls = None - if tools and finish_reason == "stop": - parser = FunctionCallParser( - tools=TypeAdapter(list[Tool]).validate_python(tools), - tool_call_parser="qwen25", - ) - message_content, parsed_calls = parser.parse_non_stream(process_result.text) - if parsed_calls: - finish_reason = "tool_calls" - tool_calls = [ - { - "id": f"call{i:05d}", - "type": "function", - "function": {"name": call.name, "arguments": call.parameters or "{}"}, - } - for i, call in enumerate(parsed_calls) - ] - else: - message_content = process_result.text - - output_token_logprobs = [(-1 / 128 * i, tid) for i, tid in enumerate(output_ids)] - - choice = { - "index": 0, - "message": { - "role": "assistant", - "content": message_content, - "tool_calls": tool_calls, - }, - "logprobs": {"content": logprobs_content}, - "finish_reason": finish_reason, - "meta_info": { - "output_token_logprobs": output_token_logprobs, - "completion_tokens": len(output_ids), - **process_result.meta_info.to_dict(), - }, - } - if prompt_ids is not None: - choice["prompt_token_ids"] = prompt_ids - - return { - "id": f"chatcmpl-{uuid.uuid4().hex[:8]}", - "object": "chat.completion", - "created": int(time.time()), - "model": "mock-model", - "choices": [choice], - } - - -class Counter: - def __init__(self): - self._current = 0 - self._max = 0 - - @property - def max_value(self) -> int: - return self._max - - def reset(self): - self._current = 0 - self._max = 0 - - @contextmanager - def track(self): - self._current += 1 - self._max = max(self._max, self._current) - try: - yield - finally: - self._current -= 1 - - -def default_process_fn(prompt: str) -> ProcessResult: - match = re.search(r"What is 1\+(\d+)\?", prompt) - if match: - num = int(match.group(1)) - ans = 1 + num - return ProcessResult(text=f"\\boxed{{{ans}}}", finish_reason="stop") - return ProcessResult(text="I don't understand.", finish_reason="stop") - - -@contextmanager -def with_mock_server( - model_name: str = "Qwen/Qwen3-0.6B", - process_fn: ProcessFn = default_process_fn, - host: str = "127.0.0.1", - port: int | None = None, - latency: float = 0.0, -): - server = MockSGLangServer( - model_name=model_name, - process_fn=process_fn, - host=host, - port=port, - latency=latency, - ) - try: - server.start() - yield server - finally: - server.stop() diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/mock_trajectories.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/mock_trajectories.py deleted file mode 100644 index a077fcc..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/mock_trajectories.py +++ /dev/null @@ -1,1198 +0,0 @@ -"""Multi-turn trajectory definitions for testing. - -Each trajectory class defines a complete multi-turn conversation with tool calls. -Used by: -- tests/fast/rollout/generate_hub/test_pretokenized_chat.py (chat template verification) -- tests/fast/router/test_session_pretokenized_e2e.py (session proxy e2e) - -Class attributes consumed by chat_template_verify: - -- ``APPEND_ROLES: frozenset[str]`` — non-assistant roles that appear *after* - the first assistant message (``tool`` / ``user`` / ``system``). These are - the roles the session must allow to be appended on top of an assistant- - stopped prefix; drives ``--tito-allowed-append-roles`` filtering. -- ``IS_THINKING: bool`` — ``True`` iff at least one assistant message carries - ``reasoning_content``. Drives ``--thinking`` filtering and whether the - ``enable_thinking`` chat-template kwarg is passed. - -Both are declared explicitly on each class so readers can see a trajectory's -verify-layer classification without having to execute the module. -""" - -from __future__ import annotations - -from copy import deepcopy -from dataclasses import dataclass -from typing import Any - -from tito_gateway.vendor.miles_compat.utils.test_utils.mock_sglang_server import ProcessFn, ProcessResult - -# --------------------------------------------------------------------------- -# Shared tool definitions -# --------------------------------------------------------------------------- - -WEATHER_TOOLS = [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get weather information for a location", - "parameters": { - "type": "object", - "properties": { - "city": {"type": "string", "description": "City name"}, - "unit": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "Temperature unit", - }, - }, - "required": ["city"], - }, - }, - } -] - -DATE_TOOL = { - "type": "function", - "function": { - "name": "get_date", - "description": "Get the current date and time for a timezone", - "parameters": { - "type": "object", - "properties": { - "timezone": { - "type": "string", - "description": "Timezone name (e.g. Asia/Shanghai, UTC)", - }, - }, - "required": ["timezone"], - }, - }, -} - -ALL_TOOLS = WEATHER_TOOLS + [DATE_TOOL] - - -# --------------------------------------------------------------------------- -# Data structures -# --------------------------------------------------------------------------- - - -@dataclass -class Turn: - """One turn in a multi-turn trajectory.""" - - request_messages: list[dict[str, Any]] - assistant_message: dict[str, Any] - response_text: str = "" # raw text returned by process_fn (computed by build_trajectory) - - -@dataclass -class Trajectory: - """A fully resolved multi-turn trajectory ready for testing.""" - - tools: list[dict[str, Any]] | None - turns: list[Turn] - full_messages: list[dict[str, Any]] - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def last_user_index(messages: list[dict[str, Any]]) -> int: - """Return the index of the last user message in *messages*.""" - for i in range(len(messages) - 1, -1, -1): - if messages[i]["role"] == "user": - return i - raise ValueError("No user message found") - - -# --------------------------------------------------------------------------- -# Trajectory classes (by scenario) -# --------------------------------------------------------------------------- - - -class SingleToolTrajectory: - """sys, user, assistant(tool_call), tool — 1 turn""" - - TOOLS = WEATHER_TOOLS - PRETOKENIZE_POSITIONS = [3] - APPEND_ROLES = frozenset({"tool"}) - IS_THINKING = False - MESSAGES = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What's the weather in Beijing?"}, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "get_weather", - "arguments": {"city": "Beijing", "unit": "celsius"}, - }, - } - ], - }, - { - "role": "tool", - "content": '{"temperature": 25, "condition": "sunny"}', - "tool_call_id": "call_1", - }, - ] - - -class MultiTurnTrajectory: - """sys, user, ass(tool), tool, ass(tool), tool — 2 turns""" - - TOOLS = WEATHER_TOOLS - PRETOKENIZE_POSITIONS = [3, 5] - APPEND_ROLES = frozenset({"tool"}) - IS_THINKING = False - MESSAGES = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What's the weather in Beijing and Shanghai?"}, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "get_weather", - "arguments": {"city": "Beijing"}, - }, - } - ], - }, - { - "role": "tool", - "content": '{"temperature": 25, "condition": "sunny"}', - "tool_call_id": "call_1", - }, - { - "role": "assistant", - "content": "Beijing is 25C. Let me check Shanghai.", - "tool_calls": [ - { - "id": "call_2", - "type": "function", - "function": { - "name": "get_weather", - "arguments": {"city": "Shanghai"}, - }, - } - ], - }, - { - "role": "tool", - "content": '{"temperature": 30, "condition": "cloudy"}', - "tool_call_id": "call_2", - }, - ] - - -class MultiToolSingleTurnTrajectory: - """sys, user, assistant(2 tool_calls: weather+date), tool, tool — 1 turn""" - - TOOLS = ALL_TOOLS - PRETOKENIZE_POSITIONS = [3] - APPEND_ROLES = frozenset({"tool"}) - IS_THINKING = False - MESSAGES = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What's the weather in Beijing and what date is it?"}, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "get_weather", - "arguments": {"city": "Beijing"}, - }, - }, - { - "id": "call_2", - "type": "function", - "function": { - "name": "get_date", - "arguments": {"timezone": "Asia/Shanghai"}, - }, - }, - ], - }, - { - "role": "tool", - "content": '{"temperature": 25}', - "tool_call_id": "call_1", - }, - { - "role": "tool", - "content": '{"date": "2025-03-15", "time": "14:30:00"}', - "tool_call_id": "call_2", - }, - ] - - -class ParallelToolsTrajectory: - """sys, user, assistant(3 parallel tool_calls), tool, tool, tool — 1 turn""" - - TOOLS = WEATHER_TOOLS - PRETOKENIZE_POSITIONS = [3] - APPEND_ROLES = frozenset({"tool"}) - IS_THINKING = False - MESSAGES = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Compare weather in Beijing, Shanghai, and Guangzhou"}, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "get_weather", - "arguments": {"city": "Beijing"}, - }, - }, - { - "id": "call_2", - "type": "function", - "function": { - "name": "get_weather", - "arguments": {"city": "Shanghai"}, - }, - }, - { - "id": "call_3", - "type": "function", - "function": { - "name": "get_weather", - "arguments": {"city": "Guangzhou"}, - }, - }, - ], - }, - { - "role": "tool", - "content": '{"temperature": 25, "condition": "sunny"}', - "tool_call_id": "call_1", - }, - { - "role": "tool", - "content": '{"temperature": 30, "condition": "cloudy"}', - "tool_call_id": "call_2", - }, - { - "role": "tool", - "content": '{"temperature": 35, "condition": "rainy"}', - "tool_call_id": "call_3", - }, - ] - - -class LongChainTrajectory: - """sys, user, ass(tool), tool, ass(tool:date), tool, ass(tool), tool — 3 turns""" - - TOOLS = ALL_TOOLS - PRETOKENIZE_POSITIONS = [3, 5, 7] - APPEND_ROLES = frozenset({"tool"}) - IS_THINKING = False - MESSAGES = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Do a multi-step task"}, - { - "role": "assistant", - "content": "Step 1", - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "get_weather", - "arguments": {"city": "Beijing"}, - }, - } - ], - }, - { - "role": "tool", - "content": '{"temperature": 25}', - "tool_call_id": "call_1", - }, - { - "role": "assistant", - "content": "Step 2", - "tool_calls": [ - { - "id": "call_2", - "type": "function", - "function": { - "name": "get_date", - "arguments": {"timezone": "UTC"}, - }, - } - ], - }, - { - "role": "tool", - "content": '{"date": "2025-03-15", "time": "12:00:00"}', - "tool_call_id": "call_2", - }, - { - "role": "assistant", - "content": "Step 3", - "tool_calls": [ - { - "id": "call_3", - "type": "function", - "function": { - "name": "get_weather", - "arguments": {"city": "Guangzhou"}, - }, - } - ], - }, - { - "role": "tool", - "content": '{"temperature": 35}', - "tool_call_id": "call_3", - }, - ] - - -class RetrySystemTrajectory: - """sys, user, ass(tool), tool, system_retry, ass(tool), tool — 2 turns with mid-conversation system message. - - Simulates an agent that injects a system-level retry prompt when the model - fails to produce a useful tool call on the first attempt. - """ - - TOOLS = WEATHER_TOOLS - PRETOKENIZE_POSITIONS = [3, 6] - APPEND_ROLES = frozenset({"tool", "system"}) - IS_THINKING = False - MESSAGES = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What's the weather in Beijing and Shanghai?"}, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "get_weather", - "arguments": {"city": "Beijing"}, - }, - } - ], - }, - { - "role": "tool", - "content": '{"temperature": 25, "condition": "sunny"}', - "tool_call_id": "call_1", - }, - {"role": "system", "content": "You still need to check Shanghai. Please call get_weather for Shanghai."}, - { - "role": "assistant", - "content": "Let me check Shanghai.", - "tool_calls": [ - { - "id": "call_2", - "type": "function", - "function": { - "name": "get_weather", - "arguments": {"city": "Shanghai"}, - }, - } - ], - }, - { - "role": "tool", - "content": '{"temperature": 30, "condition": "cloudy"}', - "tool_call_id": "call_2", - }, - ] - - -class MultiUserToolChainTrajectory: - """sys, user1, ass(tool), tool, ass, user2, ass(tool), tool, ass(tool:date), tool - - NOTE: LinearTrajectory can carry multiple user messages when - ``allowed_append_roles`` includes ``"user"``; this trajectory exercises - that path. The distribution may still deviate from the chat template - behavior, causing high tito_session_mismatch_rate. - """ - - TOOLS = ALL_TOOLS - PRETOKENIZE_POSITIONS = [3, 5, 7, 9] - APPEND_ROLES = frozenset({"tool", "user"}) - IS_THINKING = False - MESSAGES = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What's the weather in Beijing?"}, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "get_weather", - "arguments": {"city": "Beijing"}, - }, - } - ], - }, - { - "role": "tool", - "content": '{"temperature": 25, "condition": "sunny"}', - "tool_call_id": "call_1", - }, - { - "role": "assistant", - "content": "Beijing is 25C and sunny.", - }, - {"role": "user", "content": "Now check Shanghai and what date is it?"}, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_2", - "type": "function", - "function": { - "name": "get_weather", - "arguments": {"city": "Shanghai"}, - }, - } - ], - }, - { - "role": "tool", - "content": '{"temperature": 30, "condition": "cloudy"}', - "tool_call_id": "call_2", - }, - { - "role": "assistant", - "content": "Shanghai is 30C. Let me check the date.", - "tool_calls": [ - { - "id": "call_3", - "type": "function", - "function": { - "name": "get_date", - "arguments": {"timezone": "Asia/Shanghai"}, - }, - } - ], - }, - { - "role": "tool", - "content": '{"date": "2025-03-15", "time": "22:30:00"}', - "tool_call_id": "call_3", - }, - ] - - -class SimpleNoToolTrajectory: - """sys, user, asst, system_reminder, user2, asst2 — no tools, with system/user append boundaries. - - Codifies the synthetic 'single_system' case the old CI used to manually - append: at N=3 prefix ends at the first asst, append starts with the - system_reminder, exercising the system-append boundary on a no-tool model. - """ - - TOOLS = None - PRETOKENIZE_POSITIONS = [3, 4, 5, 6] - APPEND_ROLES = frozenset({"user", "system"}) - IS_THINKING = False - MESSAGES = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Hello!"}, - {"role": "assistant", "content": "Hi there! How can I help?"}, - {"role": "system", "content": "Please answer in one short sentence."}, - {"role": "user", "content": "What's 2+2?"}, - {"role": "assistant", "content": "Four."}, - ] - - -class MultiTurnNoToolTrajectory: - """sys, user, assistant, user (no tools) — multi-turn plain conversation""" - - TOOLS = None - PRETOKENIZE_POSITIONS = [3, 5] - APPEND_ROLES = frozenset({"user"}) - IS_THINKING = False - MESSAGES = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What is the capital of France?"}, - {"role": "assistant", "content": "The capital of France is Paris."}, - {"role": "user", "content": "And what about Germany?"}, - ] - - -class MultiTurnNoToolThinkingTrajectory: - """sys, user, assistant(reasoning_content), user (no tools) — multi-turn with thinking""" - - TOOLS = None - PRETOKENIZE_POSITIONS = [3, 5] - APPEND_ROLES = frozenset({"user"}) - IS_THINKING = True - MESSAGES = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What is the capital of France?"}, - { - "role": "assistant", - "reasoning_content": "The user is asking about geography. The capital of France is Paris.", - "content": "The capital of France is Paris.", - }, - {"role": "user", "content": "And what about Germany?"}, - ] - - -# --------------------------------------------------------------------------- -# Thinking variants -# --------------------------------------------------------------------------- - - -class SingleToolThinkingTrajectory: - """sys, user, ass(think+tool), tool, user2, ass(think+tool), tool, user3, ass — multi-role alternating with thinking. - - Codifies the synthetic 'alternating_user_tool' case the old CI used to - manually append: prefix cuts at N=4/N=7 exercise the user-after-tool - boundary on a thinking model. - """ - - TOOLS = WEATHER_TOOLS - PRETOKENIZE_POSITIONS = [3, 4, 5, 6, 7, 8] - APPEND_ROLES = frozenset({"tool", "user"}) - IS_THINKING = True - MESSAGES = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What's the weather in Beijing?"}, - { - "role": "assistant", - "reasoning_content": "The user wants to know the weather in Beijing. I should call the get_weather function.", - "content": None, - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "get_weather", - "arguments": {"city": "Beijing", "unit": "celsius"}, - }, - } - ], - }, - { - "role": "tool", - "content": '{"temperature": 25, "condition": "sunny"}', - "tool_call_id": "call_1", - }, - {"role": "user", "content": "Now check Shanghai too."}, - { - "role": "assistant", - "reasoning_content": "Now the user wants Shanghai's weather. Calling get_weather again.", - "content": None, - "tool_calls": [ - { - "id": "call_2", - "type": "function", - "function": { - "name": "get_weather", - "arguments": {"city": "Shanghai", "unit": "celsius"}, - }, - } - ], - }, - { - "role": "tool", - "content": '{"temperature": 30, "condition": "cloudy"}', - "tool_call_id": "call_2", - }, - {"role": "user", "content": "And tell me the date as well."}, - { - "role": "assistant", - "reasoning_content": "The user is asking for the date. I'll answer based on what I know.", - "content": "Beijing is 25°C and sunny; Shanghai is 30°C and cloudy. I don't have access to the current date.", - }, - ] - - -class MultiTurnThinkingTrajectory: - """sys, user, ass(thinking+tool), tool, ass(thinking+tool), tool — 2 turns""" - - TOOLS = WEATHER_TOOLS - PRETOKENIZE_POSITIONS = [3, 5] - APPEND_ROLES = frozenset({"tool"}) - IS_THINKING = True - MESSAGES = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What's the weather in Beijing and Shanghai?"}, - { - "role": "assistant", - "reasoning_content": "Let me check Beijing first.", - "content": None, - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "get_weather", - "arguments": {"city": "Beijing"}, - }, - } - ], - }, - { - "role": "tool", - "content": '{"temperature": 25, "condition": "sunny"}', - "tool_call_id": "call_1", - }, - { - "role": "assistant", - "reasoning_content": "Beijing is 25C. Now let me check Shanghai.", - "content": "Beijing is 25C. Let me check Shanghai.", - "tool_calls": [ - { - "id": "call_2", - "type": "function", - "function": { - "name": "get_weather", - "arguments": {"city": "Shanghai"}, - }, - } - ], - }, - { - "role": "tool", - "content": '{"temperature": 30, "condition": "cloudy"}', - "tool_call_id": "call_2", - }, - ] - - -class LongChainThinkingTrajectory: - """sys, user, ass(thinking+tool), tool, ass(thinking+tool), tool, ass(thinking+tool), tool — 3 turns""" - - TOOLS = ALL_TOOLS - PRETOKENIZE_POSITIONS = [3, 5, 7] - APPEND_ROLES = frozenset({"tool"}) - IS_THINKING = True - MESSAGES = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Do a multi-step task"}, - { - "role": "assistant", - "reasoning_content": "Starting step 1, checking Beijing weather.", - "content": "Step 1", - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "get_weather", - "arguments": {"city": "Beijing"}, - }, - } - ], - }, - { - "role": "tool", - "content": '{"temperature": 25}', - "tool_call_id": "call_1", - }, - { - "role": "assistant", - "reasoning_content": "Got Beijing result. Now step 2, checking the date.", - "content": "Step 2", - "tool_calls": [ - { - "id": "call_2", - "type": "function", - "function": { - "name": "get_date", - "arguments": {"timezone": "UTC"}, - }, - } - ], - }, - { - "role": "tool", - "content": '{"date": "2025-03-15", "time": "12:00:00"}', - "tool_call_id": "call_2", - }, - { - "role": "assistant", - "reasoning_content": "Got date result. Now step 3, checking Guangzhou.", - "content": "Step 3", - "tool_calls": [ - { - "id": "call_3", - "type": "function", - "function": { - "name": "get_weather", - "arguments": {"city": "Guangzhou"}, - }, - } - ], - }, - { - "role": "tool", - "content": '{"temperature": 35}', - "tool_call_id": "call_3", - }, - ] - - -class MultiUserTurnThinkingTrajectory: - """sys, user1, ass(thinking+tool), tool, ass(thinking), user2, ass(thinking+tool), tool - - Cross-user-turn with thinking. Tests that thinking content from user turn 1 - is not compressed/modified when rendering the full conversation including - user turn 2. - """ - - TOOLS = WEATHER_TOOLS - PRETOKENIZE_POSITIONS = [7] - APPEND_ROLES = frozenset({"tool", "user"}) - IS_THINKING = True - MESSAGES = [ - {"role": "system", "content": "You are a helpful assistant."}, - # --- user turn 1 --- - {"role": "user", "content": "What's the weather in Beijing?"}, - { - "role": "assistant", - "reasoning_content": "User wants Beijing weather, let me check.", - "content": None, - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "get_weather", - "arguments": {"city": "Beijing"}, - }, - } - ], - }, - { - "role": "tool", - "content": '{"temperature": 25, "condition": "sunny"}', - "tool_call_id": "call_1", - }, - { - "role": "assistant", - "reasoning_content": "Beijing is 25C and sunny. I should tell the user.", - "content": "Beijing is 25°C and sunny!", - }, - # --- user turn 2 --- - {"role": "user", "content": "Now check Shanghai too."}, - { - "role": "assistant", - "reasoning_content": "User wants Shanghai weather now. Let me call the tool.", - "content": None, - "tool_calls": [ - { - "id": "call_2", - "type": "function", - "function": { - "name": "get_weather", - "arguments": {"city": "Shanghai"}, - }, - } - ], - }, - { - "role": "tool", - "content": '{"temperature": 30, "condition": "cloudy"}', - "tool_call_id": "call_2", - }, - ] - - -# --------------------------------------------------------------------------- -# Intermediate system message variants -# --------------------------------------------------------------------------- - - -class IntermediateSystemTrajectory: - """sys, user, ass(tool), tool, system, ass(tool:date), tool, system, ass(tool), tool — 3 turns with system""" - - TOOLS = ALL_TOOLS - PRETOKENIZE_POSITIONS = [3, 6, 9] - APPEND_ROLES = frozenset({"tool", "system"}) - IS_THINKING = False - MESSAGES = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Do a multi-step task"}, - { - "role": "assistant", - "content": "Step 1", - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "get_weather", - "arguments": {"city": "Beijing"}, - }, - } - ], - }, - { - "role": "tool", - "content": '{"temperature": 25}', - "tool_call_id": "call_1", - }, - {"role": "system", "content": "Step 1 complete. Proceed to step 2."}, - { - "role": "assistant", - "content": "Step 2", - "tool_calls": [ - { - "id": "call_2", - "type": "function", - "function": { - "name": "get_date", - "arguments": {"timezone": "UTC"}, - }, - } - ], - }, - { - "role": "tool", - "content": '{"date": "2025-03-15", "time": "12:00:00"}', - "tool_call_id": "call_2", - }, - {"role": "system", "content": "Step 2 complete. Proceed to step 3."}, - { - "role": "assistant", - "content": "Step 3", - "tool_calls": [ - { - "id": "call_3", - "type": "function", - "function": { - "name": "get_weather", - "arguments": {"city": "Guangzhou"}, - }, - } - ], - }, - { - "role": "tool", - "content": '{"temperature": 35}', - "tool_call_id": "call_3", - }, - ] - - -class IntermediateSystemThinkingTrajectory: - """sys, user, ass(t+tool), tool, system, ass(t+tool:date), tool, system, ass(t+tool), tool — 3 turns with system+thinking""" - - TOOLS = ALL_TOOLS - PRETOKENIZE_POSITIONS = [3, 6, 9] - APPEND_ROLES = frozenset({"tool", "system"}) - IS_THINKING = True - MESSAGES = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Do a multi-step task"}, - { - "role": "assistant", - "reasoning_content": "Starting step 1, checking Beijing weather.", - "content": "Step 1", - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "get_weather", - "arguments": {"city": "Beijing"}, - }, - } - ], - }, - { - "role": "tool", - "content": '{"temperature": 25}', - "tool_call_id": "call_1", - }, - {"role": "system", "content": "Step 1 complete. Proceed to step 2."}, - { - "role": "assistant", - "reasoning_content": "Got Beijing result. Now step 2, checking the date.", - "content": "Step 2", - "tool_calls": [ - { - "id": "call_2", - "type": "function", - "function": { - "name": "get_date", - "arguments": {"timezone": "UTC"}, - }, - } - ], - }, - { - "role": "tool", - "content": '{"date": "2025-03-15", "time": "12:00:00"}', - "tool_call_id": "call_2", - }, - {"role": "system", "content": "Step 2 complete. Proceed to step 3."}, - { - "role": "assistant", - "reasoning_content": "Got date result. Now step 3, checking Guangzhou.", - "content": "Step 3", - "tool_calls": [ - { - "id": "call_3", - "type": "function", - "function": { - "name": "get_weather", - "arguments": {"city": "Guangzhou"}, - }, - } - ], - }, - { - "role": "tool", - "content": '{"temperature": 35}', - "tool_call_id": "call_3", - }, - ] - - -class MultiRoleSequenceTrajectory: - """sys, user, asst+tool, tool, user2, asst+tool, system_reminder, tool, asst-final. - - Fills the {thinking=False, append_roles={tool, user, system}} matrix cell - that GLM47's tool+user+system SUPPORTED_TEMPLATES row otherwise has no - fixture for. Cuts exercise three boundaries: tool-append (N=3), user-after-tool - (N=4), system-after-asst (N=6). Cuts at N=5/N=7/N=8 are intentionally not - listed — they only exercise generation-prompt-only or repeat tool-append - which other trajectories already cover. - """ - - TOOLS = ALL_TOOLS - PRETOKENIZE_POSITIONS = [3, 4, 6] - APPEND_ROLES = frozenset({"tool", "user", "system"}) - IS_THINKING = False - MESSAGES = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What's the weather in Beijing today?"}, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_w1", - "type": "function", - "function": { - "name": "get_weather", - "arguments": {"city": "Beijing", "unit": "celsius"}, - }, - } - ], - }, - { - "role": "tool", - "content": '{"temperature": 22, "condition": "sunny"}', - "tool_call_id": "call_w1", - }, - {"role": "user", "content": "Also tell me the date in Beijing."}, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_d1", - "type": "function", - "function": { - "name": "get_date", - "arguments": {"timezone": "Asia/Shanghai"}, - }, - } - ], - }, - {"role": "system", "content": "Please answer in one short sentence."}, - { - "role": "tool", - "content": '{"date": "2026-04-28"}', - "tool_call_id": "call_d1", - }, - { - "role": "assistant", - "content": "Beijing is 22°C and sunny on 2026-04-28.", - }, - ] - - -# --------------------------------------------------------------------------- -# Helpers: build Trajectory and process_fn from a trajectory class -# --------------------------------------------------------------------------- - - -def _split_turns(messages: list[dict[str, Any]]) -> list[tuple[list[dict], dict]]: - """Split a full message sequence into (request_messages, assistant_message) pairs. - - Each assistant message marks the end of one turn. - """ - turns: list[tuple[list[dict], dict]] = [] - accumulated: list[dict] = [] - - i = 0 - # Collect non-assistant prefix (system, user) - while i < len(messages) and messages[i]["role"] != "assistant": - accumulated.append(messages[i]) - i += 1 - - while i < len(messages): - assert messages[i]["role"] == "assistant", f"Expected assistant at index {i}, got {messages[i]['role']}" - assistant_msg = messages[i] - turns.append((list(accumulated), assistant_msg)) - accumulated.append(assistant_msg) - i += 1 - - # Collect subsequent tool (or user for multi-user) messages - while i < len(messages) and messages[i]["role"] != "assistant": - accumulated.append(messages[i]) - i += 1 - - return turns - - -def build_trajectory( - tokenizer: Any, - trajectory_cls: type, - chat_template: str | None = None, -) -> Trajectory: - """Build a Trajectory from a trajectory class. - - Splits messages into turns and computes response_text for each turn - using the diff method: render(msgs + [assistant], gen_prompt=False) - render(msgs, gen_prompt=True). - - Args: - tokenizer: HuggingFace tokenizer. - trajectory_cls: A trajectory class with MESSAGES and TOOLS attributes. - chat_template: Optional chat template string. If None, uses tokenizer's default. - """ - messages = deepcopy(trajectory_cls.MESSAGES) - tools = trajectory_cls.TOOLS - - raw_turns = _split_turns(messages) - - def _render(msgs: list[dict], add_generation_prompt: bool) -> str: - if chat_template is not None: - from tito_gateway.vendor.miles_compat.utils.chat_template_utils.template import apply_chat_template_from_str - - return apply_chat_template_from_str( - chat_template, msgs, add_generation_prompt=add_generation_prompt, tools=tools - ) - return tokenizer.apply_chat_template( - msgs, tokenize=False, add_generation_prompt=add_generation_prompt, tools=tools - ) - - turns: list[Turn] = [] - for request_msgs, assistant_msg in raw_turns: - prompt_text = _render(request_msgs, add_generation_prompt=True) - with_assistant_text = _render(request_msgs + [assistant_msg], add_generation_prompt=False) - assert with_assistant_text.startswith(prompt_text), ( - f"Assistant text does not extend prompt text.\n" - f"prompt[-100:]: {prompt_text[-100:]!r}\n" - f"with_ass[:len(prompt)+50]: {with_assistant_text[:len(prompt_text)+50]!r}" - ) - response_text = with_assistant_text[len(prompt_text) :] - - turns.append( - Turn( - request_messages=request_msgs, - assistant_message=assistant_msg, - response_text=response_text, - ) - ) - - return Trajectory(tools=tools, turns=turns, full_messages=messages) - - -def build_process_fn( - trajectory: Trajectory, - tokenizer: Any, - chat_template: str | None = None, -) -> ProcessFn: - """Build a process_fn that maps rendered prompt strings to response texts. - - Pre-computes the exact prompt string for each turn and returns the - corresponding response_text. - """ - tools = trajectory.tools - - def _render(msgs: list[dict]) -> str: - if chat_template is not None: - from tito_gateway.vendor.miles_compat.utils.chat_template_utils.template import apply_chat_template_from_str - - return apply_chat_template_from_str(chat_template, msgs, add_generation_prompt=True, tools=tools) - return tokenizer.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True, tools=tools) - - # Build prompt → response mapping - prompt_response_map: dict[str, str] = {} - for turn in trajectory.turns: - prompt_str = _render(turn.request_messages) - prompt_response_map[prompt_str] = turn.response_text - - def process_fn(prompt: str) -> ProcessResult: - for expected_prompt, response_text in prompt_response_map.items(): - if prompt == expected_prompt: - return ProcessResult(text=response_text, finish_reason="stop") - raise ValueError( - f"Unexpected prompt (length={len(prompt)}).\n" - f"Known prompts: {[len(p) for p in prompt_response_map]}\n" - f"Prompt tail: {prompt[-200:]!r}" - ) - - return process_fn - - -class SequentialProcessFn: - """A process_fn that returns response texts in turn order. - - Unlike build_process_fn which does exact prompt matching, this version - simply returns the next turn's response_text on each call. Useful for - e2e tests where the actual prompt may differ from pre-computed prompts - (e.g., tool_call IDs and argument formats differ between trajectory - definitions and mock server responses). - - Call reset() between test runs to restart from the first turn. - """ - - def __init__(self, trajectory: Trajectory): - self._response_texts = [turn.response_text for turn in trajectory.turns] - self._call_count = 0 - - def reset(self): - self._call_count = 0 - - def __call__(self, prompt: str) -> ProcessResult: - idx = self._call_count - if idx >= len(self._response_texts): - raise ValueError( - f"Sequential process_fn exhausted: called {idx + 1} times " - f"but only {len(self._response_texts)} turns defined" - ) - self._call_count += 1 - return ProcessResult(text=self._response_texts[idx], finish_reason="stop") diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/session_verify_agent.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/session_verify_agent.py deleted file mode 100644 index 5b3c55f..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/session_verify_agent.py +++ /dev/null @@ -1,460 +0,0 @@ -"""Custom-generate / custom-agent driver for TITO session-server verification. - -Wired through ``--custom-generate-function-path`` / -``--custom-agent-function-path``; consumed by -``tests/e2e/sglang/test_session_server_multi_role/`` (one test file per -model family) and ``scripts/tools/verify_session_tito_tokenizer.py``. -""" - -from __future__ import annotations - -import json -import logging -import os -from enum import Enum -try: - from enum import StrEnum -except ImportError: - class StrEnum(str, Enum): - pass - -import httpx - -from miles.rollout.base_types import GenerateFnInput, GenerateFnOutput -from miles.rollout.generate_hub.agentic_tool_call import generate as _base_generate - -logger = logging.getLogger(__name__) - - -class DriverAction(Enum): - TOOL_RESULT = "tool_result" - USER_FOLLOWUP = "user_followup" - SYSTEM_REMINDER = "system_reminder" - ROLLBACK = "rollback" - FORCE_FINAL = "force_final" - - -_T = DriverAction.TOOL_RESULT -_U = DriverAction.USER_FOLLOWUP -_S = DriverAction.SYSTEM_REMINDER -_R = DriverAction.ROLLBACK -_F = DriverAction.FORCE_FINAL - - -class ToolCallFailureMode(StrEnum): - """Recovery strategy when a TOOL_RESULT step finds the assistant emitted no tool_calls. - - APPEND_TOOL : Splice a sentinel ``tool`` message and continue. Works on - lenient templates; strict templates that hard-assert any - ``tool`` role must follow an assistant with ``tool_calls`` - (e.g. MiniMax-M2.7) will reject the next request at server-side. - APPEND_USER : Splice a ``user`` message carrying the same failure text as - APPEND_TOOL. Requires "user" in ``allowed_append_roles`` — - raises ValueError at agent start otherwise, so misconfig is - immediately visible instead of silently downgrading. - ROLLBACK : Pop the offending assistant and let the loop's chat call at - the bottom re-inference. Universal — no role-surface - dependency — and the default. - """ - - APPEND_TOOL = "append_tool" - APPEND_USER = "append_user" - ROLLBACK = "rollback" - - -DEFAULT_TOOL_CALL_FAILURE_MODE = ToolCallFailureMode.ROLLBACK - -# Cap consecutive ROLLBACK retries — same context every time, so a model that -# never tool-calls would loop forever. -MAX_CONSECUTIVE_TOOL_CALL_FAILURE_ROLLBACKS = 3 - -# Same body for both APPEND_TOOL and APPEND_USER fallbacks; only the role of -# the spliced message differs between the two modes. -TOOL_CALL_PARSE_FAILURE_TEXT = ( - "Tool call parsing failed: the previous assistant turn did not emit a " - "parseable tool_call. Please retry with a valid tool invocation." -) - -# Mismatch tiers reported by the session-server's per-sample comparator -# (sessions.py:83). Any occurrence of these "hard" types in a sample's -# tito_session_mismatch indicates a TITO bug and fails the sample. The -# soft `assistant_text` tier is excluded — it is aggregated across samples -# and gated by a ratio threshold instead. -_FORBIDDEN_MISMATCH_TYPES: frozenset[str] = frozenset( - {"special_token_count", "special_token_type", "non_assistant_text"} -) - -# Override per call: ``--session-verify-cycles N`` (CLI) or ``cycles=N`` -# (pytest via ``run_session_verify``). Smaller-context models with a 4K -# response budget should drop to 2 to avoid context overflow. -DEFAULT_CYCLES = 3 - -_SUPPORTED_ROLE_SURFACES: tuple[frozenset[str], ...] = ( - frozenset({"tool"}), - frozenset({"tool", "user"}), - frozenset({"tool", "user", "system"}), -) - - -def _build_cycle(role_surface: frozenset[str]) -> list[DriverAction]: - cycle: list[DriverAction] = [_T] - if "user" in role_surface: - cycle.append(_U) - cycle.append(_T) - if "system" in role_surface: - cycle.append(_S) - cycle.append(_R) - return cycle - - -# English-only on purpose: matches the production agentic flows tokenization -# and tool-call parsing are tuned against. -USER_FOLLOWUP_TEXT = "Now check the weather in Shanghai." -SYSTEM_REMINDER_TEXT = "Note: from now on, answer in a single sentence; skip all pleasantries." -FORCE_FINAL_TEXT = "Please summarize all results inside ... tags." - -TOOLS = [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather for a given city.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city name, e.g. Beijing", - }, - }, - "required": ["location"], - }, - }, - }, -] - -MOCK_TOOL_RESULTS = [ - '{"temperature_celsius": 22, "condition": "sunny"}', - '{"temperature_celsius": 15, "condition": "cloudy"}', - '{"temperature_celsius": 30, "condition": "rainy"}', - '{"temperature_celsius": 8, "condition": "snowy"}', -] - - -INITIAL_SYSTEM_PROMPT = ( - "You are a weather assistant. Use the get_weather tool when the user asks " - "about a city's weather. Answer one question at a time and wait for the " - "next user message; do not summarize until the user explicitly asks you " - "to. When asked to summarize, wrap the final summary in " - "... tags." -) -INITIAL_USER_PROMPT = "What's the weather in Beijing?" - - -def select_schedule(allowed_roles, *, cycles: int = DEFAULT_CYCLES) -> list[DriverAction]: - """Pick the schedule for ``frozenset(allowed_roles)``; raises on unregistered.""" - key = frozenset(allowed_roles) - if key not in _SUPPORTED_ROLE_SURFACES: - registered = sorted(sorted(k) for k in _SUPPORTED_ROLE_SURFACES) - raise ValueError(f"No schedule registered for allowed_roles={sorted(key)}. Registered: {registered}") - if cycles < 1: - raise ValueError(f"cycles must be >= 1, got {cycles}") - cycle = _build_cycle(key) - # Extra R after the first cycle exercises consecutive-rollback adjacency, - # which cycle-repeat alone never produces. - schedule = list(cycle) + [_R] + cycle * (cycles - 1) - if "user" in key: - schedule.append(_F) - return schedule - - -def build_initial_messages() -> list[dict]: - """The fixed (system, user) prompt all schedules start from.""" - return [ - {"role": "system", "content": INITIAL_SYSTEM_PROMPT}, - {"role": "user", "content": INITIAL_USER_PROMPT}, - ] - - -async def _chat(client, base_url, messages, request_kwargs, *, label): - payload = {"messages": messages, "tools": TOOLS, **request_kwargs} - resp = await client.post(f"{base_url}/v1/chat/completions", json=payload) - assert resp.status_code == 200, f"{label} failed ({resp.status_code}): {resp.text}" - return resp.json() - - -async def run_agent(base_url, prompt, request_kwargs, metadata, **kwargs): - """Custom-agent entry point. Returns ``{"driver_events": [...], **counters}``. - - ``allowed_append_roles`` must be present in ``metadata`` (the ``generate`` - wrapper below injects it from ``args.tito_allowed_append_roles``). - ``prompt`` is ignored — the driver synthesizes its own initial conversation - from ``build_initial_messages`` so runs are reproducible. - """ - allowed_roles = metadata.get("allowed_append_roles") - if allowed_roles is None: - raise ValueError( - "session_verify_agent.run_agent requires allowed_append_roles in metadata; " - "the generate wrapper should inject it from args.tito_allowed_append_roles." - ) - cycles = metadata.get("session_verify_cycles", DEFAULT_CYCLES) - schedule = select_schedule(allowed_roles, cycles=cycles) - - failure_mode = ToolCallFailureMode(metadata.get("tool_call_failure_mode", DEFAULT_TOOL_CALL_FAILURE_MODE)) - # APPEND_USER injects a user message — only valid if 'user' is in - # allowed_append_roles. Refuse up front instead of silently downgrading. - if failure_mode is ToolCallFailureMode.APPEND_USER and "user" not in allowed_roles: - raise ValueError( - f"tool_call_failure_mode=APPEND_USER requires 'user' in allowed_append_roles, " - f"got {sorted(allowed_roles)}. Pick ROLLBACK (universal) or APPEND_TOOL " - "(lenient-template) for tool-only surfaces." - ) - - rk = {k: v for k, v in request_kwargs.items() if k not in ("tools",)} - messages = build_initial_messages() - events: list[str] = [] - counters = { - "rollback_count": 0, - "user_count": 0, - "system_count": 0, - "tool_result_count": 0, - "tool_call_count": 0, - } - # Streak of TOOL_RESULT steps that fell into the ROLLBACK fallback without - # the model recovering to a real tool_call. Reset on any successful - # tool_call; gated by MAX_CONSECUTIVE_TOOL_CALL_FAILURE_ROLLBACKS to keep - # silently-stuck samples from burning wall-time. - consecutive_failure_rollbacks = 0 - - async with httpx.AsyncClient(timeout=180) as client: - # Initial completion — no driver action yet. - resp = await _chat(client, base_url, messages, rk, label="Initial") - assistant = resp["choices"][0]["message"] - messages.append(assistant) - events.append("initial") - counters["tool_call_count"] += len(assistant.get("tool_calls") or []) - - for step_idx, action in enumerate(schedule): - label = f"Step {step_idx + 1} {action.value}" - - if action is DriverAction.TOOL_RESULT: - tool_calls = assistant.get("tool_calls") or [] - if tool_calls: - consecutive_failure_rollbacks = 0 - for i, tc in enumerate(tool_calls): - result_idx = (counters["tool_result_count"] + i) % len(MOCK_TOOL_RESULTS) - messages.append( - { - "role": "tool", - "content": MOCK_TOOL_RESULTS[result_idx], - "tool_call_id": tc["id"], - } - ) - counters["tool_result_count"] += len(tool_calls) - events.append("append_tool") - else: - # Model emitted no tool_calls — apply the configured fallback. - # Templates differ on what role may follow a "no tool_calls" - # assistant: - # - GLM / Nemotron (lenient): a tool message is fine -> APPEND_TOOL. - # - Kimi: a tool message must carry the id from a valid - # tool_call, which we don't have -> APPEND_TOOL not OK. - # - MiniMax: a tool message must follow an assistant with - # non-empty tool_calls -> APPEND_TOOL not OK. - # If APPEND_TOOL not ok, use APPEND_USER as instead. - match failure_mode: - case ToolCallFailureMode.APPEND_TOOL: - messages.append( - { - "role": "tool", - "tool_call_id": "none", - "content": TOOL_CALL_PARSE_FAILURE_TEXT, - } - ) - events.append("tool_call_failure_append_tool") - case ToolCallFailureMode.APPEND_USER: - messages.append({"role": "user", "content": TOOL_CALL_PARSE_FAILURE_TEXT}) - counters["user_count"] += 1 - events.append("tool_call_failure_append_user") - case ToolCallFailureMode.ROLLBACK: - # Same as the schedule's ROLLBACK - assert messages and messages[-1]["role"] == "assistant", ( - f"tool_call_failure_mode=ROLLBACK: tail role is " - f"{messages[-1]['role'] if messages else 'EMPTY'}, expected assistant" - ) - consecutive_failure_rollbacks += 1 - if consecutive_failure_rollbacks > MAX_CONSECUTIVE_TOOL_CALL_FAILURE_ROLLBACKS: - raise AssertionError( - f"ROLLBACK fallback hit {consecutive_failure_rollbacks} consecutive " - f"tool_call failures (limit={MAX_CONSECUTIVE_TOOL_CALL_FAILURE_ROLLBACKS}). " - "Model is not tool-calling on this prompt — check sampling temperature, " - "the tool spec, or switch tool_call_failure_mode to APPEND_TOOL/APPEND_USER " - "if sentinel-driven retry is preferred." - ) - messages.pop() - counters["rollback_count"] += 1 - events.append("tool_call_failure_rollback") - case _: - raise AssertionError(f"Unknown ToolCallFailureMode {failure_mode!r}") - - elif action is DriverAction.USER_FOLLOWUP: - messages.append({"role": "user", "content": USER_FOLLOWUP_TEXT}) - counters["user_count"] += 1 - events.append("append_user") - - elif action is DriverAction.SYSTEM_REMINDER: - messages.append({"role": "system", "content": SYSTEM_REMINDER_TEXT}) - counters["system_count"] += 1 - events.append("append_system") - - elif action is DriverAction.ROLLBACK: - # Pop the last assistant from our local copy. The next - # request therefore has one fewer message than what the - # server has stored, which is the trigger for its - # ``_detect_and_rollback`` path — the server rewinds its - # state, then re-inferences. - if not messages or messages[-1]["role"] != "assistant": - raise AssertionError( - f"Cannot rollback at step {step_idx}: tail role is " - f"{messages[-1]['role'] if messages else 'EMPTY'}, expected assistant" - ) - messages.pop() - counters["rollback_count"] += 1 - events.append("rollback") - - elif action is DriverAction.FORCE_FINAL: - messages.append({"role": "user", "content": FORCE_FINAL_TEXT}) - events.append("force_final") - - else: - raise AssertionError(f"Unknown DriverAction {action!r}") - - resp = await _chat(client, base_url, messages, rk, label=label) - assistant = resp["choices"][0]["message"] - messages.append(assistant) - counters["tool_call_count"] += len(assistant.get("tool_calls") or []) - - logger.info("Agent done: events=%s counters=%s", events, counters) - - return {"driver_events": events, **counters} - - -async def generate(input: GenerateFnInput) -> GenerateFnOutput: - """Custom-generate wrapper that asserts driver-action coverage. - - - Per-sample: every sample must contain ``rollback``, plus ``append_user`` - / ``append_system`` when those roles are allowed. - - Cross-sample: at least one sample must contain ``append_tool`` - (model-dependent on emitting a tool_call). - """ - allowed_roles = list(input.args.tito_allowed_append_roles) - cycles = getattr(input.args, "session_verify_cycles", DEFAULT_CYCLES) - failure_mode = getattr(input.args, "tool_call_failure_mode", DEFAULT_TOOL_CALL_FAILURE_MODE) - # Sample.metadata is mutable even when the outer dataclass is frozen. - input.sample.metadata["allowed_append_roles"] = allowed_roles - input.sample.metadata["session_verify_cycles"] = cycles - input.sample.metadata["tool_call_failure_mode"] = failure_mode - - output = await _base_generate(input) - - samples = output.samples if isinstance(output.samples, list) else [output.samples] - events_per_sample = [s.metadata.get("driver_events", []) for s in samples] - metrics_path = os.environ.get("MILES_SESSION_VERIFY_METRICS_PATH") - - required_per_sample = ["rollback"] - if "user" in allowed_roles: - required_per_sample.append("append_user") - if "system" in allowed_roles: - required_per_sample.append("append_system") - - for i, events in enumerate(events_per_sample): - missing = [req for req in required_per_sample if req not in events] - if missing: - raise AssertionError( - f"Session multi-role e2e: sample {i} missing required driver events " - f"{missing}. allowed_roles={allowed_roles}, events={events}" - ) - - if not metrics_path and not any("append_tool" in events for events in events_per_sample): - raise AssertionError( - "Session multi-role e2e: no sample produced an append_tool action — " - f"the model may not be tool-calling. events_per_sample={events_per_sample}" - ) - - for i, sample in enumerate(samples): - mismatches = sample.metadata.get("tito_session_mismatch") - if mismatches is None: - raise AssertionError( - f"Session multi-role e2e: sample {i} has no tito_session_mismatch " - f"in metadata. The session-server's compute_session_mismatch raised " - f"TokenizationError (sessions.py:83 swallows it) — this always " - f"indicates a TITO subclass / setup bug, not a real PASS." - ) - forbidden = [m for m in mismatches if m.get("type") in _FORBIDDEN_MISMATCH_TYPES] - if forbidden: - raise AssertionError( - f"Session multi-role e2e: sample {i} has forbidden mismatches " - f"{forbidden}. allowed_roles={allowed_roles}. These types must be 0 " - f"for any TITO-correct setup." - ) - if metrics_path: - assistant_mismatches = [m for m in mismatches if m.get("type") == "assistant_text"] - had_assistant_mismatch = bool(assistant_mismatches) - example = None - if assistant_mismatches: - first = assistant_mismatches[0] - example = { - "segment_index": first.get("segment_index"), - "expected_text": (first.get("expected_text") or "")[:300], - "actual_text": (first.get("actual_text") or "")[:300], - } - with open(metrics_path, "a") as f: - f.write( - json.dumps( - { - "sample_index": i, - "driver_events": events_per_sample[i], - "had_assistant_mismatch": had_assistant_mismatch, - "total_mismatches": len(mismatches), - "assistant_mismatch_count": len(assistant_mismatches), - "assistant_mismatch_example": example, - } - ) - + "\n" - ) - - logger.info( - "Multi-role coverage verified: per_sample=%s, samples=%d, events=%s", - required_per_sample, - len(samples), - events_per_sample, - ) - return output - - -def _add_arguments(parser): - _base_generate.add_arguments(parser) - parser.add_argument( - "--session-verify-cycles", - type=int, - default=DEFAULT_CYCLES, - help="Number of driver schedule cycles per sample for session-server " - "TITO verification. Each cycle exercises every action in the role " - "surface plus a rollback; more cycles stress the TITO accumulator " - "longer but expand context length. Drop to 2 on tighter-context " - "models (e.g. Qwen3 32K with 4K response budget).", - ) - parser.add_argument( - "--tool-call-failure-mode", - type=str, - default=DEFAULT_TOOL_CALL_FAILURE_MODE.value, - choices=[m.value for m in ToolCallFailureMode], - help="Recovery mode when a TOOL_RESULT step sees no tool_calls on the " - "assistant. 'rollback' (default, universal) pops the assistant and " - "re-inferences. 'append_tool' splices a sentinel tool message (only " - "works on lenient templates). 'append_user' splices a user message " - "with the same failure text — requires 'user' in allowed_append_roles.", - ) - - -generate.add_arguments = _add_arguments diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/session_verify_runner.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/session_verify_runner.py deleted file mode 100644 index cea07cf..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/session_verify_runner.py +++ /dev/null @@ -1,332 +0,0 @@ -"""Boot a real ``miles`` rollout pipeline + run the multi-role TITO driver. - -Used by both consumers: - -- pytest e2e: ``tests/e2e/sglang/test_session_server_multi_role/`` (one test - file per model family) -- CLI: ``scripts/tools/verify_session_tito_tokenizer.py`` - -Both forms run the same ``execute_train(--debug-rollout-only)`` path: full miles -pipeline (sglang + miles-router with session support) is launched, ``train`` is -skipped, and the rollout drives ``session_verify_agent.run_agent`` against the -session server. - -Args flow through miles' canonical ``parse_args`` Namespace. - -# Backend choice - -``execute_train`` asserts ``("--train-backend fsdp" in train_args) == (megatron_model_type is None)``, -so the ``fsdp`` + ``None`` pair is the only consistent way to skip megatron init -in ``--debug-rollout-only`` mode. We use that. -""" - -from __future__ import annotations - -import argparse -import json -import logging -import os -import shutil -import tempfile -from typing import Any - -from miles.utils.chat_template_utils import resolve_reasoning_and_tool_call_parser - -logger = logging.getLogger(__name__) - -# Soft cap on how many samples may report any assistant_text mismatch. Hard -# mismatch types (special_token_count / special_token_type / non_assistant_text) -# are asserted per-sample inside the agent wrapper — those must be 0. -ASSISTANT_TEXT_MISMATCH_RATIO_THRESHOLD = 0.2 - -PROMPT_DATA_PATH = "/root/datasets/session_multi_role_verify.jsonl" -LOCAL_MODELS_ROOT = "/root/models" - -# The driver agent synthesizes its own initial conversation, but the rollout -# pipeline still needs a non-empty prompt-data file as input. This placeholder -# matches the agent's own initial prompt so the prompt is well-formed even if -# something downstream inspects it. -_PLACEHOLDER_PROMPT_RECORD = { - "messages": [ - {"role": "system", "content": "You are a weather assistant."}, - {"role": "user", "content": "What's the weather in Beijing?"}, - ], -} - -_PROXY_ENV_VARS = ("http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY") - -SESSION_VERIFY_INVARIANT_ARGS: dict[str, Any] = { - "prompt_data": PROMPT_DATA_PATH, - "input_key": "messages", - "num_rollout": 1, - "rollout_batch_size": 16, - "rollout_max_response_len": 8192, - "rollout_temperature": 0.7, - "global_batch_size": 64, - "rm_type": "random", - "custom_generate_function_path": "miles.utils.test_utils.session_verify_agent.generate", - "custom_agent_function_path": "miles.utils.test_utils.session_verify_agent.run_agent", - "use_session_server": True, - "debug_rollout_only": True, - "ci_test": True, - "colocate": True, - "train_backend": "fsdp", - "sglang_expert_parallel_size": 1, -} - - -def _command_utils() -> Any: - """Load Miles training command helpers only for the optional e2e path.""" - try: - import miles.utils.external_utils.command_utils as command_utils - except Exception as exc: - raise RuntimeError( - "verify-session-tito-tokenizer requires Miles training/e2e command " - "utilities. Install the optional Miles/SGLang training stack before " - "running the session verifier." - ) from exc - return command_utils - - -def session_verify_extras(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: - """``add_custom_arguments`` hook for ``miles.utils.arguments.parse_args``. - - Adds the wrapper-only ``--assistant-text-threshold`` knob (a post-process - gate on the per-sample metrics JSONL, NOT in ``train_args``) and applies - session-verify invariants as parser defaults — user CLI still overrides - these via the canonical miles flags. - """ - parser.add_argument( - "--assistant-text-threshold", - type=float, - default=ASSISTANT_TEXT_MISMATCH_RATIO_THRESHOLD, - help=( - f"Soft threshold for assistant_text mismatch ratio. Default {ASSISTANT_TEXT_MISMATCH_RATIO_THRESHOLD}. " - "Raise to 1.0 for families whose upstream sglang reasoning parser " - "is known to roundtrip imperfectly (e.g. nemotron_3 keeps a " - "trailing newline in reasoning_content) — hard mismatches still " - "gate. Post-process gate on per-sample JSONL metrics; not " - "forwarded to ``train_args``." - ), - ) - parser.set_defaults(**SESSION_VERIFY_INVARIANT_ARGS) - return parser - - -def _ensure_prompt_data() -> str: - os.makedirs(os.path.dirname(PROMPT_DATA_PATH), exist_ok=True) - with open(PROMPT_DATA_PATH, "w") as f: - f.write(json.dumps(_PLACEHOLDER_PROMPT_RECORD) + "\n") - return PROMPT_DATA_PATH - - -def _ensure_model_downloaded(hf_checkpoint: str) -> str: - """Return a local model path, downloading HF repos when needed. - - Lets callers pass either a HuggingFace repo id (downloaded under - ``/root/models/``) or an existing local checkpoint path - (returned as-is, no download). - """ - if os.path.exists(hf_checkpoint): - return hf_checkpoint - - short = hf_checkpoint.split("/")[-1] - local_dir = os.path.join(LOCAL_MODELS_ROOT, short) - os.makedirs(LOCAL_MODELS_ROOT, exist_ok=True) - _command_utils().exec_command(f"hf download {hf_checkpoint} --local-dir {local_dir}") - return local_dir - - -def _clear_proxy_env() -> dict[str, str | None]: - previous = {proxy_var: os.environ.get(proxy_var) for proxy_var in _PROXY_ENV_VARS} - for proxy_var in _PROXY_ENV_VARS: - os.environ.pop(proxy_var, None) - return previous - - -def _restore_proxy_env(previous: dict[str, str | None]) -> None: - for proxy_var, value in previous.items(): - if value is None: - os.environ.pop(proxy_var, None) - else: - os.environ[proxy_var] = value - - -def namespace_to_train_args(ns: argparse.Namespace) -> str: - """Serialize a fully-shaped Namespace into the ``train_args`` string. - - Reads miles-canonical field names off ``ns``; emits the exact flag set - ``execute_train`` re-parses downstream. ``actor_num_nodes`` is written - explicitly from ``ns.actor_num_nodes`` (NOT defaulted at the serializer - level) so the runner stays pinned to whatever the caller's Namespace - declared, regardless of any drift in miles' upstream default. - """ - allowed_roles_arg = " ".join(ns.tito_allowed_append_roles) - parts: list[str] = [ - f"--hf-checkpoint {ns.hf_checkpoint}", - f"--prompt-data {ns.prompt_data}", - f"--input-key {ns.input_key}", - f"--num-rollout {ns.num_rollout}", - f"--rollout-batch-size {ns.rollout_batch_size}", - f"--n-samples-per-prompt {ns.n_samples_per_prompt}", - f"--rollout-max-response-len {ns.rollout_max_response_len}", - f"--rollout-temperature {ns.rollout_temperature}", - f"--global-batch-size {ns.global_batch_size}", - f"--custom-generate-function-path {ns.custom_generate_function_path}", - f"--custom-agent-function-path {ns.custom_agent_function_path}", - f"--session-verify-cycles {ns.session_verify_cycles}", - f"--tool-call-failure-mode {ns.tool_call_failure_mode}", - f"--tito-model {ns.tito_model}", - f"--tito-allowed-append-roles {allowed_roles_arg}", - f"--rollout-num-gpus-per-engine {ns.rollout_num_gpus_per_engine}", - f"--sglang-reasoning-parser {ns.sglang_reasoning_parser}", - f"--rm-type {ns.rm_type}", - f"--actor-num-nodes {ns.actor_num_nodes}", - f"--actor-num-gpus-per-node {ns.actor_num_gpus_per_node}", - f"--train-backend {ns.train_backend}", - ] - if ns.sglang_tool_call_parser: - parts.append(f"--sglang-tool-call-parser {ns.sglang_tool_call_parser}") - # DeepSeek V3.2 (and other NSA/MoE archs) requires expert-parallel > 1 in - # sglang; the default is 1, which is fatal at engine init. Only emit the - # flag when the caller asks for ep>1 so single-expert models stay untouched. - if ns.sglang_expert_parallel_size > 1: - parts.append(f"--sglang-expert-parallel-size {ns.sglang_expert_parallel_size}") - if ns.use_session_server: - parts.append("--use-session-server") - if ns.debug_rollout_only: - parts.append("--debug-rollout-only") - if ns.ci_test: - parts.append("--ci-test") - if ns.colocate: - parts.append("--colocate") - return " ".join(parts) + " " - - -def run_session_verify(args: argparse.Namespace) -> None: - """Boot ``miles`` rollout pipeline and run the multi-role driver. - - Returns nothing on success; raises ``AssertionError`` on TITO mismatch - (HTTP 500 from server-side prefix check) or coverage shortfall (raised by - ``session_verify_agent.generate``). - - ``args`` MUST be a fully-shaped Namespace carrying miles-canonical field - names plus the session-verify-specific fields (``session_verify_cycles``, - ``tool_call_failure_mode``, ``assistant_text_threshold``). Build it via - ``parse_args(add_custom_arguments=session_verify_extras)`` for the CLI - path or by spreading ``SESSION_VERIFY_INVARIANT_ARGS`` into - ``argparse.Namespace(...)`` for tests. - - Mutates ``args`` in three places before composing train_args: - - ``args.sglang_reasoning_parser`` / ``args.sglang_tool_call_parser`` are - resolved against the TITO subclass's bound values via - ``resolve_reasoning_and_tool_call_parser`` — caller-passed values that - disagree with the bound values raise ``ValueError`` here, before any - GPU work starts. - - ``args.hf_checkpoint`` is replaced with the local download path so the - composed train_args points at the downloaded model, not the HF id. - - ``args.tito_allowed_append_roles`` is normalized (lowercase, dedup, - ensure ``'tool'`` is in) to match the schedule contract in - ``session_verify_agent._SUPPORTED_ROLE_SURFACES``. - """ - args.sglang_reasoning_parser, args.sglang_tool_call_parser = resolve_reasoning_and_tool_call_parser( - args.tito_model, args.sglang_reasoning_parser, args.sglang_tool_call_parser - ) - args.tito_allowed_append_roles = sorted(set(r.lower() for r in args.tito_allowed_append_roles) | {"tool"}) - - _ensure_prompt_data() - proxy_env = _clear_proxy_env() - try: - args.hf_checkpoint = _ensure_model_downloaded(args.hf_checkpoint) - - train_args = namespace_to_train_args(args) - - # Per-sample token-seq metrics file: rollout workers append one JSONL line - # per sample inside session_verify_agent.generate; we aggregate after - # execute_train returns to apply the assistant_text soft threshold. - metrics_fd, metrics_path = tempfile.mkstemp(prefix="session_verify_metrics_", suffix=".jsonl") - os.close(metrics_fd) - - try: - _command_utils().execute_train( - train_args=train_args, - num_gpus_per_node=args.actor_num_gpus_per_node, - megatron_model_type=None, - extra_env_vars={ - "MILES_EXPERIMENTAL_ROLLOUT_REFACTOR": "1", - "MILES_TITO_MODEL": args.tito_model, - "MILES_SESSION_VERIFY_METRICS_PATH": metrics_path, - }, - ) - try: - assert_session_verify_metrics(metrics_path, assistant_text_threshold=args.assistant_text_threshold) - except AssertionError: - preserved_metrics_path = metrics_path + ".failed" - shutil.copy(metrics_path, preserved_metrics_path) - logger.error("Preserved per-sample mismatch payloads at %s for post-mortem", preserved_metrics_path) - raise - finally: - try: - os.unlink(metrics_path) - except OSError: - pass - finally: - _restore_proxy_env(proxy_env) - - -def assert_session_verify_metrics(metrics_path: str, *, assistant_text_threshold: float) -> None: - """Read per-sample JSONL metrics and assert cross-sample verifier gates. - - Forbidden mismatch types (special_*, non_assistant_text) are caught - per-sample in the agent wrapper and would have already raised by now. - Here we only cross-check the soft assistant_text rate against the - caller-provided threshold (per-model: some upstream sglang reasoning - parsers — notably ``nemotron_3`` — leave a trailing ``\\n`` in - ``reasoning_content`` that breaks the canonical roundtrip until the - parser is patched, so those families ride at threshold=1.0). - """ - samples_with_mismatch = 0 - total_samples = 0 - has_append_tool = False - with open(metrics_path) as f: - for line in f: - line = line.strip() - if not line: - continue - entry = json.loads(line) - total_samples += 1 - has_append_tool = has_append_tool or "append_tool" in entry.get("driver_events", []) - if entry.get("had_assistant_mismatch"): - samples_with_mismatch += 1 - - if total_samples == 0: - raise AssertionError( - f"Session multi-role e2e: no per-sample metrics found at {metrics_path}. " - "Either the rollout produced 0 samples, or the agent wrapper failed to " - "run before any sample completed. Check rollout logs." - ) - - if not has_append_tool: - raise AssertionError( - "Session multi-role e2e: no sample produced an append_tool action — " - "the model may not be tool-calling. Check sampling temperature, " - "the tool spec, or parser configuration." - ) - - ratio = samples_with_mismatch / total_samples - logger.info( - "Token-seq metric summary: samples=%d, with_assistant_text_mismatch=%d, ratio=%.3f, threshold=%.3f", - total_samples, - samples_with_mismatch, - ratio, - assistant_text_threshold, - ) - if ratio > assistant_text_threshold: - raise AssertionError( - f"Session multi-role e2e: assistant_text mismatch ratio " - f"{samples_with_mismatch}/{total_samples}={ratio:.3f} exceeds " - f"threshold {assistant_text_threshold}. TITO " - "tokenization for assistant content has drifted from the chat " - "template's canonical render — investigate via " - "verify_session_tito_tokenizer.py + sample-level mismatch logs." - ) diff --git a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/uvicorn_thread_server.py b/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/uvicorn_thread_server.py deleted file mode 100644 index 904343c..0000000 --- a/sidecars/tito/tito_gateway/vendor/miles_compat/utils/test_utils/uvicorn_thread_server.py +++ /dev/null @@ -1,49 +0,0 @@ -import asyncio -import socket -import threading -import time - -import uvicorn - - -class UvicornThreadServer: - def __init__(self, app, host: str, port: int): - self._app = app - self.host = host - self.port = port - self._server: uvicorn.Server | None = None - self._thread: threading.Thread | None = None - - @property - def url(self) -> str: - return f"http://{self.host}:{self.port}" - - def start(self) -> None: - config = uvicorn.Config(self._app, host=self.host, port=self.port, log_level="info") - self._server = uvicorn.Server(config) - - def run() -> None: - asyncio.run(self._server.serve()) - - self._thread = threading.Thread(target=run, daemon=True) - self._thread.start() - self._wait_for_port_open() - - def stop(self) -> None: - if self._server is not None: - self._server.should_exit = True - if self._thread is not None and self._thread.is_alive(): - self._thread.join(timeout=2.0) - - def _wait_for_port_open(self) -> None: - for _ in range(50): - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - result = sock.connect_ex((self.host, self.port)) - sock.close() - if result == 0: - return - except Exception: - pass - time.sleep(0.1) - raise RuntimeError(f"Failed to start server on {self.url}") diff --git a/sidecars/tito/tito_gateway/verify_chat_template.py b/sidecars/tito/tito_gateway/verify_chat_template.py deleted file mode 100644 index 5e4ffc0..0000000 --- a/sidecars/tito/tito_gateway/verify_chat_template.py +++ /dev/null @@ -1,125 +0,0 @@ -"""CLI implementation for Miles chat-template append-only verification.""" - -from __future__ import annotations - -from typing import Any - - -def run_from_args(args: Any) -> int: - """Run the vendored Miles chat-template verifier from an argparse namespace.""" - if args.model is None and args.template is None: - raise ValueError("one of --model or --template is required") - if args.tito_model is not None and args.model is None: - raise ValueError("--tito-model requires --model so the TITO verifier can load the tokenizer") - - extra_template_kwargs = dict(args.chat_template_kwargs or {}) - allowed_roles = set(args.tito_allowed_append_roles) | {"tool"} - use_tito_instance = args.tito_model is not None - - if use_tito_instance: - from miles.utils.chat_template_utils import resolve_fixed_chat_template - from miles.utils.processing_utils import load_tokenizer - - fixed_path, resolved_kwargs = resolve_fixed_chat_template(args.tito_model, sorted(allowed_roles)) - for key, value in resolved_kwargs.items(): - if key in extra_template_kwargs: - continue - extra_template_kwargs[key] = value - print(f"Auto-set --chat-template-kwargs {key}={value!r} (from --tito-model={args.tito_model})") - - template_path = args.template or fixed_path - tokenizer = load_tokenizer(args.model, chat_template_path=template_path, trust_remote_code=True) - if args.template: - source_desc = f"template override via TITO: {args.template}" - elif fixed_path: - source_desc = f"fixed template via TITO: {fixed_path}" - elif getattr(tokenizer, "chat_template", None) is not None: - source_desc = f"HuggingFace via TITO: {args.model}" - else: - source_desc = f"TITO encoder: {args.tito_model}" - chat_template = None - elif args.template: - with open(args.template) as f: - chat_template = f.read() - source_desc = f"file: {args.template}" - tokenizer = None - else: - from miles.utils.chat_template_utils.template import load_hf_chat_template - - chat_template = load_hf_chat_template(args.model) - source_desc = f"HuggingFace: {args.model}" - tokenizer = None - - from miles.utils.test_utils.chat_template_verify import ( - ALL_CASES, - check_coverage, - run_all_checks, - run_all_checks_via_tito, - select_cases, - ) - - is_thinking_filter = {"off": False, "on": True, "both": None}[args.thinking] - selected = select_cases(allowed_append_roles=allowed_roles, is_thinking=is_thinking_filter) - - print(f"Template source: {source_desc}") - print(f"Allowed append roles: {sorted(allowed_roles)}") - print(f"Thinking mode: {args.thinking}") - if extra_template_kwargs: - print(f"Template kwargs: {extra_template_kwargs}") - print(f"Selected trajectories: {len(selected)} of {len(ALL_CASES)} (after filtering)") - print() - - coverage = check_coverage() - if coverage.missing: - print("Trajectory coverage gaps ((thinking, append_roles \\ {tool}) with no trajectory):") - for is_thinking, roles in coverage.missing: - label = "thinking " if is_thinking else "non-thinking" - roles_str = "{" + ", ".join(roles) + "}" if roles else "{}" - print(f" - {label} x {roles_str}") - print() - - if use_tito_instance: - results = run_all_checks_via_tito( - tokenizer, - args.tito_model, - allowed_append_roles=allowed_roles, - thinking=args.thinking, - extra_template_kwargs=extra_template_kwargs, - ) - else: - results = run_all_checks( - chat_template, - allowed_append_roles=allowed_roles, - thinking=args.thinking, - extra_template_kwargs=extra_template_kwargs, - ) - - passed = sum(1 for r in results if r.passed) - failed = sum(1 for r in results if not r.passed) - max_name_len = max((len(r.case_name) for r in results), default=0) - - for r in results: - status = "PASS" if r.passed else "FAIL" - line = f" [{status}] {r.case_name:<{max_name_len}}" - if r.error: - first_line = r.error.split("\n")[0] - if len(first_line) > 80: - first_line = first_line[:77] + "..." - line += f" -- {first_line}" - print(line) - - print() - print(f"Results: {passed}/{len(results)} passed, {failed} failed") - - if failed: - if use_tito_instance: - print("\nVerdict: FAIL - TITO incremental tokenization did NOT match standard render") - else: - print("\nVerdict: FAIL - template is NOT append-only after last user message") - return 1 - - if use_tito_instance: - print("\nVerdict: PASS - TITO incremental tokenization matched standard render") - else: - print("\nVerdict: PASS - template IS append-only after last user message") - return 0 diff --git a/sidecars/tito/tito_gateway/verify_session_tito_tokenizer.py b/sidecars/tito/tito_gateway/verify_session_tito_tokenizer.py deleted file mode 100644 index 79de7cb..0000000 --- a/sidecars/tito/tito_gateway/verify_session_tito_tokenizer.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Dependency-gated entrypoint for Miles session-server TITO e2e verification.""" - -from __future__ import annotations - -from typing import Any - - -def run_from_args(args: Any) -> int: - """Run the optional session TITO verifier when its e2e stack is available.""" - try: - from miles.utils.test_utils.session_verify_runner import run_session_verify - except Exception as exc: - print( - "verify-session-tito-tokenizer requires the optional Miles/SGLang " - "session e2e runner.", - ) - print(f"Missing runner detail: {type(exc).__name__}: {exc}") - return 2 - - try: - run_session_verify(args=args) - except AssertionError as exc: - print("verify-session-tito-tokenizer failed TITO/session verification.") - print(f"Verification detail: {exc}") - return 1 - except (ImportError, ModuleNotFoundError, RuntimeError, FileNotFoundError, OSError, ValueError) as exc: - print( - "verify-session-tito-tokenizer requires the optional Miles/SGLang " - "training stack and a valid verifier configuration.", - ) - print(f"Dependency/runtime detail: {type(exc).__name__}: {exc}") - return 2 - return 0 From 12a022bd5f1238d01030370046f3d1e559e20bed Mon Sep 17 00:00:00 2001 From: FatPigeorz Date: Wed, 1 Jul 2026 00:11:14 +0800 Subject: [PATCH 10/11] chore: remove docs --- ARCHITECTURE.md | 208 ------------------------------------------------ REFACTOR.md | 132 ------------------------------ ROADMAP.md | 164 -------------------------------------- 3 files changed, 504 deletions(-) delete mode 100644 ARCHITECTURE.md delete mode 100644 REFACTOR.md delete mode 100644 ROADMAP.md diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md deleted file mode 100644 index 8764d45..0000000 --- a/ARCHITECTURE.md +++ /dev/null @@ -1,208 +0,0 @@ -# Agentix Architecture - -Agentix is a framework for **agent evaluation**, **RL rollout -execution**, and **training-data collection**. Host-side trainers and -eval scripts orchestrate sandboxes; sandbox-side code is ordinary Python -(agents, bash, scorers). [`abridge`](plugins/abridge/README.md) correlates -traces into rollout logs for RL buffers. The design goal is a lower -integration tax than bespoke rollout servers — see the README comparison -with [ProRL-Agent-Server](https://github.com/NVIDIA-NeMo/ProRL-Agent-Server). - -## The two pieces - -Everything reduces to two operations, and the split between them is the -whole mental model: - -1. **Bundle** — `agentix build [path]` packages one Python project (the - framework, your code, integration modules, dependencies, optional - system binaries) into one deploy-ready runtime image. *The bundle - decides what code and dependencies exist in the sandbox.* -2. **Remote call** — `client.remote(fn, ...)` runs a Python callable - inside that image from host-side Python and returns its value. *The - remote call decides which callable runs.* - -```text -Bundle = what code and dependencies exist in the sandbox -client.remote(fn) = which importable function to call -Worker = where user code executes -agentix.sio = host ↔ sandbox side channels (trace, log, plugins) -SandboxProvider = where the bundle image runs -``` - -## Programming model - -Pass a normal Python callable. The provider hands you a `Sandbox` with a -`remote(...)` method; `RuntimeClient` is the lower-level handle it wraps. - -```python -from app import run - -async with provider.session(config) as sandbox: - result = await sandbox.remote(run, input="hello") -``` - -Importing the module first gives Agentix the same callable object: - -```python -import app - -result = await sandbox.remote(app.run, input="hello") -``` - -The host encodes the callable as an import-path `RemoteCallable` -(`module::qualname`). Lambdas, bound methods, partials, and other -non-importable callables are rejected at the host before the call leaves. - -## Bundle - -`agentix build [path]` takes one Python project and produces a -deploy-ready image. - -```text -my-project/ -├── pyproject.toml -├── src/app.py -└── default.nix # optional, for system binaries -``` - -Python dependencies come from the project's `pyproject.toml` — installing -the project pulls in everything the sandbox needs: - -```toml -[project] -name = "my-project" -version = "0.1.0" -dependencies = [ - "agentixx>=0.1.0", - "agentix-runtime-basic>=0.1.0", # agentix.bash, file ops - "agentix-dataset-swe>=0.1.0", # agentix.plugins.datasets.swe -] -``` - -The build splits along one hard line — **uv owns Python, Nix owns system -binaries; there is no uv2nix.** Inside the build container Agentix creates -the runtime venv and installs the full (non-editable) dependency closure -with uv: - -```bash -uv venv /nix/runtime/venv -uv sync # the project + direct + transitive deps + integration modules -``` - -If the project ships a `default.nix`, a Nix builder stage materializes its -derivation closure and symlinks `bin/*` into `/nix/runtime/bin`. The -result is one merged tree, mounted at `/nix`: - -```text -/nix/runtime/ -├── bootstrap.sh # container entry point (provider backends exec this) -├── bin/ # symlinkJoin of every Nix closure (e.g. git, rg) -└── venv/ - └── lib/python3.11/site-packages/ - ├── agentix/ - ├── agentix/bash/ - ├── agentix/plugins/datasets/swe/ - └── app.py -``` - -Worker processes inherit the runtime-server environment, with the bundle -venv and Nix bins prepended to `PATH`: - -```text -/nix/runtime/venv/bin:/nix/runtime/bin:${PATH} -``` - -So sandbox code can call tools by name: - -```python -await asyncio.create_subprocess_exec("git", "status") -await asyncio.create_subprocess_exec("claude", "-p", instruction) -``` - -## Remote calls - -`sandbox.remote(fn, ...)` runs one callable in the sandbox and returns its -value. The host: - -1. builds a `RemoteCallable` from `fn.__module__` and `fn.__qualname__` -2. pickles `(args, kwargs)` with stdlib pickle -3. sends both over Socket.IO on the `/` namespace - -```python -from agentix.plugins.datasets import swe - -score = await sandbox.remote(swe.score, instance=inst, patch=patch) -``` - -becomes a wire payload like: - -```python -{ - "call_id": "…uuid…", - "callable": "agentix.plugins.datasets.swe::score", - "arguments": pickle.dumps(((), {"instance": inst, "patch": patch})), -} -``` - -Sync and async functions both work as targets; the worker awaits when the -return value is awaitable. Args and return values round-trip as pickle -blobs — the runtime does not run pydantic validation on the wire today. - -## Flow - -```text -Host - sandbox.remote(fn, ...) - RemoteCallable._resolve(fn) -> module::qualname - pickle.dumps((args, kwargs)) - | - v Socket.IO `/` — call / call:result / call:error / cancel -Sandbox - /nix/runtime/bootstrap.sh -> uvicorn -> agentix.runtime.server.app:app - | - v length-prefixed msgpack frames on a private pipe -Single runtime worker process - RemoteCallable.resolve() -> import fn - pickle.loads(arguments) - call fn(*args, **kwargs) (awaiting when needed) - pickle.dumps(result) -``` - -Side channels share the same Socket.IO connection: - -- `/trace` — span lifecycle from sandbox to host -- `/log` — stdlib logging records from sandbox to host -- `/` — plugin namespaces registered via `agentix.sio` - -## Worker model - -The runtime server owns **one** worker subprocess that handles all remote -calls. The worker uses the same `/nix/runtime` venv as the server, so -anything installed into the bundle can be imported. For each call it: - -1. resolves the `RemoteCallable` import path -2. unpickles `(args, kwargs)` -3. calls the callable (awaiting when needed) -4. pickles the return value - -The single-worker model is intentional for now — it keeps runtime state -and debugging simple while the public API settles. It is an -implementation detail: future runtimes may use worker pools or per-call -isolation without changing `sandbox.remote(...)`. - -## End-to-end example - -```python -from agentix.bash import run as bash_run -from agentix.plugins.datasets import swe -from my_project.tasks import generate_patch - -async with provider.session(config) as sandbox: - await sandbox.remote(bash_run, command="git clone ...") - patch = await sandbox.remote(generate_patch, prompt="fix the bug") - score = await sandbox.remote(swe.score, patch=patch) -``` - -All three calls run inside the same bundle image. They target different -modules, but those modules all come from the same installed runtime -environment. diff --git a/REFACTOR.md b/REFACTOR.md deleted file mode 100644 index f7baac0..0000000 --- a/REFACTOR.md +++ /dev/null @@ -1,132 +0,0 @@ -# Convergence refactor — what we're building - -Design baseline. Some items done, the rest planned (see Work below). - -## Principles - -- **Programming framework.** Make `async def rollout(sandbox, task) -> R` - pleasant and typed. The hero is the user's rollout function. -- **Two surfaces.** Typed code (rollout authors: `Sandbox` / - `SandboxConfig` / `SandboxProvider` / `remote()` / plugins / `trace`) - and a string CLI (`agentix build` / `agentix deploy` + registries). - `deploy` is a CLI process; the bundle ref enters code as a `str`; - providers are constructed by typed import. -- **Type what we own.** Public API, `Result[T]`, plugin contracts, and - build-generated stubs are fully typed by construction. - -## Reliability contract - -The framework guarantees the integrity of *information about* each call. - -- Every `remote()` resolves to `Result` = **`Ok | Failed`** — a - truthful, unique terminal state. It never hangs and runs `fn` - at-most-once. -- **`resume`** re-fetches a terminal state (no re-run). **`retry`** is a - new `call_id` (a new rollout); the caller owns retry. - -Per channel: - -| Channel | Guarantee | -|---|---| -| **RPC** | at-most-once execution; terminal state delivered exactly once; an undeliverable result is a `Failed` | -| **trace** (the dataset) | at-least-once + host-side dedup + durable sink | -| **log** | durable on disk + best-effort live stream | - -## Target architecture - -- **One transport:** Socket.IO `/rpc`. HTTP serves `/health`. -- **Namespaces** stay pipe-forwarded and plugin-agnostic. -- **RPC and trace** each carry their reliability; share one retain/ack - buffer where it is genuinely cleaner. -- **log** is stdout/stderr line capture (per-sandbox file + best-effort - stream on the `/log` namespace) — a plain forwarded namespace, no - `ReliableStream` and no structured `LogRecord` bridge. -- **One typed plugin primitive** (below). -- **Typing is Python-native:** ParamSpec + `Result[T]` + build-time - codegen. The build step is our compile step. - -## Plugin primitive - -One declarative, typed surface: define the contract once, implement one -side, call typed from the other. Default is request/response; streaming -is an explicit opt-in. - -```python -# contract.py — one typed interface, shared by both sides -class Solve(BaseModel): task: str -class Solution(BaseModel): patch: str - -# host side — implement, fully typed -class Solver(Plugin): - name = "solver" # -> /solver - @on - async def solve(self, req: Solve) -> Solution: # typed in/out - return Solution(patch=await call_model(req.task)) - - @stream # at-least-once opt-in - async def span(self, ev: SpanEvent) -> None: ... - -# sandbox target fn — typed call -async def rollout(): - sol = await plugins.solver.solve(Solve(task="...")) - return sol.patch -``` - -- One base (`Plugin`), one registration - (`provider.session(..., plugins=[Solver()])`). -- The method *is* the op (decorator-discovered); pydantic payloads; - framework-managed `request_id` and error envelope. -- abridge is a specialization on top (op names = HTTP paths + an - in-sandbox FastAPI tunnel). -- Decide: the sandbox-side caller handle stays typed via a - contract-typed handle (preferred) over a metaclass-routed one class. - -## Build-time typed remote - -`@remote` annotation → `agentix build` emits typed client stubs + a -closed dispatch manifest + a schema'd codec. The generated boundary is -fully typed by construction. `sandbox.remote(fn, ...)` keeps its -ParamSpec `(P) -> R`; codegen adds closed-set validation, a real schema, -and (when wanted) non-Python clients. - -## Work - -### Done — branch `refactor/single-transport` - -1. **Single transport.** Removed the HTTP `/call` fast-path; every - `c.remote()` rides Socket.IO `/rpc`. ruff + pyright clean; 285 tests - pass. -2. **No silent loss.** A `resume` for an evicted/unknown `call_id` - returns a definite `call:error` (`ResultUnavailable`); terminal - states are `{result, error}`. -3. **Never-hang guarantee satisfied.** The worker already fails every - in-flight call on death (`WorkerProcessExited`), fails fast on a - closed worker, turns an oversized result frame into a `FrameTooLarge` - error, and cancels idempotently — `remote()` always reaches a - terminal state (or `CallTimeout`). -4. **Narrowed the top-level surface.** Moved `providers`, - `register_provider`, `BundleDeployer`, `DeployedBundle` off - `agentix.__all__` (still in `agentix.provider.base`); stripped the - codec to plain msgpack; collapsed the reconnection options to - socketio's defaults. -5. **`Result[T]` API.** `Ok | Failed` in `agentix.runtime.client.result`, - exported at top level; `remote()` still raises, `try_remote()` - returns `Result[R]` for exhaustive `match`. -6. **log → Ray-style capture.** The worker captures its stdout *and* - stderr (stdlib `logging` writes to stderr, so it's captured too), - appends to a sandbox-side `sandbox.log`, and streams each line - best-effort on `/log`; the host replays under `agentix.sandbox.{stdout, - stderr}`. Deleted the structured `LogRecord` bridge (`WorkerLogHandler`, - `emit_worker_record`, the host `_replay_record`) and `/log`'s - `ReliableStream` use. `configure_logging` stays as the local-logging - helper (host/runtime/worker). - -### To do — in order - -1. **trace reshape.** Prompt per-span at-least-once emit + host durable - sink. -2. **Shared retain/ack buffer** for RPC + trace, where it is cleaner. -3. **Plugin primitive.** Typed `Plugin` + `@on` / `@stream`; abridge as - a specialization. -4. **Build-time typed remote bindings.** `@remote` → build emits stubs + - manifest + schema codec. diff --git a/ROADMAP.md b/ROADMAP.md deleted file mode 100644 index ba82fb6..0000000 --- a/ROADMAP.md +++ /dev/null @@ -1,164 +0,0 @@ -# Roadmap - -Agentix keeps two user-facing concepts: - -- **Remote calls.** `c.remote(fn, ...)` calls a callable target inside a - sandbox. The callable is encoded as an import-path `RemoteCallable`; - args and kwargs travel as a pickle blob. -- **Bundle.** `agentix build [path]` packages one project root and its - declared dependencies into a deploy-ready runtime image. - -Everything below should preserve that surface. Internal worker topology, -transport choice, and provider backend details should remain opaque to -downstream users of the library. - -## v0.1.0 — RPC + Bundle - -Current architecture: - -- [x] `RuntimeClient.remote(fn, ...)` runs an importable callable in the - sandbox and returns its value. -- [x] One runtime server per sandbox image. -- [x] One worker subprocess per runtime server. -- [x] Import-path `RemoteCallable` for function identity; pickle for - args, kwargs, and return values. -- [x] Callable invocation inside `agentix.runtime.server`; targets are not - required to be pure functions. If Python can resolve the callable - from the requested target, Agentix should be able to invoke it. -- [x] Single-spec `agentix build`; integrations arrive through normal - Python dependencies. -- [x] One merged `/nix/runtime` venv containing the framework, user - project, integrations, and transitive dependencies. -- [x] SandboxProvider backend plugin axis via `agentix.provider`. -- [x] Side channels over the same Socket.IO connection: `/trace`, `/log`, - and plugin namespaces via `agentix.sio`. - -The single-worker model is intentional for now. It keeps runtime state -and debugging simple while the public API is still being shaped. - -## Architectural Direction - -### Worker Model - -Keep one worker process as the default near-term runtime model. - -Future improvements may add: - -- worker pools -- per-call worker isolation -- concurrency limits -- CPU-bound call offloading -- restart and health policies - -These changes must be opaque to downstream users. Code written as: - -```python -result = await client.remote(run, input="hello") -``` - -should not change if the runtime later moves from one worker to many -workers. - -### Callable Targets - -Agentix should not require targets to be pure functions. - -The runtime may call any resolved callable target, including callables -that close over module state, mutate sandbox-local state, call CLIs, -read/write files, or interact with benchmark harnesses. Purity is a user -or integration concern, not a framework constraint. - -The framework's responsibility is narrower: - -- encode importable callables as `RemoteCallable` -- unpickle args/kwargs and invoke the target inside the sandbox -- pickle the return value back -- surface errors in-band through the runtime protocol - -Future work may add optional annotation-driven validation/coercion on -top of pickle without changing the default path. - -### Transport Strategy - -`c.remote()` and side channels share one Socket.IO connection. HTTP is -kept only for `/health` and the internal `/call` fast-path used by -`RuntimeClient.remote` to skip a SIO round-trip for short-running -calls. - -`c.remote()` uses the `/` namespace (`call`, `call:result`, -`call:error`, `cancel`, plus `resume`/`ack` for reconnect-safe -delivery). Trace, log, and plugin traffic use dedicated namespaces -bridged through the worker pipe via `agentix.sio`; the core `/log` -and `/trace` namespaces ride `agentix.sio.ReliableStream` for -at-least-once delivery across reconnects. - -Remaining transport work: - -- optional annotation-driven msgpack codec path alongside pickle -- collapse event naming if the current `call:*` family becomes noisy - -## Plugins - -Plugins live in this monorepo under [`plugins/`](plugins) as separate -workspace members — each its own PyPI package, all updated in lockstep -with Agentix HEAD while the design is still moving quickly. - -- [`agentix-runtime-basic`](plugins/runtime-basic) — `bash` and `files` - modules. -- [`agentix-provider-docker`](plugins/providers/docker) / - [`-daytona`](plugins/providers/daytona) / - [`-e2b`](plugins/providers/e2b) / - [`-apptainer`](plugins/providers/apptainer) — sandbox backends. -- [`agentix-runner`](plugins/runner) — `run_rollouts(...)` batch - orchestration. -- [`agentix-dataset-swe`](plugins/datasets/swebench) — SWE-bench task - images and harness scoring. -- [`agentix-agent-*`](plugins/agents) — agent adapters (Claude Code, - mini-swe-agent, Qwen Code). -- [`agentix-bridge`](plugins/abridge) — model translation and host-side - rollout-to-RL-buffer capture (abridge). -- [`agentix-trace-otel`](plugins/trace-otel) — OTLP trace export. - -## Later - -Future directions, listed so the framework can avoid architectural -dead-ends without expanding the current API prematurely. - -- **~~OpenTelemetry trace export~~ (shipped — `agentix-trace-otel`).** ship `agentix.utils.trace` spans to a - production observability platform (Datadog, Jaeger, Tempo, Honeycomb, - any OTLP-compatible backend). Implementation should not change the - `agentix.utils.trace` public API. Plan: - - - Keep `agentix.utils.trace` (`Trace`, `Span`, `Processor`) as the - user surface. Sandbox code stays unchanged. - - The sandbox already streams `/trace` via `ReliableStream`; the host - receives via `HostTraceNamespace` and fans out through the existing - provider, so a new `Processor` is the right plug-in point. - - Ship as a separate plugin package `agentix-trace-otel` to keep - `opentelemetry-*` out of core dependencies (matches the current - plugin-axis style of providers / runtime-basic / agents). - - Map `agentix.Span` → OTel `ReadableSpan`: `trace_id` / `span_id` / - `parent_id` / `attrs` / `started_at` / `ended_at` / `status` / - `events` are 1:1; only the id-length normalization and timestamp - units (ns) need adapters. - - Export from the **host** by default (sandboxes are ephemeral; host - owns the long-lived collector connection). A sandbox-side exporter - is possible later for cases where the sandbox can reach the - collector directly. - - User surface: - ```python - from agentix.utils import trace - from agentix.utils.trace.otel import OTelExporter - - trace.add_processor(OTelExporter(endpoint="...", headers={...})) - ``` - -- **Trace pub/sub** — remote functions emit structured rollout events; - subscribers receive rollout-scoped fan-out. -- **RolloutPool** — warm sandbox pool for batched RL rollouts. -- **LLM proxy** — transparent proxy for API calls from remote functions, - enabling token-level trajectory capture, cost tracking, and replay. -- **Checkpoint / partial rollout** — snapshot a sandbox filesystem and - loaded runtime state, then fork to explore alternative continuations. -- **K8s provider backend** — `SandboxProvider` implementation using the - same bundle-image contract, likely shipping as `agentix-provider-k8s`. From 90f970a46cafd9591e0dace123e0f76ef04e1ddc Mon Sep 17 00:00:00 2001 From: FatPigeorz Date: Wed, 1 Jul 2026 00:27:29 +0800 Subject: [PATCH 11/11] ci: refresh uv.lock + clear ruff lint across tito/abridge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - uv.lock: regenerate so `uv lock --check` passes (was stale vs pyproject changes — transformers/tokenizers/uv/etc). - ruff: sort imports, PEP585/604 annotations, StrEnum for MismatchType/TITOTokenizerType, wrap over-long lines, export SessionForward in agentix.bridge.__all__. - ruff config: exclude vendored `sidecars/` (own upstream repos/CI, mirrors the pyright exclude). Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/abridge/agentix/bridge/__init__.py | 1 + plugins/abridge/tests/test_sidecar_forward.py | 3 +- .../providers/uv/tests/test_uv_provider.py | 2 +- plugins/tito/agentix/tito/config.py | 3 +- plugins/tito/agentix/tito/discovery.py | 1 - plugins/tito/agentix/tito/engine/compare.py | 14 +- plugins/tito/agentix/tito/engine/render.py | 6 +- .../tito/agentix/tito/engine/session_app.py | 2 +- plugins/tito/agentix/tito/gateway.py | 2 +- plugins/tito/agentix/tito/tokenizer.py | 4 +- plugins/tito/tests/package/test_cli.py | 1 - .../tests/package/test_config_discovery.py | 3 +- plugins/tito/tests/package/test_engine.py | 13 +- .../tito/tests/package/test_import_surface.py | 2 +- plugins/tito/tests/test_pool_routing.py | 1 - pyproject.toml | 3 + uv.lock | 234 ++++++++++++++++-- 17 files changed, 250 insertions(+), 45 deletions(-) diff --git a/plugins/abridge/agentix/bridge/__init__.py b/plugins/abridge/agentix/bridge/__init__.py index ef0095c..e96b774 100644 --- a/plugins/abridge/agentix/bridge/__init__.py +++ b/plugins/abridge/agentix/bridge/__init__.py @@ -61,6 +61,7 @@ "NAMESPACE", "Proxy", "Request", + "SessionForward", "Sidecar", "SidecarError", "TunnelHandle", diff --git a/plugins/abridge/tests/test_sidecar_forward.py b/plugins/abridge/tests/test_sidecar_forward.py index fa489ed..261ef9a 100644 --- a/plugins/abridge/tests/test_sidecar_forward.py +++ b/plugins/abridge/tests/test_sidecar_forward.py @@ -456,9 +456,10 @@ async def test_session_forward_through_live_sidecar(tmp_path) -> None: async def test_tunnel_rejects_non_object_body(monkeypatch) -> None: """A present-but-non-object JSON body (an array) is a 400 at the tunnel, not a silent coercion to {}.""" - import agentix as agentix_mod import agentix.bridge.proxy as proxy_mod + import agentix as agentix_mod + monkeypatch.setattr(agentix_mod, "register_namespace", lambda ns: None) monkeypatch.setattr(proxy_mod, "_namespace_singleton", None) diff --git a/plugins/providers/uv/tests/test_uv_provider.py b/plugins/providers/uv/tests/test_uv_provider.py index 461e64d..bb51b96 100644 --- a/plugins/providers/uv/tests/test_uv_provider.py +++ b/plugins/providers/uv/tests/test_uv_provider.py @@ -14,9 +14,9 @@ import sys import pytest +from agentix.provider.uv import UvProvider, UvProviderConfig from agentix.provider.base import SandboxConfig, SandboxProvider -from agentix.provider.uv import UvProvider, UvProviderConfig def _reuse_venv() -> str: diff --git a/plugins/tito/agentix/tito/config.py b/plugins/tito/agentix/tito/config.py index 1431835..ba84593 100644 --- a/plugins/tito/agentix/tito/config.py +++ b/plugins/tito/agentix/tito/config.py @@ -6,7 +6,6 @@ from .discovery import DEFAULT_BACKEND_PROBE_CANDIDATES - _VALID_APPEND_ROLES = frozenset({"tool", "user", "system"}) @@ -59,7 +58,7 @@ def from_cli_values( router_timeout: float, backend_probe_candidates: list[str] | None = None, backend_probe_timeout: float = 0.25, - ) -> "TITOGatewayConfig": + ) -> TITOGatewayConfig: return cls( hf_checkpoint=hf_checkpoint, backend_url=backend_url, diff --git a/plugins/tito/agentix/tito/discovery.py b/plugins/tito/agentix/tito/discovery.py index 9741a28..afc8b69 100644 --- a/plugins/tito/agentix/tito/discovery.py +++ b/plugins/tito/agentix/tito/discovery.py @@ -8,7 +8,6 @@ from urllib.error import HTTPError, URLError from urllib.request import Request, urlopen - DEFAULT_BACKEND_ENV_VARS = ("TITO_BACKEND_URL", "OPENAI_BASE_URL", "SGLANG_BASE_URL") DEFAULT_BACKEND_PROBE_CANDIDATES = ( "http://127.0.0.1:8000", diff --git a/plugins/tito/agentix/tito/engine/compare.py b/plugins/tito/agentix/tito/engine/compare.py index 8c20ed3..7981a58 100644 --- a/plugins/tito/agentix/tito/engine/compare.py +++ b/plugins/tito/agentix/tito/engine/compare.py @@ -8,12 +8,12 @@ from __future__ import annotations -from dataclasses import dataclass, field -from enum import Enum +from dataclasses import dataclass +from enum import StrEnum from typing import Any -class MismatchType(str, Enum): +class MismatchType(StrEnum): # Segment count or special/content pattern differs — structural break. SPECIAL_TOKEN_COUNT = "special_token_count" # Aligned special-token segment holds a different special token. @@ -67,7 +67,9 @@ def __init__( ) -> None: self.tokenizer = tokenizer self._assistant_start_str = assistant_start_str - self._special_ids = set(special_token_ids) if special_token_ids is not None else self.collect_special_ids(tokenizer) + self._special_ids = ( + set(special_token_ids) if special_token_ids is not None else self.collect_special_ids(tokenizer) + ) self._trim_trailing_ids = set(trim_trailing_ids) if trim_trailing_ids else None @staticmethod @@ -140,7 +142,9 @@ def _check_segment_structure(self, exp_segs: list[Segment], act_segs: list[Segme detail=detail, ) - def _compare_single_segment(self, idx: int, exp: Segment, act: Segment, *, is_assistant_content: bool) -> Mismatch | None: + def _compare_single_segment( + self, idx: int, exp: Segment, act: Segment, *, is_assistant_content: bool + ) -> Mismatch | None: if exp.is_special: if exp.token_ids != act.token_ids: return Mismatch( diff --git a/plugins/tito/agentix/tito/engine/render.py b/plugins/tito/agentix/tito/engine/render.py index 6d22f83..ee69863 100644 --- a/plugins/tito/agentix/tito/engine/render.py +++ b/plugins/tito/agentix/tito/engine/render.py @@ -11,7 +11,7 @@ import copy import json -from typing import Any, Literal, Optional +from typing import Any, Literal from jinja2 import TemplateError from pydantic import BaseModel, TypeAdapter @@ -19,8 +19,8 @@ class _Function(BaseModel): name: str - description: Optional[str] = None - parameters: Optional[dict[str, Any]] = None + description: str | None = None + parameters: dict[str, Any] | None = None class Tool(BaseModel): diff --git a/plugins/tito/agentix/tito/engine/session_app.py b/plugins/tito/agentix/tito/engine/session_app.py index 384bbcb..61ea609 100644 --- a/plugins/tito/agentix/tito/engine/session_app.py +++ b/plugins/tito/agentix/tito/engine/session_app.py @@ -23,8 +23,8 @@ from starlette.responses import Response from .errors import SessionError, SessionNotFoundError, TokenizationError, UpstreamResponseError -from .processing import load_tokenizer from .pretokenize import get_tito_tokenizer +from .processing import load_tokenizer from .trajectory import GetSessionResponse, SessionRecord, SessionRegistry logger = logging.getLogger(__name__) diff --git a/plugins/tito/agentix/tito/gateway.py b/plugins/tito/agentix/tito/gateway.py index 18eb393..942bcb9 100644 --- a/plugins/tito/agentix/tito/gateway.py +++ b/plugins/tito/agentix/tito/gateway.py @@ -34,7 +34,7 @@ def __init__(self, config: TITOGatewayConfig): self._register_health_alias() @classmethod - def from_server(cls, *, hf_checkpoint: str, backend_url: str | None = None, **kwargs) -> "TITOGateway": + def from_server(cls, *, hf_checkpoint: str, backend_url: str | None = None, **kwargs) -> TITOGateway: return cls(TITOGatewayConfig(hf_checkpoint=hf_checkpoint, backend_url=backend_url, **kwargs)) def _register_health_alias(self) -> None: diff --git a/plugins/tito/agentix/tito/tokenizer.py b/plugins/tito/agentix/tito/tokenizer.py index ec26fc9..bf3fa11 100644 --- a/plugins/tito/agentix/tito/tokenizer.py +++ b/plugins/tito/agentix/tito/tokenizer.py @@ -2,13 +2,13 @@ from __future__ import annotations -from enum import Enum +from enum import StrEnum from typing import Any from .engine.pretokenize import get_tito_tokenizer as _engine_get_tito_tokenizer -class TITOTokenizerType(str, Enum): +class TITOTokenizerType(StrEnum): """Tokenizer families the native engine supports. Other models are a small subclass + a fixed chat template — see agentix.tito.engine.pretokenize.""" diff --git a/plugins/tito/tests/package/test_cli.py b/plugins/tito/tests/package/test_cli.py index 925e995..0bf0722 100644 --- a/plugins/tito/tests/package/test_cli.py +++ b/plugins/tito/tests/package/test_cli.py @@ -1,5 +1,4 @@ import pytest - from agentix.tito.cli import build_parser, main diff --git a/plugins/tito/tests/package/test_config_discovery.py b/plugins/tito/tests/package/test_config_discovery.py index 7bc1a02..9b6862a 100644 --- a/plugins/tito/tests/package/test_config_discovery.py +++ b/plugins/tito/tests/package/test_config_discovery.py @@ -1,7 +1,6 @@ import pytest - -from agentix.tito.config import TITOGatewayConfig from agentix.tito import discovery +from agentix.tito.config import TITOGatewayConfig def test_explicit_backend_url_wins_over_environment_and_probe(monkeypatch): diff --git a/plugins/tito/tests/package/test_engine.py b/plugins/tito/tests/package/test_engine.py index 18404cb..77c01f2 100644 --- a/plugins/tito/tests/package/test_engine.py +++ b/plugins/tito/tests/package/test_engine.py @@ -9,13 +9,12 @@ from __future__ import annotations import pytest -from tokenizers import Tokenizer, models, pre_tokenizers -from transformers import PreTrainedTokenizerFast - from agentix.tito.engine.compare import MismatchType, TokenSeqComparator from agentix.tito.engine.messages import assert_messages_append_only_with_allowed_role, message_matches from agentix.tito.engine.pretokenize import Qwen3TITOTokenizer, get_tito_tokenizer from agentix.tito.engine.trajectory import LinearTrajectory, SessionRegistry +from tokenizers import Tokenizer, models, pre_tokenizers +from transformers import PreTrainedTokenizerFast @pytest.fixture(scope="module") @@ -126,12 +125,16 @@ def test_trajectory_rollback_to_assistant_checkpoint(tok): a0 = {"role": "assistant", "content": "ok"} tr.prepare_pretokenized(sys, None, tito_tokenizer=tt) - tr.update_pretokenized_state(sys, a0, tt.render_messages(sys + [a0], add_generation_prompt=False, tokenize=True), [], tt.max_trim_tokens) + tr.update_pretokenized_state( + sys, a0, tt.render_messages(sys + [a0], add_generation_prompt=False, tokenize=True), [], tt.max_trim_tokens + ) m1 = sys + [a0, {"role": "tool", "content": "391"}] a1 = {"role": "assistant", "content": "done"} tr.prepare_pretokenized(m1, None, tito_tokenizer=tt) - tr.update_pretokenized_state(m1, a1, tt.render_messages(m1 + [a1], add_generation_prompt=False, tokenize=True), [], tt.max_trim_tokens) + tr.update_pretokenized_state( + m1, a1, tt.render_messages(m1 + [a1], add_generation_prompt=False, tokenize=True), [], tt.max_trim_tokens + ) assert tr.num_assistant == 2 assert reg.compute_session_mismatch(tr) == [] # clean chain → no mismatch diff --git a/plugins/tito/tests/package/test_import_surface.py b/plugins/tito/tests/package/test_import_surface.py index 3bb0fa0..820f0dc 100644 --- a/plugins/tito/tests/package/test_import_surface.py +++ b/plugins/tito/tests/package/test_import_surface.py @@ -3,7 +3,7 @@ def test_public_import_surface(): import agentix.tito - from agentix.tito import TITOGateway, TITOGatewayConfig, SessionServer, get_tito_tokenizer + from agentix.tito import SessionServer, TITOGateway, TITOGatewayConfig, get_tito_tokenizer assert agentix.tito.TITOGateway is TITOGateway assert agentix.tito.TITOGatewayConfig is TITOGatewayConfig diff --git a/plugins/tito/tests/test_pool_routing.py b/plugins/tito/tests/test_pool_routing.py index 73dcc08..6933764 100644 --- a/plugins/tito/tests/test_pool_routing.py +++ b/plugins/tito/tests/test_pool_routing.py @@ -9,7 +9,6 @@ import types import pytest - from agentix.tito.pool import BackendPool from agentix.tito.server import SessionServer, _session_id_from_path diff --git a/pyproject.toml b/pyproject.toml index 039d2de..902e88f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -98,6 +98,9 @@ exclude = ["examples/*"] [tool.ruff] line-length = 120 target-version = "py311" +# Vendored sidecar sources (their own upstream repos with their own CI / +# formatting) — kept out of our lint scope, same as the pyright exclude. +extend-exclude = ["sidecars"] [tool.ruff.lint] select = ["E", "F", "I", "W", "UP"] diff --git a/uv.lock b/uv.lock index fa4ff2c..91d7df6 100644 --- a/uv.lock +++ b/uv.lock @@ -27,8 +27,10 @@ members = [ "agentix-provider-daytona", "agentix-provider-docker", "agentix-provider-e2b", + "agentix-provider-uv", "agentix-runner", "agentix-runtime-basic", + "agentix-tito", "agentix-trace-otel", "agentixx", ] @@ -171,6 +173,21 @@ dependencies = [ [package.metadata] requires-dist = [{ name = "agentixx", editable = "." }] +[[package]] +name = "agentix-provider-uv" +version = "0.1.0" +source = { editable = "plugins/providers/uv" } +dependencies = [ + { name = "agentixx" }, + { name = "uv" }, +] + +[package.metadata] +requires-dist = [ + { name = "agentixx", editable = "." }, + { name = "uv", specifier = ">=0.5" }, +] + [[package]] name = "agentix-runner" version = "0.1.0" @@ -193,6 +210,47 @@ dependencies = [ [package.metadata] requires-dist = [{ name = "agentixx", editable = "." }] +[[package]] +name = "agentix-tito" +version = "0.1.0" +source = { editable = "plugins/tito" } +dependencies = [ + { name = "agentixx" }, + { name = "fastapi" }, + { name = "httpx" }, + { name = "huggingface-hub" }, + { name = "jinja2" }, + { name = "pydantic", version = "2.11.10", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and python_full_version < '3.14'" }, + { name = "pydantic", version = "2.13.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12' or python_full_version >= '3.14'" }, + { name = "setproctitle" }, + { name = "tokenizers" }, + { name = "transformers" }, + { name = "uvicorn" }, +] + +[package.optional-dependencies] +test = [ + { name = "pytest" }, + { name = "pytest-asyncio" }, +] + +[package.metadata] +requires-dist = [ + { name = "agentixx", editable = "." }, + { name = "fastapi", specifier = ">=0.110" }, + { name = "httpx", specifier = ">=0.27" }, + { name = "huggingface-hub", specifier = ">=0.23" }, + { name = "jinja2", specifier = ">=3.1" }, + { name = "pydantic", specifier = ">=2" }, + { name = "pytest", marker = "extra == 'test'", specifier = ">=8" }, + { name = "pytest-asyncio", marker = "extra == 'test'", specifier = ">=0.23" }, + { name = "setproctitle", specifier = ">=1.3" }, + { name = "tokenizers", specifier = ">=0.19" }, + { name = "transformers", specifier = ">=4.44" }, + { name = "uvicorn", specifier = ">=0.29" }, +] +provides-extras = ["test"] + [[package]] name = "agentix-trace-otel" version = "0.1.0" @@ -3291,6 +3349,101 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9b/36/9c015cd052fca743dae8cb2aeb16b551444787467db42ceab0fc968865af/ruff-0.15.13-py3-none-win_arm64.whl", hash = "sha256:2471da9bd1068c8c064b5fd9c0c4b6dddffd6369cb1cd68b29993b1709ff1b21", size = 11179336, upload-time = "2026-05-14T13:44:33.026Z" }, ] +[[package]] +name = "safetensors" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/06/f955dbbb1859e3bd23c8ac6141af5106e7ad5fedec4a3a6e3d60f94b7001/safetensors-0.8.0.tar.gz", hash = "sha256:fabaf3e0f18a6618d9b36560682562157f77c2b71fcffc7b432be2baed9d753d", size = 325846, upload-time = "2026-06-09T07:52:25.563Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/a0/f718cda65b05407d228f97602cf60dca269c979867aa5beb25410de26cd3/safetensors-0.8.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c554f85858e05226d3c2828e32395e677434685d6d94594a41643361c5e837f0", size = 473568, upload-time = "2026-06-09T07:52:18.829Z" }, + { url = "https://files.pythonhosted.org/packages/f5/b1/fa7c600e7dceae12e9606c7578cbc9ff1e1ed55844883ee5c92205e86226/safetensors-0.8.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:c80201d22cbf405b80647a60ada77bba06c8fba2da2743ba1e89cdcc39a81f25", size = 484562, upload-time = "2026-06-09T07:52:17.518Z" }, + { url = "https://files.pythonhosted.org/packages/09/7d/65a7de0af421317bb36a067241e4235fff194eed60b961ed6d3f59a3fc60/safetensors-0.8.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235", size = 502844, upload-time = "2026-06-09T07:52:07.624Z" }, + { url = "https://files.pythonhosted.org/packages/91/4f/3175c9d75634e0e0dda0082794193521035edd7c70a6f212bf33ca06ddf4/safetensors-0.8.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4124502b78f03534117c848f87a39b8f31e577b15eff423bf8bfb95f2a8c30d0", size = 511823, upload-time = "2026-06-09T07:52:09.565Z" }, + { url = "https://files.pythonhosted.org/packages/20/87/846c289e7aa2299eff406335717cf43ce8777194ece8aad75772e0411615/safetensors-0.8.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7bc0a787ba8a35be368ee3574edfa2b1ad389eebd0a72e482ae275490e3f6c98", size = 633461, upload-time = "2026-06-09T07:52:11.128Z" }, + { url = "https://files.pythonhosted.org/packages/76/22/8d64d9df2c45d5ded401df889d0ad90882804ca172d79ec4f0df8f727fe0/safetensors-0.8.0-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:040070828e36dc8e122178bbbd5830ff9e97920affb84cbe0f46442497bed358", size = 545148, upload-time = "2026-06-09T07:52:13.603Z" }, + { url = "https://files.pythonhosted.org/packages/28/50/f203ff3a3ddfe19308efc83c5a3a29ed02bf786732ec35e68bf9162f3365/safetensors-0.8.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd6f3f93c9a0a7cc2788ee63fb763353d4bd2e89b0751bc78fcf7dda00bea774", size = 516040, upload-time = "2026-06-09T07:52:16.29Z" }, + { url = "https://files.pythonhosted.org/packages/46/fb/cdaed17ceb2948784fd9c36b6fd3e951b608547cea81a48e8ee6f8cfdfcb/safetensors-0.8.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:fcdd41ec4628fee5799f807c73c353629130fbd942aa23d83c623dd6c9d52d78", size = 513832, upload-time = "2026-06-09T07:52:12.37Z" }, + { url = "https://files.pythonhosted.org/packages/0d/49/1e15de264dcc3b77943d2d0c56a95809956883b1c2d6d585c792523f180b/safetensors-0.8.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8e9f537aa183a38ace122d27303dcd986b26bd2a7591f9181d7f0c396f4677ca", size = 559930, upload-time = "2026-06-09T07:52:14.743Z" }, + { url = "https://files.pythonhosted.org/packages/2a/43/bf38443278eab4b1be1fce2931e2b012ad9cb7df52ada751d0aab8f7659a/safetensors-0.8.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:87eec7ffed2b809f05a398a8becb7d013f19f7837cd15d9748580d6cf30dbaf4", size = 678670, upload-time = "2026-06-09T07:52:20.032Z" }, + { url = "https://files.pythonhosted.org/packages/72/e3/68cd3fa5b48488e84add63e04cb12f3bc28ae4638c06d4508c6e88823d0e/safetensors-0.8.0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4a95ae2b05d7726d751da4ebf626a2ca782b706e101bd894c95bc2450b1cffcc", size = 786679, upload-time = "2026-06-09T07:52:21.322Z" }, + { url = "https://files.pythonhosted.org/packages/29/4b/1c19c509d56e01f4fbb3d0a2e597450f6cc04d1d56cf52defb0a62dfd715/safetensors-0.8.0-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:3ae091f16662658bdc019a4ff6cb4c085bb7d725eb5978b183ffd265863b6d2d", size = 765683, upload-time = "2026-06-09T07:52:22.594Z" }, + { url = "https://files.pythonhosted.org/packages/27/43/41c1621732edd934d868a00d1b891584c892a7b62a9aab82ea5a0a5623ee/safetensors-0.8.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8e080062fcde23be189565e1c3305d16751a218ecf9412c8601e64204eb6f846", size = 722361, upload-time = "2026-06-09T07:52:23.924Z" }, + { url = "https://files.pythonhosted.org/packages/8e/3f/73ccf82579412b4a71c4ca673f10b5f1f888d7cf5af7fe24f27d30307be4/safetensors-0.8.0-cp310-abi3-win32.whl", hash = "sha256:2ddf52eac562eda224f99acfa7889d02968c1fd59a5b011ae7d8137c37e9c02d", size = 342401, upload-time = "2026-06-09T07:52:28.895Z" }, + { url = "https://files.pythonhosted.org/packages/1b/6d/3fba214c1e5e0f69991677ec3bc17023f0421776975e1de0c682dca475e2/safetensors-0.8.0-cp310-abi3-win_amd64.whl", hash = "sha256:096ec1a98435df7beb08853bb5aa9081a84f23d0adc67ed1a0a10550f608373f", size = 355540, upload-time = "2026-06-09T07:52:27.832Z" }, + { url = "https://files.pythonhosted.org/packages/8d/fc/7eedc3510d97878876e32774eebbeb61c43f148a96e915c84229a3e967aa/safetensors-0.8.0-cp310-abi3-win_arm64.whl", hash = "sha256:f7838e5135a406ad3e02efdcb8cf2e5397d368b0154537c4fec682dbc544d452", size = 340500, upload-time = "2026-06-09T07:52:26.745Z" }, +] + +[[package]] +name = "setproctitle" +version = "1.3.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8d/48/49393a96a2eef1ab418b17475fb92b8fcfad83d099e678751b05472e69de/setproctitle-1.3.7.tar.gz", hash = "sha256:bc2bc917691c1537d5b9bca1468437176809c7e11e5694ca79a9ca12345dcb9e", size = 27002, upload-time = "2025-09-05T12:51:25.278Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/cd/1b7ba5cad635510720ce19d7122154df96a2387d2a74217be552887c93e5/setproctitle-1.3.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a600eeb4145fb0ee6c287cb82a2884bd4ec5bbb076921e287039dcc7b7cc6dd0", size = 18085, upload-time = "2025-09-05T12:49:22.183Z" }, + { url = "https://files.pythonhosted.org/packages/8f/1a/b2da0a620490aae355f9d72072ac13e901a9fec809a6a24fc6493a8f3c35/setproctitle-1.3.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:97a090fed480471bb175689859532709e28c085087e344bca45cf318034f70c4", size = 13097, upload-time = "2025-09-05T12:49:23.322Z" }, + { url = "https://files.pythonhosted.org/packages/18/2e/bd03ff02432a181c1787f6fc2a678f53b7dacdd5ded69c318fe1619556e8/setproctitle-1.3.7-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1607b963e7b53e24ec8a2cb4e0ab3ae591d7c6bf0a160feef0551da63452b37f", size = 32191, upload-time = "2025-09-05T12:49:24.567Z" }, + { url = "https://files.pythonhosted.org/packages/28/78/1e62fc0937a8549f2220445ed2175daacee9b6764c7963b16148119b016d/setproctitle-1.3.7-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a20fb1a3974e2dab857870cf874b325b8705605cb7e7e8bcbb915bca896f52a9", size = 33203, upload-time = "2025-09-05T12:49:25.871Z" }, + { url = "https://files.pythonhosted.org/packages/a0/3c/65edc65db3fa3df400cf13b05e9d41a3c77517b4839ce873aa6b4043184f/setproctitle-1.3.7-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8d961bba676e07d77665204f36cffaa260f526e7b32d07ab3df6a2c1dfb44ba", size = 34963, upload-time = "2025-09-05T12:49:27.044Z" }, + { url = "https://files.pythonhosted.org/packages/a1/32/89157e3de997973e306e44152522385f428e16f92f3cf113461489e1e2ee/setproctitle-1.3.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:db0fd964fbd3a9f8999b502f65bd2e20883fdb5b1fae3a424e66db9a793ed307", size = 32398, upload-time = "2025-09-05T12:49:28.909Z" }, + { url = "https://files.pythonhosted.org/packages/4a/18/77a765a339ddf046844cb4513353d8e9dcd8183da9cdba6e078713e6b0b2/setproctitle-1.3.7-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:db116850fcf7cca19492030f8d3b4b6e231278e8fe097a043957d22ce1bdf3ee", size = 33657, upload-time = "2025-09-05T12:49:30.323Z" }, + { url = "https://files.pythonhosted.org/packages/6b/63/f0b6205c64d74d2a24a58644a38ec77bdbaa6afc13747e75973bf8904932/setproctitle-1.3.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:316664d8b24a5c91ee244460bdaf7a74a707adaa9e14fbe0dc0a53168bb9aba1", size = 31836, upload-time = "2025-09-05T12:49:32.309Z" }, + { url = "https://files.pythonhosted.org/packages/ba/51/e1277f9ba302f1a250bbd3eedbbee747a244b3cc682eb58fb9733968f6d8/setproctitle-1.3.7-cp311-cp311-win32.whl", hash = "sha256:b74774ca471c86c09b9d5037c8451fff06bb82cd320d26ae5a01c758088c0d5d", size = 12556, upload-time = "2025-09-05T12:49:33.529Z" }, + { url = "https://files.pythonhosted.org/packages/b6/7b/822a23f17e9003dfdee92cd72758441ca2a3680388da813a371b716fb07f/setproctitle-1.3.7-cp311-cp311-win_amd64.whl", hash = "sha256:acb9097213a8dd3410ed9f0dc147840e45ca9797785272928d4be3f0e69e3be4", size = 13243, upload-time = "2025-09-05T12:49:34.553Z" }, + { url = "https://files.pythonhosted.org/packages/fb/f0/2dc88e842077719d7384d86cc47403e5102810492b33680e7dadcee64cd8/setproctitle-1.3.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2dc99aec591ab6126e636b11035a70991bc1ab7a261da428491a40b84376654e", size = 18049, upload-time = "2025-09-05T12:49:36.241Z" }, + { url = "https://files.pythonhosted.org/packages/f0/b4/50940504466689cda65680c9e9a1e518e5750c10490639fa687489ac7013/setproctitle-1.3.7-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cdd8aa571b7aa39840fdbea620e308a19691ff595c3a10231e9ee830339dd798", size = 13079, upload-time = "2025-09-05T12:49:38.088Z" }, + { url = "https://files.pythonhosted.org/packages/d0/99/71630546b9395b095f4082be41165d1078204d1696c2d9baade3de3202d0/setproctitle-1.3.7-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2906b6c7959cdb75f46159bf0acd8cc9906cf1361c9e1ded0d065fe8f9039629", size = 32932, upload-time = "2025-09-05T12:49:39.271Z" }, + { url = "https://files.pythonhosted.org/packages/50/22/cee06af4ffcfb0e8aba047bd44f5262e644199ae7527ae2c1f672b86495c/setproctitle-1.3.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6915964a6dda07920a1159321dcd6d94fc7fc526f815ca08a8063aeca3c204f1", size = 33736, upload-time = "2025-09-05T12:49:40.565Z" }, + { url = "https://files.pythonhosted.org/packages/5c/00/a5949a8bb06ef5e7df214fc393bb2fb6aedf0479b17214e57750dfdd0f24/setproctitle-1.3.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cff72899861c765bd4021d1ff1c68d60edc129711a2fdba77f9cb69ef726a8b6", size = 35605, upload-time = "2025-09-05T12:49:42.362Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3a/50caca532a9343828e3bf5778c7a84d6c737a249b1796d50dd680290594d/setproctitle-1.3.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b7cb05bd446687ff816a3aaaf831047fc4c364feff7ada94a66024f1367b448c", size = 33143, upload-time = "2025-09-05T12:49:43.515Z" }, + { url = "https://files.pythonhosted.org/packages/ca/14/b843a251296ce55e2e17c017d6b9f11ce0d3d070e9265de4ecad948b913d/setproctitle-1.3.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:3a57b9a00de8cae7e2a1f7b9f0c2ac7b69372159e16a7708aa2f38f9e5cc987a", size = 34434, upload-time = "2025-09-05T12:49:45.31Z" }, + { url = "https://files.pythonhosted.org/packages/c8/b7/06145c238c0a6d2c4bc881f8be230bb9f36d2bf51aff7bddcb796d5eed67/setproctitle-1.3.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d8828b356114f6b308b04afe398ed93803d7fca4a955dd3abe84430e28d33739", size = 32795, upload-time = "2025-09-05T12:49:46.419Z" }, + { url = "https://files.pythonhosted.org/packages/ef/dc/ef76a81fac9bf27b84ed23df19c1f67391a753eed6e3c2254ebcb5133f56/setproctitle-1.3.7-cp312-cp312-win32.whl", hash = "sha256:b0304f905efc845829ac2bc791ddebb976db2885f6171f4a3de678d7ee3f7c9f", size = 12552, upload-time = "2025-09-05T12:49:47.635Z" }, + { url = "https://files.pythonhosted.org/packages/e2/5b/a9fe517912cd6e28cf43a212b80cb679ff179a91b623138a99796d7d18a0/setproctitle-1.3.7-cp312-cp312-win_amd64.whl", hash = "sha256:9888ceb4faea3116cf02a920ff00bfbc8cc899743e4b4ac914b03625bdc3c300", size = 13247, upload-time = "2025-09-05T12:49:49.16Z" }, + { url = "https://files.pythonhosted.org/packages/5d/2f/fcedcade3b307a391b6e17c774c6261a7166aed641aee00ed2aad96c63ce/setproctitle-1.3.7-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:c3736b2a423146b5e62230502e47e08e68282ff3b69bcfe08a322bee73407922", size = 18047, upload-time = "2025-09-05T12:49:50.271Z" }, + { url = "https://files.pythonhosted.org/packages/23/ae/afc141ca9631350d0a80b8f287aac79a76f26b6af28fd8bf92dae70dc2c5/setproctitle-1.3.7-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3384e682b158d569e85a51cfbde2afd1ab57ecf93ea6651fe198d0ba451196ee", size = 13073, upload-time = "2025-09-05T12:49:51.46Z" }, + { url = "https://files.pythonhosted.org/packages/87/ed/0a4f00315bc02510395b95eec3d4aa77c07192ee79f0baae77ea7b9603d8/setproctitle-1.3.7-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0564a936ea687cd24dffcea35903e2a20962aa6ac20e61dd3a207652401492dd", size = 33284, upload-time = "2025-09-05T12:49:52.741Z" }, + { url = "https://files.pythonhosted.org/packages/fc/e4/adf3c4c0a2173cb7920dc9df710bcc67e9bcdbf377e243b7a962dc31a51a/setproctitle-1.3.7-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a5d1cb3f81531f0eb40e13246b679a1bdb58762b170303463cb06ecc296f26d0", size = 34104, upload-time = "2025-09-05T12:49:54.416Z" }, + { url = "https://files.pythonhosted.org/packages/52/4f/6daf66394152756664257180439d37047aa9a1cfaa5e4f5ed35e93d1dc06/setproctitle-1.3.7-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a7d159e7345f343b44330cbba9194169b8590cb13dae940da47aa36a72aa9929", size = 35982, upload-time = "2025-09-05T12:49:56.295Z" }, + { url = "https://files.pythonhosted.org/packages/1b/62/f2c0595403cf915db031f346b0e3b2c0096050e90e0be658a64f44f4278a/setproctitle-1.3.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0b5074649797fd07c72ca1f6bff0406f4a42e1194faac03ecaab765ce605866f", size = 33150, upload-time = "2025-09-05T12:49:58.025Z" }, + { url = "https://files.pythonhosted.org/packages/a0/29/10dd41cde849fb2f9b626c846b7ea30c99c81a18a5037a45cc4ba33c19a7/setproctitle-1.3.7-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:61e96febced3f61b766115381d97a21a6265a0f29188a791f6df7ed777aef698", size = 34463, upload-time = "2025-09-05T12:49:59.424Z" }, + { url = "https://files.pythonhosted.org/packages/71/3c/cedd8eccfaf15fb73a2c20525b68c9477518917c9437737fa0fda91e378f/setproctitle-1.3.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:047138279f9463f06b858e579cc79580fbf7a04554d24e6bddf8fe5dddbe3d4c", size = 32848, upload-time = "2025-09-05T12:50:01.107Z" }, + { url = "https://files.pythonhosted.org/packages/d1/3e/0a0e27d1c9926fecccfd1f91796c244416c70bf6bca448d988638faea81d/setproctitle-1.3.7-cp313-cp313-win32.whl", hash = "sha256:7f47accafac7fe6535ba8ba9efd59df9d84a6214565108d0ebb1199119c9cbbd", size = 12544, upload-time = "2025-09-05T12:50:15.81Z" }, + { url = "https://files.pythonhosted.org/packages/36/1b/6bf4cb7acbbd5c846ede1c3f4d6b4ee52744d402e43546826da065ff2ab7/setproctitle-1.3.7-cp313-cp313-win_amd64.whl", hash = "sha256:fe5ca35aeec6dc50cabab9bf2d12fbc9067eede7ff4fe92b8f5b99d92e21263f", size = 13235, upload-time = "2025-09-05T12:50:16.89Z" }, + { url = "https://files.pythonhosted.org/packages/e6/a4/d588d3497d4714750e3eaf269e9e8985449203d82b16b933c39bd3fc52a1/setproctitle-1.3.7-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:10e92915c4b3086b1586933a36faf4f92f903c5554f3c34102d18c7d3f5378e9", size = 18058, upload-time = "2025-09-05T12:50:02.501Z" }, + { url = "https://files.pythonhosted.org/packages/05/77/7637f7682322a7244e07c373881c7e982567e2cb1dd2f31bd31481e45500/setproctitle-1.3.7-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:de879e9c2eab637f34b1a14c4da1e030c12658cdc69ee1b3e5be81b380163ce5", size = 13072, upload-time = "2025-09-05T12:50:03.601Z" }, + { url = "https://files.pythonhosted.org/packages/52/09/f366eca0973cfbac1470068d1313fa3fe3de4a594683385204ec7f1c4101/setproctitle-1.3.7-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c18246d88e227a5b16248687514f95642505000442165f4b7db354d39d0e4c29", size = 34490, upload-time = "2025-09-05T12:50:04.948Z" }, + { url = "https://files.pythonhosted.org/packages/71/36/611fc2ed149fdea17c3677e1d0df30d8186eef9562acc248682b91312706/setproctitle-1.3.7-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7081f193dab22df2c36f9fc6d113f3793f83c27891af8fe30c64d89d9a37e152", size = 35267, upload-time = "2025-09-05T12:50:06.015Z" }, + { url = "https://files.pythonhosted.org/packages/88/a4/64e77d0671446bd5a5554387b69e1efd915274686844bea733714c828813/setproctitle-1.3.7-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9cc9b901ce129350637426a89cfd650066a4adc6899e47822e2478a74023ff7c", size = 37376, upload-time = "2025-09-05T12:50:07.484Z" }, + { url = "https://files.pythonhosted.org/packages/89/bc/ad9c664fe524fb4a4b2d3663661a5c63453ce851736171e454fa2cdec35c/setproctitle-1.3.7-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:80e177eff2d1ec172188d0d7fd9694f8e43d3aab76a6f5f929bee7bf7894e98b", size = 33963, upload-time = "2025-09-05T12:50:09.056Z" }, + { url = "https://files.pythonhosted.org/packages/ab/01/a36de7caf2d90c4c28678da1466b47495cbbad43badb4e982d8db8167ed4/setproctitle-1.3.7-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:23e520776c445478a67ee71b2a3c1ffdafbe1f9f677239e03d7e2cc635954e18", size = 35550, upload-time = "2025-09-05T12:50:10.791Z" }, + { url = "https://files.pythonhosted.org/packages/dd/68/17e8aea0ed5ebc17fbf03ed2562bfab277c280e3625850c38d92a7b5fcd9/setproctitle-1.3.7-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:5fa1953126a3b9bd47049d58c51b9dac72e78ed120459bd3aceb1bacee72357c", size = 33727, upload-time = "2025-09-05T12:50:12.032Z" }, + { url = "https://files.pythonhosted.org/packages/b2/33/90a3bf43fe3a2242b4618aa799c672270250b5780667898f30663fd94993/setproctitle-1.3.7-cp313-cp313t-win32.whl", hash = "sha256:4a5e212bf438a4dbeece763f4962ad472c6008ff6702e230b4f16a037e2f6f29", size = 12549, upload-time = "2025-09-05T12:50:13.074Z" }, + { url = "https://files.pythonhosted.org/packages/0b/0e/50d1f07f3032e1f23d814ad6462bc0a138f369967c72494286b8a5228e40/setproctitle-1.3.7-cp313-cp313t-win_amd64.whl", hash = "sha256:cf2727b733e90b4f874bac53e3092aa0413fe1ea6d4f153f01207e6ce65034d9", size = 13243, upload-time = "2025-09-05T12:50:14.146Z" }, + { url = "https://files.pythonhosted.org/packages/89/c7/43ac3a98414f91d1b86a276bc2f799ad0b4b010e08497a95750d5bc42803/setproctitle-1.3.7-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:80c36c6a87ff72eabf621d0c79b66f3bdd0ecc79e873c1e9f0651ee8bf215c63", size = 18052, upload-time = "2025-09-05T12:50:17.928Z" }, + { url = "https://files.pythonhosted.org/packages/cd/2c/dc258600a25e1a1f04948073826bebc55e18dbd99dc65a576277a82146fa/setproctitle-1.3.7-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:b53602371a52b91c80aaf578b5ada29d311d12b8a69c0c17fbc35b76a1fd4f2e", size = 13071, upload-time = "2025-09-05T12:50:19.061Z" }, + { url = "https://files.pythonhosted.org/packages/ab/26/8e3bb082992f19823d831f3d62a89409deb6092e72fc6940962983ffc94f/setproctitle-1.3.7-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fcb966a6c57cf07cc9448321a08f3be6b11b7635be502669bc1d8745115d7e7f", size = 33180, upload-time = "2025-09-05T12:50:20.395Z" }, + { url = "https://files.pythonhosted.org/packages/f1/af/ae692a20276d1159dd0cf77b0bcf92cbb954b965655eb4a69672099bb214/setproctitle-1.3.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:46178672599b940368d769474fe13ecef1b587d58bb438ea72b9987f74c56ea5", size = 34043, upload-time = "2025-09-05T12:50:22.454Z" }, + { url = "https://files.pythonhosted.org/packages/34/b2/6a092076324dd4dac1a6d38482bedebbff5cf34ef29f58585ec76e47bc9d/setproctitle-1.3.7-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7f9e9e3ff135cbcc3edd2f4cf29b139f4aca040d931573102742db70ff428c17", size = 35892, upload-time = "2025-09-05T12:50:23.937Z" }, + { url = "https://files.pythonhosted.org/packages/1c/1a/8836b9f28cee32859ac36c3df85aa03e1ff4598d23ea17ca2e96b5845a8f/setproctitle-1.3.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:14c7eba8d90c93b0e79c01f0bd92a37b61983c27d6d7d5a3b5defd599113d60e", size = 32898, upload-time = "2025-09-05T12:50:25.617Z" }, + { url = "https://files.pythonhosted.org/packages/ef/22/8fabdc24baf42defb599714799d8445fe3ae987ec425a26ec8e80ea38f8e/setproctitle-1.3.7-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:9e64e98077fb30b6cf98073d6c439cd91deb8ebbf8fc62d9dbf52bd38b0c6ac0", size = 34308, upload-time = "2025-09-05T12:50:26.827Z" }, + { url = "https://files.pythonhosted.org/packages/15/1b/b9bee9de6c8cdcb3b3a6cb0b3e773afdb86bbbc1665a3bfa424a4294fda2/setproctitle-1.3.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b91387cc0f02a00ac95dcd93f066242d3cca10ff9e6153de7ee07069c6f0f7c8", size = 32536, upload-time = "2025-09-05T12:50:28.5Z" }, + { url = "https://files.pythonhosted.org/packages/37/0c/75e5f2685a5e3eda0b39a8b158d6d8895d6daf3ba86dec9e3ba021510272/setproctitle-1.3.7-cp314-cp314-win32.whl", hash = "sha256:52b054a61c99d1b72fba58b7f5486e04b20fefc6961cd76722b424c187f362ed", size = 12731, upload-time = "2025-09-05T12:50:43.955Z" }, + { url = "https://files.pythonhosted.org/packages/d2/ae/acddbce90d1361e1786e1fb421bc25baeb0c22ef244ee5d0176511769ec8/setproctitle-1.3.7-cp314-cp314-win_amd64.whl", hash = "sha256:5818e4080ac04da1851b3ec71e8a0f64e3748bf9849045180566d8b736702416", size = 13464, upload-time = "2025-09-05T12:50:45.057Z" }, + { url = "https://files.pythonhosted.org/packages/01/6d/20886c8ff2e6d85e3cabadab6aab9bb90acaf1a5cfcb04d633f8d61b2626/setproctitle-1.3.7-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:6fc87caf9e323ac426910306c3e5d3205cd9f8dcac06d233fcafe9337f0928a3", size = 18062, upload-time = "2025-09-05T12:50:29.78Z" }, + { url = "https://files.pythonhosted.org/packages/9a/60/26dfc5f198715f1343b95c2f7a1c16ae9ffa45bd89ffd45a60ed258d24ea/setproctitle-1.3.7-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6134c63853d87a4897ba7d5cc0e16abfa687f6c66fc09f262bb70d67718f2309", size = 13075, upload-time = "2025-09-05T12:50:31.604Z" }, + { url = "https://files.pythonhosted.org/packages/21/9c/980b01f50d51345dd513047e3ba9e96468134b9181319093e61db1c47188/setproctitle-1.3.7-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1403d2abfd32790b6369916e2313dffbe87d6b11dca5bbd898981bcde48e7a2b", size = 34744, upload-time = "2025-09-05T12:50:32.777Z" }, + { url = "https://files.pythonhosted.org/packages/86/b4/82cd0c86e6d1c4538e1a7eb908c7517721513b801dff4ba3f98ef816a240/setproctitle-1.3.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e7c5bfe4228ea22373e3025965d1a4116097e555ee3436044f5c954a5e63ac45", size = 35589, upload-time = "2025-09-05T12:50:34.13Z" }, + { url = "https://files.pythonhosted.org/packages/8a/4f/9f6b2a7417fd45673037554021c888b31247f7594ff4bd2239918c5cd6d0/setproctitle-1.3.7-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:585edf25e54e21a94ccb0fe81ad32b9196b69ebc4fc25f81da81fb8a50cca9e4", size = 37698, upload-time = "2025-09-05T12:50:35.524Z" }, + { url = "https://files.pythonhosted.org/packages/20/92/927b7d4744aac214d149c892cb5fa6dc6f49cfa040cb2b0a844acd63dcaf/setproctitle-1.3.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:96c38cdeef9036eb2724c2210e8d0b93224e709af68c435d46a4733a3675fee1", size = 34201, upload-time = "2025-09-05T12:50:36.697Z" }, + { url = "https://files.pythonhosted.org/packages/0a/0c/fd4901db5ba4b9d9013e62f61d9c18d52290497f956745cd3e91b0d80f90/setproctitle-1.3.7-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:45e3ef48350abb49cf937d0a8ba15e42cee1e5ae13ca41a77c66d1abc27a5070", size = 35801, upload-time = "2025-09-05T12:50:38.314Z" }, + { url = "https://files.pythonhosted.org/packages/e7/e3/54b496ac724e60e61cc3447f02690105901ca6d90da0377dffe49ff99fc7/setproctitle-1.3.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:1fae595d032b30dab4d659bece20debd202229fce12b55abab978b7f30783d73", size = 33958, upload-time = "2025-09-05T12:50:39.841Z" }, + { url = "https://files.pythonhosted.org/packages/ea/a8/c84bb045ebf8c6fdc7f7532319e86f8380d14bbd3084e6348df56bdfe6fd/setproctitle-1.3.7-cp314-cp314t-win32.whl", hash = "sha256:02432f26f5d1329ab22279ff863c83589894977063f59e6c4b4845804a08f8c2", size = 12745, upload-time = "2025-09-05T12:50:41.377Z" }, + { url = "https://files.pythonhosted.org/packages/08/b6/3a5a4f9952972791a9114ac01dfc123f0df79903577a3e0a7a404a695586/setproctitle-1.3.7-cp314-cp314t-win_amd64.whl", hash = "sha256:cbc388e3d86da1f766d8fc2e12682e446064c01cea9f88a88647cfe7c011de6a", size = 13469, upload-time = "2025-09-05T12:50:42.67Z" }, + { url = "https://files.pythonhosted.org/packages/c3/5b/5e1c117ac84e3cefcf8d7a7f6b2461795a87e20869da065a5c087149060b/setproctitle-1.3.7-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:b1cac6a4b0252b8811d60b6d8d0f157c0fdfed379ac89c25a914e6346cf355a1", size = 12587, upload-time = "2025-09-05T12:51:21.195Z" }, + { url = "https://files.pythonhosted.org/packages/73/02/b9eadc226195dcfa90eed37afe56b5dd6fa2f0e5220ab8b7867b8862b926/setproctitle-1.3.7-pp311-pypy311_pp73-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f1704c9e041f2b1dc38f5be4552e141e1432fba3dd52c72eeffd5bc2db04dc65", size = 14286, upload-time = "2025-09-05T12:51:22.61Z" }, + { url = "https://files.pythonhosted.org/packages/28/26/1be1d2a53c2a91ec48fa2ff4a409b395f836798adf194d99de9c059419ea/setproctitle-1.3.7-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:b08b61976ffa548bd5349ce54404bf6b2d51bd74d4f1b241ed1b0f25bce09c3a", size = 13282, upload-time = "2025-09-05T12:51:24.094Z" }, +] + [[package]] name = "shellingham" version = "1.5.4" @@ -3484,29 +3637,28 @@ wheels = [ [[package]] name = "tokenizers" -version = "0.23.1" +version = "0.22.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "huggingface-hub" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c1/60/21f715d9faba5f5407ff759472ade058ec4a507ad62bcea47cb847239a73/tokenizers-0.23.1.tar.gz", hash = "sha256:1feeeadf865a7915adc25445dea30e9933e593c31bb96c277cee36de227c8bfa", size = 365748, upload-time = "2026-04-27T14:43:25.606Z" } +sdist = { url = "https://files.pythonhosted.org/packages/73/6f/f80cfef4a312e1fb34baf7d85c72d4411afde10978d4657f8cdd811d3ccc/tokenizers-0.22.2.tar.gz", hash = "sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917", size = 372115, upload-time = "2026-01-05T10:45:15.988Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/87/39/b87a87d5bb9470610b80a2d31df42fcffeaf35118b8b97952b2aff598cc7/tokenizers-0.23.1-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:e03d6ffcbe0d56ee9c1ccd070e70a13fa750727c0277e138152acbc0252c2224", size = 3146732, upload-time = "2026-04-27T14:43:15.427Z" }, - { url = "https://files.pythonhosted.org/packages/e2/6a/068ed9f6e444c9d7e9d55ce134181325700f3d7f30410721bdc8f848d727/tokenizers-0.23.1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:e0948bbb1ac1d7cdfc9fb6d62c596e3b7550036ad60ecd654a66ad273326324e", size = 3054954, upload-time = "2026-04-27T14:43:13.745Z" }, - { url = "https://files.pythonhosted.org/packages/6c/36/e006edf031154cba92b8416057d92c3abe3635e4c4b0aa0b5b9bb39dde70/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bf13402aff9bc533c89cb849ec3b412dc3fbeacc9744840e423d7bf3f7dc0e3", size = 3374081, upload-time = "2026-04-27T14:43:01.241Z" }, - { url = "https://files.pythonhosted.org/packages/a2/ef/7735d226f9c7f874a6bee5e3f27fb25ecabdf207d37b8cf45286d0795893/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f836ca703b89ae07919a309f9651f7a88fd5a33d5f718ba5ad0870ec0256bad6", size = 3247641, upload-time = "2026-04-27T14:43:03.856Z" }, - { url = "https://files.pythonhosted.org/packages/b9/d9/24827036f6e21297bfffda0768e58eb6096a4f411e932964a01707857931/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae848657742035523fdf261773630cb819a26995fcd3d9ecae0c1daf6e5a4959", size = 3585624, upload-time = "2026-04-27T14:43:10.664Z" }, - { url = "https://files.pythonhosted.org/packages/0c/9a/22f3582b3a4f49358293a5206e25317621ee4526bfe9cdaa0f07a12e770e/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:53b09e85775d5187941e7bab30e941b4134ab4a7dd8c68e783d231fb7ca27c51", size = 3844062, upload-time = "2026-04-27T14:43:05.643Z" }, - { url = "https://files.pythonhosted.org/packages/7e/65/b8f8814eef95800f20721384136d9a1d22241d50b2874357cb70542c392f/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea5a0ce170074329faaa8ea3f6400ecde604b6678192688533af80980daae71a", size = 3460098, upload-time = "2026-04-27T14:43:08.854Z" }, - { url = "https://files.pythonhosted.org/packages/0d/d5/1353e5f677ec27c2494fb6a6725e82d56c985f53e90ec511369e7e4f02c6/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5075b405006415ea148a992d093699c66eb01952bf59f4d5727089a98bda45a4", size = 3346235, upload-time = "2026-04-27T14:43:12.377Z" }, - { url = "https://files.pythonhosted.org/packages/71/89/39b6b8fc073fb6d413d0147aa333dc7eff7be65639ac9d19930a0b21bf33/tokenizers-0.23.1-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:56f3a77de629917652f876294dc9fe6bad4a0c43bc229dc72e59bb23a0f4729a", size = 3426398, upload-time = "2026-04-27T14:43:07.264Z" }, - { url = "https://files.pythonhosted.org/packages/0f/80/127c854da64827e5b79264ce524993a90dddcb320e5cd42412c5c02f9e8a/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9d10a6d957ef01896dc274e890eee27d41bd0e74ef31e60616f0fc311345184e", size = 9823279, upload-time = "2026-04-27T14:43:17.222Z" }, - { url = "https://files.pythonhosted.org/packages/fe/ba/44c2502feb1a058f096ddfb4e0996ef3225a01a388e1a9b094e91689fe93/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:1974288a609c343774f1b897c8b482c791ab17b75ab5c8c2b1737565c1d82288", size = 9644986, upload-time = "2026-04-27T14:43:19.45Z" }, - { url = "https://files.pythonhosted.org/packages/9e/c1/464019a9fb059870bfe4eebb4ba12208f3042035e258bf5e782906bd3847/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:120468fb4c24faf0543c835a4fabafa4deb3f20a035c9b6e83d0b553a97615d4", size = 9976181, upload-time = "2026-04-27T14:43:21.463Z" }, - { url = "https://files.pythonhosted.org/packages/79/94/3ac1432bda31626071e9b6a12709b97ae05131c804b94c8f3ac622c5da32/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e3d8f40ea6268047de7046906326abed5134f27d4e8447b23763afe5808c8a96", size = 10113853, upload-time = "2026-04-27T14:43:23.617Z" }, - { url = "https://files.pythonhosted.org/packages/6a/dd/631b21433c771b1382535326f0eca80b9c9cee2e64961dd993bc9ac4669e/tokenizers-0.23.1-cp310-abi3-win32.whl", hash = "sha256:93120a930b919416da7cd10a2f606ac9919cc69cacae7980fa2140e277660948", size = 2536263, upload-time = "2026-04-27T14:43:29.888Z" }, - { url = "https://files.pythonhosted.org/packages/97/c9/2553f72aaf65a2797d4229e37fa7fbe38ffbf3e32912d31bdd78b3323e59/tokenizers-0.23.1-cp310-abi3-win_amd64.whl", hash = "sha256:e7bfaf995c1bdbbd21d13539decb6650967013759318627d85daeb7881af16b7", size = 2798223, upload-time = "2026-04-27T14:43:28.51Z" }, - { url = "https://files.pythonhosted.org/packages/cd/2b/2be299bab55fc595e3d38567edb1a87f86e594842968fa9515a07bdcf422/tokenizers-0.23.1-cp310-abi3-win_arm64.whl", hash = "sha256:a26197957d8e4425dfba746315f3c425ea00cfa8367c5fbc4ec73447893dcea9", size = 2664127, upload-time = "2026-04-27T14:43:26.949Z" }, + { url = "https://files.pythonhosted.org/packages/92/97/5dbfabf04c7e348e655e907ed27913e03db0923abb5dfdd120d7b25630e1/tokenizers-0.22.2-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c", size = 3100275, upload-time = "2026-01-05T10:41:02.158Z" }, + { url = "https://files.pythonhosted.org/packages/2e/47/174dca0502ef88b28f1c9e06b73ce33500eedfac7a7692108aec220464e7/tokenizers-0.22.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001", size = 2981472, upload-time = "2026-01-05T10:41:00.276Z" }, + { url = "https://files.pythonhosted.org/packages/d6/84/7990e799f1309a8b87af6b948f31edaa12a3ed22d11b352eaf4f4b2e5753/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7", size = 3290736, upload-time = "2026-01-05T10:40:32.165Z" }, + { url = "https://files.pythonhosted.org/packages/78/59/09d0d9ba94dcd5f4f1368d4858d24546b4bdc0231c2354aa31d6199f0399/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd", size = 3168835, upload-time = "2026-01-05T10:40:38.847Z" }, + { url = "https://files.pythonhosted.org/packages/47/50/b3ebb4243e7160bda8d34b731e54dd8ab8b133e50775872e7a434e524c28/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5", size = 3521673, upload-time = "2026-01-05T10:40:56.614Z" }, + { url = "https://files.pythonhosted.org/packages/e0/fa/89f4cb9e08df770b57adb96f8cbb7e22695a4cb6c2bd5f0c4f0ebcf33b66/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e", size = 3724818, upload-time = "2026-01-05T10:40:44.507Z" }, + { url = "https://files.pythonhosted.org/packages/64/04/ca2363f0bfbe3b3d36e95bf67e56a4c88c8e3362b658e616d1ac185d47f2/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b", size = 3379195, upload-time = "2026-01-05T10:40:51.139Z" }, + { url = "https://files.pythonhosted.org/packages/2e/76/932be4b50ef6ccedf9d3c6639b056a967a86258c6d9200643f01269211ca/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67", size = 3274982, upload-time = "2026-01-05T10:40:58.331Z" }, + { url = "https://files.pythonhosted.org/packages/1d/28/5f9f5a4cc211b69e89420980e483831bcc29dade307955cc9dc858a40f01/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4", size = 9478245, upload-time = "2026-01-05T10:41:04.053Z" }, + { url = "https://files.pythonhosted.org/packages/6c/fb/66e2da4704d6aadebf8cb39f1d6d1957df667ab24cff2326b77cda0dcb85/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a", size = 9560069, upload-time = "2026-01-05T10:45:10.673Z" }, + { url = "https://files.pythonhosted.org/packages/16/04/fed398b05caa87ce9b1a1bb5166645e38196081b225059a6edaff6440fac/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a", size = 9899263, upload-time = "2026-01-05T10:45:12.559Z" }, + { url = "https://files.pythonhosted.org/packages/05/a1/d62dfe7376beaaf1394917e0f8e93ee5f67fea8fcf4107501db35996586b/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5", size = 10033429, upload-time = "2026-01-05T10:45:14.333Z" }, + { url = "https://files.pythonhosted.org/packages/fd/18/a545c4ea42af3df6effd7d13d250ba77a0a86fb20393143bbb9a92e434d4/tokenizers-0.22.2-cp39-abi3-win32.whl", hash = "sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92", size = 2502363, upload-time = "2026-01-05T10:45:20.593Z" }, + { url = "https://files.pythonhosted.org/packages/65/71/0670843133a43d43070abeb1949abfdef12a86d490bea9cd9e18e37c5ff7/tokenizers-0.22.2-cp39-abi3-win_amd64.whl", hash = "sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48", size = 2747786, upload-time = "2026-01-05T10:45:18.411Z" }, + { url = "https://files.pythonhosted.org/packages/72/f4/0de46cfa12cdcbcd464cc59fde36912af405696f687e53a091fb432f694c/tokenizers-0.22.2-cp39-abi3-win_arm64.whl", hash = "sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc", size = 2612133, upload-time = "2026-01-05T10:45:17.232Z" }, ] [[package]] @@ -3584,6 +3736,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374, upload-time = "2026-02-03T17:35:50.982Z" }, ] +[[package]] +name = "transformers" +version = "5.12.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "regex" }, + { name = "safetensors" }, + { name = "tokenizers" }, + { name = "tqdm" }, + { name = "typer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/aa/7c/8240f612819718100a9346dc28dea6a11370c3ca9c8c6eabadd3dea4ef29/transformers-5.12.1.tar.gz", hash = "sha256:679ee731c8225347889ad4fb3b2c926a62e9da3b7d284e9d12c791da7272466b", size = 8924054, upload-time = "2026-06-15T17:27:50.604Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/56/bbd60dd8668055803bf8ba55a81f9b8a8b31497f620109a9671d26a2076d/transformers-5.12.1-py3-none-any.whl", hash = "sha256:2a5e109d2021265df7098ffbb738295acaf5ad256f12cbc586db2ea4dcbb1a8a", size = 11150587, upload-time = "2026-06-15T17:27:46.679Z" }, +] + [[package]] name = "typer" version = "0.25.1" @@ -3700,6 +3872,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, ] +[[package]] +name = "uv" +version = "0.11.26" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/cb/5efc713948ddb10b00abfb51bfd429221c720175557f9c7965fea2448fe4/uv-0.11.26.tar.gz", hash = "sha256:2a433ece2ace088dd572d8abb0e6bd9a4ecb0e10bc9856447bbb37545f384f29", size = 4331220, upload-time = "2026-06-30T14:52:03.77Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/71/86dbffac9e26df28a16639c426cf4ba572aaf43d9231463e0dca337895b2/uv-0.11.26-py3-none-linux_armv6l.whl", hash = "sha256:fb97bf04512dfe16d86084e75d8129701fc8da9fb40de8746b73c3aa617c5897", size = 25197324, upload-time = "2026-06-30T14:50:51.75Z" }, + { url = "https://files.pythonhosted.org/packages/ec/80/525b73c8188e7052343e7109466a08fcd5195055aff4b0346ce3622e48cb/uv-0.11.26-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:a58a06e5a4b0035538d3ab4160ad74c716076ea7148eb3317171c6276ac020b4", size = 24179172, upload-time = "2026-06-30T14:50:56.52Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5e/cf7b94ed3b1932c2a62573dcd388ad6c1da5c52111cd71ab7f20faa4a0aa/uv-0.11.26-py3-none-macosx_11_0_arm64.whl", hash = "sha256:7b6d078d2ce83897884c2330c0676f27be4bf3d223fb2a409460f579fb5f0a98", size = 22949576, upload-time = "2026-06-30T14:51:00.538Z" }, + { url = "https://files.pythonhosted.org/packages/bf/fd/71fa021f6909c4139d8354bea623b5e0ef0ce4a08da250da1a1645528da2/uv-0.11.26-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:1cd9ba4951681ce17f1703106266fcbe27aaa7d37f07d53cce8b5686d68a8755", size = 24936673, upload-time = "2026-06-30T14:51:04.496Z" }, + { url = "https://files.pythonhosted.org/packages/7d/5e/273425e58a8812423e3d1f6c5da1015e636fbf13a83d104317ca37e16304/uv-0.11.26-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:e4f4c3268e69ac96f01972274a62f5f930c03cbc680adba6f21e63237ba3a639", size = 24719617, upload-time = "2026-06-30T14:51:08.419Z" }, + { url = "https://files.pythonhosted.org/packages/81/f8/1601e2acc7c54963814b4831eab996d8599e690712722c5acec5114860be/uv-0.11.26-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:efcbe0e187846f5ddba23bcaed17e4f9cd2463da5c45bdb5869616f686d713ff", size = 24734176, upload-time = "2026-06-30T14:51:12.685Z" }, + { url = "https://files.pythonhosted.org/packages/88/d2/a8a422e54c08cf4b8d51bedb9dbdd3cc233aa290ad8b3ee0438c0c02a3a5/uv-0.11.26-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:120ab2de93164d08cf5950f7fe18cbebe3ff670865ae41a292452bab2346477f", size = 26158780, upload-time = "2026-06-30T14:51:16.514Z" }, + { url = "https://files.pythonhosted.org/packages/db/e6/647fe5fdc888a3d27f79977877ce4e88052fe9be5398371e51bb134fc262/uv-0.11.26-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9052bf27c7ee426901f35a48715fa9288ce631c1878b91c9a6c950288f4b8633", size = 27009550, upload-time = "2026-06-30T14:51:20.659Z" }, + { url = "https://files.pythonhosted.org/packages/72/c2/85d8e762ad83b0f14fae2255b0578c4fd7dc915746f81b64ed786342627a/uv-0.11.26-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:efdddfcc9b1b790c5f7985c5c183c851682ced165b44ffa914f4947f5cad1fbf", size = 26183777, upload-time = "2026-06-30T14:51:24.715Z" }, + { url = "https://files.pythonhosted.org/packages/d3/00/478c3a870dcac690b8c337ee950a60a952e817f574945e85155c3cc0ab34/uv-0.11.26-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4dcf4e0b5b5cbdc242dcb002f1f8d99e7cf8c043609869228a9ce15e095c0b18", size = 26260589, upload-time = "2026-06-30T14:51:28.809Z" }, + { url = "https://files.pythonhosted.org/packages/a7/51/e4e43e106fb8cdc026b97491ea4600f4194a9c4da0b4e4e30c2a7dceb268/uv-0.11.26-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:866ae8d28f7381c15de0906a284c1e97916424c635bf40f7960b3fc889cd725e", size = 25073850, upload-time = "2026-06-30T14:51:32.717Z" }, + { url = "https://files.pythonhosted.org/packages/f2/c2/e772b7e6c8a835e8bf6739a391cdfc8e8e244c5c496d9b40625068b59ff4/uv-0.11.26-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:22f6d62e794b252ff3a1e2dfe5010cc76208f90b2c906e54971a0223ad6f16bc", size = 25682609, upload-time = "2026-06-30T14:51:36.888Z" }, + { url = "https://files.pythonhosted.org/packages/1a/69/ea77209a224a23a399cb7f6414f77ef032bd9e083e01199a0ebebf0d3ff2/uv-0.11.26-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:edd0c12b75141a6d830d138a91e366ad66e630f1c1dcaf83b8325b80cbacfcbb", size = 25800556, upload-time = "2026-06-30T14:51:40.937Z" }, + { url = "https://files.pythonhosted.org/packages/77/60/b6c0c03d2538a016b6624fa251960012e564ea02f841e958c7d60e974685/uv-0.11.26-py3-none-musllinux_1_1_i686.whl", hash = "sha256:af6a45b11a569cc4d2437e89a25a53dcf753f2a02a8f2de96be09b9b942cb3ec", size = 25385658, upload-time = "2026-06-30T14:51:45.103Z" }, + { url = "https://files.pythonhosted.org/packages/8d/e7/46881ff9164aa2e7c649901837d58eee3c57beb3b0fcc0fea6a4e40cf8f3/uv-0.11.26-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:c28822517d03aebbe9549aaaecc88ad580e4b2b6a927abffe5774a74d6ba09f6", size = 26551013, upload-time = "2026-06-30T14:51:49.062Z" }, + { url = "https://files.pythonhosted.org/packages/d6/94/380dad6c2bbe12417025aacd12cfc08322ed4c9dd8f760bff7035b86f22d/uv-0.11.26-py3-none-win32.whl", hash = "sha256:79e5c1b3410047e1962290c3b7b8f512d2c1bb95200c60b016f7729287cf34c0", size = 23947180, upload-time = "2026-06-30T14:51:53.065Z" }, + { url = "https://files.pythonhosted.org/packages/d0/13/9c588226d5b478328d739e654944430719f3ffe8999d6a24d425ec9664ab/uv-0.11.26-py3-none-win_amd64.whl", hash = "sha256:d95567e9470dc48ff03265f420c3c6973f6437f18a79d5e00b6eb4b2d9379907", size = 26909320, upload-time = "2026-06-30T14:51:57.235Z" }, + { url = "https://files.pythonhosted.org/packages/21/1d/ea66b12813878797126e2b3aca124b1c9c5ef53120702d1c00172f90a21d/uv-0.11.26-py3-none-win_arm64.whl", hash = "sha256:7e69d1569afbb936e7bf4e4ab2f72d606405f4a68f380f088a0b2233e84e056a", size = 25176820, upload-time = "2026-06-30T14:52:01.05Z" }, +] + [[package]] name = "uvicorn" version = "0.47.0"