diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a173c6501..c4875f554 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -271,6 +271,13 @@ jobs: . .venv-langgraph-agui/bin/activate python -m pip install --requirement agent-langgraph-agui/requirements.txt --requirement agent-langgraph-agui/requirements-test.txt python -m pytest agent-langgraph-agui/tests -q + - name: LlamaIndex model-choice regression + run: | + set -euo pipefail + python -m venv .venv-llamaindex + . .venv-llamaindex/bin/activate + python -m pip install --requirement agent-llamaindex/requirements.txt --requirement agent-llamaindex/requirements-test.txt + python -m pytest agent-llamaindex/tests -q - run: bun install --frozen-lockfile - run: bun test tests/compose.test.ts - run: docker compose --env-file /dev/null --profile harness config --format json >/dev/null diff --git a/CHANGELOG.md b/CHANGELOG.md index d178b1634..a5e46567c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,18 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. ## Unreleased +### The LlamaIndex Bot answers with the model the setup screen chose + +The LlamaIndex Bot built an OpenAI client from `BOT_MODEL` and ignored `BOT_PROVIDER`, so it only +worked with an OpenAI model name sent to OpenAI. Picked with an Anthropic key, every run failed with +`Unknown model 'claude-sonnet-4-5'`; picked with an OpenAI-compatible endpoint, every run failed +with `Unknown model` for that endpoint's model, and an OpenAI model name went to api.openai.com +instead of the address given, because the client read `OPENAI_API_BASE` and not the +`OPENAI_BASE_URL` Compose passes. It now reaches the model through LiteLLM as `provider/model`, the +way the Agno Bot does, so all three choices answer. A model LiteLLM does not know is treated as able +to call tools, which the AG-UI workflow requires, and parameters a model does not accept, such as +the temperature LlamaIndex sends to a reasoning model, are dropped rather than refused. + ### The Audit page says "not enforced" only under a dry-run refusal that went ahead On a deployment in `dry-run`, the Audit page printed "dry-run: recorded, not enforced" under every diff --git a/agent-llamaindex/requirements-test.txt b/agent-llamaindex/requirements-test.txt new file mode 100644 index 000000000..a91d60c68 --- /dev/null +++ b/agent-llamaindex/requirements-test.txt @@ -0,0 +1,2 @@ +httpx==0.28.1 +pytest==9.0.2 diff --git a/agent-llamaindex/requirements.txt b/agent-llamaindex/requirements.txt index aeda0194d..e7e7dac57 100644 --- a/agent-llamaindex/requirements.txt +++ b/agent-llamaindex/requirements.txt @@ -1,6 +1,6 @@ llama-index-core llama-index-protocols-ag-ui -llama-index-llms-openai +llama-index-llms-litellm fastapi python-multipart uvicorn[standard] diff --git a/agent-llamaindex/src/main.py b/agent-llamaindex/src/main.py index ed9c0015d..026016454 100644 --- a/agent-llamaindex/src/main.py +++ b/agent-llamaindex/src/main.py @@ -6,9 +6,10 @@ import os +import litellm from fastapi import FastAPI, Request from fastapi.responses import JSONResponse -from llama_index.llms.openai import OpenAI +from llama_index.llms.litellm import LiteLLM from llama_index.protocols.ag_ui.router import get_ag_ui_workflow_router TOKEN_HEADER = "x-openbot-agent-token" @@ -20,6 +21,21 @@ def _model_id() -> str: return model if "/" in model else f"{provider}/{model}" +def _llm() -> LiteLLM: + model = _model_id() + if not litellm.supports_function_calling(model): + litellm.register_model( + { + model: { + "litellm_provider": model.split("/", 1)[0], + "mode": "chat", + "supports_function_calling": True, + } + } + ) + return LiteLLM(model=model, additional_kwargs={"drop_params": True}) + + app = FastAPI() @@ -39,4 +55,4 @@ async def health(): return {"ok": True, "harness": "llamaindex"} -app.include_router(get_ag_ui_workflow_router(llm=OpenAI(model=(os.environ.get("BOT_MODEL") or "gpt-4o-mini")))) +app.include_router(get_ag_ui_workflow_router(llm=_llm())) diff --git a/agent-llamaindex/tests/test_main.py b/agent-llamaindex/tests/test_main.py new file mode 100644 index 000000000..c357332f4 --- /dev/null +++ b/agent-llamaindex/tests/test_main.py @@ -0,0 +1,184 @@ +import importlib +import json +import socket +import sys +import threading +import time +from pathlib import Path + +import pytest +import uvicorn +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse, StreamingResponse +from fastapi.testclient import TestClient + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +TOKEN = "test-token" +RUN = { + "threadId": "thread-1", + "runId": "run-1", + "state": {}, + "messages": [{"id": "m1", "role": "user", "content": "Say hello"}], + "tools": [], + "context": [], + "forwardedProps": {}, +} + + +def _sse(events): + async def stream(): + for event in events: + yield event + + return StreamingResponse(stream(), media_type="text/event-stream") + + +def _provider_app(seen): + app = FastAPI() + + @app.post("/v1/chat/completions") + async def openai_chat(request: Request): + body = await request.json() + seen.append(("openai", body["model"])) + if not body.get("stream"): + return JSONResponse( + { + "id": "c", + "object": "chat.completion", + "created": 0, + "model": body["model"], + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": "hello"}, + } + ], + } + ) + chunk = { + "id": "c", + "object": "chat.completion.chunk", + "created": 0, + "model": body["model"], + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": "hello"}, + "finish_reason": None, + } + ], + } + done = {**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]} + return _sse( + [f"data: {json.dumps(chunk)}\n\n", f"data: {json.dumps(done)}\n\n", "data: [DONE]\n\n"] + ) + + @app.post("/v1/messages") + async def anthropic_messages(request: Request): + body = await request.json() + seen.append(("anthropic", body["model"])) + message = { + "id": "msg", + "type": "message", + "role": "assistant", + "model": body["model"], + "stop_sequence": None, + } + if not body.get("stream"): + return JSONResponse( + { + **message, + "content": [{"type": "text", "text": "hello"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + ) + events = [ + ("message_start", {"type": "message_start", "message": {**message, "content": [], "stop_reason": None, "usage": {"input_tokens": 1, "output_tokens": 0}}}), + ("content_block_start", {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}), + ("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "hello"}}), + ("content_block_stop", {"type": "content_block_stop", "index": 0}), + ("message_delta", {"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": None}, "usage": {"output_tokens": 1}}), + ("message_stop", {"type": "message_stop"}), + ] + return _sse([f"event: {name}\ndata: {json.dumps(data)}\n\n" for name, data in events]) + + return app + + +@pytest.fixture +def provider(): + seen = [] + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + port = probe.getsockname()[1] + server = uvicorn.Server( + uvicorn.Config(_provider_app(seen), host="127.0.0.1", port=port, log_level="error") + ) + thread = threading.Thread(target=server.run, daemon=True) + thread.start() + deadline = time.monotonic() + 10 + while not server.started and time.monotonic() < deadline: + time.sleep(0.01) + yield f"http://127.0.0.1:{port}", seen + server.should_exit = True + thread.join(timeout=10) + + +CHOICES = { + "an Anthropic key": ( + lambda base: { + "BOT_PROVIDER": "anthropic", + "BOT_MODEL": "claude-sonnet-4-5", + "ANTHROPIC_API_KEY": "test-key", + "ANTHROPIC_BASE_URL": base, + "OPENAI_API_KEY": "", + "OPENAI_BASE_URL": "", + }, + ("anthropic", "claude-sonnet-4-5"), + ), + "an OpenAI-compatible endpoint": ( + lambda base: { + "BOT_PROVIDER": "", + "BOT_MODEL": "local-model", + "OPENAI_API_KEY": "no-key-needed", + "OPENAI_BASE_URL": f"{base}/v1", + "ANTHROPIC_API_KEY": "", + }, + ("openai", "local-model"), + ), + "an OpenAI key": ( + lambda base: { + "BOT_PROVIDER": "", + "BOT_MODEL": "gpt-5.5", + "OPENAI_API_KEY": "test-key", + "OPENAI_BASE_URL": f"{base}/v1", + "ANTHROPIC_API_KEY": "", + }, + ("openai", "gpt-5.5"), + ), +} + + +@pytest.mark.parametrize("choice", list(CHOICES)) +def test_a_run_reaches_the_model_the_setup_screen_chose(monkeypatch, provider, choice): + base, seen = provider + environment, expected = CHOICES[choice] + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setenv("MANAGED_AGENT_TOKEN", TOKEN) + for key, value in environment(base).items(): + monkeypatch.setenv(key, value) + + from src import main + + main = importlib.reload(main) + response = TestClient(main.app).post( + "/run", json=RUN, headers={"x-openbot-agent-token": TOKEN} + ) + + assert response.status_code == 200 + assert '"RUN_FINISHED"' in response.text + assert '"RUN_ERROR"' not in response.text + assert seen == [expected]