From 28a354c4560bd591c66c6e5bbec34a4a1247ead6 Mon Sep 17 00:00:00 2001 From: Eric Curtin Date: Mon, 31 Aug 2026 02:06:37 +0100 Subject: [PATCH] feat: add llmman as an OpenAI-compatible provider llmman (https://github.com/llmmanorg/llmman) runs local models distributed as OCI artifacts and serves an OpenAI-compatible API on port 17434. It needs no API key, so it follows the Ollama entry rather than the hosted ones. LLMMAN_HOST is a bare host:port value, so it also joins the /v1 base-URL normalization; that helper is renamed since it is no longer Ollama-specific. Routing is registered in LLM as well as the provider registry, so both llmman/ and provider="llmman" reach OpenAICompatibleCompletion instead of falling back to LiteLLM. Closes #7216 Signed-off-by: Eric Curtin --- lib/cli/src/crewai_cli/constants.py | 8 +++ lib/crewai/src/crewai/llm.py | 7 ++- .../providers/openai_compatible/completion.py | 20 ++++--- .../test_openai_compatible.py | 53 ++++++++++++++++--- 4 files changed, 73 insertions(+), 15 deletions(-) diff --git a/lib/cli/src/crewai_cli/constants.py b/lib/cli/src/crewai_cli/constants.py index c587147b3f..2ff02f289d 100644 --- a/lib/cli/src/crewai_cli/constants.py +++ b/lib/cli/src/crewai_cli/constants.py @@ -58,6 +58,12 @@ "API_BASE": "http://localhost:11434", } ], + "llmman": [ + { + "default": True, + "API_BASE": "http://localhost:17434", + } + ], "bedrock": [ { "prompt": "Enter your AWS Access Key ID (press Enter to skip)", @@ -123,6 +129,7 @@ "groq", "huggingface", "ollama", + "llmman", "watson", "bedrock", "azure", @@ -269,6 +276,7 @@ "groq/gemma-7b-it", ], "ollama": ["ollama/llama3.1", "ollama/mixtral"], + "llmman": ["llmman/qwen3.8", "llmman/gemma4"], "watson": [ "watsonx/meta-llama/llama-3-1-70b-instruct", "watsonx/meta-llama/llama-3-1-8b-instruct", diff --git a/lib/crewai/src/crewai/llm.py b/lib/crewai/src/crewai/llm.py index 1eb52255fb..0b6a61e53c 100644 --- a/lib/crewai/src/crewai/llm.py +++ b/lib/crewai/src/crewai/llm.py @@ -341,6 +341,7 @@ def _ensure_litellm() -> bool: "deepseek", "ollama", "ollama_chat", + "llmman", "hosted_vllm", "cerebras", "dashscope", @@ -446,6 +447,7 @@ def __new__(cls, model: str, is_litellm: bool = False, **kwargs: Any) -> LLM: "deepseek": "deepseek", "ollama": "ollama", "ollama_chat": "ollama_chat", + "llmman": "llmman", "hosted_vllm": "hosted_vllm", "cerebras": "cerebras", "dashscope": "dashscope", @@ -561,8 +563,8 @@ def _matches_provider_pattern(cls, model: str, provider: str) -> bool: if provider == "deepseek": return model_lower.startswith("deepseek") - if provider == "ollama" or provider == "ollama_chat": - # Ollama accepts any local model name + if provider in ("ollama", "ollama_chat", "llmman"): + # Local servers accept any model name they can resolve return True if provider == "hosted_vllm": @@ -704,6 +706,7 @@ def _get_native_provider(cls, provider: str) -> type | None: "deepseek", "ollama", "ollama_chat", + "llmman", "hosted_vllm", "cerebras", "dashscope", diff --git a/lib/crewai/src/crewai/llms/providers/openai_compatible/completion.py b/lib/crewai/src/crewai/llms/providers/openai_compatible/completion.py index da4cfd03db..2139e8c3d4 100644 --- a/lib/crewai/src/crewai/llms/providers/openai_compatible/completion.py +++ b/lib/crewai/src/crewai/llms/providers/openai_compatible/completion.py @@ -70,6 +70,13 @@ class ProviderConfig: api_key_required=False, default_api_key="ollama", ), + "llmman": ProviderConfig( + base_url="http://localhost:17434/v1", + api_key_env="LLMMAN_API_KEY", + base_url_env="LLMMAN_HOST", + api_key_required=False, + default_api_key="llmman", + ), "hosted_vllm": ProviderConfig( base_url="http://localhost:8000/v1", api_key_env="VLLM_API_KEY", @@ -92,11 +99,11 @@ class ProviderConfig: } -def _normalize_ollama_base_url(base_url: str) -> str: - """Normalize Ollama base URL to ensure it ends with /v1. +def _normalize_local_base_url(base_url: str) -> str: + """Ensure a local server base URL ends with /v1. - Ollama uses OLLAMA_HOST which may not include the /v1 suffix, - but the OpenAI-compatible endpoint requires it. + Hosts configured via OLLAMA_HOST or LLMMAN_HOST are bare host:port values, + but the OpenAI-compatible endpoint requires the /v1 suffix. Args: base_url: The base URL, potentially without /v1 suffix. @@ -122,6 +129,7 @@ class OpenAICompatibleCompletion(OpenAICompletion): - deepseek: DeepSeek (https://deepseek.com) - ollama: Ollama local server (https://ollama.ai) - ollama_chat: Alias for ollama + - llmman: llmman local server (https://github.com/llmmanorg/llmman) - hosted_vllm: vLLM server (https://github.com/vllm-project/vllm) - cerebras: Cerebras (https://cerebras.ai) - dashscope: Alibaba Dashscope/Qwen (https://dashscope.aliyun.com) @@ -222,8 +230,8 @@ def _resolve_base_url( else: resolved = config.base_url - if provider in ("ollama", "ollama_chat"): - resolved = _normalize_ollama_base_url(resolved) + if provider in ("ollama", "ollama_chat", "llmman"): + resolved = _normalize_local_base_url(resolved) return resolved diff --git a/lib/crewai/tests/llms/openai_compatible/test_openai_compatible.py b/lib/crewai/tests/llms/openai_compatible/test_openai_compatible.py index ce856a5334..1c5305f363 100644 --- a/lib/crewai/tests/llms/openai_compatible/test_openai_compatible.py +++ b/lib/crewai/tests/llms/openai_compatible/test_openai_compatible.py @@ -10,7 +10,7 @@ OPENAI_COMPATIBLE_PROVIDERS, OpenAICompatibleCompletion, ProviderConfig, - _normalize_ollama_base_url, + _normalize_local_base_url, ) @@ -73,6 +73,24 @@ def test_ollama_chat_is_alias(self): assert ollama.base_url == ollama_chat.base_url assert ollama.api_key_required == ollama_chat.api_key_required + def test_llmman_config(self): + """Test llmman provider configuration.""" + config = OPENAI_COMPATIBLE_PROVIDERS["llmman"] + assert config.base_url == "http://localhost:17434/v1" + assert config.api_key_env == "LLMMAN_API_KEY" + assert config.base_url_env == "LLMMAN_HOST" + assert config.api_key_required is False + assert config.default_api_key == "llmman" + + def test_llmman_base_url_does_not_collide(self): + """llmman must not reuse another provider's default endpoint.""" + others = [ + cfg.base_url + for name, cfg in OPENAI_COMPATIBLE_PROVIDERS.items() + if name != "llmman" + ] + assert OPENAI_COMPATIBLE_PROVIDERS["llmman"].base_url not in others + def test_hosted_vllm_config(self): """Test hosted_vllm provider configuration.""" config = OPENAI_COMPATIBLE_PROVIDERS["hosted_vllm"] @@ -96,24 +114,24 @@ def test_dashscope_config(self): assert config.api_key_required is True -class TestNormalizeOllamaBaseUrl: - """Tests for _normalize_ollama_base_url helper.""" +class TestNormalizeLocalBaseUrl: + """Tests for _normalize_local_base_url helper.""" def test_adds_v1_suffix(self): """Test that /v1 is added when missing.""" - assert _normalize_ollama_base_url("http://localhost:11434") == "http://localhost:11434/v1" + assert _normalize_local_base_url("http://localhost:11434") == "http://localhost:11434/v1" def test_preserves_existing_v1(self): """Test that existing /v1 is preserved.""" - assert _normalize_ollama_base_url("http://localhost:11434/v1") == "http://localhost:11434/v1" + assert _normalize_local_base_url("http://localhost:11434/v1") == "http://localhost:11434/v1" def test_strips_trailing_slash(self): """Test that trailing slash is handled.""" - assert _normalize_ollama_base_url("http://localhost:11434/") == "http://localhost:11434/v1" + assert _normalize_local_base_url("http://localhost:11434/") == "http://localhost:11434/v1" def test_handles_v1_with_trailing_slash(self): """Test /v1/ is normalized.""" - assert _normalize_ollama_base_url("http://localhost:11434/v1/") == "http://localhost:11434/v1" + assert _normalize_local_base_url("http://localhost:11434/v1/") == "http://localhost:11434/v1" class TestOpenAICompatibleCompletion: @@ -197,6 +215,12 @@ def test_ollama_base_url_normalized(self): completion = OpenAICompatibleCompletion(model="llama3", provider="ollama") assert completion.base_url == "http://custom-ollama:11434/v1" + def test_llmman_base_url_normalized(self): + """Test llmman base URL is normalized to include /v1.""" + with patch.dict(os.environ, {"LLMMAN_HOST": "http://custom-llmman:17434"}): + completion = OpenAICompatibleCompletion(model="qwen3.8", provider="llmman") + assert completion.base_url == "http://custom-llmman:17434/v1" + def test_openrouter_headers(self): """Test OpenRouter has HTTP-Referer header.""" with patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-key"}): @@ -242,6 +266,21 @@ def test_llm_creates_openai_compatible_for_ollama(self): assert llm.provider == "ollama" assert llm.model == "llama3" + def test_llm_creates_openai_compatible_for_llmman(self): + """Test LLM factory creates OpenAICompatibleCompletion for llmman.""" + llm = LLM(model="llmman/qwen3.8") + assert isinstance(llm, OpenAICompatibleCompletion) + assert llm.provider == "llmman" + assert llm.model == "qwen3.8" + assert llm.base_url == "http://localhost:17434/v1" + + def test_llm_creates_openai_compatible_for_explicit_llmman(self): + """Test LLM factory routes an explicit llmman provider without a prefix.""" + llm = LLM(model="qwen3.8", provider="llmman") + assert isinstance(llm, OpenAICompatibleCompletion) + assert llm.provider == "llmman" + assert llm.model == "qwen3.8" + def test_llm_creates_openai_compatible_for_openrouter(self): """Test LLM factory creates OpenAICompatibleCompletion for OpenRouter.""" with patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-key"}):