From 7cc2425c08660820a1e5c9ff8eeb71551664ed37 Mon Sep 17 00:00:00 2001 From: "M. Mansour" <3020010+marazik@users.noreply.github.com> Date: Wed, 3 Jun 2026 17:41:56 -0400 Subject: [PATCH 1/3] feature(litellm): integrate docker profile and config overrides --- api/config/embedder.litellm.json | 27 ++++ api/config/generator.litellm.json | 220 ++++++++++++++++++++++++++++++ docker-compose-litellm.yml | 3 + 3 files changed, 250 insertions(+) create mode 100644 api/config/embedder.litellm.json create mode 100644 api/config/generator.litellm.json diff --git a/api/config/embedder.litellm.json b/api/config/embedder.litellm.json new file mode 100644 index 000000000..bf23bc8ed --- /dev/null +++ b/api/config/embedder.litellm.json @@ -0,0 +1,27 @@ +{ + "embedder_ollama": { + "client_class": "OllamaClient", + "model_kwargs": { + "model": "nomic-embed-text" + } + }, + "embedder": { + "client_class": "LiteLLMClient", + "initialize_kwargs": { + "api_key": "${LITELLM_API_KEY}", + "base_url": "${LITELLM_BASE_URL}" + }, + "batch_size": 10, + "model_kwargs": { + "model": "nomic-embed-text" + } + }, + "retriever": { + "top_k": 20 + }, + "text_splitter": { + "split_by": "word", + "chunk_size": 350, + "chunk_overlap": 100 + } +} diff --git a/api/config/generator.litellm.json b/api/config/generator.litellm.json new file mode 100644 index 000000000..7cb15b687 --- /dev/null +++ b/api/config/generator.litellm.json @@ -0,0 +1,220 @@ +{ + "default_provider": "litellm", + "providers": { + "dashscope": { + "default_model": "qwen-plus", + "supportsCustomModel": true, + "models": { + "qwen-plus": { + "temperature": 0.7, + "top_p": 0.8 + }, + "qwen-turbo": { + "temperature": 0.7, + "top_p": 0.8 + }, + "deepseek-r1": { + "temperature": 0.7, + "top_p": 0.8 + } + } + }, + "google": { + "default_model": "gemini-2.5-flash", + "supportsCustomModel": true, + "models": { + "gemini-2.5-flash": { + "temperature": 1.0, + "top_p": 0.8, + "top_k": 20 + }, + "gemini-2.5-flash-lite": { + "temperature": 1.0, + "top_p": 0.8, + "top_k": 20 + }, + "gemini-2.5-pro": { + "temperature": 1.0, + "top_p": 0.8, + "top_k": 20 + } + } + }, + "openai": { + "default_model": "gpt-5-nano", + "supportsCustomModel": true, + "models": { + "gpt-5": { + "temperature": 1.0 + }, + "gpt-5-nano": { + "temperature": 1.0 + }, + "gpt-5-mini": { + "temperature": 1.0 + }, + "gpt-4o": { + "temperature": 0.7, + "top_p": 0.8 + }, + "gpt-4.1": { + "temperature": 0.7, + "top_p": 0.8 + }, + "o1": { + "temperature": 0.7, + "top_p": 0.8 + }, + "o3": { + "temperature": 1.0 + }, + "o4-mini": { + "temperature": 1.0 + } + } + }, + "litellm": { + "default_model": "qwen3:1.7b", + "supportsCustomModel": true, + "models": { + "qwen3:1.7b": { + "temperature": 0.7, + "top_p": 0.8, + "num_ctx": 32000 + }, + "llama3:8b": { + "temperature": 0.7, + "top_p": 0.8, + "num_ctx": 8000 + }, + "qwen3:8b": { + "temperature": 0.7, + "top_p": 0.8, + "num_ctx": 32000 + } + } + }, + "openrouter": { + "default_model": "openai/gpt-5-nano", + "supportsCustomModel": true, + "models": { + "openai/gpt-5-nano": { + "temperature": 0.7, + "top_p": 0.8 + }, + "openai/gpt-4o": { + "temperature": 0.7, + "top_p": 0.8 + }, + "deepseek/deepseek-r1": { + "temperature": 0.7, + "top_p": 0.8 + }, + "openai/gpt-4.1": { + "temperature": 0.7, + "top_p": 0.8 + }, + "openai/o1": { + "temperature": 0.7, + "top_p": 0.8 + }, + "openai/o3": { + "temperature": 1.0 + }, + "openai/o4-mini": { + "temperature": 1.0 + }, + "anthropic/claude-3.7-sonnet": { + "temperature": 0.7, + "top_p": 0.8 + }, + "anthropic/claude-3.5-sonnet": { + "temperature": 0.7, + "top_p": 0.8 + } + } + }, + "ollama": { + "default_model": "qwen3:1.7b", + "supportsCustomModel": true, + "models": { + "qwen3:1.7b": { + "options": { + "temperature": 0.7, + "top_p": 0.8, + "num_ctx": 32000 + } + }, + "llama3:8b": { + "options": { + "temperature": 0.7, + "top_p": 0.8, + "num_ctx": 8000 + } + }, + "qwen3:8b": { + "options": { + "temperature": 0.7, + "top_p": 0.8, + "num_ctx": 32000 + } + } + } + }, + "bedrock": { + "client_class": "BedrockClient", + "default_model": "anthropic.claude-3-sonnet-20240229-v1:0", + "supportsCustomModel": true, + "models": { + "anthropic.claude-3-sonnet-20240229-v1:0": { + "temperature": 0.7, + "top_p": 0.8 + }, + "anthropic.claude-3-haiku-20240307-v1:0": { + "temperature": 0.7, + "top_p": 0.8 + }, + "anthropic.claude-3-opus-20240229-v1:0": { + "temperature": 0.7, + "top_p": 0.8 + }, + "amazon.titan-text-express-v1": { + "temperature": 0.7, + "top_p": 0.8 + }, + "cohere.command-r-v1:0": { + "temperature": 0.7, + "top_p": 0.8 + }, + "ai21.j2-ultra-v1": { + "temperature": 0.7, + "top_p": 0.8 + } + } + }, + "azure": { + "client_class": "AzureAIClient", + "default_model": "gpt-4o", + "supportsCustomModel": true, + "models": { + "gpt-4o": { + "temperature": 0.7, + "top_p": 0.8 + }, + "gpt-4": { + "temperature": 0.7, + "top_p": 0.8 + }, + "gpt-35-turbo": { + "temperature": 0.7, + "top_p": 0.8 + }, + "gpt-4-turbo": { + "temperature": 0.7, + "top_p": 0.8 + } + } + } + } +} + diff --git a/docker-compose-litellm.yml b/docker-compose-litellm.yml index 6f90878a2..a082aff81 100644 --- a/docker-compose-litellm.yml +++ b/docker-compose-litellm.yml @@ -53,6 +53,9 @@ services: volumes: - ~/.adalflow:/root/.adalflow # Persist repository and embedding data - ./api/logs:/app/api/logs # Persist log files across container restarts + # LiteLLM-specific config overrides + - ./api/config/generator.litellm.json:/app/api/config/generator.json:ro + - ./api/config/embedder.litellm.json:/app/api/config/embedder.json:ro # Resource limits for docker-compose up (not Swarm mode) mem_limit: 6g mem_reservation: 2g From 4f6161dd438b429da451f0c4c606b23ef51bf98a Mon Sep 17 00:00:00 2001 From: "M. Mansour" <3020010+marazik@users.noreply.github.com> Date: Wed, 19 Aug 2026 19:50:09 -0400 Subject: [PATCH 2/3] Removed the context window size (`num_ctx`) parameter --- api/config/generator.litellm.json | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/api/config/generator.litellm.json b/api/config/generator.litellm.json index 7cb15b687..9d28054b7 100644 --- a/api/config/generator.litellm.json +++ b/api/config/generator.litellm.json @@ -78,19 +78,16 @@ "supportsCustomModel": true, "models": { "qwen3:1.7b": { - "temperature": 0.7, - "top_p": 0.8, - "num_ctx": 32000 + "temperature": 0.7, + "top_p": 0.8 }, "llama3:8b": { - "temperature": 0.7, - "top_p": 0.8, - "num_ctx": 8000 + "temperature": 0.7, + "top_p": 0.8 }, "qwen3:8b": { - "temperature": 0.7, - "top_p": 0.8, - "num_ctx": 32000 + "temperature": 0.7, + "top_p": 0.8 } } }, From 9b5f246c05e958045f6a20c4312510ac56dd5948 Mon Sep 17 00:00:00 2001 From: "M. Mansour" <3020010+marazik@users.noreply.github.com> Date: Wed, 19 Aug 2026 20:02:17 -0400 Subject: [PATCH 3/3] Removed the redundant `initialize_kwargs` block from LiteLLM embedder config --- api/config/embedder.litellm.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/api/config/embedder.litellm.json b/api/config/embedder.litellm.json index bf23bc8ed..d10e6d253 100644 --- a/api/config/embedder.litellm.json +++ b/api/config/embedder.litellm.json @@ -7,10 +7,6 @@ }, "embedder": { "client_class": "LiteLLMClient", - "initialize_kwargs": { - "api_key": "${LITELLM_API_KEY}", - "base_url": "${LITELLM_BASE_URL}" - }, "batch_size": 10, "model_kwargs": { "model": "nomic-embed-text"