diff --git a/python/litellm-gateway/.gitignore b/python/litellm-gateway/.gitignore deleted file mode 100644 index 824ac067d5..0000000000 --- a/python/litellm-gateway/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -node_modules/ -.next/ -__pycache__/ -*.pyc -.env -.env.local -.vercel diff --git a/python/litellm-gateway/README.md b/python/litellm-gateway/README.md deleted file mode 100644 index ae6987e897..0000000000 --- a/python/litellm-gateway/README.md +++ /dev/null @@ -1,109 +0,0 @@ -# LiteLLM Gateway on Vercel - -Self-hosted AI gateway using [LiteLLM](https://www.litellm.ai/) deployed on Vercel with [Services](https://vercel.com/docs/services). A Next.js chat frontend talks to a LiteLLM proxy backend — both deployed as a single Vercel project. - -[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fvercel%2Fexamples%2Ftree%2Fmain%2Fpython%2Flitellm-gateway&env=VERCEL_AI_GATEWAY_API_KEY&envDescription=Vercel%20AI%20Gateway%20API%20key%20from%20your%20dashboard&envLink=https%3A%2F%2Fvercel.com%2Fdocs%2Fai-gateway) - -## Architecture - -Two services deployed together via `experimentalServices` in `vercel.json`: - -- **frontend** (Next.js) at `/` — Chat UI using the [AI SDK](https://sdk.vercel.ai) -- **gateway** (LiteLLM/FastAPI) at `/gateway` — OpenAI-compatible proxy that routes to any LLM provider - -``` -Browser → /api/chat (Next.js) → GATEWAY_URL/v1/chat/completions (LiteLLM) → Vercel AI Gateway → Provider -``` - -Vercel Services automatically generates a `GATEWAY_URL` environment variable so the frontend can reach the gateway without hardcoded URLs. - -## Project structure - -``` -litellm-gateway/ -├── gateway/ -│ ├── app.py # LiteLLM proxy entrypoint -│ ├── litellm_config.yaml # Model + provider config -│ └── pyproject.toml # Python dependencies -├── frontend/ -│ ├── app/ -│ │ ├── api/chat/route.ts # Proxies to LiteLLM via GATEWAY_URL -│ │ ├── layout.tsx -│ │ ├── page.tsx # Chat UI -│ │ └── globals.css -│ ├── package.json -│ └── next.config.js -└── vercel.json # Services configuration -``` - -## Setup - -### 1. Configure models - -Edit `gateway/litellm_config.yaml` to define your model routing. The default config routes through the [Vercel AI Gateway](https://vercel.com/docs/ai-gateway): - -```yaml -model_list: - - model_name: gpt-4o-mini - litellm_params: - model: vercel_ai_gateway/openai/gpt-4o-mini - api_key: os.environ/VERCEL_AI_GATEWAY_API_KEY -``` - -You can also route directly to providers (OpenAI, Anthropic, etc.) — see the [LiteLLM provider docs](https://docs.litellm.ai/docs/providers/) and the [config reference](https://docs.litellm.ai/docs/proxy/configs). - -### 2. Set environment variables - -| Variable | Required | Description | -| --------------------------- | ------------------------ | --------------------------------------------------------------- | -| `VERCEL_AI_GATEWAY_API_KEY` | Yes (for default config) | [Vercel AI Gateway](https://vercel.com/docs/ai-gateway) API key | -| `LITELLM_MASTER_KEY` | No | Require auth for LiteLLM proxy endpoints | - -If routing directly to providers instead, set their API keys (`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, etc.) and update `litellm_config.yaml`. - -### 3. Update the frontend model list - -The chat UI has a hardcoded model list in `frontend/app/page.tsx`. Update the `MODELS` array to match your `litellm_config.yaml`. - -### 4. Deploy - -Set the project framework to **Services** in your Vercel project settings, then: - -```bash -vercel deploy -``` - -## Local development - -Install frontend dependencies: - -```bash -cd frontend -npm install -``` - -Run all services together: - -```bash -cd .. -vercel dev -L -``` - -Open http://localhost:3000 to use the chat UI. The LiteLLM gateway runs at `/gateway` — try `/gateway/health/liveliness` to verify it's up. - -## How it works - -The Next.js API route at `/api/chat` creates an OpenAI-compatible client pointed at the LiteLLM gateway: - -```ts -const litellm = createOpenAI({ - baseURL: `${process.env.GATEWAY_URL}/v1`, - apiKey: process.env.LITELLM_MASTER_KEY || 'not-needed', -}) -``` - -`GATEWAY_URL` is auto-generated by Vercel Services — no hardcoded URLs needed. The AI SDK's `streamText` function handles streaming the LLM response back to the browser. - -## Tuning - -The gateway service is configured with `maxDuration: 120` (seconds) in `vercel.json`. You can also set `memory` (128–10240 MB) if your config requires more resources. See the [Services docs](https://vercel.com/docs/services) for all options. diff --git a/python/litellm-gateway/frontend/app/api/chat/route.ts b/python/litellm-gateway/frontend/app/api/chat/route.ts deleted file mode 100644 index a1f00b76cb..0000000000 --- a/python/litellm-gateway/frontend/app/api/chat/route.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { createOpenAI } from '@ai-sdk/openai' -import { streamText, convertToModelMessages, type UIMessage } from 'ai' -import { headers } from 'next/headers' - -export const maxDuration = 120 - -export async function POST(req: Request) { - const { - messages, - model = 'gpt-4o-mini', - }: { messages: UIMessage[]; model: string } = await req.json() - - // Forward auth cookies so the request passes Vercel's deployment protection - const requestHeaders = await headers() - const cookie = requestHeaders.get('cookie') || '' - - const litellm = createOpenAI({ - baseURL: `${process.env.GATEWAY_URL}/v1`, - apiKey: process.env.LITELLM_MASTER_KEY || 'not-needed', - headers: { cookie }, - }) - - const result = streamText({ - model: litellm(model), - messages: await convertToModelMessages(messages), - }) - - return result.toUIMessageStreamResponse() -} diff --git a/python/litellm-gateway/frontend/app/globals.css b/python/litellm-gateway/frontend/app/globals.css deleted file mode 100644 index 59c2e67412..0000000000 --- a/python/litellm-gateway/frontend/app/globals.css +++ /dev/null @@ -1,236 +0,0 @@ -* { - margin: 0; - padding: 0; - box-sizing: border-box; -} - -body { - font-family: - -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, - Cantarell, sans-serif; - background: #000; - color: #fff; - line-height: 1.6; - height: 100vh; - overflow: hidden; -} - -.container { - display: flex; - flex-direction: column; - height: 100vh; -} - -/* Nav */ - -header { - border-bottom: 1px solid #222; - flex-shrink: 0; -} - -nav { - display: flex; - align-items: center; - padding: 0.75rem 1rem; - gap: 0.75rem; - max-width: 800px; - margin: 0 auto; - width: 100%; -} - -.logo { - font-weight: 600; - font-size: 0.9rem; -} - -.new-chat { - background: #111; - border: 1px solid #333; - color: #fff; - width: 32px; - height: 32px; - border-radius: 6px; - cursor: pointer; - font-size: 1.1rem; - display: flex; - align-items: center; - justify-content: center; - transition: background 0.15s; -} - -.new-chat:hover { - background: #222; -} - -.model-select { - margin-left: auto; - background: #111; - border: 1px solid #333; - color: #fff; - padding: 0.4rem 0.6rem; - border-radius: 6px; - font-size: 0.8rem; - cursor: pointer; -} - -/* Main */ - -main { - flex: 1; - overflow-y: auto; - padding: 1rem; -} - -.empty-state { - display: flex; - flex-direction: column; - align-items: center; - justify-content: center; - height: 100%; - text-align: center; - gap: 1rem; -} - -.empty-state h1 { - font-size: 2.5rem; - font-weight: 700; - background: linear-gradient(to right, #fff, #888); - -webkit-background-clip: text; - -webkit-text-fill-color: transparent; - background-clip: text; -} - -.empty-state p { - color: #888; - font-size: 0.95rem; - max-width: 500px; -} - -.empty-state a { - color: #999; - text-decoration: underline; - text-underline-offset: 2px; -} - -.empty-state a:hover { - color: #fff; -} - -/* Messages */ - -.messages { - max-width: 800px; - margin: 0 auto; - display: flex; - flex-direction: column; - gap: 1.5rem; - padding-bottom: 1rem; -} - -.message { - display: flex; - flex-direction: column; - gap: 0.25rem; -} - -.message-role { - font-size: 0.7rem; - font-weight: 600; - color: #555; - text-transform: uppercase; - letter-spacing: 0.05em; -} - -.message.user .message-content { - background: #111; - border: 1px solid #222; - border-radius: 8px; - padding: 0.75rem 1rem; -} - -.message.assistant .message-content { - color: #ccc; - line-height: 1.7; - white-space: pre-wrap; -} - -/* Error */ - -.error { - max-width: 800px; - margin: 0 auto 0.5rem; - padding: 0.75rem 1rem; - background: #1a0000; - border: 1px solid #4a0000; - border-radius: 8px; - color: #ff6b6b; - font-size: 0.85rem; - width: calc(100% - 2rem); -} - -/* Input */ - -footer { - border-top: 1px solid #222; - padding: 1rem; - flex-shrink: 0; -} - -form { - max-width: 800px; - margin: 0 auto; -} - -.input-row { - display: flex; - gap: 0.5rem; - background: #111; - border: 1px solid #333; - border-radius: 10px; - padding: 0.5rem; -} - -.input-row input { - flex: 1; - background: transparent; - border: none; - color: #fff; - font-size: 0.9rem; - padding: 0.4rem 0.5rem; - outline: none; -} - -.input-row input::placeholder { - color: #555; -} - -.send { - background: #fff; - color: #000; - border: none; - padding: 0.4rem 1rem; - border-radius: 6px; - font-size: 0.85rem; - font-weight: 500; - cursor: pointer; - transition: opacity 0.15s; -} - -.send:disabled { - opacity: 0.3; - cursor: not-allowed; -} - -.send:not(:disabled):hover { - opacity: 0.9; -} - -@media (max-width: 640px) { - .empty-state h1 { - font-size: 1.8rem; - } - - nav { - padding: 0.5rem 0.75rem; - } -} diff --git a/python/litellm-gateway/frontend/app/layout.tsx b/python/litellm-gateway/frontend/app/layout.tsx deleted file mode 100644 index ef0907fcdf..0000000000 --- a/python/litellm-gateway/frontend/app/layout.tsx +++ /dev/null @@ -1,18 +0,0 @@ -import './globals.css' - -export const metadata = { - title: 'LiteLLM Gateway on Vercel', - description: 'A self-hosted AI gateway using LiteLLM and Vercel Services.', -} - -export default function RootLayout({ - children, -}: { - children: React.ReactNode -}) { - return ( - - {children} - - ) -} diff --git a/python/litellm-gateway/frontend/app/page.tsx b/python/litellm-gateway/frontend/app/page.tsx deleted file mode 100644 index bdb385d1ed..0000000000 --- a/python/litellm-gateway/frontend/app/page.tsx +++ /dev/null @@ -1,134 +0,0 @@ -'use client' - -import { useChat } from '@ai-sdk/react' -import { useState, useRef, useEffect } from 'react' - -const MODELS = [ - { id: 'gpt-4o-mini', name: 'GPT-4o Mini' }, - { id: 'claude-3-5-haiku', name: 'Claude 3.5 Haiku' }, -] - -export default function Home() { - const [model, setModel] = useState(MODELS[0].id) - const [input, setInput] = useState('') - const messagesEndRef = useRef(null) - - const { messages, sendMessage, error, status, setMessages, stop } = useChat() - - const isStreaming = status === 'streaming' - - useEffect(() => { - messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' }) - }, [messages]) - - function handleSubmit(e: React.FormEvent) { - e.preventDefault() - if (!input.trim() || isStreaming) return - sendMessage({ text: input }, { body: { model } }) - setInput('') - } - - function handleNewChat() { - stop() - setMessages([]) - setInput('') - } - - return ( -
-
- -
- -
- {messages.length === 0 ? ( -
-

LiteLLM Gateway

-

- Self-hosted AI gateway powered by{' '} - - LiteLLM - {' '} - and deployed on{' '} - - Vercel Services - - . -

-
- ) : ( -
- {messages.map((m) => ( -
-
- {m.role === 'user' ? 'You' : model} -
-
- {m.parts.map((part, i) => { - if (part.type === 'text') { - return {part.text} - } - return null - })} -
-
- ))} -
-
- )} -
- - {error && ( -
- {error.message || - 'Something went wrong. Check your API keys and try again.'} -
- )} - - -
- ) -} diff --git a/python/litellm-gateway/frontend/next.config.js b/python/litellm-gateway/frontend/next.config.js deleted file mode 100644 index d09bdc66ec..0000000000 --- a/python/litellm-gateway/frontend/next.config.js +++ /dev/null @@ -1,4 +0,0 @@ -/** @type {import("next").NextConfig} */ -const nextConfig = {} - -module.exports = nextConfig diff --git a/python/litellm-gateway/frontend/package.json b/python/litellm-gateway/frontend/package.json deleted file mode 100644 index c2b5f2f9b1..0000000000 --- a/python/litellm-gateway/frontend/package.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "name": "litellm-gateway-frontend", - "private": true, - "scripts": { - "dev": "next dev", - "build": "next build", - "start": "next start" - }, - "dependencies": { - "@ai-sdk/openai": "latest", - "@ai-sdk/react": "latest", - "ai": "latest", - "next": "latest", - "react": "latest", - "react-dom": "latest" - }, - "devDependencies": { - "@types/node": "latest", - "@types/react": "latest", - "@types/react-dom": "latest", - "typescript": "latest" - } -} diff --git a/python/litellm-gateway/frontend/tsconfig.json b/python/litellm-gateway/frontend/tsconfig.json deleted file mode 100644 index 0f0e975fd6..0000000000 --- a/python/litellm-gateway/frontend/tsconfig.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2017", - "lib": ["dom", "dom.iterable", "esnext"], - "allowJs": true, - "skipLibCheck": true, - "strict": true, - "noEmit": true, - "esModuleInterop": true, - "module": "esnext", - "moduleResolution": "bundler", - "resolveJsonModule": true, - "isolatedModules": true, - "jsx": "preserve", - "incremental": true, - "plugins": [{ "name": "next" }], - "paths": { - "@/*": ["./*"] - } - }, - "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"], - "exclude": ["node_modules"] -} diff --git a/python/litellm-gateway/gateway/.python-version b/python/litellm-gateway/gateway/.python-version deleted file mode 100644 index 24ee5b1be9..0000000000 --- a/python/litellm-gateway/gateway/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.13 diff --git a/python/litellm-gateway/gateway/app.py b/python/litellm-gateway/gateway/app.py deleted file mode 100644 index 3050105c7a..0000000000 --- a/python/litellm-gateway/gateway/app.py +++ /dev/null @@ -1,3 +0,0 @@ -from litellm.proxy import proxy_server - -app = proxy_server.app diff --git a/python/litellm-gateway/gateway/litellm_config.yaml b/python/litellm-gateway/gateway/litellm_config.yaml deleted file mode 100644 index 8d05014ec4..0000000000 --- a/python/litellm-gateway/gateway/litellm_config.yaml +++ /dev/null @@ -1,10 +0,0 @@ -model_list: - - model_name: gpt-4o-mini - litellm_params: - model: vercel_ai_gateway/openai/gpt-4o-mini - api_key: os.environ/VERCEL_AI_GATEWAY_API_KEY - - - model_name: claude-3-5-haiku - litellm_params: - model: vercel_ai_gateway/anthropic/claude-3-5-haiku-20241022 - api_key: os.environ/VERCEL_AI_GATEWAY_API_KEY diff --git a/python/litellm-gateway/gateway/pyproject.toml b/python/litellm-gateway/gateway/pyproject.toml deleted file mode 100644 index 5c325d01c3..0000000000 --- a/python/litellm-gateway/gateway/pyproject.toml +++ /dev/null @@ -1,5 +0,0 @@ -[project] -name = "litellm-gateway" -version = "0.0.0" -requires-python = "~=3.13.0" -dependencies = ["litellm[proxy]", "fastapi<1.0.0"] diff --git a/python/litellm-gateway/vercel.json b/python/litellm-gateway/vercel.json deleted file mode 100644 index 7698fbe56f..0000000000 --- a/python/litellm-gateway/vercel.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "$schema": "https://openapi.vercel.sh/vercel.json", - "experimentalServices": { - "frontend": { - "entrypoint": "frontend", - "routePrefix": "/", - "framework": "nextjs" - }, - "gateway": { - "entrypoint": "gateway/app.py", - "routePrefix": "/gateway", - "maxDuration": 120 - } - } -}