diff --git a/apisix/plugins/ai-providers/base.lua b/apisix/plugins/ai-providers/base.lua index 41e3bffaaf98..826e6d6486de 100644 --- a/apisix/plugins/ai-providers/base.lua +++ b/apisix/plugins/ai-providers/base.lua @@ -283,6 +283,9 @@ function _M.build_request(self, conf, body, opts) -- Passed through as given: a string goes out verbatim (the client's own -- bytes), a table is encoded by the transport. body = body, + -- Providers that route through a redirect declare it, so the transport + -- follows the hop instead of handing the 3xx back downstream. + follow_redirects = self.follow_redirects, } -- AWS SigV4 signing (must be last — signs the finalized body) diff --git a/apisix/plugins/ai-providers/schema.lua b/apisix/plugins/ai-providers/schema.lua index d8be7b6ebb0d..9a50e875aca8 100644 --- a/apisix/plugins/ai-providers/schema.lua +++ b/apisix/plugins/ai-providers/schema.lua @@ -21,7 +21,7 @@ local _M = {} _M.providers = { "openai", "deepseek", "aimlapi", "anthropic", "openai-compatible", "azure-openai", "openrouter", - "gemini", "vertex-ai", "bedrock", + "gemini", "vertex-ai", "bedrock", "thegrid", } return _M diff --git a/apisix/plugins/ai-providers/thegrid.lua b/apisix/plugins/ai-providers/thegrid.lua new file mode 100644 index 000000000000..e34673626da5 --- /dev/null +++ b/apisix/plugins/ai-providers/thegrid.lua @@ -0,0 +1,59 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +-- The Grid accepts `max_completion_tokens` like OpenAI, so the override is +-- mapped to that field rather than the legacy `max_tokens`. +local function rewrite_chat_request_body(body, override, force) + if override.max_tokens then + if force or (body.max_completion_tokens == nil and body.max_tokens == nil) then + body.max_completion_tokens = override.max_tokens + body.max_tokens = nil + end + end +end + + +local function rewrite_responses_request_body(body, override, force) + if override.max_tokens then + if force or body.max_output_tokens == nil then + body.max_output_tokens = override.max_tokens + end + end +end + +return require("apisix.plugins.ai-providers.base").new( + { + host = "api.thegrid.ai", + port = 443, + -- The Consumption API validates the request and then answers with a + -- documented 307 to its routing layer on the same host, which is where + -- inference is actually fulfilled. The transport follows that hop so + -- the Plugin observes the real response, and token usage stays visible + -- to ai-rate-limiting and to ai-proxy-multi's fallback. + follow_redirects = true, + capabilities = { + ["openai-chat"] = { + path = "/v1/chat/completions", + rewrite_request_body = rewrite_chat_request_body, + }, + ["openai-responses"] = { + path = "/v1/responses", + rewrite_request_body = rewrite_responses_request_body, + }, + }, + } +) diff --git a/apisix/plugins/ai-transport/http.lua b/apisix/plugins/ai-transport/http.lua index ee72fd7a333b..5ca32a054168 100644 --- a/apisix/plugins/ai-transport/http.lua +++ b/apisix/plugins/ai-transport/http.lua @@ -20,12 +20,14 @@ local core = require("apisix.core") local http_client = require("apisix.utils.http") +local url = require("socket.url") local ngx_now = ngx.now local pairs = pairs local ipairs = ipairs local pcall = pcall local type = type local str_lower = string.lower +local tonumber = tonumber local tostring = tostring local attr_schema = { @@ -121,6 +123,82 @@ local function encode_body(body) end +-- Redirect statuses that preserve the method and the body, so the request can +-- be replayed unchanged. 301/302/303 are deliberately absent: they permit the +-- method to be rewritten to GET, which would silently drop the prompt. +local REDIRECT_STATUSES = { + [307] = true, + [308] = true, +} + + +--- Follow one same-origin 307/308 redirect, reusing the open connection. +-- Providers that route through a redirect (The Grid's Consumption API answers +-- with a documented 307 to its routing layer) need it followed inside the +-- gateway: returning the 3xx downstream would leave the Plugin blind to the +-- real response, so token usage, retries and instance fallback would all see a +-- request that never produced any tokens. +-- +-- Only same-origin redirects are followed. A cross-origin hop would replay the +-- provider credentials in `Authorization` against a host chosen by the +-- response, so it is refused rather than followed. +-- @param httpc table Connected HTTP client +-- @param params table Request parameters that produced `res` +-- @param res table The redirect response +-- @return table|nil New response object +-- @return string|nil Error message +local function follow_redirect(httpc, params, res) + local location = res.headers["Location"] or res.headers["location"] + if not location then + return nil, "redirect response carried no Location header" + end + + local parsed = url.parse(location) + if not parsed or not parsed.path then + return nil, "could not parse Location header" + end + + -- A Location without a host is relative, and therefore same-origin already. + if parsed.host then + local scheme = parsed.scheme or params.scheme + local port = tonumber(parsed.port) + or (scheme == "https" and 443 or 80) + if parsed.host ~= params.host + or scheme ~= params.scheme + or port ~= tonumber(params.port) then + return nil, "refusing to follow cross-origin redirect to " .. location + end + end + + -- The connection is reused for the replay, so the redirect's own body has + -- to be drained off the wire first. + local _, read_err = res:read_body() + if read_err then + return nil, "failed to drain redirect body: " .. read_err + end + + local next_params = {} + for k, v in pairs(params) do + next_params[k] = v + end + next_params.path = parsed.path + -- A redirect target that carries its own query string replaces the original + -- one, which belonged to the hop that has already been answered. Without + -- one, the original is kept, so providers that authenticate through + -- `auth.query` still reach the target authenticated. + if parsed.query then + next_params.query = parsed.query + end + + local next_res, err = httpc:request(next_params) + if not next_res then + return nil, "redirect request: " .. (err or "unknown") + end + + return next_res +end + + --- Send an HTTP request to an AI service. -- Handles the full lifecycle: create client, connect, encode body, -- send request, and return the response object. @@ -196,6 +274,22 @@ function _M.request(params, timeout) } end + if params.follow_redirects and REDIRECT_STATUSES[res.status] then + local redirected, redirect_err = follow_redirect(httpc, params, res) + if not redirected then + httpc:close() + return nil, redirect_err, { + upstream_addr = upstream_addr, + upstream_host = upstream_host, + upstream_scheme = upstream_scheme, + upstream_uri = params.path, + connect_time = connect_time, + t0 = t0, + } + end + res = redirected + end + local header_time = (ngx_now() - t0) * 1000 -- Attach httpc and upstream metadata to res diff --git a/docs/en/latest/plugins/ai-proxy-multi.md b/docs/en/latest/plugins/ai-proxy-multi.md index 425771e3024a..7efea05239b8 100644 --- a/docs/en/latest/plugins/ai-proxy-multi.md +++ b/docs/en/latest/plugins/ai-proxy-multi.md @@ -77,7 +77,7 @@ When an instance's `provider` is set to `bedrock`, the Plugin expects requests i | balancer.key | string | False | | | Used when `type` is `chash`. When `hash_on` is set to `header` or `cookie`, `key` is required. When `hash_on` is set to `consumer`, `key` is not required as the consumer name will be used as the key automatically. | | instances | array[object] | True | | | LLM instance configurations. | | instances.name | string | True | | | Name of the LLM service instance. It must be unique within `instances`, since it identifies the instance in the balancer, in its health checker, and in other Plugins that reference it, such as `ai-rate-limiting`. | -| instances.provider | string | True | | [openai, deepseek, azure-openai, aimlapi, anthropic, openrouter, gemini, vertex-ai, bedrock, openai-compatible] | LLM service provider. When set to `openai`, the Plugin will proxy the request to `api.openai.com`. When set to `deepseek`, the Plugin will proxy the request to `api.deepseek.com`. When set to `aimlapi`, the Plugin uses the OpenAI-compatible driver and proxies the request to `api.aimlapi.com` by default. When set to `anthropic`, the Plugin will proxy the request to `api.anthropic.com` by default. When set to `openrouter`, the Plugin uses the OpenAI-compatible driver and proxies the request to `openrouter.ai` by default. When set to `gemini`, the Plugin uses the OpenAI-compatible driver and proxies the request to `generativelanguage.googleapis.com` by default. When set to `vertex-ai`, the Plugin will proxy the request to `aiplatform.googleapis.com` by default and requires `provider_conf` or `override`. When set to `bedrock`, the Plugin proxies the request to Amazon Bedrock's Converse API at `bedrock-runtime.{region}.amazonaws.com` and signs the request with AWS SigV4. Requires `provider_conf.region` and `auth.aws`. When set to `openai-compatible`, the Plugin will proxy the request to the custom endpoint configured in `override`. | +| instances.provider | string | True | | [openai, deepseek, azure-openai, aimlapi, anthropic, openrouter, gemini, vertex-ai, bedrock, thegrid, openai-compatible] | LLM service provider. When set to `openai`, the Plugin will proxy the request to `api.openai.com`. When set to `deepseek`, the Plugin will proxy the request to `api.deepseek.com`. When set to `aimlapi`, the Plugin uses the OpenAI-compatible driver and proxies the request to `api.aimlapi.com` by default. When set to `anthropic`, the Plugin will proxy the request to `api.anthropic.com` by default. When set to `openrouter`, the Plugin uses the OpenAI-compatible driver and proxies the request to `openrouter.ai` by default. When set to `gemini`, the Plugin uses the OpenAI-compatible driver and proxies the request to `generativelanguage.googleapis.com` by default. When set to `vertex-ai`, the Plugin will proxy the request to `aiplatform.googleapis.com` by default and requires `provider_conf` or `override`. When set to `bedrock`, the Plugin proxies the request to Amazon Bedrock's Converse API at `bedrock-runtime.{region}.amazonaws.com` and signs the request with AWS SigV4. Requires `provider_conf.region` and `auth.aws`. When set to `thegrid`, the Plugin uses the OpenAI-compatible driver and proxies the request to `api.thegrid.ai` by default; The Grid's model ids are market instruments such as `text-standard` or `agent-max` rather than fixed models. The Consumption API answers with a `307` redirect to its routing layer on the same host; the Plugin follows it, so the response — and the token usage that `ai-rate-limiting` and instance fallback depend on — stays visible to the gateway. When set to `openai-compatible`, the Plugin will proxy the request to the custom endpoint configured in `override`. | | instances.provider_conf | object | False | | | Configuration for the specific provider. Required when `provider` is set to `vertex-ai` and `override` is not configured. Required when `provider` is set to `bedrock`. | | instances.provider_conf.project_id | string | True | | | Google Cloud Project ID. | | instances.provider_conf.region | string | True (depending on provider) | | minLength = 1 (for Bedrock) | When `provider` is `vertex-ai`, this is the Google Cloud Region. When `provider` is `bedrock`, this is the AWS region used to construct the Bedrock endpoint and to sign the request with SigV4 (required, must be non-empty). | diff --git a/docs/en/latest/plugins/ai-proxy.md b/docs/en/latest/plugins/ai-proxy.md index 005f3108e4b0..90d0c4419fb0 100644 --- a/docs/en/latest/plugins/ai-proxy.md +++ b/docs/en/latest/plugins/ai-proxy.md @@ -67,7 +67,7 @@ When `provider` is set to `bedrock`, the Plugin expects requests in the [Bedrock | Name | Type | Required | Default | Valid values | Description | |--------------------|--------|----------|---------|------------------------------------------|-------------| -| provider | string | True | | [openai, deepseek, azure-openai, aimlapi, anthropic, openrouter, gemini, vertex-ai, bedrock, openai-compatible] | LLM service provider. When set to `openai`, the Plugin will proxy the request to `https://api.openai.com/chat/completions`. When set to `deepseek`, the Plugin will proxy the request to `https://api.deepseek.com/chat/completions`. When set to `aimlapi`, the Plugin uses the OpenAI-compatible driver and proxies the request to `https://api.aimlapi.com/v1/chat/completions` by default. When set to `anthropic`, the Plugin will proxy the request to `https://api.anthropic.com/v1/chat/completions` by default. When set to `openrouter`, the Plugin uses the OpenAI-compatible driver and proxies the request to `https://openrouter.ai/api/v1/chat/completions` by default. When set to `gemini`, the Plugin uses the OpenAI-compatible driver and proxies the request to `https://generativelanguage.googleapis.com/v1beta/openai/chat/completions` by default. When set to `vertex-ai`, the Plugin will proxy the request to `https://aiplatform.googleapis.com` by default and requires `provider_conf` or `override`. When set to `bedrock`, the Plugin will proxy the request to the AWS Bedrock Converse API (`https://bedrock-runtime..amazonaws.com`) and signs the request with AWS SigV4. When set to `openai-compatible`, the Plugin will proxy the request to the custom endpoint configured in `override`. | +| provider | string | True | | [openai, deepseek, azure-openai, aimlapi, anthropic, openrouter, gemini, vertex-ai, bedrock, thegrid, openai-compatible] | LLM service provider. When set to `openai`, the Plugin will proxy the request to `https://api.openai.com/chat/completions`. When set to `deepseek`, the Plugin will proxy the request to `https://api.deepseek.com/chat/completions`. When set to `aimlapi`, the Plugin uses the OpenAI-compatible driver and proxies the request to `https://api.aimlapi.com/v1/chat/completions` by default. When set to `anthropic`, the Plugin will proxy the request to `https://api.anthropic.com/v1/chat/completions` by default. When set to `openrouter`, the Plugin uses the OpenAI-compatible driver and proxies the request to `https://openrouter.ai/api/v1/chat/completions` by default. When set to `gemini`, the Plugin uses the OpenAI-compatible driver and proxies the request to `https://generativelanguage.googleapis.com/v1beta/openai/chat/completions` by default. When set to `vertex-ai`, the Plugin will proxy the request to `https://aiplatform.googleapis.com` by default and requires `provider_conf` or `override`. When set to `bedrock`, the Plugin will proxy the request to the AWS Bedrock Converse API (`https://bedrock-runtime..amazonaws.com`) and signs the request with AWS SigV4. When set to `thegrid`, the Plugin uses the OpenAI-compatible driver and proxies the request to `api.thegrid.ai` by default; The Grid's model ids are market instruments such as `text-standard` or `agent-max` rather than fixed models. The Consumption API answers with a `307` redirect to its routing layer on the same host; the Plugin follows it, so the response — and the token usage that `ai-rate-limiting` and instance fallback depend on — stays visible to the gateway. When set to `openai-compatible`, the Plugin will proxy the request to the custom endpoint configured in `override`. | | provider_conf | object | False | | | Configuration for the specific provider. Required when `provider` is set to `vertex-ai` and `override` is not configured. Required when `provider` is set to `bedrock`. | | provider_conf.project_id | string | True | | | Google Cloud Project ID. | | provider_conf.region | string | True (depending on provider) | | minLength = 1 (for Bedrock) | When `provider` is `vertex-ai`, this is the Google Cloud Region. When `provider` is `bedrock`, this is the AWS region used to construct the Bedrock endpoint and to sign the request with SigV4 (required, must be non-empty). | @@ -117,6 +117,7 @@ The table below shows, for each `provider` and target API endpoint, the upstream | `deepseek` | `max_tokens` | — | — | | `aimlapi` | `max_tokens` | — | — | | `openrouter` | `max_tokens` | — | — | +| `thegrid` | `max_completion_tokens` | `max_output_tokens` | — | | `gemini` | `max_completion_tokens` | — | — | | `vertex-ai` | `max_completion_tokens` | — | — | | `anthropic` | `max_tokens` | — | `max_tokens` | diff --git a/docs/en/latest/plugins/ai-request-rewrite.md b/docs/en/latest/plugins/ai-request-rewrite.md index cdeddb008403..c9938f15026e 100644 --- a/docs/en/latest/plugins/ai-request-rewrite.md +++ b/docs/en/latest/plugins/ai-request-rewrite.md @@ -44,7 +44,7 @@ The `ai-request-rewrite` Plugin processes client requests by forwarding them to | --- | --- | --- | --- | --- | --- | | `max_req_body_size` | integer | False | 67108864 | >= 1 | Maximum request body size in bytes buffered into memory. Requests with a larger body are rejected. | | `prompt` | string | True | | | The prompt to send to the LLM service for rewriting the client request. | -| `provider` | string | True | | [openai, deepseek, azure-openai, aimlapi, gemini, vertex-ai, anthropic, openrouter, openai-compatible] | LLM service provider. When set to `aimlapi`, the Plugin uses the OpenAI-compatible driver and proxies the request to `https://api.aimlapi.com/v1/chat/completions`. When set to `openai-compatible`, the Plugin proxies requests to the custom endpoint configured in `override`. When set to `azure-openai`, the Plugin also proxies requests to the custom endpoint configured in `override` and additionally omits the `model` parameter from the request body sent to Azure OpenAI. | +| `provider` | string | True | | [openai, deepseek, azure-openai, aimlapi, gemini, vertex-ai, anthropic, openrouter, thegrid, openai-compatible] | LLM service provider. When set to `aimlapi`, the Plugin uses the OpenAI-compatible driver and proxies the request to `https://api.aimlapi.com/v1/chat/completions`. When set to `openai-compatible`, the Plugin proxies requests to the custom endpoint configured in `override`. When set to `azure-openai`, the Plugin also proxies requests to the custom endpoint configured in `override` and additionally omits the `model` parameter from the request body sent to Azure OpenAI. | | `auth` | object | True | | | Authentication configurations. | | `auth.header` | object | False | | | Authentication headers. Key must match pattern `^[a-zA-Z0-9._-]+$`. At least one of `header` and `query` should be configured. | | `auth.query` | object | False | | | Authentication query parameters. Key must match pattern `^[a-zA-Z0-9._-]+$`. At least one of `header` and `query` should be configured. | diff --git a/docs/zh/latest/plugins/ai-proxy-multi.md b/docs/zh/latest/plugins/ai-proxy-multi.md index dd50b025930c..9912cfe52d9d 100644 --- a/docs/zh/latest/plugins/ai-proxy-multi.md +++ b/docs/zh/latest/plugins/ai-proxy-multi.md @@ -77,7 +77,7 @@ import TabItem from '@theme/TabItem'; | balancer.key | string | 否 | | | 当 `type` 为 `chash` 时使用。当 `hash_on` 设置为 `header` 或 `cookie` 时，需要 `key`。当 `hash_on` 设置为 `consumer` 时，不需要 `key`，因为消费者名称将自动用作键。 | | instances | array[object] | 是 | | | LLM 实例配置。 | | instances.name | string | 是 | | | LLM 服务实例的名称。该名称在 `instances` 中必须唯一，因为负载均衡、健康检查以及 `ai-rate-limiting` 等引用该实例的插件都以它作为实例标识。 | -| instances.provider | string | 是 | | [openai, deepseek, azure-openai, aimlapi, anthropic, openrouter, gemini, vertex-ai, bedrock, openai-compatible] | LLM 服务提供商。设置为 `openai` 时，插件将代理请求到 `api.openai.com`。设置为 `deepseek` 时，插件将代理请求到 `api.deepseek.com`。设置为 `aimlapi` 时，插件使用 OpenAI 兼容驱动程序，默认将请求代理到 `api.aimlapi.com`。设置为 `anthropic` 时，插件使用 OpenAI 兼容驱动程序，默认将请求代理到 `api.anthropic.com`。设置为 `openrouter` 时，插件使用 OpenAI 兼容驱动程序，默认将请求代理到 `openrouter.ai`。设置为 `gemini` 时，插件使用 OpenAI 兼容驱动程序，默认将请求代理到 `generativelanguage.googleapis.com`。设置为 `vertex-ai` 时，插件默认将请求代理到 `aiplatform.googleapis.com`，且需要配置 `provider_conf` 或 `override`。设置为 `bedrock` 时，插件将代理请求到 AWS Bedrock Converse API（`bedrock-runtime..amazonaws.com`），并使用 AWS SigV4 对请求进行签名。设置为 `openai-compatible` 时，插件将代理请求到在 `override` 中配置的自定义端点。 | +| instances.provider | string | 是 | | [openai, deepseek, azure-openai, aimlapi, anthropic, openrouter, gemini, vertex-ai, bedrock, thegrid, openai-compatible] | LLM 服务提供商。设置为 `openai` 时，插件将代理请求到 `api.openai.com`。设置为 `deepseek` 时，插件将代理请求到 `api.deepseek.com`。设置为 `aimlapi` 时，插件使用 OpenAI 兼容驱动程序，默认将请求代理到 `api.aimlapi.com`。设置为 `anthropic` 时，插件使用 OpenAI 兼容驱动程序，默认将请求代理到 `api.anthropic.com`。设置为 `openrouter` 时，插件使用 OpenAI 兼容驱动程序，默认将请求代理到 `openrouter.ai`。设置为 `gemini` 时，插件使用 OpenAI 兼容驱动程序，默认将请求代理到 `generativelanguage.googleapis.com`。设置为 `vertex-ai` 时，插件默认将请求代理到 `aiplatform.googleapis.com`，且需要配置 `provider_conf` 或 `override`。设置为 `bedrock` 时，插件将代理请求到 AWS Bedrock Converse API（`bedrock-runtime..amazonaws.com`），并使用 AWS SigV4 对请求进行签名。设置为 `thegrid` 时，插件使用 OpenAI 兼容驱动程序，默认将请求代理到 `api.thegrid.ai`；The Grid 的模型 ID 是诸如 `text-standard` 或 `agent-max` 的市场合约，而非固定模型。Consumption API 会返回 `307` 重定向到同一主机上的路由层；插件会跟随该重定向，因此响应以及 `ai-rate-limiting` 和实例回退所依赖的 token 用量对网关始终可见。设置为 `openai-compatible` 时，插件将代理请求到在 `override` 中配置的自定义端点。 | | instances.provider_conf | object | 否 | | | 特定提供商的配置。当 `provider` 设置为 `vertex-ai` 且未配置 `override` 时必填。当 `provider` 设置为 `bedrock` 时必填。 | | instances.provider_conf.project_id | string | 是 | | | Google Cloud 项目 ID。 | | instances.provider_conf.region | string | 视提供商而定 | | minLength = 1（Bedrock 时） | 当 `provider` 为 `vertex-ai` 时，此项为 Google Cloud 区域。当 `provider` 为 `bedrock` 时，此项为用于构造 Bedrock 端点并使用 SigV4 对请求进行签名的 AWS 区域（必填，不能为空）。 | diff --git a/docs/zh/latest/plugins/ai-proxy.md b/docs/zh/latest/plugins/ai-proxy.md index 62a1b4eae41e..06dff7c569a2 100644 --- a/docs/zh/latest/plugins/ai-proxy.md +++ b/docs/zh/latest/plugins/ai-proxy.md @@ -67,7 +67,7 @@ import TabItem from '@theme/TabItem'; | 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 | |--------------------|--------|----------|---------|------------------------------------------|-------------| -| provider | string | 是 | | [openai, deepseek, azure-openai, aimlapi, anthropic, openrouter, gemini, vertex-ai, bedrock, openai-compatible] | LLM 服务提供商。当设置为 `openai` 时，插件将代理请求到 `https://api.openai.com/chat/completions`。当设置为 `deepseek` 时，插件将代理请求到 `https://api.deepseek.com/chat/completions`。当设置为 `aimlapi` 时，插件使用 OpenAI 兼容驱动程序，默认将请求代理到 `https://api.aimlapi.com/v1/chat/completions`。当设置为 `anthropic` 时，插件将代理请求到 `https://api.anthropic.com/v1/chat/completions`。当设置为 `openrouter` 时，插件使用 OpenAI 兼容驱动程序，默认将请求代理到 `https://openrouter.ai/api/v1/chat/completions`。当设置为 `gemini` 时，插件使用 OpenAI 兼容驱动程序，默认将请求代理到 `https://generativelanguage.googleapis.com/v1beta/openai/chat/completions`。当设置为 `vertex-ai` 时，插件默认将请求代理到 `https://aiplatform.googleapis.com`，需要配置 `provider_conf` 或 `override`。当设置为 `bedrock` 时，插件将代理请求到 AWS Bedrock Converse API（`https://bedrock-runtime..amazonaws.com`），并使用 AWS SigV4 对请求进行签名。当设置为 `openai-compatible` 时，插件将代理请求到在 `override` 中配置的自定义端点。当设置为 `azure-openai` 时，插件同样将请求代理到 `override` 中配置的自定义端点，并会额外移除用户请求中的 `model` 参数。 | +| provider | string | 是 | | [openai, deepseek, azure-openai, aimlapi, anthropic, openrouter, gemini, vertex-ai, bedrock, thegrid, openai-compatible] | LLM 服务提供商。当设置为 `openai` 时，插件将代理请求到 `https://api.openai.com/chat/completions`。当设置为 `deepseek` 时，插件将代理请求到 `https://api.deepseek.com/chat/completions`。当设置为 `aimlapi` 时，插件使用 OpenAI 兼容驱动程序，默认将请求代理到 `https://api.aimlapi.com/v1/chat/completions`。当设置为 `anthropic` 时，插件将代理请求到 `https://api.anthropic.com/v1/chat/completions`。当设置为 `openrouter` 时，插件使用 OpenAI 兼容驱动程序，默认将请求代理到 `https://openrouter.ai/api/v1/chat/completions`。当设置为 `gemini` 时，插件使用 OpenAI 兼容驱动程序，默认将请求代理到 `https://generativelanguage.googleapis.com/v1beta/openai/chat/completions`。当设置为 `vertex-ai` 时，插件默认将请求代理到 `https://aiplatform.googleapis.com`，需要配置 `provider_conf` 或 `override`。当设置为 `bedrock` 时，插件将代理请求到 AWS Bedrock Converse API（`https://bedrock-runtime..amazonaws.com`），并使用 AWS SigV4 对请求进行签名。当设置为 `thegrid` 时，插件使用 OpenAI 兼容驱动，默认将请求代理到 `api.thegrid.ai`；The Grid 的模型 ID 是诸如 `text-standard` 或 `agent-max` 的市场合约，而非固定模型。Consumption API 会返回 `307` 重定向到同一主机上的路由层；插件会跟随该重定向，因此响应以及 `ai-rate-limiting` 和实例回退所依赖的 token 用量对网关始终可见。 当设置为 `openai-compatible` 时，插件将代理请求到在 `override` 中配置的自定义端点。当设置为 `azure-openai` 时，插件同样将请求代理到 `override` 中配置的自定义端点，并会额外移除用户请求中的 `model` 参数。 | | provider_conf | object | 否 | | | 特定提供商的配置。当 `provider` 设置为 `vertex-ai` 且未配置 `override` 时必填。当 `provider` 设置为 `bedrock` 时必填。 | | provider_conf.project_id | string | 是 | | | Google Cloud 项目 ID。 | | provider_conf.region | string | 视提供商而定 | | minLength = 1（Bedrock 时） | 当 `provider` 为 `vertex-ai` 时，此项为 Google Cloud 区域。当 `provider` 为 `bedrock` 时，此项为用于构造 Bedrock 端点并使用 SigV4 对请求进行签名的 AWS 区域（必填，不能为空）。 | @@ -116,6 +116,7 @@ import TabItem from '@theme/TabItem'; | `deepseek` | `max_tokens` | — | — | | `aimlapi` | `max_tokens` | — | — | | `openrouter` | `max_tokens` | — | — | +| `thegrid` | `max_completion_tokens` | `max_output_tokens` | — | | `gemini` | `max_completion_tokens` | — | — | | `vertex-ai` | `max_completion_tokens` | — | — | | `anthropic` | `max_tokens` | — | `max_tokens` | diff --git a/docs/zh/latest/plugins/ai-request-rewrite.md b/docs/zh/latest/plugins/ai-request-rewrite.md index bd0c1c0585c3..5dd2c2620fa1 100644 --- a/docs/zh/latest/plugins/ai-request-rewrite.md +++ b/docs/zh/latest/plugins/ai-request-rewrite.md @@ -43,7 +43,7 @@ import TabItem from '@theme/TabItem'; | 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 | | --- | --- | --- | --- | --- | --- | | `prompt` | string | 是 | | | 发送到 LLM 服务用于重写客户端请求的提示词。 | -| `provider` | string | 是 | | [openai, deepseek, azure-openai, aimlapi, gemini, vertex-ai, anthropic, openrouter, openai-compatible] | LLM 服务提供商。设置为 `aimlapi` 时，插件使用 OpenAI 兼容驱动并将请求代理到 `https://api.aimlapi.com/v1/chat/completions`。设置为 `openai-compatible` 时，插件将请求代理到 `override` 中配置的自定义端点。设置为 `azure-openai` 时，插件同样将请求代理到 `override` 中配置的自定义端点，并会额外移除用户请求中的 `model` 参数。 | +| `provider` | string | 是 | | [openai, deepseek, azure-openai, aimlapi, gemini, vertex-ai, anthropic, openrouter, thegrid, openai-compatible] | LLM 服务提供商。设置为 `aimlapi` 时，插件使用 OpenAI 兼容驱动并将请求代理到 `https://api.aimlapi.com/v1/chat/completions`。设置为 `openai-compatible` 时，插件将请求代理到 `override` 中配置的自定义端点。设置为 `azure-openai` 时，插件同样将请求代理到 `override` 中配置的自定义端点，并会额外移除用户请求中的 `model` 参数。 | | `auth` | object | 是 | | | 身份验证配置。 | | `auth.header` | object | 否 | | | 身份验证请求头。键必须匹配模式 `^[a-zA-Z0-9._-]+$`。`header` 和 `query` 至少需要配置其中一个。 | | `auth.query` | object | 否 | | | 身份验证查询参数。键必须匹配模式 `^[a-zA-Z0-9._-]+$`。`header` 和 `query` 至少需要配置其中一个。 | diff --git a/t/lib/server.lua b/t/lib/server.lua index c21975ff6691..f1ca7a793213 100644 --- a/t/lib/server.lua +++ b/t/lib/server.lua @@ -819,6 +819,61 @@ local function ai_fixture_dispatch() require("lib.fixture_loader").dispatch() end +-- Routing-redirect endpoints, modelling a provider whose API validates the +-- request and then answers with a 307 to the host that fulfills it (The Grid's +-- Consumption API works this way). The redirect target is on the same origin. +function _M.redirect_v1_chat_completions() + ngx.status = 307 + ngx.header["Location"] = + "http://127.0.0.1:1980/r/v1/chat/completions?token=mock-routing-token" + ngx.say("redirecting") +end + + +-- Same, but pointing at another origin, which must never be followed because +-- the replay would carry the provider credentials to a host named by the +-- response rather than by the configuration. +function _M.redirect_cross_v1_chat_completions() + ngx.status = 307 + ngx.header["Location"] = + "http://localhost:1980/r/v1/chat/completions?token=mock-routing-token" + ngx.say("redirecting") +end + + +-- The redirect target. Asserts that the replay preserved the method, the body +-- and the request headers, then serves the requested fixture. +function _M.r_v1_chat_completions() + if ngx.req.get_method() ~= "POST" then + ngx.status = 400 + ngx.say([[{"error":"redirect replay changed the method"}]]) + return + end + + ngx.req.read_body() + local body = ngx.req.get_body_data() + if not body or body == "" then + ngx.status = 400 + ngx.say([[{"error":"redirect replay dropped the body"}]]) + return + end + + if ngx.req.get_uri_args()["token"] ~= "mock-routing-token" then + ngx.status = 400 + ngx.say([[{"error":"redirect replay dropped the target query"}]]) + return + end + + if not ngx.req.get_headers()["authorization"] then + ngx.status = 400 + ngx.say([[{"error":"redirect replay dropped the authorization header"}]]) + return + end + + ai_fixture_dispatch() +end + + function _M.v1_chat_completions() local json = require("cjson.safe") local fixture = ngx.req.get_headers()["x-ai-fixture"] diff --git a/t/plugin/ai-proxy-thegrid.t b/t/plugin/ai-proxy-thegrid.t new file mode 100644 index 000000000000..b469d9d33a54 --- /dev/null +++ b/t/plugin/ai-proxy-thegrid.t @@ -0,0 +1,290 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +use t::APISIX 'no_plan'; + +log_level("info"); +repeat_each(1); +no_long_string(); +no_root_location(); + + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!defined $block->request) { + $block->set_value("request", "GET /t"); + } +}); + +run_tests(); + +__DATA__ + +=== TEST 1: set route with thegrid provider +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/anything", + "plugins": { + "ai-proxy": { + "provider": "thegrid", + "auth": { + "header": { + "Authorization": "Bearer token" + } + }, + "options": { + "model": "text-standard", + "max_tokens": 512, + "temperature": 1.0 + }, + "override": { + "endpoint": "http://127.0.0.1:1980/v1/chat/completions" + }, + "ssl_verify": false + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 2: send request +--- request +POST /anything +{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] } +--- more_headers +Authorization: Bearer token +X-AI-Fixture: openai/chat-basic.json +--- error_code: 200 +--- response_body eval +qr/\{ "content": "1 \+ 1 = 2\.", "role": "assistant" \}/ + + + +=== TEST 3: set route with ai-proxy-multi +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/anything", + "plugins": { + "ai-proxy-multi": { + "instances": [ + { + "name": "thegrid", + "provider": "thegrid", + "weight": 1, + "auth": { + "header": { + "Authorization": "Bearer token" + } + }, + "options": { + "model": "agent-standard", + "max_tokens": 512 + }, + "override": { + "endpoint": "http://127.0.0.1:1980/v1/chat/completions" + } + } + ], + "ssl_verify": false + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 4: send request through ai-proxy-multi +--- request +POST /anything +{ "messages": [ { "role": "system", "content": "You are a mathematician" }, { "role": "user", "content": "What is 1+1?"} ] } +--- more_headers +Authorization: Bearer token +X-AI-Fixture: openai/chat-basic.json +--- error_code: 200 +--- response_body eval +qr/\{ "content": "1 \+ 1 = 2\.", "role": "assistant" \}/ + + + +=== TEST 5: reject an unknown provider name +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/anything", + "plugins": { + "ai-proxy": { + "provider": "thegrid-typo", + "auth": { + "header": { + "Authorization": "Bearer token" + } + }, + "options": { + "model": "text-standard" + } + } + } + }]] + ) + + ngx.status = code + ngx.say(body) + } + } +--- error_code: 400 +--- response_body_like eval +qr/provider/ + + + +=== TEST 6: follow the routing redirect and return the final response +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/anything", + "plugins": { + "ai-proxy": { + "provider": "thegrid", + "auth": { + "header": { + "Authorization": "Bearer token" + } + }, + "options": { + "model": "text-standard" + }, + "override": { + "endpoint": "http://127.0.0.1:1980/redirect/v1/chat/completions" + }, + "ssl_verify": false + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 7: redirected request completes with the model output +--- request +POST /anything +{ "messages": [ { "role": "user", "content": "What is 1+1?"} ] } +--- more_headers +Authorization: Bearer token +X-AI-Fixture: openai/chat-basic.json +--- error_code: 200 +--- response_body eval +qr/\{ "content": "1 \+ 1 = 2\.", "role": "assistant" \}/ + + + +=== TEST 8: set route whose redirect leaves the origin +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/anything", + "plugins": { + "ai-proxy": { + "provider": "thegrid", + "auth": { + "header": { + "Authorization": "Bearer token" + } + }, + "options": { + "model": "text-standard" + }, + "override": { + "endpoint": "http://127.0.0.1:1980/redirect-cross/v1/chat/completions" + }, + "ssl_verify": false + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 9: a cross-origin redirect is refused, not followed +--- request +POST /anything +{ "messages": [ { "role": "user", "content": "What is 1+1?"} ] } +--- more_headers +Authorization: Bearer token +X-AI-Fixture: openai/chat-basic.json +--- error_code: 500 +--- error_log +refusing to follow cross-origin redirect