From 189a3b3bb8cad4d3c8f163ee5e7589acb3a0fda1 Mon Sep 17 00:00:00 2001 From: Saadi Myftija Date: Tue, 11 Aug 2026 12:10:43 +0200 Subject: [PATCH] feat(webapp): separate rate limit budget for deployment endpoints Most deploy-flow API calls (build-time env var resolution and sync, env key exchange, preview branches) drew from the same general API rate limit bucket as an environment's runtime traffic, so a busy environment could starve its own deployments; the /api/v*/deployments endpoints themselves were fully exempt from rate limits as a stopgap. The whole group now goes through a dedicated limiter with its own token bucket, keyed per environment for environment API keys and per token for PATs/OATs, configurable via DEPLOYMENT_RATE_LIMIT_* env vars. The general API limiter whitelists the group via a shared path list. /api/v1/deployments/current is runtime SDK surface and stays exempt from rate limits as before, outside the deploy budget. --- .../deployment-rate-limit-budget.md | 6 ++ apps/webapp/app/entry.server.tsx | 1 + apps/webapp/app/env.server.ts | 8 +++ .../app/services/apiRateLimit.server.ts | 4 +- .../app/services/deploymentApiPaths.server.ts | 13 ++++ .../services/deploymentRateLimit.server.ts | 53 ++++++++++++++ apps/webapp/server.ts | 2 + apps/webapp/test/deploymentApiPaths.test.ts | 72 +++++++++++++++++++ docs/self-hosting/env/webapp.mdx | 6 ++ 9 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 .server-changes/deployment-rate-limit-budget.md create mode 100644 apps/webapp/app/services/deploymentApiPaths.server.ts create mode 100644 apps/webapp/app/services/deploymentRateLimit.server.ts create mode 100644 apps/webapp/test/deploymentApiPaths.test.ts diff --git a/.server-changes/deployment-rate-limit-budget.md b/.server-changes/deployment-rate-limit-budget.md new file mode 100644 index 00000000000..9171813f21f --- /dev/null +++ b/.server-changes/deployment-rate-limit-budget.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +Deployment-related API endpoints now draw from their own generous rate limit budget, configurable via the `DEPLOYMENT_RATE_LIMIT_*` environment variables, so runtime API traffic no longer competes with deployments for the same per-environment budget. diff --git a/apps/webapp/app/entry.server.tsx b/apps/webapp/app/entry.server.tsx index 4511deb9966..4f3bb289f0c 100644 --- a/apps/webapp/app/entry.server.tsx +++ b/apps/webapp/app/entry.server.tsx @@ -302,6 +302,7 @@ singleton("SentryTenantContextProcessor", () => { }); export { apiRateLimiter } from "./services/apiRateLimit.server"; +export { deploymentRateLimiter } from "./services/deploymentRateLimit.server"; export { engineRateLimiter } from "./services/engineRateLimit.server"; export { otlpRateLimiter } from "./services/otlpRateLimit.server"; export { runWithHttpContext } from "./services/httpAsyncStorage.server"; diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 5393fbb2148..b7a99e4458c 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -614,6 +614,14 @@ const EnvironmentSchema = z API_RATE_LIMIT_JWT_WINDOW: z.string().default("1m"), API_RATE_LIMIT_JWT_TOKENS: z.coerce.number().int().default(60), + // Separate budget for deploy-flow endpoints, see deploymentRateLimit.server.ts + DEPLOYMENT_RATE_LIMIT_REFILL_INTERVAL: z.string().default("10s"), + DEPLOYMENT_RATE_LIMIT_MAX: z.coerce.number().int().default(1500), + DEPLOYMENT_RATE_LIMIT_REFILL_RATE: z.coerce.number().int().default(500), + DEPLOYMENT_RATE_LIMIT_REQUEST_LOGS_ENABLED: z.string().default("0"), + DEPLOYMENT_RATE_LIMIT_REJECTION_LOGS_ENABLED: z.string().default("1"), + DEPLOYMENT_RATE_LIMIT_LIMITER_LOGS_ENABLED: z.string().default("0"), + // Per-IP rate limit for the unauthenticated OTLP ingestion endpoints // (/otel/*). Bounds unauthenticated request rates. Opt-in // (disabled by default): because it keys on the source IP, it is only diff --git a/apps/webapp/app/services/apiRateLimit.server.ts b/apps/webapp/app/services/apiRateLimit.server.ts index 435c82f606c..2f25b498566 100644 --- a/apps/webapp/app/services/apiRateLimit.server.ts +++ b/apps/webapp/app/services/apiRateLimit.server.ts @@ -4,6 +4,7 @@ import { resolvePrivateApiKeyRateLimitScope } from "~/models/runtimeEnvironment. import { batchStreamGrants } from "~/runEngine/concerns/batchStreamGrantsInstance.server"; import { authenticateAuthorizationHeader } from "./apiAuth.server"; import { authorizationRateLimitMiddleware } from "./authorizationRateLimitMiddleware.server"; +import { deploymentApiPaths } from "./deploymentApiPaths.server"; import type { Duration } from "./rateLimiter.server"; const BATCH_STREAM_ITEMS_PATH = /^\/api\/v3\/batches\/([^/]+)\/items$/; @@ -91,7 +92,8 @@ export const apiRateLimiter = authorizationRateLimitMiddleware({ "/api/v1/auth/jwt/claims", /^\/api\/v1\/runs\/[^/]+\/attempts$/, // /api/v1/runs/$runFriendlyId/attempts /^\/api\/v1\/waitpoints\/tokens\/[^/]+\/callback\/[^/]+$/, // /api/v1/waitpoints/tokens/$waitpointFriendlyId/callback/$hash - /^\/api\/v\d+\/deployments/, // /api/v{1,2,3,n}/deployments/* + ...deploymentApiPaths, // rate limited separately by deploymentRateLimiter + /^\/api\/v\d+\/deployments\/current$/, // runtime SDK surface, exempt as before the deploy budget split // Internal SDK plumbing — packets are presigned-URL handshakes for // payload uploads (v2 PUT) and downloads (v1 GET), authenticated via // run-scoped JWT, called once per task/turn boundary by the runtime. diff --git a/apps/webapp/app/services/deploymentApiPaths.server.ts b/apps/webapp/app/services/deploymentApiPaths.server.ts new file mode 100644 index 00000000000..c044394b244 --- /dev/null +++ b/apps/webapp/app/services/deploymentApiPaths.server.ts @@ -0,0 +1,13 @@ +// Deploy-flow endpoints, rate limited by deploymentRateLimiter with a separate +// budget instead of the general per-environment buckets. +export const deploymentApiPaths: (RegExp | string)[] = [ + // /current is runtime SDK surface, kept out of the deploy budget + /^\/api\/v\d+\/deployments(?!\/current$)(\/|$)/, + /^\/api\/v1\/projects\/[^/]+\/(dev|staging|prod|preview)$/, + /^\/api\/v1\/projects\/[^/]+\/envvars$/, + /^\/api\/v1\/projects\/[^/]+\/envvars\/[^/]+\/import$/, + /^\/api\/v1\/projects\/[^/]+\/branches$/, + /^\/api\/v1\/projects\/[^/]+\/branches\/archive$/, + "/api/v1/remote-build-provider-status", + "/api/v1/artifacts", +]; diff --git a/apps/webapp/app/services/deploymentRateLimit.server.ts b/apps/webapp/app/services/deploymentRateLimit.server.ts new file mode 100644 index 00000000000..2c2fe395d7a --- /dev/null +++ b/apps/webapp/app/services/deploymentRateLimit.server.ts @@ -0,0 +1,53 @@ +import { env } from "~/env.server"; +import { resolvePrivateApiKeyRateLimitScope } from "~/models/runtimeEnvironment.server"; +import { authorizationRateLimitMiddleware } from "./authorizationRateLimitMiddleware.server"; +import { deploymentApiPaths } from "./deploymentApiPaths.server"; +import type { Duration } from "./rateLimiter.server"; + +export const deploymentRateLimiter = authorizationRateLimitMiddleware({ + redis: { + port: env.RATE_LIMIT_REDIS_PORT, + host: env.RATE_LIMIT_REDIS_HOST, + username: env.RATE_LIMIT_REDIS_USERNAME, + password: env.RATE_LIMIT_REDIS_PASSWORD, + tlsDisabled: env.RATE_LIMIT_REDIS_TLS_DISABLED === "true", + clusterMode: env.RATE_LIMIT_REDIS_CLUSTER_MODE_ENABLED === "1", + }, + keyPrefix: "deployment", + defaultLimiter: { + type: "tokenBucket", + refillRate: env.DEPLOYMENT_RATE_LIMIT_REFILL_RATE, + interval: env.DEPLOYMENT_RATE_LIMIT_REFILL_INTERVAL as Duration, + maxTokens: env.DEPLOYMENT_RATE_LIMIT_MAX, + }, + limiterCache: { + fresh: 60_000 * 10, + stale: 60_000 * 20, + maxItems: 1000, + }, + limiterConfigOverride: async (authorizationValue) => { + const rawApiKey = authorizationValue.replace(/^Bearer /, ""); + + if (!rawApiKey.startsWith("tr_")) { + return; + } + + const scope = await resolvePrivateApiKeyRateLimitScope(rawApiKey); + + if (!scope) { + return; + } + + // Identifier only: the org's apiRateLimiterConfig governs the general API + // limiter, not the deploy budget. + return { + identifier: scope.environmentId, + }; + }, + pathMatchers: deploymentApiPaths, + log: { + rejections: env.DEPLOYMENT_RATE_LIMIT_REJECTION_LOGS_ENABLED === "1", + requests: env.DEPLOYMENT_RATE_LIMIT_REQUEST_LOGS_ENABLED === "1", + limiter: env.DEPLOYMENT_RATE_LIMIT_LIMITER_LOGS_ENABLED === "1", + }, +}); diff --git a/apps/webapp/server.ts b/apps/webapp/server.ts index a5dbfb06b2a..3e8a8cb287d 100644 --- a/apps/webapp/server.ts +++ b/apps/webapp/server.ts @@ -183,6 +183,7 @@ async function startServer() { const socketIo: { io: IoServer } | undefined = build.entry.module.socketIo; const wss: WebSocketServer | undefined = build.entry.module.wss; const apiRateLimiter: RateLimitMiddleware = build.entry.module.apiRateLimiter; + const deploymentRateLimiter: RateLimitMiddleware = build.entry.module.deploymentRateLimiter; const engineRateLimiter: RateLimitMiddleware = build.entry.module.engineRateLimiter; const otlpRateLimiter: RequestHandler = build.entry.module.otlpRateLimiter; const runWithHttpContext: RunWithHttpContextFunction = build.entry.module.runWithHttpContext; @@ -235,6 +236,7 @@ async function startServer() { } app.use(apiRateLimiter); + app.use(deploymentRateLimiter); app.use(engineRateLimiter); app.use(otlpRateLimiter); diff --git a/apps/webapp/test/deploymentApiPaths.test.ts b/apps/webapp/test/deploymentApiPaths.test.ts new file mode 100644 index 00000000000..8e1fc84b17d --- /dev/null +++ b/apps/webapp/test/deploymentApiPaths.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from "vitest"; +import { deploymentApiPaths } from "../app/services/deploymentApiPaths.server.js"; + +// Same matching semantics as authorizationRateLimitMiddleware's pathMatchers/pathWhiteList +function matchesAnyPath(path: string, matchers: (RegExp | string)[]): boolean { + return matchers.some((matcher) => + matcher instanceof RegExp ? matcher.test(path) : path === matcher + ); +} + +describe("deploymentApiPaths", () => { + it("matches every endpoint the deploy flow calls", () => { + const deployFlowPaths = [ + "/api/v1/deployments", + "/api/v1/deployments/latest", + "/api/v1/deployments/deployment_123", + "/api/v1/deployments/deployment_123/progress", + "/api/v1/deployments/deployment_123/fail", + "/api/v1/deployments/deployment_123/cancel", + "/api/v1/deployments/deployment_123/background-workers", + "/api/v1/deployments/deployment_123/generate-registry-credentials", + "/api/v1/deployments/20260811.1/promote", + "/api/v3/deployments/deployment_123/finalize", + "/api/v1/projects/proj_abc123/dev", + "/api/v1/projects/proj_abc123/staging", + "/api/v1/projects/proj_abc123/prod", + "/api/v1/projects/proj_abc123/preview", + "/api/v1/projects/proj_abc123/envvars", + "/api/v1/projects/proj_abc123/envvars/prod/import", + "/api/v1/projects/proj_abc123/branches", + "/api/v1/projects/proj_abc123/branches/archive", + "/api/v1/remote-build-provider-status", + "/api/v1/artifacts", + ]; + + for (const path of deployFlowPaths) { + expect( + matchesAnyPath(path, deploymentApiPaths), + `expected ${path} to be a deployment API path` + ).toBe(true); + } + }); + + it("does not match runtime API surface", () => { + const runtimePaths = [ + "/api/v1/deployments/current", + "/api/v1/deploymentsfoo", + "/api/v1/whoami", + "/api/v2/whoami", + "/api/v1/tasks/my-task/trigger", + "/api/v1/tasks/batch", + "/api/v2/runs/run_123", + "/api/v1/runs/run_123/replay", + "/api/v3/runs/run_123/trace", + "/api/v1/projects", + "/api/v1/projects/proj_abc123", + "/api/v1/projects/proj_abc123/dev-status", + "/api/v1/projects/proj_abc123/prod/jwt", + "/api/v1/projects/proj_abc123/envvars/prod", + "/api/v1/projects/proj_abc123/envvars/prod/MY_VAR", + "/api/v1/schedules", + "/api/v1/queues/queue_123", + ]; + + for (const path of runtimePaths) { + expect( + matchesAnyPath(path, deploymentApiPaths), + `expected ${path} not to be a deployment API path` + ).toBe(false); + } + }); +}); diff --git a/docs/self-hosting/env/webapp.mdx b/docs/self-hosting/env/webapp.mdx index f8820886e49..191db34c757 100644 --- a/docs/self-hosting/env/webapp.mdx +++ b/docs/self-hosting/env/webapp.mdx @@ -76,6 +76,12 @@ mode: "wide" | `API_RATE_LIMIT_LIMITER_LOGS_ENABLED` | No | 0 | API rate limit limiter logs. | | `API_RATE_LIMIT_JWT_WINDOW` | No | 1m | API rate limit JWT window. | | `API_RATE_LIMIT_JWT_TOKENS` | No | 60 | API rate limit JWT tokens. | +| `DEPLOYMENT_RATE_LIMIT_REFILL_INTERVAL` | No | 10s | Deployment endpoints rate limit refill interval. | +| `DEPLOYMENT_RATE_LIMIT_MAX` | No | 1500 | Deployment endpoints rate limit max. | +| `DEPLOYMENT_RATE_LIMIT_REFILL_RATE` | No | 500 | Deployment endpoints rate limit refill rate. | +| `DEPLOYMENT_RATE_LIMIT_REQUEST_LOGS_ENABLED` | No | 0 | Deployment endpoints rate limit request logs. | +| `DEPLOYMENT_RATE_LIMIT_REJECTION_LOGS_ENABLED` | No | 1 | Deployment endpoints rate limit rejection logs. | +| `DEPLOYMENT_RATE_LIMIT_LIMITER_LOGS_ENABLED` | No | 0 | Deployment endpoints rate limit limiter logs. | | **Deploy & Registry** | | | | | `DEPLOY_REGISTRY_HOST` | Yes | — | Deploy registry host. | | `DEPLOY_REGISTRY_USERNAME` | No | — | Deploy registry username. |