feat(api): separate rate limit budget for deployment endpoints

.server-changes/deployment-rate-limit-budget.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: improvement
+---
+
+Deployment-related API endpoints now draw from their own generous rate limit budget, configurable via the `DEPLOYMENT_RATE_LIMIT_*` environment variables, so runtime API traffic no longer competes with deployments for the same per-environment budget.

apps/webapp/app/entry.server.tsx

@@ -302,6 +302,7 @@ singleton("SentryTenantContextProcessor", () => {
 });
 
 export { apiRateLimiter } from "./services/apiRateLimit.server";
+export { deploymentRateLimiter } from "./services/deploymentRateLimit.server";
 export { engineRateLimiter } from "./services/engineRateLimit.server";
 export { otlpRateLimiter } from "./services/otlpRateLimit.server";
 export { runWithHttpContext } from "./services/httpAsyncStorage.server";

apps/webapp/app/env.server.ts

@@ -614,6 +614,14 @@ const EnvironmentSchema = z
     API_RATE_LIMIT_JWT_WINDOW: z.string().default("1m"),
     API_RATE_LIMIT_JWT_TOKENS: z.coerce.number().int().default(60),
 
+    // Separate budget for deploy-flow endpoints, see deploymentRateLimit.server.ts
+    DEPLOYMENT_RATE_LIMIT_REFILL_INTERVAL: z.string().default("10s"),
+    DEPLOYMENT_RATE_LIMIT_MAX: z.coerce.number().int().default(1500),
+    DEPLOYMENT_RATE_LIMIT_REFILL_RATE: z.coerce.number().int().default(500),
+    DEPLOYMENT_RATE_LIMIT_REQUEST_LOGS_ENABLED: z.string().default("0"),
+    DEPLOYMENT_RATE_LIMIT_REJECTION_LOGS_ENABLED: z.string().default("1"),
+    DEPLOYMENT_RATE_LIMIT_LIMITER_LOGS_ENABLED: z.string().default("0"),
+
     // Per-IP rate limit for the unauthenticated OTLP ingestion endpoints
     // (/otel/*). Bounds unauthenticated request rates. Opt-in
     // (disabled by default): because it keys on the source IP, it is only

apps/webapp/app/services/apiRateLimit.server.ts

@@ -4,6 +4,7 @@ import { resolvePrivateApiKeyRateLimitScope } from "~/models/runtimeEnvironment.
 import { batchStreamGrants } from "~/runEngine/concerns/batchStreamGrantsInstance.server";
 import { authenticateAuthorizationHeader } from "./apiAuth.server";
 import { authorizationRateLimitMiddleware } from "./authorizationRateLimitMiddleware.server";
+import { deploymentApiPaths } from "./deploymentApiPaths.server";
 import type { Duration } from "./rateLimiter.server";
 
 const BATCH_STREAM_ITEMS_PATH = /^\/api\/v3\/batches\/([^/]+)\/items$/;
@@ -91,7 +92,8 @@ export const apiRateLimiter = authorizationRateLimitMiddleware({
     "/api/v1/auth/jwt/claims",
     /^\/api\/v1\/runs\/[^/]+\/attempts$/, // /api/v1/runs/$runFriendlyId/attempts
     /^\/api\/v1\/waitpoints\/tokens\/[^/]+\/callback\/[^/]+$/, // /api/v1/waitpoints/tokens/$waitpointFriendlyId/callback/$hash
-    /^\/api\/v\d+\/deployments/, // /api/v{1,2,3,n}/deployments/*
+    ...deploymentApiPaths, // rate limited separately by deploymentRateLimiter
+    /^\/api\/v\d+\/deployments\/current$/, // runtime SDK surface, exempt as before the deploy budget split
     // Internal SDK plumbing — packets are presigned-URL handshakes for
     // payload uploads (v2 PUT) and downloads (v1 GET), authenticated via
     // run-scoped JWT, called once per task/turn boundary by the runtime.

apps/webapp/app/services/deploymentApiPaths.server.ts

@@ -0,0 +1,13 @@
+// Deploy-flow endpoints, rate limited by deploymentRateLimiter with a separate
+// budget instead of the general per-environment buckets.
+export const deploymentApiPaths: (RegExp | string)[] = [
+  // /current is runtime SDK surface, kept out of the deploy budget
+  /^\/api\/v\d+\/deployments(?!\/current$)(\/|$)/,
+  /^\/api\/v1\/projects\/[^/]+\/(dev|staging|prod|preview)$/,
+  /^\/api\/v1\/projects\/[^/]+\/envvars$/,
+  /^\/api\/v1\/projects\/[^/]+\/envvars\/[^/]+\/import$/,
+  /^\/api\/v1\/projects\/[^/]+\/branches$/,
+  /^\/api\/v1\/projects\/[^/]+\/branches\/archive$/,
+  "/api/v1/remote-build-provider-status",
+  "/api/v1/artifacts",
+];

apps/webapp/app/services/deploymentRateLimit.server.ts

@@ -0,0 +1,53 @@
+import { env } from "~/env.server";
+import { resolvePrivateApiKeyRateLimitScope } from "~/models/runtimeEnvironment.server";
+import { authorizationRateLimitMiddleware } from "./authorizationRateLimitMiddleware.server";
+import { deploymentApiPaths } from "./deploymentApiPaths.server";
+import type { Duration } from "./rateLimiter.server";
+
+export const deploymentRateLimiter = authorizationRateLimitMiddleware({
+  redis: {
+    port: env.RATE_LIMIT_REDIS_PORT,
+    host: env.RATE_LIMIT_REDIS_HOST,
+    username: env.RATE_LIMIT_REDIS_USERNAME,
+    password: env.RATE_LIMIT_REDIS_PASSWORD,
+    tlsDisabled: env.RATE_LIMIT_REDIS_TLS_DISABLED === "true",
+    clusterMode: env.RATE_LIMIT_REDIS_CLUSTER_MODE_ENABLED === "1",
+  },
+  keyPrefix: "deployment",
+  defaultLimiter: {
+    type: "tokenBucket",
+    refillRate: env.DEPLOYMENT_RATE_LIMIT_REFILL_RATE,
+    interval: env.DEPLOYMENT_RATE_LIMIT_REFILL_INTERVAL as Duration,
+    maxTokens: env.DEPLOYMENT_RATE_LIMIT_MAX,
+  },
+  limiterCache: {
+    fresh: 60_000 * 10,
+    stale: 60_000 * 20,
+    maxItems: 1000,
+  },
+  limiterConfigOverride: async (authorizationValue) => {
+    const rawApiKey = authorizationValue.replace(/^Bearer /, "");
+
+    if (!rawApiKey.startsWith("tr_")) {
+      return;
+    }
+
+    const scope = await resolvePrivateApiKeyRateLimitScope(rawApiKey);
+
+    if (!scope) {
+      return;
+    }
+
+    // Identifier only: the org's apiRateLimiterConfig governs the general API
+    // limiter, not the deploy budget.
+    return {
+      identifier: scope.environmentId,
+    };
+  },
+  pathMatchers: deploymentApiPaths,
+  log: {
+    rejections: env.DEPLOYMENT_RATE_LIMIT_REJECTION_LOGS_ENABLED === "1",
+    requests: env.DEPLOYMENT_RATE_LIMIT_REQUEST_LOGS_ENABLED === "1",
+    limiter: env.DEPLOYMENT_RATE_LIMIT_LIMITER_LOGS_ENABLED === "1",
+  },
+});

apps/webapp/server.ts

@@ -183,6 +183,7 @@ async function startServer() {
     const socketIo: { io: IoServer } | undefined = build.entry.module.socketIo;
     const wss: WebSocketServer | undefined = build.entry.module.wss;
     const apiRateLimiter: RateLimitMiddleware = build.entry.module.apiRateLimiter;
+    const deploymentRateLimiter: RateLimitMiddleware = build.entry.module.deploymentRateLimiter;
     const engineRateLimiter: RateLimitMiddleware = build.entry.module.engineRateLimiter;
     const otlpRateLimiter: RequestHandler = build.entry.module.otlpRateLimiter;
     const runWithHttpContext: RunWithHttpContextFunction = build.entry.module.runWithHttpContext;
@@ -235,6 +236,7 @@ async function startServer() {
       }
 
       app.use(apiRateLimiter);
+      app.use(deploymentRateLimiter);
       app.use(engineRateLimiter);
       app.use(otlpRateLimiter);
 

apps/webapp/test/deploymentApiPaths.test.ts

@@ -0,0 +1,72 @@
+import { describe, expect, it } from "vitest";
+import { deploymentApiPaths } from "../app/services/deploymentApiPaths.server.js";
+
+// Same matching semantics as authorizationRateLimitMiddleware's pathMatchers/pathWhiteList
+function matchesAnyPath(path: string, matchers: (RegExp | string)[]): boolean {
+  return matchers.some((matcher) =>
+    matcher instanceof RegExp ? matcher.test(path) : path === matcher
+  );
+}
+
+describe("deploymentApiPaths", () => {
+  it("matches every endpoint the deploy flow calls", () => {
+    const deployFlowPaths = [
+      "/api/v1/deployments",
+      "/api/v1/deployments/latest",
+      "/api/v1/deployments/deployment_123",
+      "/api/v1/deployments/deployment_123/progress",
+      "/api/v1/deployments/deployment_123/fail",
+      "/api/v1/deployments/deployment_123/cancel",
+      "/api/v1/deployments/deployment_123/background-workers",
+      "/api/v1/deployments/deployment_123/generate-registry-credentials",
+      "/api/v1/deployments/20260811.1/promote",
+      "/api/v3/deployments/deployment_123/finalize",
+      "/api/v1/projects/proj_abc123/dev",
+      "/api/v1/projects/proj_abc123/staging",
+      "/api/v1/projects/proj_abc123/prod",
+      "/api/v1/projects/proj_abc123/preview",
+      "/api/v1/projects/proj_abc123/envvars",
+      "/api/v1/projects/proj_abc123/envvars/prod/import",
+      "/api/v1/projects/proj_abc123/branches",
+      "/api/v1/projects/proj_abc123/branches/archive",
+      "/api/v1/remote-build-provider-status",
+      "/api/v1/artifacts",
+    ];
+
+    for (const path of deployFlowPaths) {
+      expect(
+        matchesAnyPath(path, deploymentApiPaths),
+        `expected ${path} to be a deployment API path`
+      ).toBe(true);
+    }
+  });
+
+  it("does not match runtime API surface", () => {
+    const runtimePaths = [
+      "/api/v1/deployments/current",
+      "/api/v1/deploymentsfoo",
+      "/api/v1/whoami",
+      "/api/v2/whoami",
+      "/api/v1/tasks/my-task/trigger",
+      "/api/v1/tasks/batch",
+      "/api/v2/runs/run_123",
+      "/api/v1/runs/run_123/replay",
+      "/api/v3/runs/run_123/trace",
+      "/api/v1/projects",
+      "/api/v1/projects/proj_abc123",
+      "/api/v1/projects/proj_abc123/dev-status",
+      "/api/v1/projects/proj_abc123/prod/jwt",
+      "/api/v1/projects/proj_abc123/envvars/prod",
+      "/api/v1/projects/proj_abc123/envvars/prod/MY_VAR",
+      "/api/v1/schedules",
+      "/api/v1/queues/queue_123",
+    ];
+
+    for (const path of runtimePaths) {
+      expect(
+        matchesAnyPath(path, deploymentApiPaths),
+        `expected ${path} not to be a deployment API path`
+      ).toBe(false);
+    }
+  });
+});

docs/self-hosting/env/webapp.mdx

@@ -76,6 +76,12 @@ mode: "wide"
 | `API_RATE_LIMIT_LIMITER_LOGS_ENABLED`            | No       | 0                     | API rate limit limiter logs.                                                                                       |
 | `API_RATE_LIMIT_JWT_WINDOW`                      | No       | 1m                    | API rate limit JWT window.                                                                                         |
 | `API_RATE_LIMIT_JWT_TOKENS`                      | No       | 60                    | API rate limit JWT tokens.                                                                                         |
+| `DEPLOYMENT_RATE_LIMIT_REFILL_INTERVAL`          | No       | 10s                   | Deployment endpoints rate limit refill interval.                                                                   |
+| `DEPLOYMENT_RATE_LIMIT_MAX`                      | No       | 1500                  | Deployment endpoints rate limit max.                                                                               |
+| `DEPLOYMENT_RATE_LIMIT_REFILL_RATE`              | No       | 500                   | Deployment endpoints rate limit refill rate.                                                                       |
+| `DEPLOYMENT_RATE_LIMIT_REQUEST_LOGS_ENABLED`     | No       | 0                     | Deployment endpoints rate limit request logs.                                                                      |
+| `DEPLOYMENT_RATE_LIMIT_REJECTION_LOGS_ENABLED`   | No       | 1                     | Deployment endpoints rate limit rejection logs.                                                                    |
+| `DEPLOYMENT_RATE_LIMIT_LIMITER_LOGS_ENABLED`     | No       | 0                     | Deployment endpoints rate limit limiter logs.                                                                      |
 | **Deploy & Registry**                            |          |                       |                                                                                                                    |
 | `DEPLOY_REGISTRY_HOST`                           | Yes      | —                     | Deploy registry host.                                                                                              |
 | `DEPLOY_REGISTRY_USERNAME`                       | No       | —                     | Deploy registry username.                                                                                          |