From d1de4e70ad4bf1f359102a63951c2bcecefd54b2 Mon Sep 17 00:00:00 2001 From: Jonathan Baldie Date: Fri, 25 Sep 2026 10:48:41 +0100 Subject: [PATCH] refactor: extract payload ingestion and validation module from HTTP handler (#139) --- mutation/mutasaurus_ci.ts | 1 + scripts/messcript.mjs | 1 + src/handler.ts | 173 +++--------------------- src/payload.ts | 187 ++++++++++++++++++++++++++ tests/payload_test.ts | 267 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 473 insertions(+), 156 deletions(-) create mode 100644 src/payload.ts create mode 100644 tests/payload_test.ts diff --git a/mutation/mutasaurus_ci.ts b/mutation/mutasaurus_ci.ts index f7fff3a..0acdb60 100644 --- a/mutation/mutasaurus_ci.ts +++ b/mutation/mutasaurus_ci.ts @@ -42,6 +42,7 @@ const mutasaurus = new Mutasaurus({ "./tests/e2e_test.ts", "./tests/handler_test.ts", "./tests/manager_test.ts", + "./tests/payload_test.ts", "./tests/persist_test.ts", "./tests/rate_limiter_test.ts", "./tests/router_test.ts", diff --git a/scripts/messcript.mjs b/scripts/messcript.mjs index 12c01f6..9796ba4 100644 --- a/scripts/messcript.mjs +++ b/scripts/messcript.mjs @@ -18,6 +18,7 @@ const productionUnits = new Map([ ["rate-limiter", ["src/rate_limiter.ts"]], ["queue-manager", ["src/manager.ts"]], ["persist-engine", ["src/persist.ts"]], + ["payload", ["src/payload.ts"]], ["http-handler", ["src/handler.ts"]], ["entrypoint", ["main.ts"]], ]); diff --git a/src/handler.ts b/src/handler.ts index 11d849a..95a76e8 100644 --- a/src/handler.ts +++ b/src/handler.ts @@ -2,149 +2,10 @@ import QueueManager, { QueueNameTooLongError } from "./manager.ts"; import { RateLimiter } from "./rate_limiter.ts"; import { withAuth, withRateLimit } from "./middleware.ts"; import { RouteHandler, Router } from "./router.ts"; +import * as Payload from "./payload.ts"; -const MAX_BODY_SIZE = 1024 * 1024; // 1 MB const LOG_ENCODER = Reflect.construct(TextEncoder, []); -class UnsupportedNumberError extends Error { - constructor() { - super("Payload contains an unsupported number"); - } -} - -class JsonNestingTooDeepError extends Error {} - -function canonicalJsonNumber(source: string): string { - const match = /^(-?)(\d+)(?:\.(\d+))?(?:[eE]([+-]?\d+))?$/.exec(source); - if (match === null) { - return source; - } - - const sign = match[1] === "-" ? "-" : ""; - let digits = `${match[2]}${match[3] ?? ""}`.replace(/^0+/, ""); - if (digits === "") { - return "0"; - } - - let exponent = Number(match[4] ?? "0") - (match[3]?.length ?? 0); - const digitsWithoutTrailingZeros = digits.replace(/0+$/, ""); - exponent += digits.length - digitsWithoutTrailingZeros.length; - digits = digitsWithoutTrailingZeros; - return `${sign}${digits}e${exponent}`; -} - -function isUnsupportedNumber(value: number, source: string): boolean { - if (!Number.isFinite(value)) { - return true; - } - - const serializedValue = JSON.stringify(value)!; - return canonicalJsonNumber(source) !== canonicalJsonNumber(serializedValue); -} - -// Matches strings, containers, and number literals in valid JSON. Scanning -// the source avoids invoking a JSON.parse reviver once per JSON value. -const JSON_TOKEN = /"[^"\\]*(?:\\.[^"\\]*)*"|[[{]|[\]}]|-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?/g; -const EXACT_INTEGER = /^-?\d{1,15}$/; -const MAX_JSON_DEPTH = 3000; - -function rejectUnsupportedNumber(source: string): void { - // Every integer with at most 15 digits is below Number.MAX_SAFE_INTEGER. - if (EXACT_INTEGER.test(source)) { - return; - } - if (isUnsupportedNumber(Number(source), source)) { - throw new UnsupportedNumberError(); - } -} - -function validateJsonSource(source: string): void { - let depth = 0; - for (const match of source.matchAll(JSON_TOKEN)) { - const token = match[0]; - if (token === "[" || token === "{") { - depth++; - if (depth > MAX_JSON_DEPTH) { - throw new JsonNestingTooDeepError(); - } - } else if (token === "]" || token === "}") { - depth--; - } else if (token[0] !== '"') { - rejectUnsupportedNumber(token); - } - } -} - -function parseJsonBody(body: string) { - try { - const json = JSON.parse(body); - validateJsonSource(body); - return json; - } catch (error) { - // Deeply nested JSON can overflow the native parser's stack; treat it - // as unparseable input rather than a server fault. - if (error instanceof RangeError) { - throw new JsonNestingTooDeepError(); - } - throw error; - } -} - -async function readRequestBody(request: Request): Promise { - const contentLength = request.headers.get("content-length"); - if (contentLength && parseInt(contentLength) > MAX_BODY_SIZE) { - return new Response("Payload too large", { status: 413 }); - } - - if (request.body === null) { - return ""; - } - - const reader = request.body.getReader(); - const chunks: Uint8Array[] = []; - let bodySize = 0; - try { - while (true) { - const { done, value } = await reader.read(); - if (done) { - break; - } - bodySize += value.byteLength; - if (bodySize > MAX_BODY_SIZE) { - await reader.cancel(); - return new Response("Payload too large", { status: 413 }); - } - chunks.push(value); - } - } catch { - try { - await reader.cancel(); - } catch (error) { - // The stream may already be closed or errored. - void error; - } - return new Response("Payload too large", { status: 413 }); - } finally { - reader.releaseLock(); - } - - const body = new Uint8Array(bodySize); - let offset = 0; - for (const chunk of chunks) { - body.set(chunk, offset); - offset += chunk.byteLength; - } - return decodeUtf8Body(body); -} - -function decodeUtf8Body(body: Uint8Array): string | Response { - try { - return new TextDecoder("utf-8", { fatal: true }).decode(body); - } catch { - return new Response("Invalid JSON", { status: 400 }); - } -} - function extractQueueName(match: Parameters[1]): { name: string } | { error: Response } { const raw = match.pathname.groups.queue; if (raw === undefined) { @@ -167,25 +28,19 @@ function enqueueHandler(mgr: QueueManager): RouteHandler { return queueResult.error; } const queueName = queueResult.name; - const body = await readRequestBody(request); - if (body instanceof Response) { - return body; - } try { - const json = parseJsonBody(body); - if (json === null || typeof json !== "object") { - return new Response("Missing payload key", { status: 400 }); - } - if (!("payload" in json)) { - return new Response("Missing payload key", { status: 400 }); - } - if (json.payload === null) { - return new Response("Null payload not allowed", { status: 400 }); + const contentLength = request.headers.get("content-length"); + if (contentLength && parseInt(contentLength) > Payload.DEFAULT_MAX_PAYLOAD_SIZE) { + return new Response("Payload too large", { status: 413 }); } + const payload = await Payload.readAndValidatePayload( + request.body, + Payload.DEFAULT_MAX_PAYLOAD_SIZE, + ); if (!mgr.canEnqueue(queueName)) { return new Response("Queue full or too many queues", { status: 507 }); } - mgr.enqueue(queueName, json.payload); + mgr.enqueue(queueName, payload); return new Response(`Payload successfully queued onto ${queueName}.`); } catch (error) { return enqueueErrorResponse(error); @@ -194,10 +49,16 @@ function enqueueHandler(mgr: QueueManager): RouteHandler { } function enqueueErrorResponse(error: unknown): Response { - if (error instanceof SyntaxError || error instanceof JsonNestingTooDeepError) { + if (error instanceof Payload.PayloadTooLargeError) { + return new Response(error.message, { status: 413 }); + } + if (error instanceof Payload.UnsupportedNumberError) { + return new Response(error.message, { status: 400 }); + } + if (error instanceof Payload.JsonNestingTooDeepError) { return new Response("Invalid JSON", { status: 400 }); } - if (error instanceof UnsupportedNumberError) { + if (error instanceof Payload.InvalidPayloadError) { return new Response(error.message, { status: 400 }); } return queueNameErrorResponse(error); diff --git a/src/payload.ts b/src/payload.ts new file mode 100644 index 0000000..b3033b4 --- /dev/null +++ b/src/payload.ts @@ -0,0 +1,187 @@ +export class InvalidPayloadError extends Error { + constructor(message: string = "Invalid JSON") { + super(message); + this.name = "InvalidPayloadError"; + } +} + +export class PayloadTooLargeError extends Error { + constructor(message: string = "Payload too large") { + super(message); + this.name = "PayloadTooLargeError"; + } +} + +export const DEFAULT_MAX_PAYLOAD_SIZE = 1024 * 1024; // 1 MB + +export class JsonNestingTooDeepError extends Error { + constructor(message: string = "Invalid JSON") { + super(message); + this.name = "JsonNestingTooDeepError"; + } +} + +export class UnsupportedNumberError extends Error { + constructor(message: string = "Payload contains an unsupported number") { + super(message); + this.name = "UnsupportedNumberError"; + } +} + +export const MAX_JSON_DEPTH = 3000; + +function canonicalJsonNumber(source: string): string { + const match = /^(-?)(\d+)(?:\.(\d+))?(?:[eE]([+-]?\d+))?$/.exec(source); + if (match === null) { + return source; + } + + const sign = match[1] === "-" ? "-" : ""; + let digits = `${match[2]}${match[3] ?? ""}`.replace(/^0+/, ""); + if (digits === "") { + return "0"; + } + + let exponent = Number(match[4] ?? "0") - (match[3]?.length ?? 0); + const digitsWithoutTrailingZeros = digits.replace(/0+$/, ""); + exponent += digits.length - digitsWithoutTrailingZeros.length; + digits = digitsWithoutTrailingZeros; + return `${sign}${digits}e${exponent}`; +} + +function isUnsupportedNumber(value: number, source: string): boolean { + if (!Number.isFinite(value)) { + return true; + } + + const serializedValue = JSON.stringify(value)!; + return canonicalJsonNumber(source) !== canonicalJsonNumber(serializedValue); +} + +// Matches strings, containers, and number literals in valid JSON. Scanning +// the source avoids invoking a JSON.parse reviver once per JSON value. +const JSON_TOKEN = /"[^"\\]*(?:\\.[^"\\]*)*"|[[{]|[\]}]|-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?/g; +const EXACT_INTEGER = /^-?\d{1,15}$/; + +function rejectUnsupportedNumber(source: string): void { + // Every integer with at most 15 digits is below Number.MAX_SAFE_INTEGER. + if (EXACT_INTEGER.test(source)) { + return; + } + if (isUnsupportedNumber(Number(source), source)) { + throw new UnsupportedNumberError(); + } +} + +function validateJsonSource(source: string): void { + let depth = 0; + for (const match of source.matchAll(JSON_TOKEN)) { + const token = match[0]; + if (token === "[" || token === "{") { + depth++; + if (depth > MAX_JSON_DEPTH) { + throw new JsonNestingTooDeepError(); + } + } else if (token === "]" || token === "}") { + depth--; + } else if (token[0] !== '"') { + rejectUnsupportedNumber(token); + } + } +} + +function decodePayloadBody(body: string | Uint8Array): string { + if (typeof body === "string") { + return body; + } + try { + return new TextDecoder("utf-8", { fatal: true }).decode(body); + } catch { + throw new InvalidPayloadError("Invalid JSON"); + } +} + +function parseAndValidateJson(text: string): unknown { + try { + const json = JSON.parse(text); + validateJsonSource(text); + return json; + } catch (error) { + if (error instanceof RangeError || error instanceof JsonNestingTooDeepError) { + throw new JsonNestingTooDeepError(); + } + if (error instanceof UnsupportedNumberError || error instanceof InvalidPayloadError) { + throw error; + } + throw new InvalidPayloadError("Invalid JSON"); + } +} + +function extractPayloadValue(json: unknown): T { + if (json === null || typeof json !== "object") { + throw new InvalidPayloadError("Missing payload key"); + } + if (!("payload" in json)) { + throw new InvalidPayloadError("Missing payload key"); + } + const payload = (json as Record).payload; + if (payload === null) { + throw new InvalidPayloadError("Null payload not allowed"); + } + return payload as T; +} + +export function parsePayloadBody(body: string | Uint8Array): T { + const text = decodePayloadBody(body); + const json = parseAndValidateJson(text); + return extractPayloadValue(json); +} + +export async function readAndValidatePayload( + stream: ReadableStream | null, + maxBytes: number = DEFAULT_MAX_PAYLOAD_SIZE, +): Promise { + if (stream === null) { + return parsePayloadBody(""); + } + + const reader = stream.getReader(); + const chunks: Uint8Array[] = []; + let bodySize = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + bodySize += value.byteLength; + if (bodySize > maxBytes) { + await reader.cancel(); + throw new PayloadTooLargeError(); + } + chunks.push(value); + } + } catch (error) { + if (error instanceof PayloadTooLargeError) { + throw error; + } + try { + await reader.cancel(); + } catch (cancelError) { + // The stream may already be closed or errored. + void cancelError; + } + throw new PayloadTooLargeError(); + } finally { + reader.releaseLock(); + } + + const body = new Uint8Array(bodySize); + let offset = 0; + for (const chunk of chunks) { + body.set(chunk, offset); + offset += chunk.byteLength; + } + return parsePayloadBody(body); +} + diff --git a/tests/payload_test.ts b/tests/payload_test.ts new file mode 100644 index 0000000..8bea36c --- /dev/null +++ b/tests/payload_test.ts @@ -0,0 +1,267 @@ +import { assertEquals, assertRejects, assertThrows } from "jsr:@std/assert@1.0"; +import { + DEFAULT_MAX_PAYLOAD_SIZE, + InvalidPayloadError, + JsonNestingTooDeepError, + MAX_JSON_DEPTH, + parsePayloadBody, + PayloadTooLargeError, + readAndValidatePayload, + UnsupportedNumberError, +} from "../src/payload.ts"; + +Deno.test("payload: parses valid payload from JSON string", () => { + const result = parsePayloadBody('{"payload":"hello world"}'); + assertEquals(result, "hello world"); +}); + +Deno.test("payload: rejects JSON without payload key", () => { + const error = assertThrows( + () => parsePayloadBody('{"data":"hello"}'), + InvalidPayloadError, + "Missing payload key", + ); + assertEquals(error.message, "Missing payload key"); +}); + +Deno.test("payload: rejects non-object JSON", () => { + const error = assertThrows( + () => parsePayloadBody('"just a string"'), + InvalidPayloadError, + "Missing payload key", + ); + assertEquals(error.message, "Missing payload key"); +}); + +Deno.test("payload: rejects null payload", () => { + const error = assertThrows( + () => parsePayloadBody('{"payload":null}'), + InvalidPayloadError, + "Null payload not allowed", + ); + assertEquals(error.message, "Null payload not allowed"); +}); + +Deno.test("payload: rejects malformed JSON string", () => { + const error = assertThrows( + () => parsePayloadBody('{invalid json}'), + InvalidPayloadError, + "Invalid JSON", + ); + assertEquals(error.message, "Invalid JSON"); +}); + +Deno.test("payload: parses valid UTF-8 Uint8Array payload", () => { + const bytes = new TextEncoder().encode('{"payload":"héllo 🌍"}'); + const result = parsePayloadBody(bytes); + assertEquals(result, "héllo 🌍"); +}); + +Deno.test("payload: rejects invalid UTF-8 bytes", () => { + const invalidUtf8 = Uint8Array.of( + ...new TextEncoder().encode('{"payload":"caf'), + 0xe9, + ...new TextEncoder().encode('"}'), + ); + const error = assertThrows( + () => parsePayloadBody(invalidUtf8), + InvalidPayloadError, + "Invalid JSON", + ); + assertEquals(error.message, "Invalid JSON"); +}); + +Deno.test("payload: accepts nesting within limit", () => { + const depth = 50; + const nested = `${"[".repeat(depth)}${"]".repeat(depth)}`; + const result = parsePayloadBody(`{"payload":${nested}}`); + assertEquals(Array.isArray(result), true); +}); + +Deno.test("payload: accepts nesting at exactly MAX_JSON_DEPTH", () => { + // 1 level for outer {"payload": ...} + (MAX_JSON_DEPTH - 1) levels = MAX_JSON_DEPTH + const depth = MAX_JSON_DEPTH - 1; + const nested = `${"[".repeat(depth)}${"]".repeat(depth)}`; + const result = parsePayloadBody(`{"payload":${nested}}`); + assertEquals(Array.isArray(result), true); +}); + +Deno.test("payload: rejects nesting exceeding MAX_JSON_DEPTH", () => { + // 1 level for outer {"payload": ...} + MAX_JSON_DEPTH levels = MAX_JSON_DEPTH + 1 + const depth = MAX_JSON_DEPTH; + const nested = `${"[".repeat(depth)}${"]".repeat(depth)}`; + assertThrows( + () => parsePayloadBody(`{"payload":${nested}}`), + JsonNestingTooDeepError, + ); +}); + +Deno.test("payload: rejects nesting causing parser stack overflow (RangeError)", () => { + const depth = 100_000; + const nested = `${"[".repeat(depth)}${"]".repeat(depth)}`; + assertThrows( + () => parsePayloadBody(`{"payload":${nested}}`), + JsonNestingTooDeepError, + ); +}); + +Deno.test("payload: accepts valid and safe numbers", () => { + assertEquals(parsePayloadBody('{"payload":42}'), 42); + assertEquals(parsePayloadBody('{"payload":-123456789012345}'), -123456789012345); + assertEquals(parsePayloadBody('{"payload":9007199254740992}'), 9007199254740992); + assertEquals(parsePayloadBody('{"payload":1.5}'), 1.5); + assertEquals(parsePayloadBody('{"payload":1e3}'), 1000); +}); + +Deno.test("payload: rejects non-finite number literals", () => { + assertThrows( + () => parsePayloadBody('{"payload":1e400}'), + UnsupportedNumberError, + "Payload contains an unsupported number", + ); +}); + +Deno.test("payload: rejects unsafe integers with precision loss", () => { + assertThrows( + () => parsePayloadBody('{"payload":9007199254740993}'), + UnsupportedNumberError, + "Payload contains an unsupported number", + ); +}); + +Deno.test("payload: rejects underflowing numbers", () => { + assertThrows( + () => parsePayloadBody('{"payload":1e-400}'), + UnsupportedNumberError, + "Payload contains an unsupported number", + ); +}); + +Deno.test("payload: rejects inexact decimal numbers that lose precision", () => { + assertThrows( + () => parsePayloadBody('{"payload":1.234567890123456789}'), + UnsupportedNumberError, + "Payload contains an unsupported number", + ); +}); + +Deno.test("readAndValidatePayload: rejects null stream as InvalidPayloadError", async () => { + await assertRejects( + () => readAndValidatePayload(null), + InvalidPayloadError, + "Invalid JSON", + ); +}); + +Deno.test("readAndValidatePayload: reads and parses stream within size limit", async () => { + const bytes = new TextEncoder().encode('{"payload":"streamed-data"}'); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(bytes); + controller.close(); + }, + }); + const result = await readAndValidatePayload(stream); + assertEquals(result, "streamed-data"); +}); + +Deno.test("readAndValidatePayload: reassembles payload split across multiple chunks", async () => { + const part1 = new TextEncoder().encode('{"pay'); + const part2 = new TextEncoder().encode('load":"multi-'); + const part3 = new TextEncoder().encode('chunk"}'); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(part1); + controller.enqueue(part2); + controller.enqueue(part3); + controller.close(); + }, + }); + const result = await readAndValidatePayload(stream); + assertEquals(result, "multi-chunk"); +}); + + +Deno.test("readAndValidatePayload: respects custom maxBytes limit", async () => { + const bytes = new TextEncoder().encode('{"payload":"12345"}'); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(bytes); + controller.close(); + }, + }); + // Stream size is 19 bytes. Set maxBytes to 10. + await assertRejects( + () => readAndValidatePayload(stream, 10), + PayloadTooLargeError, + "Payload too large", + ); +}); + +Deno.test("readAndValidatePayload: rejects stream when size limit is exceeded", async () => { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(50)); + controller.enqueue(new Uint8Array(50)); + }, + }); + await assertRejects( + () => readAndValidatePayload(stream, 40), + PayloadTooLargeError, + "Payload too large", + ); +}); + +Deno.test("readAndValidatePayload: throws PayloadTooLargeError when stream errors", async () => { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(10)); + controller.error(new Error("network error")); + }, + }); + await assertRejects( + () => readAndValidatePayload(stream), + PayloadTooLargeError, + "Payload too large", + ); +}); + +Deno.test("readAndValidatePayload: accepts exactly 1 MiB payload", async () => { + const emptyPayloadBody = '{"payload":""}'; + const bodyText = `{"payload":"${"x".repeat(DEFAULT_MAX_PAYLOAD_SIZE - emptyPayloadBody.length)}"}`; + const bodyBytes = new TextEncoder().encode(bodyText); + assertEquals(bodyBytes.byteLength, DEFAULT_MAX_PAYLOAD_SIZE); + + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(bodyBytes); + controller.close(); + }, + }); + const result = await readAndValidatePayload(stream); + assertEquals(typeof result, "string"); + assertEquals(result.length, DEFAULT_MAX_PAYLOAD_SIZE - emptyPayloadBody.length); +}); + +Deno.test("readAndValidatePayload: rejects exactly 1 MiB + 1 byte", async () => { + const emptyPayloadBody = '{"payload":""}'; + const bodyText = `{"payload":"${"x".repeat(DEFAULT_MAX_PAYLOAD_SIZE - emptyPayloadBody.length + 1)}"}`; + const bodyBytes = new TextEncoder().encode(bodyText); + assertEquals(bodyBytes.byteLength, DEFAULT_MAX_PAYLOAD_SIZE + 1); + + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(bodyBytes); + controller.close(); + }, + }); + await assertRejects( + () => readAndValidatePayload(stream), + PayloadTooLargeError, + "Payload too large", + ); +}); + + + +